From 5760b6a0965b6b582040e4ebe9601e0ca885b125 Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Sat, 24 Oct 2026 07:43:38 +0100 Subject: [PATCH 1/2] #158 remove the DOM from the ParseFilters ParseFilter.filter() no longer takes a DocumentFragment and needsDOM() is removed. The parser bolts do not build a W3C DOM any more: extracting data from the markup of a document is done with the JSoupFilters, which are now applied by both the JSoupParserBolt and the Tika ParserBolt. - remove the DOM-based XPathFilter and the DebugParseFilter - remove DocumentFragmentBuilder (core) and DOMBuilder / XMLCharacterRecognizer (Tika), as well as the now unused parser.uppercase.element.names option of the Tika ParserBolt - drop the xercesImpl dependency from core - update the ParseFilter implementations, tests and documentation Co-Authored-By: Claude Opus 5.5 --- THIRD-PARTY.txt | 1 - core/pom.xml | 13 - .../stormcrawler/bolt/FeedParserBolt.java | 2 +- .../stormcrawler/bolt/JSoupParserBolt.java | 9 +- .../stormcrawler/bolt/SiteMapParserBolt.java | 2 +- .../parse/DocumentFragmentBuilder.java | 103 --- .../stormcrawler/parse/ParseFilter.java | 16 +- .../stormcrawler/parse/ParseFilters.java | 30 +- .../parse/filter/CollectionTagger.java | 3 +- .../CommaSeparatedToMultivaluedMetadata.java | 3 +- .../parse/filter/DebugParseFilter.java | 79 -- .../parse/filter/DomainParseFilter.java | 3 +- .../parse/filter/MD5SignatureParseFilter.java | 3 +- .../parse/filter/MimeTypeNormalization.java | 3 +- .../parse/filter/XPathFilter.java | 223 ------ .../parse/filter/CSVMetadataFilterTest.java | 6 +- .../parse/filter/CollectionTaggerTest.java | 6 +- ...mmaSeparatedToMultivaluedMetadataTest.java | 12 +- .../parse/filter/EmptyParentParseFilter.java | 3 +- .../filter/EmptySubDocumentParseFilter.java | 3 +- .../parse/filter/SubDocumentsParseFilter.java | 59 +- .../parse/filter/XPathFilterTest.java | 112 --- .../src/test/resources/test.parsefilters.json | 11 - docs/src/main/asciidoc/architecture.adoc | 4 +- docs/src/main/asciidoc/extending.adoc | 11 +- docs/src/main/asciidoc/internals.adoc | 15 +- .../stormcrawler/parse/filter/LanguageID.java | 3 +- .../parse/filter/JSONResourceWrapper.java | 10 +- .../opensearch/DelegateRefresherTest.java | 3 +- .../parse/filter/JSONResourceWrapperTest.java | 87 --- .../parsefilter/JsRenderingDetector.java | 12 +- .../parsefilter/JsRenderingDetectorTest.java | 7 +- .../apache/stormcrawler/tika/DOMBuilder.java | 676 ------------------ .../apache/stormcrawler/tika/ParserBolt.java | 20 +- .../tika/XMLCharacterRecognizer.java | 107 --- 35 files changed, 58 insertions(+), 1602 deletions(-) delete mode 100644 core/src/main/java/org/apache/stormcrawler/parse/DocumentFragmentBuilder.java delete mode 100644 core/src/main/java/org/apache/stormcrawler/parse/filter/DebugParseFilter.java delete mode 100644 core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java delete mode 100644 core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java delete mode 100644 external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapperTest.java delete mode 100644 external/tika/src/main/java/org/apache/stormcrawler/tika/DOMBuilder.java delete mode 100644 external/tika/src/main/java/org/apache/stormcrawler/tika/XMLCharacterRecognizer.java diff --git a/THIRD-PARTY.txt b/THIRD-PARTY.txt index 1470c3a40..996324e3b 100644 --- a/THIRD-PARTY.txt +++ b/THIRD-PARTY.txt @@ -246,7 +246,6 @@ List of third-party dependencies grouped by their license type. * Storm Shaded Deps (org.apache.storm:storm-shaded-deps:3.1.0 - https://storm.apache.org/storm-shaded-deps) * swagger-annotations-jakarta (io.swagger.core.v3:swagger-annotations-jakarta:2.2.22 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations-jakarta) * urlfrontier-API (com.github.crawler-commons:urlfrontier-API:2.6 - https://github.com/crawler-commons/url-frontier/urlfrontier-API) - * Xerces2-j (xerces:xercesImpl:2.12.2 - https://xerces.apache.org/xerces2-j/) * XmlBeans (org.apache.xmlbeans:xmlbeans:5.3.0 - https://xmlbeans.apache.org/) * zstd-kmp (com.squareup.zstd:zstd-kmp-jvm:0.4.0 - https://github.com/square/okio-zstd/) * zstd-kmp-okio (com.squareup.zstd:zstd-kmp-okio-jvm:0.4.0 - https://github.com/square/okio-zstd/) diff --git a/core/pom.xml b/core/pom.xml index 92e35533b..0f9900f82 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -40,7 +40,6 @@ under the License. 1.6 1.23.2 78.3 - 2.12.2 4.5.14 2.7 3.21.0 @@ -173,18 +172,6 @@ under the License. ${crawler-commons.version} - - xerces - xercesImpl - ${xerces.version} - - - xml-apis - xml-apis - - - - com.fasterxml.jackson.core jackson-databind diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java index 1591c14d9..dbbe5c29f 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java @@ -140,7 +140,7 @@ public void execute(Tuple tuple) { // apply the parse filters if any try { - parseFilters.filter(url, content, null, parse); + parseFilters.filter(url, content, parse); } catch (RuntimeException e) { String errorMessage = "Exception while running parse filters on " + url + ": " + e; LOG.error(errorMessage, e); diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java index c128082a6..c4fefe735 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java @@ -43,7 +43,6 @@ import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.metrics.CrawlerMetrics; import org.apache.stormcrawler.metrics.ScopedCounter; -import org.apache.stormcrawler.parse.DocumentFragmentBuilder; import org.apache.stormcrawler.parse.JSoupFilter; import org.apache.stormcrawler.parse.JSoupFilters; import org.apache.stormcrawler.parse.JSoupTextExtractor; @@ -70,7 +69,6 @@ import org.jsoup.parser.Parser; import org.jsoup.select.Elements; import org.slf4j.LoggerFactory; -import org.w3c.dom.DocumentFragment; /** * Parser for HTML documents only which uses ICU4J to detect the charset encoding. Kindly donated to @@ -477,12 +475,7 @@ public void execute(Tuple tuple) { // apply the parse filters if any try { - DocumentFragment fragment = null; - // lazy building of fragment - if (parseFilters.needsDOM()) { - fragment = DocumentFragmentBuilder.fromJsoup(jsoupDoc); - } - parseFilters.filter(url, content, fragment, parse); + parseFilters.filter(url, content, parse); } catch (RuntimeException e) { String errorMessage = "Exception while running parse filters on " + url + ": " + e; handleException(url, e, metadata, tuple, "content filtering", errorMessage); diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java index bd8b15cb4..9c9b2cd3a 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java @@ -205,7 +205,7 @@ public void execute(Tuple tuple) { // apply the parse filters if any try { - parseFilters.filter(url, content, null, parse); + parseFilters.filter(url, content, parse); } catch (RuntimeException e) { String errorMessage = "Exception while running parse filters on " + url + ": " + e; LOG.error(errorMessage); diff --git a/core/src/main/java/org/apache/stormcrawler/parse/DocumentFragmentBuilder.java b/core/src/main/java/org/apache/stormcrawler/parse/DocumentFragmentBuilder.java deleted file mode 100644 index 3b6832a12..000000000 --- a/core/src/main/java/org/apache/stormcrawler/parse/DocumentFragmentBuilder.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.parse; - -import org.apache.html.dom.HTMLDocumentImpl; -import org.jetbrains.annotations.NotNull; -import org.jsoup.nodes.Attribute; -import org.jsoup.select.NodeTraversor; -import org.jsoup.select.NodeVisitor; -import org.w3c.dom.Comment; -import org.w3c.dom.DocumentFragment; -import org.w3c.dom.Element; -import org.w3c.dom.Text; - -/** Adapted from org.jsoup.helper.W3CDom but does not transfer namespaces. * */ -public final class DocumentFragmentBuilder { - - /** Restrict instantiation. */ - private DocumentFragmentBuilder() {} - - public static DocumentFragment fromJsoup(org.jsoup.nodes.Document jsoupDocument) { - HTMLDocumentImpl htmlDoc = new HTMLDocumentImpl(); - htmlDoc.setErrorChecking(false); - DocumentFragment fragment = htmlDoc.createDocumentFragment(); - org.jsoup.nodes.Element rootEl = jsoupDocument.child(0); // skip the - // #root node - NodeTraversor.traverse(new W3CBuilder(htmlDoc, fragment), rootEl); - return fragment; - } - - /** Implements the conversion by walking the input. */ - protected static class W3CBuilder implements NodeVisitor { - private final HTMLDocumentImpl doc; - private final DocumentFragment fragment; - - private Element dest; - - public W3CBuilder(HTMLDocumentImpl doc, DocumentFragment fragment) { - this.fragment = fragment; - this.doc = doc; - } - - public void head(@NotNull org.jsoup.nodes.Node source, int depth) { - if (source instanceof org.jsoup.nodes.Element) { - org.jsoup.nodes.Element sourceEl = (org.jsoup.nodes.Element) source; - Element el = doc.createElement(sourceEl.tagName()); - copyAttributes(sourceEl, el); - if (dest == null) { // sets up the root - fragment.appendChild(el); - } else { - dest.appendChild(el); - } - dest = el; // descend - } else if (source instanceof org.jsoup.nodes.TextNode) { - org.jsoup.nodes.TextNode sourceText = (org.jsoup.nodes.TextNode) source; - Text text = doc.createTextNode(sourceText.getWholeText()); - dest.appendChild(text); - } else if (source instanceof org.jsoup.nodes.Comment) { - org.jsoup.nodes.Comment sourceComment = (org.jsoup.nodes.Comment) source; - Comment comment = doc.createComment(sourceComment.getData()); - dest.appendChild(comment); - } else if (source instanceof org.jsoup.nodes.DataNode) { - org.jsoup.nodes.DataNode sourceData = (org.jsoup.nodes.DataNode) source; - Text node = doc.createTextNode(sourceData.getWholeData()); - dest.appendChild(node); - } else { - // unhandled - } - } - - public void tail(@NotNull org.jsoup.nodes.Node source, int depth) { - if (source instanceof org.jsoup.nodes.Element - && dest.getParentNode() instanceof Element) { - dest = (Element) dest.getParentNode(); // undescend. cromulent. - } - } - - private void copyAttributes(org.jsoup.nodes.Node source, Element el) { - for (Attribute attribute : source.attributes()) { - // valid xml attribute names are: ^[a-zA-Z_:][-a-zA-Z0-9_:.] - String key = attribute.getKey().replaceAll("[^-a-zA-Z0-9_:.]", ""); - if (key.matches("[a-zA-Z_:][-a-zA-Z0-9_:.]*")) { - el.setAttribute(key, attribute.getValue()); - } - } - } - } -} diff --git a/core/src/main/java/org/apache/stormcrawler/parse/ParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilter.java index ba37c61a2..f46e88527 100644 --- a/core/src/main/java/org/apache/stormcrawler/parse/ParseFilter.java +++ b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilter.java @@ -19,7 +19,6 @@ import org.apache.storm.task.IBolt; import org.apache.stormcrawler.util.AbstractConfigurable; -import org.w3c.dom.DocumentFragment; /** * Implementations of ParseFilter are responsible for extracting custom data from the crawled @@ -34,22 +33,9 @@ public abstract class ParseFilter extends AbstractConfigurable { * * @param url the URL of the page being parsed * @param content the content being parsed - * @param doc the DOM tree resulting of the parsing of the content or null if {@link - * #needsDOM()} returns false * @param parse the metadata to be updated with the resulting of the parsing */ - public abstract void filter( - String url, byte[] content, DocumentFragment doc, ParseResult parse); - - /** - * Specifies whether this filter requires a DOM representation of the document. - * - * @return trueif this needs a DOM representation of the document, false - * otherwise. - */ - public boolean needsDOM() { - return false; - } + public abstract void filter(String url, byte[] content, ParseResult parse); /** * Might be used to clean any resources associated with this {@link ParseFilter}.- See {@link diff --git a/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java index dd58778d5..465f3eec9 100644 --- a/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java +++ b/core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.InputStream; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import org.apache.commons.cli.CommandLine; @@ -40,10 +39,7 @@ import org.apache.stormcrawler.util.Configurable; import org.apache.stormcrawler.util.URLUtil; import org.jetbrains.annotations.NotNull; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; import org.slf4j.LoggerFactory; -import org.w3c.dom.DocumentFragment; /** * Wrapper for the ParseFilters defined in a JSON configuration. @@ -119,29 +115,11 @@ public void configure(@NotNull Map stormConf, @NotNull JsonNode } @Override - public boolean needsDOM() { - for (ParseFilter filter : filters) { - boolean needsDom = filter.needsDOM(); - if (needsDom) { - return true; - } - } - return false; - } - - @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { + public void filter(String url, byte[] content, ParseResult parse) { for (ParseFilter filter : filters) { long start = System.currentTimeMillis(); - if (doc == null && filter.needsDOM()) { - LOG.info( - "ParseFilter {} needs DOM but has none to work on - skip : {}", - filter.getClass().getName(), - url); - continue; - } - filter.filter(url, content, doc, parse); + filter.filter(url, content, parse); long end = System.currentTimeMillis(); LOG.debug("ParseFilter {} took {} msec", filter.getClass().getName(), end - start); } @@ -189,9 +167,7 @@ public static void main(String[] args) throws IOException, ParseException { byte[] content = IOUtils.toByteArray(URLUtil.toURL(url).openStream()); - Document doc = Jsoup.parse(new String(content, StandardCharsets.UTF_8), url); - - filters.filter(url, content, DocumentFragmentBuilder.fromJsoup(doc), parse); + filters.filter(url, content, parse); System.out.println(parse.toString()); diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/CollectionTagger.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/CollectionTagger.java index 529806311..3ece432b4 100644 --- a/core/src/main/java/org/apache/stormcrawler/parse/filter/CollectionTagger.java +++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/CollectionTagger.java @@ -36,7 +36,6 @@ import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.w3c.dom.DocumentFragment; /** * Assigns one or more tags to the metadata of a document based on its URL matching patterns defined @@ -119,7 +118,7 @@ public void loadJSONResources(InputStream inputStream) } @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { + public void filter(String url, byte[] content, ParseResult parse) { String[] tags = collections.tag(url); if (tags.length > 0) { parse.getOrCreate(url).getMetadata().setValues(key, tags); diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java index abc75f8d4..160e8fd4e 100644 --- a/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java +++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java @@ -30,7 +30,6 @@ import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.w3c.dom.DocumentFragment; /** * Rewrites single metadata containing comma separated values into multiple values for the same key, @@ -73,7 +72,7 @@ public void configure(@NotNull Map stormConf, @NotNull JsonNode } @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { + public void filter(String url, byte[] content, ParseResult parse) { Metadata m = parse.getOrCreate(url).getMetadata(); for (String key : keys) { String val = m.getFirstValue(key); diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/DebugParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/DebugParseFilter.java deleted file mode 100644 index b48047a30..000000000 --- a/core/src/main/java/org/apache/stormcrawler/parse/filter/DebugParseFilter.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.parse.filter; - -import com.fasterxml.jackson.databind.JsonNode; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.util.Map; -import org.apache.commons.io.FileUtils; -import org.apache.stormcrawler.parse.ParseFilter; -import org.apache.stormcrawler.parse.ParseResult; -import org.apache.xml.serialize.XMLSerializer; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.DocumentFragment; - -/** Dumps the DOM representation of a document into a file. */ -public class DebugParseFilter extends ParseFilter { - - private static final Logger LOG = LoggerFactory.getLogger(DebugParseFilter.class); - - private OutputStream os; - - @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { - - try { - XMLSerializer serializer = new XMLSerializer(os, null); - serializer.serialize(doc); - os.flush(); - } catch (IOException e) { - LOG.error("Exception while serializing DOM", e); - } - } - - @Override - public void configure(@NotNull Map stormConf, @NotNull JsonNode filterParams) { - try { - File outFile = Files.createTempFile("DOMDump", ".xml").toFile(); - os = FileUtils.openOutputStream(outFile); - } catch (IOException e) { - LOG.error("Exception while configuring DebugParseFilter", e); - } - } - - @Override - public boolean needsDOM() { - return true; - } - - @Override - public void cleanup() { - if (os != null) { - try { - os.close(); - } catch (IOException e) { - LOG.error("Exception while closing output stream in DebugParseFilter", e); - } - } - } -} diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java index c054c3450..36c2732c4 100644 --- a/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java +++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java @@ -26,7 +26,6 @@ import org.apache.stormcrawler.parse.ParseResult; import org.apache.stormcrawler.util.URLPartitioner; import org.jetbrains.annotations.NotNull; -import org.w3c.dom.DocumentFragment; /** Adds domain (or host) to metadata - can be used later on for indexing. */ public class DomainParseFilter extends ParseFilter { @@ -55,7 +54,7 @@ public void configure(@NotNull Map stormConf, @NotNull JsonNode } @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { + public void filter(String url, byte[] content, ParseResult parse) { Metadata metadata = parse.getOrCreate(url).getMetadata(); String value = partitioner.getPartition(url, metadata); metadata.setValue(mdKey, value); diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/MD5SignatureParseFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/MD5SignatureParseFilter.java index ce16be1cb..10d56d7b4 100644 --- a/core/src/main/java/org/apache/stormcrawler/parse/filter/MD5SignatureParseFilter.java +++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/MD5SignatureParseFilter.java @@ -27,7 +27,6 @@ import org.apache.stormcrawler.parse.ParseFilter; import org.apache.stormcrawler.parse.ParseResult; import org.jetbrains.annotations.NotNull; -import org.w3c.dom.DocumentFragment; /** * Computes a signature for a page, based on the binary content or text. If the content is empty, @@ -55,7 +54,7 @@ public class MD5SignatureParseFilter extends ParseFilter { private String copyKeyName = null; @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { + public void filter(String url, byte[] content, ParseResult parse) { ParseData parseData = parse.getOrCreate(url); Metadata metadata = parseData.getMetadata(); if (copyKeyName != null) { diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/MimeTypeNormalization.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/MimeTypeNormalization.java index c5caa0276..c5d56ae22 100644 --- a/core/src/main/java/org/apache/stormcrawler/parse/filter/MimeTypeNormalization.java +++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/MimeTypeNormalization.java @@ -22,7 +22,6 @@ import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.parse.ParseFilter; import org.apache.stormcrawler.parse.ParseResult; -import org.w3c.dom.DocumentFragment; /** * Normalises the MimeType value e.g. text/html; charset=UTF-8 => HTML application/pdf => PDF @@ -32,7 +31,7 @@ public class MimeTypeNormalization extends ParseFilter { @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { + public void filter(String url, byte[] content, ParseResult parse) { Metadata m = parse.getOrCreate(url).getMetadata(); String ct = m.getFirstValue("parse.Content-Type"); diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java deleted file mode 100644 index 9515a1f72..000000000 --- a/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.parse.filter; - -import com.fasterxml.jackson.databind.JsonNode; -import java.io.IOException; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import javax.xml.namespace.QName; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.xml.secure.SecureXPathFactory; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.parse.ParseData; -import org.apache.stormcrawler.parse.ParseFilter; -import org.apache.stormcrawler.parse.ParseResult; -import org.apache.xml.serialize.Method; -import org.apache.xml.serialize.OutputFormat; -import org.apache.xml.serialize.XMLSerializer; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.DocumentFragment; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -/** - * Simple ParseFilter to illustrate and test the interface. Reads a XPATH pattern from the config - * file and stores the value as metadata - */ -public class XPathFilter extends ParseFilter { - - private enum EvalFunction { - NONE, - STRING, - SERIALIZE; - - public QName getReturnType() { - switch (this) { - case STRING: - return XPathConstants.STRING; - default: - return XPathConstants.NODESET; - } - } - } - - private static final Logger LOG = LoggerFactory.getLogger(XPathFilter.class); - - private XPathFactory factory = SecureXPathFactory.newDefaultInstance(); - private XPath xpath = factory.newXPath(); - - protected final Map> expressions = new HashMap<>(); - - class LabelledExpression { - - String key; - private EvalFunction evalFunction; - private XPathExpression expression; - - private LabelledExpression(String key, String expression) throws XPathExpressionException { - this.key = key; - if (expression.startsWith("string(")) { - evalFunction = EvalFunction.STRING; - } else if (expression.startsWith("serialize(")) { - expression = expression.substring(10, expression.length() - 1); - evalFunction = EvalFunction.SERIALIZE; - } else { - evalFunction = EvalFunction.NONE; - } - this.expression = xpath.compile(expression); - } - - List evaluate(DocumentFragment doc) throws XPathExpressionException, IOException { - Object evalResult = expression.evaluate(doc, evalFunction.getReturnType()); - List values = new LinkedList<>(); - switch (evalFunction) { - case STRING: - if (evalResult != null) { - String strippedValue = StringUtils.strip((String) evalResult); - values.add(strippedValue); - } - break; - case SERIALIZE: - NodeList nodesToSerialize = (NodeList) evalResult; - StringWriter out = new StringWriter(); - OutputFormat format = new OutputFormat(Method.XHTML, null, false); - format.setOmitXMLDeclaration(true); - XMLSerializer serializer = new XMLSerializer(out, format); - for (int i = 0; i < nodesToSerialize.getLength(); i++) { - Node node = nodesToSerialize.item(i); - switch (node.getNodeType()) { - case Node.ELEMENT_NODE: - serializer.serialize((Element) node); - break; - case Node.DOCUMENT_NODE: - serializer.serialize((Document) node); - break; - case Node.DOCUMENT_FRAGMENT_NODE: - serializer.serialize((DocumentFragment) node); - break; - case Node.TEXT_NODE: - String text = node.getTextContent(); - if (text.length() > 0) { - values.add(text); - } - // By pass the rest of the code since it is used to - // extract - // the value out of the serialized which isn't used in - // this case - continue; - default: - throw new IllegalStateException( - "Unexpected value: " + node.getNodeType()); - } - String serializedValue = out.toString(); - if (serializedValue.length() > 0) { - values.add(serializedValue); - } - out.getBuffer().setLength(0); - } - break; - default: - NodeList nodes = (NodeList) evalResult; - for (int i = 0; i < nodes.getLength(); i++) { - Node node = nodes.item(i); - values.add(StringUtils.strip(node.getTextContent())); - } - } - return values; - } - } - - @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { - - ParseData parseData = parse.getOrCreate(url); - Metadata metadata = parseData.getMetadata(); - - // applies the XPATH expression in the order in which they are produced - for (List leList : expressions.values()) { - for (LabelledExpression le : leList) { - try { - List values = le.evaluate(doc); - if (values == null) { - continue; - } - // blank values are skipped so that the next expression can be tried - values.removeIf(StringUtils::isBlank); - if (!values.isEmpty()) { - metadata.addValues(le.key, values); - break; - } - } catch (XPathExpressionException | IOException e) { - LOG.error("Error evaluating {}: {}", le.key, e); - } - } - } - } - - @Override - public void configure(@NotNull Map stormConf, @NotNull JsonNode filterParams) { - java.util.Iterator> iter = filterParams.fields(); - while (iter.hasNext()) { - Entry entry = iter.next(); - String key = entry.getKey(); - JsonNode node = entry.getValue(); - if (node.isArray()) { - for (JsonNode expression : node) { - addExpression(key, expression); - } - } else { - addExpression(key, entry.getValue()); - } - } - } - - private void addExpression(String key, JsonNode expression) { - String xpathvalue = expression.asText(); - try { - List lexpressionList = expressions.get(key); - if (lexpressionList == null) { - lexpressionList = new ArrayList<>(); - expressions.put(key, lexpressionList); - } - LabelledExpression lexpression = new LabelledExpression(key, xpathvalue); - lexpressionList.add(lexpression); - } catch (XPathExpressionException e) { - throw new RuntimeException("Can't compile expression : " + xpathvalue, e); - } - } - - @Override - public boolean needsDOM() { - return true; - } -} diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/CSVMetadataFilterTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/CSVMetadataFilterTest.java index 3781fe406..f39552d5e 100644 --- a/core/src/test/java/org/apache/stormcrawler/parse/filter/CSVMetadataFilterTest.java +++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/CSVMetadataFilterTest.java @@ -18,7 +18,9 @@ package org.apache.stormcrawler.parse.filter; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.bolt.JSoupParserBolt; import org.apache.stormcrawler.parse.ParsingTester; @@ -36,7 +38,9 @@ void setupParserBolt() { @Test void testMultivalued() throws IOException { - prepareParserBolt("test.parsefilters.json"); + Map parserConfig = new HashMap<>(); + parserConfig.put("jsoup.filters.config.file", "test.jsoupfilters.json"); + prepareParserBolt("test.parsefilters.json", parserConfig); parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html"); Assertions.assertEquals(1, output.getEmitted().size()); List parsedTuple = output.getEmitted().get(0); diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/CollectionTaggerTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/CollectionTaggerTest.java index e3c0c8b55..e102ed153 100644 --- a/core/src/test/java/org/apache/stormcrawler/parse/filter/CollectionTaggerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/CollectionTaggerTest.java @@ -32,16 +32,16 @@ void testCollectionTagger() throws IOException { tagger.configure(new HashMap<>(), NullNode.instance); ParseResult parse = new ParseResult(); String URL = "http://stormcrawler.net/"; - tagger.filter(URL, null, null, parse); + tagger.filter(URL, null, parse); String[] collections = parse.getOrCreate(URL).getMetadata().getValues("collections"); Assertions.assertNotNull(collections); Assertions.assertEquals(2, collections.length); URL = "http://baby.com/tiny-crawler/"; - tagger.filter(URL, null, null, parse); + tagger.filter(URL, null, parse); collections = parse.getOrCreate(URL).getMetadata().getValues("collections"); Assertions.assertNull(collections); URL = "http://nutch.apache.org/"; - tagger.filter(URL, null, null, parse); + tagger.filter(URL, null, parse); collections = parse.getOrCreate(URL).getMetadata().getValues("collections"); Assertions.assertNotNull(collections); Assertions.assertEquals(1, collections.length); diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadataTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadataTest.java index 6c00393cf..7b41d8725 100644 --- a/core/src/test/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadataTest.java +++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadataTest.java @@ -69,7 +69,7 @@ void splittingUsesABulkAppend() throws Exception { // the default cap is deliberately small; the bulk write path is what is under test here newFilter("{\"keys\": [\"parse.keywords\"], \"maxTokens\": 1000}") - .filter(url, new byte[0], null, parse); + .filter(url, new byte[0], parse); Assertions.assertEquals(1000, md.getValues("parse.keywords").length); Assertions.assertTrue( @@ -94,7 +94,7 @@ void timeLargeValue() throws Exception { long start = System.nanoTime(); // the default cap is deliberately small; the bulk write path is what is under test here newFilter("{\"keys\": [\"parse.keywords\"], \"maxTokens\": 65536}") - .filter(url, new byte[0], null, parse); + .filter(url, new byte[0], parse); long msec = (System.nanoTime() - start) / 1_000_000; System.out.println("32768 tokens took " + msec + " ms"); @@ -110,7 +110,7 @@ void splittingHandlesBlankTokens() throws Exception { md.setValue("parse.keywords", "a,,b"); ParseResult parse = new ParseResult(); parse.set(url, md); - newFilter("{\"keys\": [\"parse.keywords\"]}").filter(url, new byte[0], null, parse); + newFilter("{\"keys\": [\"parse.keywords\"]}").filter(url, new byte[0], parse); Assertions.assertArrayEquals(new String[] {"a", "b"}, md.getValues("parse.keywords")); // a value made only of empty tokens leaves no entry behind @@ -118,7 +118,7 @@ void splittingHandlesBlankTokens() throws Exception { md.setValue("parse.keywords", " , ,"); parse = new ParseResult(); parse.set(url, md); - newFilter("{\"keys\": [\"parse.keywords\"]}").filter(url, new byte[0], null, parse); + newFilter("{\"keys\": [\"parse.keywords\"]}").filter(url, new byte[0], parse); Assertions.assertNull(md.getValues("parse.keywords")); } @@ -132,7 +132,7 @@ void capAppliesAfterBlankTokensAreDropped() throws Exception { ParseResult parse = new ParseResult(); parse.set(url, md); newFilter("{\"keys\": [\"parse.keywords\"], \"maxTokens\": 2}") - .filter(url, new byte[0], null, parse); + .filter(url, new byte[0], parse); Assertions.assertArrayEquals(new String[] {"a", "b"}, md.getValues("parse.keywords")); } @@ -144,7 +144,7 @@ void defaultCapTrimsLongValues() throws Exception { md.setValue("parse.keywords", commaList(200)); ParseResult parse = new ParseResult(); parse.set(url, md); - newFilter("{\"keys\": [\"parse.keywords\"]}").filter(url, new byte[0], null, parse); + newFilter("{\"keys\": [\"parse.keywords\"]}").filter(url, new byte[0], parse); Assertions.assertEquals(128, md.getValues("parse.keywords").length); } } diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/EmptyParentParseFilter.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/EmptyParentParseFilter.java index 760d796f4..5600a3ab9 100644 --- a/core/src/test/java/org/apache/stormcrawler/parse/filter/EmptyParentParseFilter.java +++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/EmptyParentParseFilter.java @@ -21,7 +21,6 @@ import org.apache.stormcrawler.parse.ParseData; import org.apache.stormcrawler.parse.ParseFilter; import org.apache.stormcrawler.parse.ParseResult; -import org.w3c.dom.DocumentFragment; /** * Empties the ParseData of the URL being parsed: no text, no content and no metadata. Used to check @@ -31,7 +30,7 @@ public class EmptyParentParseFilter extends ParseFilter { @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { + public void filter(String url, byte[] content, ParseResult parse) { ParseData parent = parse.getOrCreate(url); parent.setText(null); parent.setContent(new byte[0]); diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/EmptySubDocumentParseFilter.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/EmptySubDocumentParseFilter.java index e3dd11c89..4c040b505 100644 --- a/core/src/test/java/org/apache/stormcrawler/parse/filter/EmptySubDocumentParseFilter.java +++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/EmptySubDocumentParseFilter.java @@ -19,7 +19,6 @@ import org.apache.stormcrawler.parse.ParseFilter; import org.apache.stormcrawler.parse.ParseResult; -import org.w3c.dom.DocumentFragment; /** * Creates an entry in the ParseResult for a URL which was never parsed and which carries no @@ -29,7 +28,7 @@ public class EmptySubDocumentParseFilter extends ParseFilter { @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { + public void filter(String url, byte[] content, ParseResult parse) { parse.getOrCreate(url + "#never-parsed"); } } diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsParseFilter.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsParseFilter.java index 9e010edd7..bd15378fd 100644 --- a/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsParseFilter.java +++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/SubDocumentsParseFilter.java @@ -17,63 +17,28 @@ package org.apache.stormcrawler.parse.filter; +import java.nio.charset.StandardCharsets; import org.apache.stormcrawler.parse.ParseFilter; import org.apache.stormcrawler.parse.ParseResult; -import org.w3c.dom.DocumentFragment; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Element; public class SubDocumentsParseFilter extends ParseFilter { @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { - if (doc == null) { + public void filter(String url, byte[] content, ParseResult parse) { + if (content == null) { return; } - NodeList links = findElements(doc, "a"); - for (int i = 0; i < links.getLength(); i++) { - Node node = links.item(i); - if (node instanceof Element) { - String href = ((Element) node).getAttribute("href"); - if (href != null && !href.isEmpty()) { - // create a sub-document for each link; the metadata entry - // makes sure that it is not skipped as an empty document - parse.put(href, "isSubDocument", "true"); - } + String html = new String(content, StandardCharsets.UTF_8); + for (Element link : Jsoup.parse(html, url).select("a[href]")) { + String href = link.attr("href"); + if (!href.isEmpty()) { + // create a sub-document for each link; the metadata entry + // makes sure that it is not skipped as an empty document + parse.put(href, "isSubDocument", "true"); } } } - - private NodeList findElements(Node node, String tagName) { - if (node instanceof org.w3c.dom.Document) { - return ((org.w3c.dom.Document) node).getElementsByTagName(tagName); - } - Node child = node.getFirstChild(); - while (child != null) { - if (child instanceof Element) { - return ((Element) child).getElementsByTagName(tagName); - } - child = child.getNextSibling(); - } - return new EmptyNodeList(); - } - - @Override - public boolean needsDOM() { - return true; - } - - private static class EmptyNodeList implements NodeList { - @Override - public Node item(int index) { - return null; - } - - @Override - public int getLength() { - return 0; - } - } } diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java deleted file mode 100644 index 856be4e6f..000000000 --- a/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.parse.filter; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import org.apache.stormcrawler.Metadata; -import org.apache.stormcrawler.bolt.JSoupParserBolt; -import org.apache.stormcrawler.parse.DocumentFragmentBuilder; -import org.apache.stormcrawler.parse.ParseResult; -import org.apache.stormcrawler.parse.ParsingTester; -import org.jsoup.Jsoup; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class XPathFilterTest extends ParsingTester { - - @BeforeEach - void setupParserBolt() { - bolt = new JSoupParserBolt(); - setupParserBolt(bolt); - } - - @Test - void testBasicExtraction() throws IOException { - prepareParserBolt("test.parsefilters.json"); - parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html"); - Assertions.assertEquals(1, output.getEmitted().size()); - List parsedTuple = output.getEmitted().get(0); - Metadata metadata = (Metadata) parsedTuple.get(2); - Assertions.assertNotNull(metadata); - String concept = metadata.getFirstValue("concept"); - Assertions.assertNotNull(concept); - concept = metadata.getFirstValue("concept2"); - Assertions.assertNotNull(concept); - } - - @Test - // https://github.com/apache/stormcrawler/issues/219 - void testScriptExtraction() throws IOException { - prepareParserBolt("test.parsefilters.json"); - parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html"); - Assertions.assertEquals(1, output.getEmitted().size()); - List parsedTuple = output.getEmitted().get(0); - Metadata metadata = (Metadata) parsedTuple.get(2); - Assertions.assertNotNull(metadata); - String[] scripts = metadata.getValues("js"); - Assertions.assertNotNull(scripts); - // the first script is external and has no content, its blank value is skipped - Assertions.assertEquals(1, scripts.length); - Assertions.assertTrue(scripts[0].contains("_paq")); - } - - // element names are in upper case in the DOM built from the JSoup document - private static final String TITLE_PARAMS = - """ - {"title": ["//TITLE", "//META[@name='title']/@content"]} - """; - - private static Metadata filterTitle(String html) throws IOException { - XPathFilter filter = new XPathFilter(); - filter.configure(new HashMap<>(), new ObjectMapper().readTree(TITLE_PARAMS)); - String url = "https://example.com"; - ParseResult parse = new ParseResult(); - org.jsoup.nodes.Document doc = Jsoup.parse(html, url); - filter.filter( - url, - html.getBytes(StandardCharsets.UTF_8), - DocumentFragmentBuilder.fromJsoup(doc), - parse); - return parse.getOrCreate(url).getMetadata(); - } - - @Test - void testBlankValueFallsBackToNextExpression() throws IOException { - Metadata metadata = - filterTitle( - " " - + "" - + ""); - Assertions.assertArrayEquals(new String[] {"Fallback title"}, metadata.getValues("title")); - } - - @Test - void testBlankValuesAreNotStored() throws IOException { - Metadata metadata = - filterTitle( - "" - + "" - + ""); - Assertions.assertNull(metadata.getValues("title")); - } -} diff --git a/core/src/test/resources/test.parsefilters.json b/core/src/test/resources/test.parsefilters.json index ceac8bde6..4d8fd204c 100644 --- a/core/src/test/resources/test.parsefilters.json +++ b/core/src/test/resources/test.parsefilters.json @@ -1,16 +1,5 @@ { "org.apache.stormcrawler.parse.ParseFilters": [ - { - "class": "org.apache.stormcrawler.parse.filter.XPathFilter", - "name": "XPathFilter", - "params": { - "concept": "//SPAN[@class=\"concept\"]", - "concept2": "//*[@class=\"concept\"]", - "js": "//SCRIPT[@type='text/javascript']", - "keywords": "//META[@name=\"keywords\"]/@content", - "title": "//TITLE" - } - }, { "class": "org.apache.stormcrawler.parse.filter.CommaSeparatedToMultivaluedMetadata", "name": "CommaSeparatedToMultivaluedMetadata", diff --git a/docs/src/main/asciidoc/architecture.adoc b/docs/src/main/asciidoc/architecture.adoc index aed5c7d9d..ec5c9aa96 100644 --- a/docs/src/main/asciidoc/architecture.adoc +++ b/docs/src/main/asciidoc/architecture.adoc @@ -76,8 +76,8 @@ processed. Parse Filters are applied within the parsing bolts, following parsing by `SiteMapParser`, `JSoupParser`, or `ParserBolt` (Tika). They can modify extracted text, metadata, and links before the content is indexed or new URLs are emitted. -For HTML documents, `JSoupParser` also applies JSoup Filters, which work directly -on the JSoup document and do not require a DOM. +For HTML documents, `JSoupParser` and `ParserBolt` also apply JSoup Filters, which work +directly on the JSoup document. Common Parse Filters include: diff --git a/docs/src/main/asciidoc/extending.adoc b/docs/src/main/asciidoc/extending.adoc index dce33e001..8f6b29bfa 100644 --- a/docs/src/main/asciidoc/extending.adoc +++ b/docs/src/main/asciidoc/extending.adoc @@ -69,7 +69,6 @@ Parse filters extract data from fetched documents and can modify metadata and ou import org.apache.stormcrawler.parse.ParseFilter; import org.apache.stormcrawler.parse.ParseResult; import com.fasterxml.jackson.databind.JsonNode; -import org.w3c.dom.DocumentFragment; import java.util.Map; public class MyParseFilter extends ParseFilter { @@ -82,13 +81,7 @@ public class MyParseFilter extends ParseFilter { } @Override - public boolean needsDOM() { - return false; // set to true if you need the DOM tree - } - - @Override - public void filter(String url, byte[] content, DocumentFragment doc, - ParseResult parse) { + public void filter(String url, byte[] content, ParseResult parse) { // Extract data and store in metadata parse.getOrCreate(url).getMetadata().setValue(metadataKey, "extracted-value"); } @@ -112,7 +105,7 @@ Register it in `parsefilters.json`: } ---- -If `needsDOM()` returns `false` for all configured parse filters, the parsing bolt skips DOM generation entirely, improving performance. +ParseFilters do not get a parsed representation of the document. To extract data from the HTML markup, implement a xref:internals.adoc#jsoupfilters[JSoupFilter] instead, which gets the JSoup document. ==== Custom Protocol diff --git a/docs/src/main/asciidoc/internals.adoc b/docs/src/main/asciidoc/internals.adoc index 6d14ad36b..b7d94918e 100644 --- a/docs/src/main/asciidoc/internals.adoc +++ b/docs/src/main/asciidoc/internals.adoc @@ -173,19 +173,16 @@ The link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org ParseFilters are called from parsing bolts such as link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java[JSoupParserBolt] and link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java[SiteMapParserBolt] to extract data from web pages. The extracted data is stored in the Metadata object. ParseFilters can also modify the Outlinks and, in that sense, act as URLFilters. -ParseFilters need to implement the interface link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/ParseFilter.java[ParseFilter], which defines three methods: +ParseFilters need to implement the interface link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/ParseFilter.java[ParseFilter], which defines two methods: [source,java] ---- -public void filter(String URL, byte[] content, DocumentFragment doc, ParseResult parse); +public void filter(String URL, byte[] content, ParseResult parse); public void configure(Map stormConf, JsonNode filterParams); - -public boolean needsDOM(); ---- * The `filter` method is where the extraction occurs. ParseResult objects contain the outlinks extracted from the document as well as a Map of String to ParseData, where the String is the URL of a subdocument or the main document itself. ParseData objects contain Metadata, binary content, and text for the subdocuments, which is useful for indexing subdocuments independently of the main document. Use link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/ParseResult.java[ParseResult.getOrCreate(URL)] to add data for a URL and ParseResult.getIfPresent(URL) for a read-only lookup. Note that the parsing bolts skip entries which carry no content, no text and no metadata, except for the URL being parsed, so a sub-document should always set at least one of the three to be emitted. -* The `needsDOM` method indicates whether the ParseFilter instance requires the DOM structure. If no ParseFilters need it, the parsing bolt will skip generating the DOM, slightly improving performance. * The `configure` method takes a JSON object loaded by the wrapper class ParseFilters. The Storm configuration map can also be used to configure the filters, as described in xref:configuration.adoc[Configuration]. The archetype includes a default link:https://github.com/apache/stormcrawler/blob/main/archetype/src/main/resources/archetype-resources/src/main/resources/parsefilters.json[JSON configuration file] for ParseFilters. The configuration allows multiple instances of the same filter class with different parameters and supports complex parameter objects. ParseFilters are executed in the order they appear in the JSON file. @@ -212,13 +209,11 @@ The archetype includes a default link:https://github.com/apache/stormcrawler/blo ---- * **CommaSeparatedToMultivaluedMetadata** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java[CommaSeparatedToMultivaluedMetadata] rewrites single metadata values containing comma-separated entries into multiple values for the same key, useful for keyword tags. The number of values produced from a single entry is bounded by the optional `maxTokens` parameter (128 by default), so that a page cannot generate an unbounded amount of metadata. -* **DebugParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/DebugParseFilter.java[DebugParseFilter] dumps an XML representation of the DOM structure to a temporary file. * **DomainParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/DomainParseFilter.java[DomainParseFilter] stores the domain or host name in the metadata for later indexing. * **MD5SignatureParseFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/MD5SignatureParseFilter.java[MD5SignatureParseFilter] generates an MD5 signature of a document based on the binary content, text, or URL (as a last resort). It can be combined with content filtering to exclude boilerplate text. * **MimeTypeNormalization** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/MimeTypeNormalization.java[MimeTypeNormalization] converts server-reported or inferred mime-type values into human-readable values such as _pdf_, _html_, or _image_ and stores them in the metadata, useful for indexing and filtering search results. -* **XPathFilter** – link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java[XPathFilter] allows extraction of data using XPath expressions and storing them in the Metadata object. -For HTML documents parsed by the JSoupParserBolt, extracting data from JSON-LD or outlinks with XPath expressions is done with xref:jsoupfilters[JSoupFilters]. +Extracting data from the markup of a document, e.g. with XPath expressions or from JSON-LD, is done with xref:jsoupfilters[JSoupFilters]. You can also implement custom ParseFilters to extend the capabilities of the parsing pipeline. For example, you might create a filter to enrich a document's metadata with additional information, such as language detection, sentiment analysis, named entity recognition, or custom tags extracted from the content. Custom filters can also modify or remove outlinks, normalize text, or integrate external data sources, allowing you to tailor the crawler to your specific processing or indexing requirements. By implementing the ParseFilter interface and configuring the filter in the JSON file, your custom logic will be seamlessly executed within the parsing bolt. @@ -226,7 +221,7 @@ By implementing the ParseFilter interface and configuring the filter in the JSON [[jsoupfilters]] ==== JSoup Filters -JSoupFilters are called by the link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java[JSoupParserBolt] only. Unlike ParseFilters, they work directly on the document produced by link:https://jsoup.org/[JSoup] and do not require a DOM, which makes them the preferred option for extracting data from HTML documents. They are called before the xref:parsefilters[ParseFilters], which can therefore use the metadata and outlinks generated by the JSoupFilters. +JSoupFilters are called by the link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java[JSoupParserBolt] and by the Tika-based ParserBolt. Unlike ParseFilters, they get the document produced by link:https://jsoup.org/[JSoup], which makes them the option to use for extracting data from the markup of a document. They are called before the xref:parsefilters[ParseFilters], which can therefore use the metadata and outlinks generated by the JSoupFilters. JSoupFilters need to implement the interface link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilter.java[JSoupFilter]: @@ -237,7 +232,7 @@ public void filter(String url, byte[] content, org.jsoup.nodes.Document doc, Par public void configure(Map stormConf, JsonNode filterParams); ---- -The `filter` method has the same role as for the ParseFilters (see above) but gets the JSoup document instead of a DOM. The `configure` method takes the JSON object loaded by the wrapper class link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java[JSoupFilters]. +The `filter` method has the same role as for the ParseFilters (see above) but also gets the JSoup document. The `configure` method takes the JSON object loaded by the wrapper class link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/parse/JSoupFilters.java[JSoupFilters]. The JSoupFilters are defined in a JSON file whose location is set with `jsoup.filters.config.file`; no JSoupFilters are used if this is not set. The archetype includes a default link:https://github.com/apache/stormcrawler/blob/main/archetype/src/main/resources/archetype-resources/src/main/resources/jsoupfilters.json[jsoupfilters.json]: diff --git a/external/langid/src/main/java/org/apache/stormcrawler/parse/filter/LanguageID.java b/external/langid/src/main/java/org/apache/stormcrawler/parse/filter/LanguageID.java index 89cdeb435..716ac29e9 100644 --- a/external/langid/src/main/java/org/apache/stormcrawler/parse/filter/LanguageID.java +++ b/external/langid/src/main/java/org/apache/stormcrawler/parse/filter/LanguageID.java @@ -39,7 +39,6 @@ import org.apache.stormcrawler.parse.ParseResult; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; -import org.w3c.dom.DocumentFragment; /** * Language identification; the language codes gets stored in the metadata.
@@ -102,7 +101,7 @@ public void configure(@NotNull Map stormConf, @NotNull JsonNode } @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { + public void filter(String url, byte[] content, ParseResult parse) { // check whether the metadata already contains a lang value // in which case we normalise its value and use it diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java index 566ff7533..e4ee68420 100644 --- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java +++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapper.java @@ -23,7 +23,6 @@ import org.apache.stormcrawler.parse.ParseFilter; import org.apache.stormcrawler.parse.ParseResult; import org.jetbrains.annotations.NotNull; -import org.w3c.dom.DocumentFragment; /** * Wraps a ParseFilter whose resources are in a JSON file that can be stored in OpenSearch. The @@ -66,13 +65,8 @@ public void configure(@NotNull Map stormConf, @NotNull JsonNode } @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) { - refresher.getDelegate().filter(url, content, doc, parse); - } - - @Override - public boolean needsDOM() { - return refresher.getDelegate().needsDOM(); + public void filter(String url, byte[] content, ParseResult parse) { + refresher.getDelegate().filter(url, content, parse); } @Override diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java index 67bcca62b..3b781b253 100644 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java +++ b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/DelegateRefresherTest.java @@ -40,7 +40,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.junit.jupiter.api.Test; -import org.w3c.dom.DocumentFragment; class DelegateRefresherTest { @@ -84,7 +83,7 @@ public void configure(@NotNull Map stormConf, @NotNull JsonNode } @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {} + public void filter(String url, byte[] content, ParseResult parse) {} @Override public String getResourceFile() { diff --git a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapperTest.java b/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapperTest.java deleted file mode 100644 index 0b988576d..000000000 --- a/external/opensearch/src/test/java/org/apache/stormcrawler/opensearch/parse/filter/JSONResourceWrapperTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.stormcrawler.opensearch.parse.filter; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; -import org.apache.stormcrawler.JSONResource; -import org.apache.stormcrawler.parse.ParseFilter; -import org.apache.stormcrawler.parse.ParseResult; -import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.Test; -import org.w3c.dom.DocumentFragment; - -class JSONResourceWrapperTest { - - private static final ObjectMapper MAPPER = new ObjectMapper(); - - /** Minimal ParseFilter + JSONResource implementation. */ - public static class StubParseFilter extends ParseFilter implements JSONResource { - - @Override - public void configure(@NotNull Map stormConf, @NotNull JsonNode params) {} - - @Override - public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {} - - @Override - public String getResourceFile() { - return "stub.json"; - } - - @Override - public void loadJSONResources(InputStream inputStream) {} - } - - /** ParseFilter which requires a DOM. */ - public static class DOMStubParseFilter extends StubParseFilter { - @Override - public boolean needsDOM() { - return true; - } - } - - private static boolean wrapperNeedsDOM(Class delegateClass) { - ObjectNode delegate = MAPPER.createObjectNode(); - delegate.put("class", delegateClass.getName()); - delegate.set("params", MAPPER.createObjectNode()); - ObjectNode params = MAPPER.createObjectNode(); - params.set("delegate", delegate); - - JSONResourceWrapper wrapper = new JSONResourceWrapper(); - wrapper.configure(new HashMap<>(), params); - try { - return wrapper.needsDOM(); - } finally { - wrapper.cleanup(); - } - } - - @Test - void needsDOMIsDelegated() { - assertTrue(wrapperNeedsDOM(DOMStubParseFilter.class)); - assertFalse(wrapperNeedsDOM(StubParseFilter.class)); - } -} diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java index c0d1f889d..d2d6b0413 100644 --- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java +++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetector.java @@ -33,7 +33,6 @@ import org.apache.stormcrawler.util.ConfUtils; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; -import org.w3c.dom.DocumentFragment; /** * Heuristic parse filter that flags URLs whose content looks like it requires JavaScript rendering, @@ -192,16 +191,7 @@ public void configure( } @Override - public boolean needsDOM() { - return false; - } - - @Override - public void filter( - final String url, - final byte[] content, - final DocumentFragment doc, - final ParseResult parse) { + public void filter(final String url, final byte[] content, final ParseResult parse) { if (content == null || content.length == 0) { return; } diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java index 937c51328..ba3f0c2d4 100644 --- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java +++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/parsefilter/JsRenderingDetectorTest.java @@ -39,7 +39,7 @@ private JsRenderingDetector detector(final String paramJson) throws Exception { private ParseResult applyTo(final JsRenderingDetector d, final String url, final String html) { final ParseResult parse = new ParseResult(); - d.filter(url, html.getBytes(StandardCharsets.UTF_8), null, parse); + d.filter(url, html.getBytes(StandardCharsets.UTF_8), parse); return parse; } @@ -137,7 +137,7 @@ void skipsIfAlreadyFetchedByPlaywright() throws Exception { parse.getOrCreate("u") .getMetadata() .setValue(HttpProtocol.MD_KEY_END, "2026-05-04T00:00:00Z"); - d.filter("u", "
".getBytes(StandardCharsets.UTF_8), null, parse); + d.filter("u", "
".getBytes(StandardCharsets.UTF_8), parse); Assertions.assertNull(parse.getOrCreate("u").getMetadata().getFirstValue("fetch.with")); } @@ -149,7 +149,6 @@ void skipsIfAlreadyFlagged() throws Exception { d.filter( "u", "

some content

".getBytes(StandardCharsets.UTF_8), - null, parse); // not overwritten, no reason added Assertions.assertEquals( @@ -216,7 +215,7 @@ void decodesWithThePrefixedContentTypeCharset() throws Exception { final Metadata md = parse.getOrCreate("u").getMetadata(); md.setValue("protocol.Content-Type", "text/html; charset=ISO-8859-15"); final String html = "

" + message + "

"; - d.filter("u", html.getBytes(Charset.forName("ISO-8859-15")), null, parse); + d.filter("u", html.getBytes(Charset.forName("ISO-8859-15")), parse); Assertions.assertEquals("playwright", md.getFirstValue("fetch.with")); } } diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/DOMBuilder.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/DOMBuilder.java deleted file mode 100644 index a5c62588c..000000000 --- a/external/tika/src/main/java/org/apache/stormcrawler/tika/DOMBuilder.java +++ /dev/null @@ -1,676 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * XXX NUTCH COMMENT - * XXX ab@apache.org: This class is copied verbatim from Xalan-J 2.6.0 - * XXX distribution, org.apache.xml.utils.DOMBuilder, in order to - * avoid dependency on Xalan. - */ - -package org.apache.stormcrawler.tika; - -import java.util.Locale; -import java.util.Stack; -import org.w3c.dom.CDATASection; -import org.w3c.dom.Comment; -import org.w3c.dom.Document; -import org.w3c.dom.DocumentFragment; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.Text; -import org.xml.sax.Attributes; -import org.xml.sax.ContentHandler; -import org.xml.sax.Locator; -import org.xml.sax.ext.LexicalHandler; - -/** - * This class takes SAX events (in addition to some extra events that SAX doesn't handle yet) and - * adds the result to a document or document fragment. - */ -public class DOMBuilder implements ContentHandler, LexicalHandler { - - private boolean upperCaseElementNames = true; - - /** Root document */ - private Document m_doc; - - /** Current node */ - private Node m_currentNode = null; - - /** First node of document fragment or null if not a DocumentFragment */ - private DocumentFragment m_docFrag = null; - - /** Vector of element nodes */ - private Stack m_elemStack = new Stack<>(); - - /** Element recorded with this namespace will be converted to Node without a namespace */ - private String defaultNamespaceURI = null; - - /** - * DOMBuilder instance constructor... it will add the DOM nodes to the document fragment. - * - * @param doc Root document - * @param node Current node - */ - public DOMBuilder(Document doc, Node node) { - m_doc = doc; - m_currentNode = node; - } - - /** - * DOMBuilder instance constructor... it will add the DOM nodes to the document fragment. - * - * @param doc Root document - * @param docFrag Document fragment - */ - DOMBuilder(Document doc, DocumentFragment docFrag) { - m_doc = doc; - m_docFrag = docFrag; - } - - /** - * DOMBuilder instance constructor... it will add the DOM nodes to the document. - * - * @param doc Root document - */ - DOMBuilder(Document doc) { - m_doc = doc; - } - - /** - * Get the root node of the DOM being created. This is either a Document or a DocumentFragment. - * - * @return The root document or document fragment if not null - */ - Node getRootNode() { - return (null != m_docFrag) ? (Node) m_docFrag : (Node) m_doc; - } - - /** - * Get the node currently being processed. - * - * @return the current node being processed - */ - Node getCurrentNode() { - return m_currentNode; - } - - /** - * Return null since there is no Writer for this class. - * - * @return null - */ - java.io.Writer getWriter() { - return null; - } - - /** - * Append a node to the current container. - * - * @param newNode New node to append - */ - protected void append(Node newNode) throws org.xml.sax.SAXException { - - Node currentNode = m_currentNode; - - if (null != currentNode) { - currentNode.appendChild(newNode); - - // System.out.println(newNode.getNodeName()); - } else if (null != m_docFrag) { - m_docFrag.appendChild(newNode); - } else { - boolean ok = true; - short type = newNode.getNodeType(); - - if (type == Node.TEXT_NODE) { - String data = newNode.getNodeValue(); - - if ((null != data) && (data.trim().length() > 0)) { - throw new org.xml.sax.SAXException( - "Warning: can't output text before document element! Ignoring..."); - } - - ok = false; - } else if (type == Node.ELEMENT_NODE) { - if (m_doc.getDocumentElement() != null) { - throw new org.xml.sax.SAXException("Can't have more than one root on a DOM!"); - } - } - - if (ok) { - m_doc.appendChild(newNode); - } - } - } - - /** - * Receive an object for locating the origin of SAX document events. - * - *

SAX parsers are strongly encouraged (though not absolutely required) to supply a locator: - * if it does so, it must supply the locator to the application by invoking this method before - * invoking any of the other methods in the ContentHandler interface. - * - *

The locator allows the application to determine the end position of any document-related - * event, even if the parser is not reporting an error. Typically, the application will use this - * information for reporting its own errors (such as character content that does not match an - * application's business rules). The information returned by the locator is probably not - * sufficient for use with a search engine. - * - *

Note that the locator will return correct information only during the invocation of the - * events in this interface. The application should not attempt to use it at any other time. - * - * @param locator An object that can return the location of any SAX document event. - * @see org.xml.sax.Locator - */ - @Override - public void setDocumentLocator(Locator locator) { - - // No action for the moment. - } - - /** - * Receive notification of the beginning of a document. - * - *

The SAX parser will invoke this method only once, before any other methods in this - * interface or in DTDHandler (except for setDocumentLocator). - */ - @Override - public void startDocument() throws org.xml.sax.SAXException { - - // No action for the moment. - } - - /** - * Receive notification of the end of a document. - * - *

The SAX parser will invoke this method only once, and it will be the last method invoked - * during the parse. The parser shall not invoke this method until it has either abandoned - * parsing (because of an unrecoverable error) or reached the end of input. - */ - @Override - public void endDocument() throws org.xml.sax.SAXException { - - // No action for the moment. - } - - /** - * Receive notification of the beginning of an element. - * - *

The Parser will invoke this method at the beginning of every element in the XML document; - * there will be a corresponding endElement() event for every startElement() event (even when - * the element is empty). All of the element's content will be reported, in order, before the - * corresponding endElement() event. - * - *

If the element name has a namespace prefix, the prefix will still be attached. Note that - * the attribute list provided will contain only attributes with explicit values (specified or - * defaulted): #IMPLIED attributes will be omitted. - * - * @param ns The namespace of the node - * @param localName The local part of the qualified name - * @param name The element name. - * @param atts The attributes attached to the element, if any. - * @see #endElement - * @see org.xml.sax.Attributes - */ - @Override - public void startElement(String ns, String localName, String name, Attributes atts) - throws org.xml.sax.SAXException { - - Element elem; - - if (upperCaseElementNames) { - name = name.toUpperCase(Locale.ROOT); - } - - // Note that the namespace-aware call must be used to correctly - // construct a Level 2 DOM, even for non-namespaced nodes. - if ((null == ns) || (ns.length() == 0) || ns.equals(defaultNamespaceURI)) { - elem = m_doc.createElementNS(null, name); - } else { - elem = m_doc.createElementNS(ns, name); - } - - append(elem); - - try { - int nAtts = atts.getLength(); - - if (0 != nAtts) { - for (int i = 0; i < nAtts; i++) { - // First handle a possible ID attribute - if (atts.getType(i).equalsIgnoreCase("ID")) { - setIDAttribute(atts.getValue(i), elem); - } - - String attrNS = atts.getURI(i); - - if ("".equals(attrNS)) { - attrNS = null; // DOM represents no-namespace as null - } - - // System.out.println("attrNS: "+attrNS+", localName: "+atts.getQName(i) - // +", qname: "+atts.getQName(i)+", value: "+atts.getValue(i)); - // Crimson won't let us set an xmlns: attribute on the DOM. - String attrQName = atts.getQName(i); - - // In SAX, xmlns: attributes have an empty namespace, while - // in DOM they should have the xmlns namespace - if (attrQName.startsWith("xmlns:")) { - attrNS = "http://www.w3.org/2000/xmlns/"; - } - - // ALWAYS use the DOM Level 2 call! - elem.setAttributeNS(attrNS, attrQName, atts.getValue(i)); - } - } - - // append(elem); - - m_elemStack.push(elem); - - m_currentNode = elem; - - // append(elem); - } catch (java.lang.Exception de) { - // de.printStackTrace(); - throw new org.xml.sax.SAXException(de); - } - } - - /** - * Receive notification of the end of an element. - * - *

The SAX parser will invoke this method at the end of every element in the XML document; - * there will be a corresponding startElement() event for every endElement() event (even when - * the element is empty). - * - *

If the element name has a namespace prefix, the prefix will still be attached to the name. - * - * @param ns the namespace of the element - * @param localName The local part of the qualified name of the element - * @param name The element name - */ - @Override - public void endElement(String ns, String localName, String name) - throws org.xml.sax.SAXException { - m_elemStack.pop(); - m_currentNode = m_elemStack.isEmpty() ? null : (Node) m_elemStack.peek(); - } - - /** - * Set an ID string to node association in the ID table. - * - * @param id The ID string. - * @param elem The associated ID. - */ - public void setIDAttribute(String id, Element elem) { - - // Do nothing. This method is meant to be overridden. - } - - /** - * Receive notification of character data. - * - *

The Parser will call this method to report each chunk of character data. SAX parsers may - * return all contiguous character data in a single chunk, or they may split it into several - * chunks; however, all of the characters in any single event must come from the same external - * entity, so that the Locator provides useful information. - * - *

The application must not attempt to read from the array outside of the specified range. - * - *

Note that some parsers will report whitespace using the ignorableWhitespace() method - * rather than this one (validating parsers must do so). - * - * @param ch The characters from the XML document. - * @param start The start position in the array. - * @param length The number of characters to read from the array. - * @see #ignorableWhitespace - * @see org.xml.sax.Locator - */ - @Override - public void characters(char[] ch, int start, int length) throws org.xml.sax.SAXException { - if (isOutsideDocElem() && XMLCharacterRecognizer.isWhiteSpace(ch, start, length)) { - return; // avoid DOM006 Hierarchy request error - } - - if (m_inCData) { - cdata(ch, start, length); - - return; - } - - String s = new String(ch, start, length); - Node childNode; - childNode = m_currentNode != null ? m_currentNode.getLastChild() : null; - if (childNode != null && childNode.getNodeType() == Node.TEXT_NODE) { - ((Text) childNode).appendData(s); - } else { - Text text = m_doc.createTextNode(s); - append(text); - } - } - - /** - * If available, when the disable-output-escaping attribute is used, output raw text without - * escaping. A PI will be inserted in front of the node with the name "lotusxsl-next-is-raw" and - * a value of "formatter-to-dom". - * - * @param ch Array containing the characters - * @param start Index to start of characters in the array - * @param length Number of characters in the array - */ - public void charactersRaw(char[] ch, int start, int length) throws org.xml.sax.SAXException { - if (isOutsideDocElem() && XMLCharacterRecognizer.isWhiteSpace(ch, start, length)) { - return; // avoid DOM006 Hierarchy request error - } - - String s = new String(ch, start, length); - - append(m_doc.createProcessingInstruction("xslt-next-is-raw", "formatter-to-dom")); - append(m_doc.createTextNode(s)); - } - - /** - * Report the beginning of an entity. - * - *

The start and end of the document entity are not reported. The start and end of the - * external DTD subset are reported using the pseudo-name "[dtd]". All other events must be - * properly nested within start/end entity events. - * - * @param name The name of the entity. If it is a parameter entity, the name will begin with - * '%'. - * @see #endEntity - * @see org.xml.sax.ext.DeclHandler#internalEntityDecl - * @see org.xml.sax.ext.DeclHandler#externalEntityDecl - */ - @Override - public void startEntity(String name) throws org.xml.sax.SAXException { - - // Almost certainly the wrong behavior... - // entityReference(name); - } - - /** - * Report the end of an entity. - * - * @param name The name of the entity that is ending. - * @see #startEntity - */ - @Override - public void endEntity(String name) throws org.xml.sax.SAXException {} - - /** - * Receive notivication of a entityReference. - * - * @param name name of the entity reference - */ - public void entityReference(String name) throws org.xml.sax.SAXException { - append(m_doc.createEntityReference(name)); - } - - /** - * Receive notification of ignorable whitespace in element content. - * - *

Validating Parsers must use this method to report each chunk of ignorable whitespace (see - * the W3C XML 1.0 recommendation, section 2.10): non-validating parsers may also use this - * method if they are capable of parsing and using content models. - * - *

SAX parsers may return all contiguous whitespace in a single chunk, or they may split it - * into several chunks; however, all of the characters in any single event must come from the - * same external entity, so that the Locator provides useful information. - * - *

The application must not attempt to read from the array outside of the specified range. - * - * @param ch The characters from the XML document. - * @param start The start position in the array. - * @param length The number of characters to read from the array. - * @see #characters - */ - @Override - public void ignorableWhitespace(char[] ch, int start, int length) - throws org.xml.sax.SAXException { - if (isOutsideDocElem()) { - return; // avoid DOM006 Hierarchy request error - } - - String s = new String(ch, start, length); - - append(m_doc.createTextNode(s)); - } - - /** - * Tell if the current node is outside the document element. - * - * @return true if the current node is outside the document element. - */ - private boolean isOutsideDocElem() { - return (null == m_docFrag) - && m_elemStack.size() == 0 - && (null == m_currentNode || m_currentNode.getNodeType() == Node.DOCUMENT_NODE); - } - - /** - * Receive notification of a processing instruction. - * - *

The Parser will invoke this method once for each processing instruction found: note that - * processing instructions may occur before or after the main document element. - * - *

A SAX parser should never report an XML declaration (XML 1.0, section 2.8) or a text - * declaration (XML 1.0, section 4.3.1) using this method. - * - * @param target The processing instruction target. - * @param data The processing instruction data, or null if none was supplied. - */ - @Override - public void processingInstruction(String target, String data) throws org.xml.sax.SAXException { - append(m_doc.createProcessingInstruction(target, data)); - } - - /** - * Report an XML comment anywhere in the document. - * - *

This callback will be used for comments inside or outside the document element, including - * comments in the external DTD subset (if read). - * - * @param ch An array holding the characters in the comment. - * @param start The starting position in the array. - * @param length The number of characters to use from the array. - */ - @Override - public void comment(char[] ch, int start, int length) throws org.xml.sax.SAXException { - // tagsoup sometimes submits invalid values here - if (ch == null || start < 0 || length >= (ch.length - start) || length < 0) { - return; - } - append(m_doc.createComment(new String(ch, start, length))); - } - - /** Flag indicating that we are processing a CData section */ - protected boolean m_inCData = false; - - /** - * Report the start of a CDATA section. - * - * @see #endCDATA - */ - @Override - public void startCDATA() throws org.xml.sax.SAXException { - m_inCData = true; - append(m_doc.createCDATASection("")); - } - - /** - * Report the end of a CDATA section. - * - * @see #startCDATA - */ - @Override - public void endCDATA() throws org.xml.sax.SAXException { - m_inCData = false; - } - - /** - * Receive notification of cdata. - * - *

The Parser will call this method to report each chunk of character data. SAX parsers may - * return all contiguous character data in a single chunk, or they may split it into several - * chunks; however, all of the characters in any single event must come from the same external - * entity, so that the Locator provides useful information. - * - *

The application must not attempt to read from the array outside of the specified range. - * - *

Note that some parsers will report whitespace using the ignorableWhitespace() method - * rather than this one (validating parsers must do so). - * - * @param ch The characters from the XML document. - * @param start The start position in the array. - * @param length The number of characters to read from the array. - * @see #ignorableWhitespace - * @see org.xml.sax.Locator - */ - public void cdata(char[] ch, int start, int length) { - if (isOutsideDocElem() && XMLCharacterRecognizer.isWhiteSpace(ch, start, length)) { - return; // avoid DOM006 Hierarchy request error - } - - String s = new String(ch, start, length); - - // XXX ab@apache.org: modified from the original, to accommodate TagSoup. - Node n = m_currentNode.getLastChild(); - if (n instanceof CDATASection) { - ((CDATASection) n).appendData(s); - } else if (n instanceof Comment) { - ((Comment) n).appendData(s); - } - } - - /** - * Report the start of DTD declarations, if any. - * - *

Any declarations are assumed to be in the internal subset unless otherwise indicated. - * - * @param name The document type name. - * @param publicId The declared public identifier for the external DTD subset, or null if none - * was declared. - * @param systemId The declared system identifier for the external DTD subset, or null if none - * was declared. - * @see #endDTD - * @see #startEntity - */ - @Override - public void startDTD(String name, String publicId, String systemId) - throws org.xml.sax.SAXException { - - // Do nothing for now. - } - - /** - * Report the end of DTD declarations. - * - * @see #startDTD - */ - @Override - public void endDTD() throws org.xml.sax.SAXException { - - // Do nothing for now. - } - - /** - * Begin the scope of a prefix-URI Namespace mapping. - * - *

The information from this event is not necessary for normal Namespace processing: the SAX - * XML reader will automatically replace prefixes for element and attribute names when the - * http://xml.org/sax/features/namespaces feature is true (the default). - * - *

There are cases, however, when applications need to use prefixes in character data or in - * attribute values, where they cannot safely be expanded automatically; the - * start/endPrefixMapping event supplies the information to the application to expand prefixes - * in those contexts itself, if necessary. - * - *

Note that start/endPrefixMapping events are not guaranteed to be properly nested relative - * to each-other: all startPrefixMapping events will occur before the corresponding startElement - * event, and all endPrefixMapping events will occur after the corresponding endElement event, - * but their order is not guaranteed. - * - * @param prefix The Namespace prefix being declared. - * @param uri The Namespace URI the prefix is mapped to. - * @see #endPrefixMapping - * @see #startElement - */ - @Override - public void startPrefixMapping(String prefix, String uri) throws org.xml.sax.SAXException { - - /* - * // Not sure if this is needed or wanted // Also, it fails in the - * stree. if((null != m_currentNode) && (m_currentNode.getNodeType() == - * Node.ELEMENT_NODE)) { String qname; if(((null != prefix) && - * (prefix.length() == 0)) || (null == prefix)) qname = "xmlns"; else - * qname = "xmlns:"+prefix; - * - * Element elem = (Element)m_currentNode; String val = - * elem.getAttribute(qname); // Obsolete, should be DOM2...? if(val == - * null) { elem.setAttributeNS("http://www.w3.org/XML/1998/namespace", - * qname, uri); } } - */ - } - - /** - * End the scope of a prefix-URI mapping. - * - *

See startPrefixMapping for details. This event will always occur after the corresponding - * endElement event, but the order of endPrefixMapping events is not otherwise guaranteed. - * - * @param prefix The prefix that was being mapping. - * @see #startPrefixMapping - * @see #endElement - */ - @Override - public void endPrefixMapping(String prefix) throws org.xml.sax.SAXException {} - - /** - * Receive notification of a skipped entity. - * - *

The Parser will invoke this method once for each entity skipped. Non-validating processors - * may skip entities if they have not seen the declarations (because, for example, the entity - * was declared in an external DTD subset). All processors may skip external entities, depending - * on the values of the http://xml.org/sax/features/external-general-entities and the - * http://xml.org/sax/features/external-parameter-entities properties. - * - * @param name The name of the skipped entity. If it is a parameter entity, the name will begin - * with '%'. - */ - @Override - public void skippedEntity(String name) throws org.xml.sax.SAXException {} - - public boolean isUpperCaseElementNames() { - return upperCaseElementNames; - } - - public void setUpperCaseElementNames(boolean upperCaseElementNames) { - this.upperCaseElementNames = upperCaseElementNames; - } - - public String getDefaultNamespaceURI() { - return defaultNamespaceURI; - } - - public void setDefaultNamespaceURI(String defaultNamespaceURI) { - this.defaultNamespaceURI = defaultNamespaceURI; - } -} diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java index be57f7b36..cbf19714a 100644 --- a/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java +++ b/external/tika/src/main/java/org/apache/stormcrawler/tika/ParserBolt.java @@ -44,7 +44,6 @@ import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParserFactory; import org.apache.commons.lang3.StringUtils; -import org.apache.html.dom.HTMLDocumentImpl; import org.apache.http.HttpHeaders; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; @@ -97,11 +96,9 @@ import org.apache.tika.sax.Link; import org.apache.tika.sax.LinkContentHandler; import org.apache.tika.sax.TeeContentHandler; -import org.apache.tika.sax.XHTMLContentHandler; import org.apache.tika.serialization.ComponentNameResolver; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; -import org.w3c.dom.DocumentFragment; import org.xml.sax.ContentHandler; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -201,7 +198,6 @@ private static SAXParserFactory newHardenedSaxParserFactory() { private ScopedCounter eventCounter; - private boolean upperCaseElementNames = true; private Class htmlMapperClass = IdentityHtmlMapper.class; private boolean extractEmbedded = false; @@ -253,8 +249,6 @@ public void prepare( jsoupFilters = JSoupFilters.fromConf(conf); - upperCaseElementNames = ConfUtils.getBoolean(conf, "parser.uppercase.element.names", true); - extractEmbedded = ConfUtils.getBoolean(conf, "parser.extract.embedded", false); String htmlmapperClassName = @@ -429,18 +423,6 @@ public void execute(Tuple tuple) { LOG.error("Exception while specifying HTMLMapper {}", url, e); } - // build a DOM if required by the parseFilters - DocumentFragment root = null; - if (parseFilters.needsDOM()) { - HTMLDocumentImpl doc = new HTMLDocumentImpl(); - doc.setErrorChecking(false); - root = doc.createDocumentFragment(); - DOMBuilder domhandler = new DOMBuilder(doc, root); - domhandler.setUpperCaseElementNames(upperCaseElementNames); - domhandler.setDefaultNamespaceURI(XHTMLContentHandler.XHTML); - handlers.add(domhandler); - } - // build a JSoup document if there are JSoupFilters JSoupBuilder jsoupBuilder = null; if (!jsoupFilters.isEmpty()) { @@ -527,7 +509,7 @@ public void execute(Tuple tuple) { // apply the parse filters if any try { - parseFilters.filter(url, content, root, parse); + parseFilters.filter(url, content, parse); } catch (RuntimeException e) { handleException(url, e, metadata, tuple, "parse filters"); return; diff --git a/external/tika/src/main/java/org/apache/stormcrawler/tika/XMLCharacterRecognizer.java b/external/tika/src/main/java/org/apache/stormcrawler/tika/XMLCharacterRecognizer.java deleted file mode 100644 index da079010c..000000000 --- a/external/tika/src/main/java/org/apache/stormcrawler/tika/XMLCharacterRecognizer.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to you under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * XXX NUTCH COMMENT - * XXX ab@apache.org: This class is copied verbatim from Xalan-J 2.6.0 - * XXX distribution, org.apache.xml.utils.DOMBuilder, in order to - * avoid dependency on Xalan. - */ - -package org.apache.stormcrawler.tika; - -/** - * Class used to verify whether the specified ch conforms to the XML 1.0 definition of - * whitespace. - */ -class XMLCharacterRecognizer { - - private XMLCharacterRecognizer() {} - - /** - * Returns whether the specified ch conforms to the XML 1.0 definition of whitespace. - * Refer to the definition of S - * for details. - * - * @param ch Character to check as XML whitespace. - * @return =true if ch is XML whitespace; otherwise =false. - */ - static boolean isWhiteSpace(char ch) { - return (ch == 0x20) || (ch == 0x09) || (ch == 0xD) || (ch == 0xA); - } - - /** - * Tell if the string is whitespace. - * - * @param ch Character array to check as XML whitespace. - * @param start Start index of characters in the array - * @param length Number of characters in the array - * @return True if the characters in the array are XML whitespace; otherwise, false. - */ - static boolean isWhiteSpace(char[] ch, int start, int length) { - - int end = start + length; - - for (int s = start; s < end; s++) { - if (!isWhiteSpace(ch[s])) { - return false; - } - } - - return true; - } - - /** - * Tell if the string is whitespace. - * - * @param buf StringBuffer to check as XML whitespace. - * @return True if characters in buffer are XML whitespace, false otherwise - */ - static boolean isWhiteSpace(StringBuffer buf) { - - int n = buf.length(); - - for (int i = 0; i < n; i++) { - if (!isWhiteSpace(buf.charAt(i))) { - return false; - } - } - - return true; - } - - /** - * Tell if the string is whitespace. - * - * @param s String to check as XML whitespace. - * @return True if characters in buffer are XML whitespace, false otherwise - */ - static boolean isWhiteSpace(String s) { - - if (null != s) { - int n = s.length(); - - for (int i = 0; i < n; i++) { - if (!isWhiteSpace(s.charAt(i))) { - return false; - } - } - } - - return true; - } -} From 16a577fbc44a73f9b639114e0c5d68eecf16c092 Mon Sep 17 00:00:00 2001 From: Julien Nioche Date: Sat, 24 Oct 2026 07:53:13 +0100 Subject: [PATCH 2/2] StackOverflowTest: use the jsoup XPathFilter now that the DOM one is gone Co-Authored-By: Claude Opus 5.5 --- .../stormcrawler/parse/StackOverflowTest.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/apache/stormcrawler/parse/StackOverflowTest.java b/core/src/test/java/org/apache/stormcrawler/parse/StackOverflowTest.java index 2eb2f997b..d422d78ac 100644 --- a/core/src/test/java/org/apache/stormcrawler/parse/StackOverflowTest.java +++ b/core/src/test/java/org/apache/stormcrawler/parse/StackOverflowTest.java @@ -20,7 +20,9 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.stormcrawler.Constants; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.bolt.JSoupParserBolt; @@ -41,7 +43,7 @@ void setupParserBolt() { @Test void testStackOverflow() throws IOException { - prepareParserBolt("test.parsefilters.json"); + prepareParserBolt("test.parsefilters.json", jsoupFiltersConfig()); Metadata metadata = new Metadata(); parse("https://stormcrawler.apache.org", "stackexception.html", metadata); Assertions.assertEquals(31, output.getEmitted(Constants.StatusStreamName).size()); @@ -52,12 +54,18 @@ void testStackOverflow() throws IOException { */ @Test void testNamespaceExtraction() throws IOException { - prepareParserBolt("test.parsefilters.json"); + prepareParserBolt("test.parsefilters.json", jsoupFiltersConfig()); Metadata metadata = new Metadata(); parse("https://stormcrawler.apache.org", "stackexception.html", metadata); Assertions.assertEquals(1, output.getEmitted().size()); List obj = output.getEmitted().get(0); Metadata m = (Metadata) obj.get(2); - assertNotNull(m.getFirstValue("title")); + assertNotNull(m.getFirstValue("title_text")); + } + + private static Map jsoupFiltersConfig() { + Map parserConfig = new HashMap<>(); + parserConfig.put("jsoup.filters.config.file", "test.xpath.jsoupfilters.json"); + return parserConfig; } }