#158 remove the DOM from the ParseFilters

THIRD-PARTY.txt

@@ -246,7 +246,6 @@ List of third-party dependencies grouped by their license type.
         * Storm Shaded Deps (org.apache.storm:storm-shaded-deps:3.1.0 - https://storm.apache.org/storm-shaded-deps)
         * swagger-annotations-jakarta (io.swagger.core.v3:swagger-annotations-jakarta:2.2.22 - https://github.com/swagger-api/swagger-core/modules/swagger-annotations-jakarta)
         * urlfrontier-API (com.github.crawler-commons:urlfrontier-API:2.6 - https://github.com/crawler-commons/url-frontier/urlfrontier-API)
-        * Xerces2-j (xerces:xercesImpl:2.12.2 - https://xerces.apache.org/xerces2-j/)
         * XmlBeans (org.apache.xmlbeans:xmlbeans:5.3.0 - https://xmlbeans.apache.org/)
         * zstd-kmp (com.squareup.zstd:zstd-kmp-jvm:0.4.0 - https://github.com/square/okio-zstd/)
         * zstd-kmp-okio (com.squareup.zstd:zstd-kmp-okio-jvm:0.4.0 - https://github.com/square/okio-zstd/)

core/pom.xml

@@ -40,7 +40,6 @@ under the License.
         <crawler-commons.version>1.6</crawler-commons.version>
         <jsoup.version>1.23.2</jsoup.version>
         <icu4j.version>78.3</icu4j.version>
-        <xerces.version>2.12.2</xerces.version>
         <httpclient.version>4.5.14</httpclient.version>
         <snakeyaml.version>2.7</snakeyaml.version>
         <commons.lang.version>3.21.0</commons.lang.version>
@@ -173,18 +172,6 @@ under the License.
             <version>${crawler-commons.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>xerces</groupId>
-            <artifactId>xercesImpl</artifactId>
-            <version>${xerces.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>xml-apis</groupId>
-                    <artifactId>xml-apis</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>

core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java

@@ -140,7 +140,7 @@ public void execute(Tuple tuple) {
 
         // apply the parse filters if any
         try {
-            parseFilters.filter(url, content, null, parse);
+            parseFilters.filter(url, content, parse);
         } catch (RuntimeException e) {
             String errorMessage = "Exception while running parse filters on " + url + ": " + e;
             LOG.error(errorMessage, e);

core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java

@@ -43,7 +43,6 @@
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.metrics.CrawlerMetrics;
 import org.apache.stormcrawler.metrics.ScopedCounter;
-import org.apache.stormcrawler.parse.DocumentFragmentBuilder;
 import org.apache.stormcrawler.parse.JSoupFilter;
 import org.apache.stormcrawler.parse.JSoupFilters;
 import org.apache.stormcrawler.parse.JSoupTextExtractor;
@@ -70,7 +69,6 @@
 import org.jsoup.parser.Parser;
 import org.jsoup.select.Elements;
 import org.slf4j.LoggerFactory;
-import org.w3c.dom.DocumentFragment;
 
 /**
  * Parser for HTML documents only which uses ICU4J to detect the charset encoding. Kindly donated to
@@ -477,12 +475,7 @@ public void execute(Tuple tuple) {
 
         // apply the parse filters if any
         try {
-            DocumentFragment fragment = null;
-            // lazy building of fragment
-            if (parseFilters.needsDOM()) {
-                fragment = DocumentFragmentBuilder.fromJsoup(jsoupDoc);
-            }
-            parseFilters.filter(url, content, fragment, parse);
+            parseFilters.filter(url, content, parse);
         } catch (RuntimeException e) {
             String errorMessage = "Exception while running parse filters on " + url + ": " + e;
             handleException(url, e, metadata, tuple, "content filtering", errorMessage);

core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java

@@ -205,7 +205,7 @@ public void execute(Tuple tuple) {
 
         // apply the parse filters if any
         try {
-            parseFilters.filter(url, content, null, parse);
+            parseFilters.filter(url, content, parse);
         } catch (RuntimeException e) {
             String errorMessage = "Exception while running parse filters on " + url + ": " + e;
             LOG.error(errorMessage);

core/src/main/java/org/apache/stormcrawler/parse/ParseFilter.java

@@ -19,7 +19,6 @@
 
 import org.apache.storm.task.IBolt;
 import org.apache.stormcrawler.util.AbstractConfigurable;
-import org.w3c.dom.DocumentFragment;
 
 /**
  * Implementations of ParseFilter are responsible for extracting custom data from the crawled
@@ -34,22 +33,9 @@ public abstract class ParseFilter extends AbstractConfigurable {
      *
      * @param url the URL of the page being parsed
      * @param content the content being parsed
-     * @param doc the DOM tree resulting of the parsing of the content or null if {@link
-     *     #needsDOM()} returns <code>false</code>
      * @param parse the metadata to be updated with the resulting of the parsing
      */
-    public abstract void filter(
-            String url, byte[] content, DocumentFragment doc, ParseResult parse);
-
-    /**
-     * Specifies whether this filter requires a DOM representation of the document.
-     *
-     * @return <code>true</code>if this needs a DOM representation of the document, <code>false
-     *     </code> otherwise.
-     */
-    public boolean needsDOM() {
-        return false;
-    }
+    public abstract void filter(String url, byte[] content, ParseResult parse);
 
     /**
      * Might be used to clean any resources associated with this {@link ParseFilter}.- See {@link

core/src/main/java/org/apache/stormcrawler/parse/ParseFilters.java

@@ -23,7 +23,6 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
 import org.apache.commons.cli.CommandLine;
@@ -40,10 +39,7 @@
 import org.apache.stormcrawler.util.Configurable;
 import org.apache.stormcrawler.util.URLUtil;
 import org.jetbrains.annotations.NotNull;
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
 import org.slf4j.LoggerFactory;
-import org.w3c.dom.DocumentFragment;
 
 /**
  * Wrapper for the ParseFilters defined in a JSON configuration.
@@ -119,29 +115,11 @@ public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode
     }
 
     @Override
-    public boolean needsDOM() {
-        for (ParseFilter filter : filters) {
-            boolean needsDom = filter.needsDOM();
-            if (needsDom) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {
+    public void filter(String url, byte[] content, ParseResult parse) {
 
         for (ParseFilter filter : filters) {
             long start = System.currentTimeMillis();
-            if (doc == null && filter.needsDOM()) {
-                LOG.info(
-                        "ParseFilter {} needs DOM but has none to work on - skip : {}",
-                        filter.getClass().getName(),
-                        url);
-                continue;
-            }
-            filter.filter(url, content, doc, parse);
+            filter.filter(url, content, parse);
             long end = System.currentTimeMillis();
             LOG.debug("ParseFilter {} took {} msec", filter.getClass().getName(), end - start);
         }
@@ -189,9 +167,7 @@ public static void main(String[] args) throws IOException, ParseException {
 
         byte[] content = IOUtils.toByteArray(URLUtil.toURL(url).openStream());
 
-        Document doc = Jsoup.parse(new String(content, StandardCharsets.UTF_8), url);
-
-        filters.filter(url, content, DocumentFragmentBuilder.fromJsoup(doc), parse);
+        filters.filter(url, content, parse);
 
         System.out.println(parse.toString());
 

core/src/main/java/org/apache/stormcrawler/parse/filter/CollectionTagger.java

@@ -36,7 +36,6 @@
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.w3c.dom.DocumentFragment;
 
 /**
  * Assigns one or more tags to the metadata of a document based on its URL matching patterns defined
@@ -119,7 +118,7 @@ public void loadJSONResources(InputStream inputStream)
     }
 
     @Override
-    public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {
+    public void filter(String url, byte[] content, ParseResult parse) {
         String[] tags = collections.tag(url);
         if (tags.length > 0) {
             parse.getOrCreate(url).getMetadata().setValues(key, tags);

core/src/main/java/org/apache/stormcrawler/parse/filter/CommaSeparatedToMultivaluedMetadata.java

@@ -30,7 +30,6 @@
 import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.w3c.dom.DocumentFragment;
 
 /**
  * Rewrites single metadata containing comma separated values into multiple values for the same key,
@@ -73,7 +72,7 @@ public void configure(@NotNull Map<String, Object> stormConf, @NotNull JsonNode
     }
 
     @Override
-    public void filter(String url, byte[] content, DocumentFragment doc, ParseResult parse) {
+    public void filter(String url, byte[] content, ParseResult parse) {
         Metadata m = parse.getOrCreate(url).getMetadata();
         for (String key : keys) {
             String val = m.getFirstValue(key);