XPathFilter: skip blank values and try the next expression

core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java

@@ -181,7 +181,8 @@ List<String> evaluate(W3CContext context) throws XPathExpressionException {
             List<String> results = new ArrayList<>();
             for (Element element : elements) {
                 String value = evalFunction.evaluate(element, attrName);
-                if (value != null) {
+                // blank values are skipped so that the next expression can be tried
+                if (value != null && !value.isBlank()) {
                     results.add(value);
                 }
             }

core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java

@@ -167,7 +167,12 @@ public void filter(String url, byte[] content, DocumentFragment doc, ParseResult
             for (LabelledExpression le : leList) {
                 try {
                     List<String> values = le.evaluate(doc);
-                    if (values != null && !values.isEmpty()) {
+                    if (values == null) {
+                        continue;
+                    }
+                    // blank values are skipped so that the next expression can be tried
+                    values.removeIf(StringUtils::isBlank);
+                    if (!values.isEmpty()) {
                         metadata.addValues(le.key, values);
                         break;
                     }

core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java

@@ -20,6 +20,7 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -28,7 +29,9 @@
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.TestUtil;
 import org.apache.stormcrawler.bolt.JSoupParserBolt;
+import org.apache.stormcrawler.parse.ParseResult;
 import org.apache.stormcrawler.parse.ParsingTester;
+import org.jsoup.Jsoup;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -73,10 +76,9 @@ void testScriptExtraction() throws IOException {
         Assertions.assertNotNull(metadata);
         String[] scripts = metadata.getValues("js");
         Assertions.assertNotNull(scripts);
-        // should be 2 of them
-        Assertions.assertEquals(2, scripts.length);
-        Assertions.assertEquals("", scripts[0].trim());
-        Assertions.assertTrue(scripts[1].contains("_paq"));
+        // the first script is external and has no content, its blank value is skipped
+        Assertions.assertEquals(1, scripts.length);
+        Assertions.assertTrue(scripts[0].contains("_paq"));
     }
 
     @Test
@@ -183,4 +185,39 @@ void testLowercaseElementNames() {
                 "//div/span/a[@href]", XPathFilter.lowercaseElementNames("//DIV/SPAN/A[@href]"));
         Assertions.assertEquals("//title", XPathFilter.lowercaseElementNames("//title"));
     }
+
+    private static final String TITLE_PARAMS =
+            """
+            {"title": ["//title", "//meta[@name='title']/@content"]}
+            """;
+
+    private static Metadata filterTitle(String html) throws IOException {
+        XPathFilter filter = new XPathFilter();
+        filter.configure(new HashMap<>(), new ObjectMapper().readTree(TITLE_PARAMS));
+        String url = "https://example.com";
+        ParseResult parse = new ParseResult();
+        org.jsoup.nodes.Document doc = Jsoup.parse(html, url);
+        filter.filter(url, html.getBytes(StandardCharsets.UTF_8), doc, parse);
+        return parse.getOrCreate(url).getMetadata();
+    }
+
+    @Test
+    void testBlankValueFallsBackToNextExpression() throws IOException {
+        Metadata metadata =
+                filterTitle(
+                        "<html><head><title> </title>"
+                                + "<meta name=\"title\" content=\"Fallback title\"></head>"
+                                + "<body></body></html>");
+        Assertions.assertArrayEquals(new String[] {"Fallback title"}, metadata.getValues("title"));
+    }
+
+    @Test
+    void testBlankValuesAreNotStored() throws IOException {
+        Metadata metadata =
+                filterTitle(
+                        "<html><head><title></title>"
+                                + "<meta name=\"title\" content=\"  \"></head>"
+                                + "<body></body></html>");
+        Assertions.assertNull(metadata.getValues("title"));
+    }
 }

core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java

@@ -17,11 +17,17 @@
 
 package org.apache.stormcrawler.parse.filter;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
 import java.util.List;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.bolt.JSoupParserBolt;
+import org.apache.stormcrawler.parse.DocumentFragmentBuilder;
+import org.apache.stormcrawler.parse.ParseResult;
 import org.apache.stormcrawler.parse.ParsingTester;
+import org.jsoup.Jsoup;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -59,9 +65,48 @@ void testScriptExtraction() throws IOException {
         Assertions.assertNotNull(metadata);
         String[] scripts = metadata.getValues("js");
         Assertions.assertNotNull(scripts);
-        // should be 2 of them
-        Assertions.assertEquals(2, scripts.length);
-        Assertions.assertEquals("", scripts[0].trim());
-        Assertions.assertTrue(scripts[1].contains("_paq"));
+        // the first script is external and has no content, its blank value is skipped
+        Assertions.assertEquals(1, scripts.length);
+        Assertions.assertTrue(scripts[0].contains("_paq"));
+    }
+
+    // element names are in upper case in the DOM built from the JSoup document
+    private static final String TITLE_PARAMS =
+            """
+            {"title": ["//TITLE", "//META[@name='title']/@content"]}
+            """;
+
+    private static Metadata filterTitle(String html) throws IOException {
+        XPathFilter filter = new XPathFilter();
+        filter.configure(new HashMap<>(), new ObjectMapper().readTree(TITLE_PARAMS));
+        String url = "https://example.com";
+        ParseResult parse = new ParseResult();
+        org.jsoup.nodes.Document doc = Jsoup.parse(html, url);
+        filter.filter(
+                url,
+                html.getBytes(StandardCharsets.UTF_8),
+                DocumentFragmentBuilder.fromJsoup(doc),
+                parse);
+        return parse.getOrCreate(url).getMetadata();
+    }
+
+    @Test
+    void testBlankValueFallsBackToNextExpression() throws IOException {
+        Metadata metadata =
+                filterTitle(
+                        "<html><head><title> </title>"
+                                + "<meta name=\"title\" content=\"Fallback title\"></head>"
+                                + "<body></body></html>");
+        Assertions.assertArrayEquals(new String[] {"Fallback title"}, metadata.getValues("title"));
+    }
+
+    @Test
+    void testBlankValuesAreNotStored() throws IOException {
+        Metadata metadata =
+                filterTitle(
+                        "<html><head><title></title>"
+                                + "<meta name=\"title\" content=\"  \"></head>"
+                                + "<body></body></html>");
+        Assertions.assertNull(metadata.getValues("title"));
     }
 }