diff --git a/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java b/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java index 7dd5c417f..4ec2d5735 100644 --- a/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java +++ b/core/src/main/java/org/apache/stormcrawler/jsoup/XPathFilter.java @@ -181,7 +181,8 @@ List evaluate(W3CContext context) throws XPathExpressionException { List results = new ArrayList<>(); for (Element element : elements) { String value = evalFunction.evaluate(element, attrName); - if (value != null) { + // blank values are skipped so that the next expression can be tried + if (value != null && !value.isBlank()) { results.add(value); } } diff --git a/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java b/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java index f1319baff..cbe18e754 100644 --- a/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java +++ b/core/src/main/java/org/apache/stormcrawler/parse/filter/XPathFilter.java @@ -167,7 +167,12 @@ public void filter(String url, byte[] content, DocumentFragment doc, ParseResult for (LabelledExpression le : leList) { try { List values = le.evaluate(doc); - if (values != null && !values.isEmpty()) { + if (values == null) { + continue; + } + // blank values are skipped so that the next expression can be tried + values.removeIf(StringUtils::isBlank); + if (!values.isEmpty()) { metadata.addValues(le.key, values); break; } diff --git a/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java b/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java index b4d6b00b0..c04626a30 100644 --- a/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java +++ b/core/src/test/java/org/apache/stormcrawler/jsoup/JSoupFiltersTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,7 +29,9 @@ import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.TestUtil; import org.apache.stormcrawler.bolt.JSoupParserBolt; +import org.apache.stormcrawler.parse.ParseResult; import org.apache.stormcrawler.parse.ParsingTester; +import org.jsoup.Jsoup; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -73,10 +76,9 @@ void testScriptExtraction() throws IOException { Assertions.assertNotNull(metadata); String[] scripts = metadata.getValues("js"); Assertions.assertNotNull(scripts); - // should be 2 of them - Assertions.assertEquals(2, scripts.length); - Assertions.assertEquals("", scripts[0].trim()); - Assertions.assertTrue(scripts[1].contains("_paq")); + // the first script is external and has no content, its blank value is skipped + Assertions.assertEquals(1, scripts.length); + Assertions.assertTrue(scripts[0].contains("_paq")); } @Test @@ -183,4 +185,39 @@ void testLowercaseElementNames() { "//div/span/a[@href]", XPathFilter.lowercaseElementNames("//DIV/SPAN/A[@href]")); Assertions.assertEquals("//title", XPathFilter.lowercaseElementNames("//title")); } + + private static final String TITLE_PARAMS = + """ + {"title": ["//title", "//meta[@name='title']/@content"]} + """; + + private static Metadata filterTitle(String html) throws IOException { + XPathFilter filter = new XPathFilter(); + filter.configure(new HashMap<>(), new ObjectMapper().readTree(TITLE_PARAMS)); + String url = "https://example.com"; + ParseResult parse = new ParseResult(); + org.jsoup.nodes.Document doc = Jsoup.parse(html, url); + filter.filter(url, html.getBytes(StandardCharsets.UTF_8), doc, parse); + return parse.getOrCreate(url).getMetadata(); + } + + @Test + void testBlankValueFallsBackToNextExpression() throws IOException { + Metadata metadata = + filterTitle( + " " + + "" + + ""); + Assertions.assertArrayEquals(new String[] {"Fallback title"}, metadata.getValues("title")); + } + + @Test + void testBlankValuesAreNotStored() throws IOException { + Metadata metadata = + filterTitle( + "" + + "" + + ""); + Assertions.assertNull(metadata.getValues("title")); + } } diff --git a/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java b/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java index 1ffbfae3f..856be4e6f 100644 --- a/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java +++ b/core/src/test/java/org/apache/stormcrawler/parse/filter/XPathFilterTest.java @@ -17,11 +17,17 @@ package org.apache.stormcrawler.parse.filter; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; import java.util.List; import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.bolt.JSoupParserBolt; +import org.apache.stormcrawler.parse.DocumentFragmentBuilder; +import org.apache.stormcrawler.parse.ParseResult; import org.apache.stormcrawler.parse.ParsingTester; +import org.jsoup.Jsoup; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -59,9 +65,48 @@ void testScriptExtraction() throws IOException { Assertions.assertNotNull(metadata); String[] scripts = metadata.getValues("js"); Assertions.assertNotNull(scripts); - // should be 2 of them - Assertions.assertEquals(2, scripts.length); - Assertions.assertEquals("", scripts[0].trim()); - Assertions.assertTrue(scripts[1].contains("_paq")); + // the first script is external and has no content, its blank value is skipped + Assertions.assertEquals(1, scripts.length); + Assertions.assertTrue(scripts[0].contains("_paq")); + } + + // element names are in upper case in the DOM built from the JSoup document + private static final String TITLE_PARAMS = + """ + {"title": ["//TITLE", "//META[@name='title']/@content"]} + """; + + private static Metadata filterTitle(String html) throws IOException { + XPathFilter filter = new XPathFilter(); + filter.configure(new HashMap<>(), new ObjectMapper().readTree(TITLE_PARAMS)); + String url = "https://example.com"; + ParseResult parse = new ParseResult(); + org.jsoup.nodes.Document doc = Jsoup.parse(html, url); + filter.filter( + url, + html.getBytes(StandardCharsets.UTF_8), + DocumentFragmentBuilder.fromJsoup(doc), + parse); + return parse.getOrCreate(url).getMetadata(); + } + + @Test + void testBlankValueFallsBackToNextExpression() throws IOException { + Metadata metadata = + filterTitle( + " " + + "" + + ""); + Assertions.assertArrayEquals(new String[] {"Fallback title"}, metadata.getValues("title")); + } + + @Test + void testBlankValuesAreNotStored() throws IOException { + Metadata metadata = + filterTitle( + "" + + "" + + ""); + Assertions.assertNull(metadata.getValues("title")); } }