diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java index 969429b9e..7908aee7b 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java @@ -289,10 +289,7 @@ private String processQueryElements(String urlToFilter) { return urlToFilter; } - // Keep parse and format: WWWFormCodec changes semicolon splitting and tilde/star - // encoding. - List pairs = - URLEncodedUtils.parse(query, StandardCharsets.UTF_8, '&', ';'); + List pairs = URLEncodedUtils.parse(query, StandardCharsets.UTF_8, '&'); Iterator pairsIterator = pairs.iterator(); while (pairsIterator.hasNext()) { NameValuePair param = pairsIterator.next(); diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java index 712521a85..7a6b91110 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java @@ -487,6 +487,35 @@ void testQueryNormalizedLikePath() throws MalformedURLException { urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/p?u=%7Euser")); } + // https://github.com/apache/stormcrawler/issues/2199 + @Test + void testSemicolonIsNotAQuerySeparator() throws MalformedURLException { + // the query is only parsed when there are elements to remove + URLFilter urlFilter = createFilter(List.of("utm_source")); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + // ';' is data: the pair must stay a single parameter + assertEquals( + "http://a.example/search?q=a%3Bb", + urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/search?q=a;b")); + assertEquals( + "http://a.example/p?a=1%3Bb%3D2", + urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/p?a=1;b=2")); + // a trailing ';' in a value must not be dropped + assertEquals( + "http://a.example/books?dq=x+%3B&hl=fr", + urlFilter.filter( + testSourceUrl, new Metadata(), "http://a.example/books?dq=x+;&hl=fr")); + // an escaped ';' is left alone + assertEquals( + "http://a.example/p?q=a%3Bb&z=1", + urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/p?q=a%3Bb&z=1")); + // a removed parameter is removed as a whole, including a ';' in its value + assertEquals( + "http://a.example/p?q=1", + urlFilter.filter( + testSourceUrl, new Metadata(), "http://a.example/p?utm_source=x;y&q=1")); + } + @Test void testHostWithUnderscoreWhenURLRebuilt() throws MalformedURLException { URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance));