From d093e2cb1e077028562aa204c423e28ad2442ed8 Mon Sep 17 00:00:00 2001 From: Elmehdi Elmouate Date: Fri, 23 Oct 2026 19:45:23 +0000 Subject: [PATCH] #2199 BasicURLNormalizer: split the query on & only, not on ; --- .../filtering/basic/BasicURLNormalizer.java | 2 +- .../filtering/BasicURLNormalizerTest.java | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java index 4e14119e5..4f4899484 100644 --- a/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java +++ b/core/src/main/java/org/apache/stormcrawler/filtering/basic/BasicURLNormalizer.java @@ -289,7 +289,7 @@ private String processQueryElements(String urlToFilter) { return urlToFilter; } - List pairs = URLEncodedUtils.parse(query, StandardCharsets.UTF_8); + List pairs = URLEncodedUtils.parse(query, StandardCharsets.UTF_8, '&'); Iterator pairsIterator = pairs.iterator(); while (pairsIterator.hasNext()) { NameValuePair param = pairsIterator.next(); diff --git a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java index 5df8b97a7..940fc736d 100644 --- a/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java +++ b/core/src/test/java/org/apache/stormcrawler/filtering/BasicURLNormalizerTest.java @@ -384,6 +384,35 @@ void testQueryNormalizedLikePath() throws MalformedURLException { urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/p?u=%7Euser")); } + // https://github.com/apache/stormcrawler/issues/2199 + @Test + void testSemicolonIsNotAQuerySeparator() throws MalformedURLException { + // the query is only parsed when there are elements to remove + URLFilter urlFilter = createFilter(List.of("utm_source")); + URL testSourceUrl = URLUtil.toURL("http://a.example/"); + // ';' is data: the pair must stay a single parameter + assertEquals( + "http://a.example/search?q=a%3Bb", + urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/search?q=a;b")); + assertEquals( + "http://a.example/p?a=1%3Bb%3D2", + urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/p?a=1;b=2")); + // a trailing ';' in a value must not be dropped + assertEquals( + "http://a.example/books?dq=x+%3B&hl=fr", + urlFilter.filter( + testSourceUrl, new Metadata(), "http://a.example/books?dq=x+;&hl=fr")); + // an escaped ';' is left alone + assertEquals( + "http://a.example/p?q=a%3Bb&z=1", + urlFilter.filter(testSourceUrl, new Metadata(), "http://a.example/p?q=a%3Bb&z=1")); + // a removed parameter is removed as a whole, including a ';' in its value + assertEquals( + "http://a.example/p?q=1", + urlFilter.filter( + testSourceUrl, new Metadata(), "http://a.example/p?utm_source=x;y&q=1")); + } + @Test void testHostWithUnderscoreWhenURLRebuilt() throws MalformedURLException { URLFilter urlFilter = createFilter(new ObjectNode(JsonNodeFactory.instance));