ensure original token is not set to type:SYNONYM in SGF

changelog/unreleased/preserve-original-token-type-managed-synonym-graph-filter.yml

@@ -0,0 +1,9 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: ManagedSynonymGraphFilterFactory no longer tags the original token as type `SYNONYM` for equivalence-group mappings (e.g. `[terma]->[terma, termb, termc]`), using Lucene's `includeOrig` flag so only the injected synonyms get that type.
+type: fixed
+authors:
+  - name: Renato Haeberli
+    nick: renatoh
+links:
+  - name: PR#4960
+    url: https://github.com/apache/solr/pull/4960

solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java

@@ -365,16 +365,24 @@ public void parse(Reader in) throws IOException, ParseException {
       boolean ignoreCase = synonymManager.getIgnoreCase();
       for (CasePreservedSynonymMappings cpsm : synonymManager.synonymMappings.values()) {
         for (Map.Entry<String, Set<String>> entry : cpsm.mappings.entrySet()) {
+          // Use includeOrig=true so the original token keeps its upstream type and only the
+          // injected synonyms are tagged as SYNONYM
+          boolean mapsToItself = entry.getValue().contains(entry.getKey());
+          CharsRef casedTerm =
+              analyze(
+                  synonymManager.applyCaseSetting(ignoreCase, entry.getKey()),
+                  new CharsRefBuilder());
           for (String mapping : entry.getValue()) {
-            // apply the case setting to match the behavior of the SynonymMap builder
-            CharsRef casedTerm =
-                analyze(
-                    synonymManager.applyCaseSetting(ignoreCase, entry.getKey()),
-                    new CharsRefBuilder());
             CharsRef casedMapping =
                 analyze(
                     synonymManager.applyCaseSetting(ignoreCase, mapping), new CharsRefBuilder());
-            add(casedTerm, casedMapping, false);
+            if (mapsToItself) {
+              if (!casedTerm.equals(casedMapping)) {
+                add(casedTerm, casedMapping, true);
+              }
+            } else {
+              add(casedTerm, casedMapping, false);
+            }
           }
         }
       }

solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java

@@ -247,6 +247,32 @@ public void testManagedSynonyms() throws Exception {
     assertJDelete(endpoint + "/cheerful/joyful", "/error/code==404");
   }
 
+  @Test
+  public void testUpstreamTokenTypeIsPreserved() throws Exception {
+    String endpoint = "/schema/analysis/synonyms/englishgraph";
+
+    List<String> equivGroup = Arrays.asList("funny", "entertaining", "whimsical");
+    assertJPut(endpoint, toJSONString(equivGroup), "/responseHeader/status==0");
+
+    assertJQ(endpoint + "/funny", "/funny==['entertaining','funny','whimsical']");
+
+    restTestHarness.reload();
+
+    String analysisUrl = "/analysis/field?analysis.fieldtype=managed_graph_en&analysis.query=funny";
+    String synFilter = "//arr[contains(@name,'SynonymGraphFilter')]";
+    assertQ(
+        analysisUrl,
+        "count("
+            + synFilter
+            + "/lst[str[@name='text'][.='funny']]/str[@name='type'][.='<ALPHANUM>']) = 1",
+        "count("
+            + synFilter
+            + "/lst[str[@name='text'][.='entertaining']]/str[@name='type'][.='SYNONYM']) = 1",
+        "count("
+            + synFilter
+            + "/lst[str[@name='text'][.='whimsical']]/str[@name='type'][.='SYNONYM']) = 1");
+  }
+
   /** Can we add and remove stopwords with umlauts */
   @Test
   public void testCanHandleDecodingAndEncodingForSynonyms() throws Exception {