NOTE: the {@code Collector} detects segments sorted according to a an {@link - * IndexWriterConfig#setIndexSort}. Also, it collects up to a specified {@code numDocsToCollect} - * from each segment, and therefore is mostly suitable for use in conjunction with collectors such - * as {@link TopDocsCollector}, and not e.g. {@link TotalHitCountCollector}. - * - *
NOTE: If you wrap a {@code TopDocsCollector} that sorts in the same order as the index
- * order, the returned {@link TopDocsCollector#topDocs() TopDocs} will be correct. However the total
- * of {@link TopDocsCollector#getTotalHits() hit count} will be vastly underestimated since not all
- * matching documents will have been collected.
- *
- * @deprecated Use {@link TopFieldCollector} and set trackTotalHits to false.
- * @lucene.experimental
- */
-@Deprecated(since = "7.2")
-final class EarlyTerminatingSortingCollector extends FilterCollector {
-
- /**
- * Returns whether collection can be early-terminated if it sorts with the provided {@link Sort}
- * and if segments are merged with the provided {@link Sort}.
- */
- public static boolean canEarlyTerminate(Sort searchSort, Sort mergePolicySort) {
- final SortField[] fields1 = searchSort.getSort();
- final SortField[] fields2 = mergePolicySort.getSort();
- // early termination is possible if fields1 is a prefix of fields2
- if (fields1.length > fields2.length) {
- return false;
- }
- return Arrays.asList(fields1).equals(Arrays.asList(fields2).subList(0, fields1.length));
- }
-
- /** Sort used to sort the search results */
- private final Sort sort;
-
- /** Number of documents to collect in each segment */
- private final int numDocsToCollect;
-
- private final AtomicBoolean terminatedEarly = new AtomicBoolean(false);
-
- /**
- * Create a new {@link EarlyTerminatingSortingCollector} instance.
- *
- * @param in the collector to wrap
- * @param sort the sort you are sorting the search results on
- * @param numDocsToCollect the number of documents to collect on each segment. When wrapping a
- * {@link TopDocsCollector}, this number should be the number of hits.
- * @throws IllegalArgumentException if the sort order doesn't allow for early termination with the
- * given merge policy.
- */
- public EarlyTerminatingSortingCollector(Collector in, Sort sort, int numDocsToCollect) {
- super(in);
- if (numDocsToCollect <= 0) {
- throw new IllegalArgumentException(
- "numDocsToCollect must always be > 0, got " + numDocsToCollect);
- }
- this.sort = sort;
- this.numDocsToCollect = numDocsToCollect;
- }
-
- @Override
- public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
- Sort segmentSort = context.reader().getMetaData().sort();
- if (segmentSort != null && canEarlyTerminate(sort, segmentSort) == false) {
- throw new IllegalStateException(
- "Cannot early terminate with sort order "
- + sort
- + " if segments are sorted with "
- + segmentSort);
- }
-
- if (segmentSort != null) {
- // segment is sorted, can early-terminate
- return new FilterLeafCollector(super.getLeafCollector(context)) {
- private int numCollected;
-
- @Override
- public void collect(int doc) throws IOException {
- super.collect(doc);
- if (++numCollected >= numDocsToCollect) {
- terminatedEarly.set(true);
- throw new CollectionTerminatedException();
- }
- }
- };
- } else {
- return super.getLeafCollector(context);
- }
- }
-
- public boolean terminatedEarly() {
- return terminatedEarly.get();
- }
-}
diff --git a/solr/core/src/java/org/apache/solr/search/QueryCommand.java b/solr/core/src/java/org/apache/solr/search/QueryCommand.java
index 8442217464b7..dee6b0646d21 100755
--- a/solr/core/src/java/org/apache/solr/search/QueryCommand.java
+++ b/solr/core/src/java/org/apache/solr/search/QueryCommand.java
@@ -211,10 +211,18 @@ public QueryCommand setTerminateEarly(boolean segmentTerminateEarly) {
}
}
+ /**
+ * @deprecated see {@link org.apache.solr.common.params.CommonParams#SEGMENT_TERMINATE_EARLY}
+ */
+ @Deprecated(since = "11.0")
public boolean getSegmentTerminateEarly() {
return (flags & SolrIndexSearcher.SEGMENT_TERMINATE_EARLY) != 0;
}
+ /**
+ * @deprecated see {@link org.apache.solr.common.params.CommonParams#SEGMENT_TERMINATE_EARLY}
+ */
+ @Deprecated(since = "11.0")
public QueryCommand setSegmentTerminateEarly(boolean segmentSegmentTerminateEarly) {
if (segmentSegmentTerminateEarly) {
return setFlags(SolrIndexSearcher.SEGMENT_TERMINATE_EARLY);
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index de4e7c601c59..f3dc756413cc 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -33,6 +33,7 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;
import java.util.function.Function;
@@ -54,10 +55,13 @@
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.CollectionTerminatedException;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.FilterCollector;
+import org.apache.lucene.search.FilterLeafCollector;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.MatchAllDocsQuery;
@@ -285,28 +289,6 @@ private Collector buildAndRunCollectorChain(
DelegatingCollector postFilter)
throws IOException {
- EarlyTerminatingSortingCollector earlyTerminatingSortingCollector = null;
- if (cmd.getSegmentTerminateEarly()) {
- final Sort cmdSort = cmd.getSort();
- final int cmdLen = cmd.getLen();
- final Sort mergeSort = core.getSolrCoreState().getMergePolicySort();
-
- if (cmdSort == null
- || cmdLen <= 0
- || mergeSort == null
- || !EarlyTerminatingSortingCollector.canEarlyTerminate(cmdSort, mergeSort)) {
- log.warn(
- "unsupported combination: segmentTerminateEarly=true cmdSort={} cmdLen={} mergeSort={}",
- cmdSort,
- cmdLen,
- mergeSort);
- } else {
- collector =
- earlyTerminatingSortingCollector =
- new EarlyTerminatingSortingCollector(collector, cmdSort, cmd.getLen());
- }
- }
-
if (cmd.shouldEarlyTerminateSearch()) {
collector = new EarlyTerminatingCollector(collector, cmd.getMaxHitsAllowed());
}
@@ -344,9 +326,6 @@ private Collector buildAndRunCollectorChain(
qr.setPartialResultsDetails(etce.getMessage());
qr.setApproximateTotalHits(etce.getApproximateTotalHits(reader.maxDoc()));
} finally {
- if (earlyTerminatingSortingCollector != null) {
- qr.setSegmentTerminatedEarly(earlyTerminatingSortingCollector.terminatedEarly());
- }
if (cmd.isQueryCancellable()) {
core.getCancellableQueryTracker().removeCancellableQuery(cmd.getQueryID());
}
@@ -1804,6 +1783,24 @@ private void populateNextCursorMarkFromTopDocs(QueryResult qr, QueryCommand qc,
*/
TopDocsCollector buildTopDocsCollector(int len, QueryCommand cmd)
throws IOException {
+ return buildTopDocsCollector(len, cmd, false);
+ }
+
+ /**
+ * @param allowNativeSegmentTerminateEarly if true and {@code cmd} has a sort, forces {@link
+ * TopFieldCollectorManager}'s totalHitsThreshold down to {@code len} so {@link
+ * TopFieldCollector} early-terminates a segment itself once its index sort matches the search
+ * sort (the same effect {@code segmentTerminateEarly=true} used to get from the now- removed
+ * {@code EarlyTerminatingSortingCollector} wrapper). Callers that pass {@code true} must not
+ * further wrap the returned collector together with a sibling collector that doesn't itself
+ * early-terminate (e.g. via {@code MultiCollector}) -- see SOLR-18363: {@code MultiCollector}
+ * only propagates {@code CollectionTerminatedException} once *every* wrapped collector has
+ * thrown it, so an early-terminating {@link TopFieldCollector} paired with a non-terminating
+ * sibling collector would silently stop counting hits early while the sibling kept counting,
+ * desyncing the two.
+ */
+ private TopDocsCollector buildTopDocsCollector(
+ int len, QueryCommand cmd, boolean allowNativeSegmentTerminateEarly) throws IOException {
int minNumFound = cmd.getMinExactCount();
Query q = cmd.getQuery();
if (q instanceof RankQuery rq) {
@@ -1819,11 +1816,74 @@ TopDocsCollector buildTopDocsCollector(int len, QueryCommand
final CursorMark cursor = cmd.getCursorMark();
final FieldDoc searchAfter = (null != cursor ? cursor.getSearchAfterFieldDoc() : null);
+ if (allowNativeSegmentTerminateEarly) {
+ minNumFound = len;
+ }
return new TopFieldCollectorManager(weightedSort, len, searchAfter, minNumFound)
.newCollector();
}
}
+ /**
+ * Whether {@link #buildTopDocsCollector(int, QueryCommand, boolean)} should be asked to force
+ * native per-segment early termination for this command. Also logs the same "unsupported
+ * combination" warning the old wrapper-based check used to log.
+ */
+ private boolean allowNativeSegmentTerminateEarly(QueryCommand cmd, int len) {
+ if (!cmd.getSegmentTerminateEarly()) {
+ return false;
+ }
+ final Sort cmdSort = cmd.getSort();
+ if (cmdSort == null || len <= 0 || cmd.getQuery() instanceof RankQuery) {
+ log.warn(
+ "unsupported combination: segmentTerminateEarly=true cmdSort={} cmdLen={} query={}",
+ cmdSort,
+ len,
+ cmd.getQuery().getClass().getSimpleName());
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Records whether a wrapped collector's per-segment collection was actually cut short by a {@link
+ * CollectionTerminatedException}. Used with {@link #buildTopDocsCollector(int, QueryCommand,
+ * boolean)}'s {@code allowNativeSegmentTerminateEarly=true} path: {@link TopFieldCollector} only
+ * ever throws that exception from its sort-compatibility-gated fast path (see {@code
+ * TopFieldCollector.TopFieldLeafCollector#thresholdCheck}), never from the routine "totalHits
+ * exceeded totalHitsThreshold" bookkeeping alone -- so observing the exception itself gives the
+ * same precise "a segment was actually skipped" signal the removed {@code
+ * EarlyTerminatingSortingCollector} tracked directly, without needing
+ * TopFieldCollector#isEarlyTerminated() (which conflates that signal with the routine case) or
+ * any of Lucene's package-private sort-compatibility check.
+ */
+ private static final class SegmentTerminatedEarlyObserver extends FilterCollector {
+ private final AtomicBoolean terminatedEarly = new AtomicBoolean(false);
+
+ SegmentTerminatedEarlyObserver(Collector in) {
+ super(in);
+ }
+
+ boolean terminatedEarly() {
+ return terminatedEarly.get();
+ }
+
+ @Override
+ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+ return new FilterLeafCollector(super.getLeafCollector(context)) {
+ @Override
+ public void collect(int doc) throws IOException {
+ try {
+ super.collect(doc);
+ } catch (CollectionTerminatedException e) {
+ terminatedEarly.set(true);
+ throw e;
+ }
+ }
+ };
+ }
+ }
+
private void getDocListNC(QueryResult qr, QueryCommand cmd) throws IOException {
final int len = cmd.getSupersetMaxDoc();
int last = len;
@@ -1895,16 +1955,28 @@ public ScoreMode scoreMode() {
final ScoreMode scoreModeUsed;
if (!MultiThreadedSearcher.allowMT(pf.postFilter, cmd, getTaskExecutor())) {
log.trace("SINGLE THREADED search, skipping collector manager in getDocListNC");
- final TopDocsCollector topCollector = buildTopDocsCollector(len, cmd);
+ final boolean nativeSegmentTerminateEarly = allowNativeSegmentTerminateEarly(cmd, len);
+ final TopDocsCollector topCollector =
+ buildTopDocsCollector(len, cmd, nativeSegmentTerminateEarly);
+ SegmentTerminatedEarlyObserver terminatedEarlyObserver = null;
+ Collector observedTopCollector = topCollector;
+ if (nativeSegmentTerminateEarly) {
+ observedTopCollector =
+ terminatedEarlyObserver = new SegmentTerminatedEarlyObserver(topCollector);
+ }
MaxScoreCollector maxScoreCollector = null;
- Collector collector = topCollector;
+ Collector collector = observedTopCollector;
if (needScores) {
maxScoreCollector = new MaxScoreCollector();
- collector = MultiCollector.wrap(topCollector, maxScoreCollector);
+ collector = MultiCollector.wrap(observedTopCollector, maxScoreCollector);
}
scoreModeUsed =
buildAndRunCollectorChain(qr, query, collector, cmd, pf.postFilter).scoreMode();
+ if (terminatedEarlyObserver != null) {
+ qr.setSegmentTerminatedEarly(terminatedEarlyObserver.terminatedEarly());
+ }
+
totalHits = topCollector.getTotalHits();
topDocs = topCollector.topDocs(0, len);
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc b/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc
index 7c8f1765246a..e757731626d1 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc
@@ -435,6 +435,12 @@ of the distributed query processing.
== segmentTerminateEarly Parameter
+WARNING: Deprecated since 11.0. Its implementation relies on internal Lucene behavior that cannot
+currently be replicated using Lucene's own public early-termination support (see SOLR-18363).
+Use <