Introduce kind: text requests for BM25 over a text index

README.md

@@ -346,6 +346,52 @@ at least two stages, `weights` needs one entry per stage, and `rrf_k` and
 `weights` need `fusion: rrf`. A single prefetch without `fusion` stays the
 two-stage rescore described above.
 
+#### BM25 over a text index
+
+A `text` request ranks points by BM25 over a payload field's text index, the
+`Query.text` variant of Qdrant 1.19.3+. `using` names the payload field, whose
+index needs `scoring: bm25` in the upload config:
+
+```yaml
+requests:
+  - kind: text
+    using: body                    # payload field with a scoring text index
+    k: 1.2                         # optional, BM25 term-frequency saturation
+    b: 0.75                        # optional, BM25 length normalization, 0..1
+    source:                        # random words, or `type: file`
+      type: random
+      vocab_size: 20000
+      min_length: 2
+      max_length: 5
+      distribution: zipf
+```
+
+```yaml
+fields:
+  - name: body
+    type: text
+    tokenizer: word
+    scoring: bm25                  # rank by this field; implies phrase matching
+    lowercase: true                # optional tokenizer options
+    stopwords: english
+    stemmer: english
+```
+
+A random source draws `word_<id>` words, the vocabulary text payloads are generated
+from, so the queries hit the corpus. `source: { type: file, path: queries.txt }`
+reads one query per line instead (blank lines skipped; `strategy: from-start` or
+`random-sample`), for a real query set. Matching is any-term: a point scores when it
+holds one of the query's words, and terms it must or must not hold belong in
+`filters`. BM25 over a text index is exact, so `--search-quality` has no exact
+reference to compare against and is refused with a `text` request, and so is
+`--prefetch`, which rescores a vector search. `lowercase`, `stopwords`, `stemmer`
+and `scoring` apply to `type: text` fields only.
+
+```bash
+bfb upload --example upload-bm25-text -n 200k --uri http://localhost:6334
+bfb search --example search-bm25-text -n 10k --search-limit 10 --uri http://localhost:6334
+```
+
 #### ACORN
 
 `--acorn` asks Qdrant to use ACORN for filtered HNSW search, and

examples/search-bm25-text.yaml

@@ -0,0 +1,21 @@
+# Example bfb search config for BM25 over a text index (Qdrant 1.19.3+), matching
+# upload-bm25-text. A point scores when it holds any of the query's words; terms
+# a point must or must not hold belong in `filters`.
+#
+#   bfb search --example search-bm25-text -n 10k --search-limit 10 -p 8 \
+#       --uri http://localhost:6334
+
+collection:
+  name: benchmark
+
+requests:
+  - kind: text
+    using: body                    # the payload field whose text index ranks points
+    # k: 1.2                       # BM25 term-frequency saturation (server default)
+    # b: 0.75                      # BM25 length normalization, 0..1 (server default)
+    source:
+      type: random                 # random | file (one query per line)
+      vocab_size: 20000
+      min_length: 2                # words per query
+      max_length: 5
+      distribution: zipf

examples/upload-bm25-text.yaml

@@ -0,0 +1,30 @@
+# Example bfb upload config for BM25 over a text index (Qdrant 1.19.3+): every
+# point carries a zipf-distributed text field indexed with `scoring: bm25`, so
+# `kind: text` search requests can rank by it (see search-bm25-text). The dense
+# vector is a placeholder the text query never reads.
+#
+#   bfb upload --example upload-bm25-text -n 200k -b 256 -p 8 -t 4 \
+#       --uri http://localhost:6334
+
+collection:
+  name: benchmark
+
+  vectors:
+    - size: 4
+      distance: dot
+      source: random
+
+  fields:
+    - name: body
+      type: text
+      tokenizer: word
+      scoring: bm25                # rank by BM25; implies phrase matching
+      # lowercase: true            # tokenizer options, as for any text index
+      # stopwords: english
+      # stemmer: english
+      source:
+        type: random
+        vocab_size: 20000
+        min_length: 20             # words per document
+        max_length: 200
+        distribution: zipf

src/collection/from_config.rs

@@ -27,7 +27,7 @@ use crate::args::Args;
 use crate::client::random_client;
 use crate::config::{
     ComparatorKind, DatatypeKind, DistanceKind, MemoryKind, ModifierKind, PayloadType, QuantKind,
-    TokenizerKind, UploadConfig, VectorConfig,
+    TextScoringKind, TokenizerKind, UploadConfig, VectorConfig,
 };
 
 fn distance_to_grpc(distance: DistanceKind) -> Distance {
@@ -442,6 +442,18 @@ async fn create_field_indices_from_config(
                 if let Some(memory) = memory {
                     params = params.memory(memory);
                 }
+                if let Some(lowercase) = pc.lowercase {
+                    params = params.lowercase(lowercase);
+                }
+                if let Some(language) = &pc.stopwords {
+                    params = params.stopwords_language(language.clone());
+                }
+                if let Some(language) = &pc.stemmer {
+                    params = params.snowball_stemmer(language.clone());
+                }
+                if let Some(TextScoringKind::Bm25) = pc.scoring {
+                    params = params.bm25_scoring();
+                }
                 CreateFieldIndexCollectionBuilder::new(name, field, FieldType::Text)
                     .field_index_params(params)
             }

src/config/examples.rs

@@ -100,6 +100,16 @@ pub static EXAMPLES: &[Example] = &[
         Search,
         "Search requests matching upload-config (dense, sparse, filters)"
     ),
+    example!(
+        "upload-bm25-text",
+        Upload,
+        "A text field indexed for BM25 ranking (Qdrant 1.19.3+)"
+    ),
+    example!(
+        "search-bm25-text",
+        Search,
+        "BM25 over a text index, matching upload-bm25-text"
+    ),
     example!(
         "search-dataset-accuracy",
         Search,

src/config/mod.rs

@@ -15,7 +15,7 @@ pub mod search;
 pub mod vector;
 
 pub use collection::{CollectionConfig, IdType, MemoryKind, QuantKind};
-pub use payload::{PayloadSource, PayloadSourceKind, PayloadType, TokenizerKind};
+pub use payload::{PayloadSource, PayloadSourceKind, PayloadType, TextScoringKind, TokenizerKind};
 pub use vector::{
     ComparatorKind, DatatypeKind, DistanceKind, DistributionKind, FileStrategy, ModifierKind,
     SparseKind, SparseSource, VectorConfig, VectorSource,
@@ -136,6 +136,33 @@ impl UploadConfig {
             if !payload_names.insert(p.name.clone()) {
                 bail!("duplicate payload field name {:?}", p.name);
             }
+            // Text index options describe a text index; anywhere else they would be
+            // silently dropped.
+            let text_options = p.lowercase.is_some()
+                || p.stopwords.is_some()
+                || p.stemmer.is_some()
+                || p.scoring.is_some();
+            if text_options && p.kind != PayloadType::Text {
+                bail!(
+                    "payload {:?}: `lowercase`, `stopwords`, `stemmer` and `scoring` apply to \
+                     `type: text` fields only",
+                    p.name
+                );
+            }
+            if text_options && !p.index {
+                bail!(
+                    "payload {:?}: text index options need `index: true`",
+                    p.name
+                );
+            }
+            for (key, value) in [("stopwords", &p.stopwords), ("stemmer", &p.stemmer)] {
+                if value.as_deref().is_some_and(str::is_empty) {
+                    bail!(
+                        "payload {:?}: `{key}` names a language and must not be empty",
+                        p.name
+                    );
+                }
+            }
             let Some(src) = &p.source else {
                 // No per-field source: values come from `payload.source` (or,
                 // absent that, are randomly generated). Nothing to validate.
@@ -280,6 +307,50 @@ fn default_custom() -> String {
 mod tests {
     use super::*;
 
+    #[test]
+    fn text_index_options_apply_to_text_fields_only() {
+        let config =
+            |field: &str| format!("collection:\n  vectors:\n    - size: 4\n  fields:\n{field}");
+        let ok = parse(
+            &config("    - name: body\n      type: text\n      lowercase: true\n      stopwords: english\n      stemmer: english\n      scoring: bm25\n      source: random\n"),
+            "test",
+        )
+        .unwrap();
+        let body = &ok.collection.fields[0];
+        assert_eq!(body.scoring, Some(TextScoringKind::Bm25));
+        assert_eq!(
+            (
+                body.stopwords.as_deref(),
+                body.stemmer.as_deref(),
+                body.lowercase
+            ),
+            (Some("english"), Some("english"), Some(true))
+        );
+        let err = parse(
+            &config("    - name: color\n      type: keyword\n      scoring: bm25\n      source: random\n"),
+            "test",
+        )
+        .unwrap_err();
+        assert!(
+            format!("{err:#}").contains("`type: text` fields only"),
+            "{err:#}"
+        );
+        let err = parse(
+            &config("    - name: body\n      type: text\n      index: false\n      scoring: bm25\n      source: random\n"),
+            "test",
+        )
+        .unwrap_err();
+        assert!(format!("{err:#}").contains("need `index: true`"), "{err:#}");
+        let err = parse(
+            &config(
+                "    - name: body\n      type: text\n      stemmer: ''\n      source: random\n",
+            ),
+            "test",
+        )
+        .unwrap_err();
+        assert!(format!("{err:#}").contains("must not be empty"), "{err:#}");
+    }
+
     #[test]
     fn parses_minimal_config() {
         let yaml = r#"

src/config/payload.rs

@@ -36,6 +36,19 @@ pub struct PayloadConfig {
     pub prefix: bool,
     /// Text payloads: tokenizer.
     pub tokenizer: Option<TokenizerKind>,
+    /// Text payloads: lowercase tokens. Server default: true.
+    #[serde(default)]
+    pub lowercase: Option<bool>,
+    /// Text payloads: drop the stopwords of this language, e.g. `english`.
+    #[serde(default)]
+    pub stopwords: Option<String>,
+    /// Text payloads: stem tokens with this language's Snowball stemmer, e.g. `english`.
+    #[serde(default)]
+    pub stemmer: Option<String>,
+    /// Text payloads: rank points by this field, so `kind: text` search requests
+    /// can query it (Qdrant 1.19.3+). Implies phrase matching.
+    #[serde(default)]
+    pub scoring: Option<TextScoringKind>,
     /// Per-field value source. May be omitted when `collection.payload.source`
     /// provides the payload object — the entry is then index-only.
     #[serde(default, deserialize_with = "option_string_or_struct")]
@@ -55,6 +68,13 @@ pub enum PayloadType {
     Datetime,
 }
 
+/// How a text index ranks points for `kind: text` queries.
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "snake_case")]
+pub enum TextScoringKind {
+    Bm25,
+}
+
 #[derive(Debug, Clone, Copy, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum TokenizerKind {

src/config/schema.rs

@@ -197,6 +197,10 @@ collection:
       prefix: false              # bool           default=false        keyword payloads: enable prefix matching
                                  #   (required for search `match_prefix` filters)
       tokenizer: null            # enum           optional (text)      [word | whitespace | prefix | multilingual]
+      lowercase: null            # bool           optional (text)      lowercase tokens (server default true)
+      stopwords: null            # string         optional (text)      stopword language, e.g. english
+      stemmer: null              # string         optional (text)      Snowball stemmer language, e.g. english
+      scoring: null              # enum           optional (text)      [bm25] rank by this field (search `kind: text`)
       # Value source (optional when `payload.source` is set — then the entry is
       # index-only). Shorthand string `random` / `random-clusters` / `now`, or a
       # map. Which keys apply depends on the payload `type`; others ignored.
@@ -303,34 +307,59 @@ mod tests {
                     source: None,
                     memory: Some(MemoryKind::Cached),
                 },
-                fields: vec![PayloadConfig {
-                    name: "color".to_string(),
-                    kind: PayloadType::Keyword,
-                    index: true,
-                    on_disk: false,
-                    memory: Some(MemoryKind::Cached),
-                    is_tenant: false,
-                    is_principal: false,
-                    range_index: true,
-                    prefix: true,
-                    tokenizer: Some(TokenizerKind::Word),
-                    source: Some(PayloadSource {
-                        kind: PayloadSourceKind::Random,
-                        dataset: None,
-                        field: None,
-                        cardinality: Some(100),
-                        length_multiplier: Some(1),
-                        values_per_point: Some(1),
-                        min: Some(0.0),
-                        max: Some(1.0),
-                        true_ratio: Some(0.5),
-                        clusters: Some(10),
-                        vocab_size: Some(1000),
-                        min_length: Some(1),
-                        max_length: Some(10),
-                        distribution: DistributionKind::Uniform,
-                    }),
-                }],
+                fields: vec![
+                    PayloadConfig {
+                        name: "color".to_string(),
+                        kind: PayloadType::Keyword,
+                        index: true,
+                        on_disk: false,
+                        memory: Some(MemoryKind::Cached),
+                        is_tenant: false,
+                        is_principal: false,
+                        range_index: true,
+                        prefix: true,
+                        tokenizer: Some(TokenizerKind::Word),
+                        lowercase: None,
+                        stopwords: None,
+                        stemmer: None,
+                        scoring: None,
+                        source: Some(PayloadSource {
+                            kind: PayloadSourceKind::Random,
+                            dataset: None,
+                            field: None,
+                            cardinality: Some(100),
+                            length_multiplier: Some(1),
+                            values_per_point: Some(1),
+                            min: Some(0.0),
+                            max: Some(1.0),
+                            true_ratio: Some(0.5),
+                            clusters: Some(10),
+                            vocab_size: Some(1000),
+                            min_length: Some(1),
+                            max_length: Some(10),
+                            distribution: DistributionKind::Uniform,
+                        }),
+                    },
+                    // The text index options are refused on any other type, so they
+                    // need a text field of their own.
+                    PayloadConfig {
+                        name: "body".to_string(),
+                        kind: PayloadType::Text,
+                        index: true,
+                        on_disk: false,
+                        memory: None,
+                        is_tenant: false,
+                        is_principal: false,
+                        range_index: true,
+                        prefix: false,
+                        tokenizer: Some(TokenizerKind::Word),
+                        lowercase: Some(true),
+                        stopwords: Some("english".to_string()),
+                        stemmer: Some("english".to_string()),
+                        scoring: Some(TextScoringKind::Bm25),
+                        source: None,
+                    },
+                ],
             },
         }
     }