From 5cc306347d4c34f95ff9c965234de63bcaaae67c Mon Sep 17 00:00:00 2001 From: Daniil Porokhnin Date: Tue, 13 Oct 2026 00:08:55 +0300 Subject: [PATCH] perf: optimize execution tree --- frac/processor/batch_eval_tree.go | 23 +- frac/processor/batch_eval_tree_test.go | 22 +- frac/processor/eval_tree.go | 7 +- frac/processor/resolver.go | 321 +++++++++++++++++++++++++ frac/processor/resolver_test.go | 215 +++++++++++++++++ frac/processor/search.go | 29 ++- 6 files changed, 584 insertions(+), 33 deletions(-) create mode 100644 frac/processor/resolver.go create mode 100644 frac/processor/resolver_test.go diff --git a/frac/processor/batch_eval_tree.go b/frac/processor/batch_eval_tree.go index 421160f2c..9d992173b 100644 --- a/frac/processor/batch_eval_tree.go +++ b/frac/processor/batch_eval_tree.go @@ -16,8 +16,6 @@ var errBatchingUnsupported = errors.New("batching unsupported") // maxBatchedTIDsPerLeaf limits number of TIDs for a single OrMulti node const maxBatchedTIDsPerLeaf = 5 -type leafTIDsCache map[parser.Token][]uint32 - // tryBuildBatchEvalTree tries to build a batched eval tree if possible. // // Returns errBatchingUnsupported when the non-batched path should be used. @@ -25,6 +23,7 @@ func tryBuildBatchEvalTree( root *parser.ASTNode, fracVer config.BinaryDataVersion, ti tokenIndex, + cache tidsCache, queryOpts QueryOptimizationConfig, minLID, maxLID uint32, stats *searchStats, @@ -43,7 +42,6 @@ func tryBuildBatchEvalTree( return nil, errBatchingUnsupported } - cache := make(leafTIDsCache) cost, err := calculateQueryIterationCost(root, ti, cache) if err != nil { return nil, err @@ -86,7 +84,7 @@ func astSupportsBatching(root *parser.ASTNode) bool { } } -func calculateQueryIterationCost(root *parser.ASTNode, ti tokenIndex, cache leafTIDsCache) (uint64, error) { +func calculateQueryIterationCost(root *parser.ASTNode, ti tokenIndex, cache tidsCache) (uint64, error) { if root == nil { return 0, fmt.Errorf("empty AST") } @@ -126,8 +124,8 @@ func calculateQueryIterationCost(root *parser.ASTNode, ti tokenIndex, cache leaf } } -func leafIterationCost(ti tokenIndex, field string, token parser.Token, cache leafTIDsCache) (uint64, error) { - tids, err := ti.GetTIDsByTokenExpr(token) +func leafIterationCost(ti tokenIndex, field string, token parser.Token, cache tidsCache) (uint64, error) { + tids, err := cache.get(ti, token) if err != nil { return 0, err } @@ -136,7 +134,6 @@ func leafIterationCost(ti tokenIndex, field string, token parser.Token, cache le if len(tids) > maxBatchedTIDsPerLeaf { return 0, errBatchingUnsupported } - cache[token] = tids if len(tids) == 0 { return 0, nil } @@ -195,7 +192,7 @@ func buildBatchEvalTree( func evalBatchLeaf( ti tokenIndex, token parser.Token, - cache leafTIDsCache, + cache tidsCache, sw *stopwatch.Stopwatch, stats *searchStats, minLID, maxLID uint32, @@ -203,13 +200,9 @@ func evalBatchLeaf( ) (node.BatchedNode, error) { stats.LeavesTotal++ - tids, ok := cache[token] - if !ok { - var err error - tids, err = ti.GetTIDsByTokenExpr(token) - if err != nil { - return nil, err - } + tids, err := cache.get(ti, token) + if err != nil { + return nil, err } if len(tids) == 0 { diff --git a/frac/processor/batch_eval_tree_test.go b/frac/processor/batch_eval_tree_test.go index 77b15dc6a..0b978c602 100644 --- a/frac/processor/batch_eval_tree_test.go +++ b/frac/processor/batch_eval_tree_test.go @@ -107,7 +107,7 @@ func TestQueryIterationCost(t *testing.T) { t.Run("leaf", func(t *testing.T) { q, err := parser.ParseSeqQL(`a:1`, nil) require.NoError(t, err) - cost, err := calculateQueryIterationCost(q.Root, index, make(leafTIDsCache)) + cost, err := calculateQueryIterationCost(q.Root, index, make(tidsCache)) require.NoError(t, err) assert.Equal(t, uint64(80_000), cost) }) @@ -115,7 +115,7 @@ func TestQueryIterationCost(t *testing.T) { t.Run("and", func(t *testing.T) { q, err := parser.ParseSeqQL(`a:1 AND b:2`, nil) require.NoError(t, err) - cost, err := calculateQueryIterationCost(q.Root, index, make(leafTIDsCache)) + cost, err := calculateQueryIterationCost(q.Root, index, make(tidsCache)) require.NoError(t, err) assert.Equal(t, uint64(80_000), cost) }) @@ -123,7 +123,7 @@ func TestQueryIterationCost(t *testing.T) { t.Run("or", func(t *testing.T) { q, err := parser.ParseSeqQL(`a:1 OR b:2`, nil) require.NoError(t, err) - cost, err := calculateQueryIterationCost(q.Root, index, make(leafTIDsCache)) + cost, err := calculateQueryIterationCost(q.Root, index, make(tidsCache)) require.NoError(t, err) assert.Equal(t, uint64(200_000), cost) }) @@ -131,7 +131,7 @@ func TestQueryIterationCost(t *testing.T) { t.Run("and not", func(t *testing.T) { q, err := parser.ParseSeqQL(`a:1 AND NOT b:2`, nil) require.NoError(t, err) - cost, err := calculateQueryIterationCost(q.Root, index, make(leafTIDsCache)) + cost, err := calculateQueryIterationCost(q.Root, index, make(tidsCache)) require.NoError(t, err) assert.Equal(t, uint64(200_000), cost) }) @@ -139,7 +139,7 @@ func TestQueryIterationCost(t *testing.T) { t.Run("nested and-or", func(t *testing.T) { q, err := parser.ParseSeqQL(`a:1 AND (b:2 OR c:3)`, nil) require.NoError(t, err) - cost, err := calculateQueryIterationCost(q.Root, index, make(leafTIDsCache)) + cost, err := calculateQueryIterationCost(q.Root, index, make(tidsCache)) require.NoError(t, err) assert.Equal(t, uint64(80_000), cost) }) @@ -175,7 +175,7 @@ func TestTryBuildBatchEvalTree(t *testing.T) { q, err := parser.ParseSeqQL(`a:1 AND b:2`, nil) require.NoError(t, err) stats := &searchStats{} - tree, err := tryBuildBatchEvalTree(q.Root, config.BinaryDataV6, denseIndex, queryOpt, 1, 100, stats, seq.DocsOrderDesc, sw) + tree, err := tryBuildBatchEvalTree(q.Root, config.BinaryDataV6, denseIndex, make(tidsCache), queryOpt, 1, 100, stats, seq.DocsOrderDesc, sw) require.NoError(t, err) assert.NotNil(t, tree) assert.Equal(t, 2, stats.LeavesTotal) @@ -187,7 +187,7 @@ func TestTryBuildBatchEvalTree(t *testing.T) { q, err := parser.ParseSeqQL(`a:1 AND b:2`, nil) require.NoError(t, err) stats := &searchStats{} - tree, err := tryBuildBatchEvalTree(q.Root, config.BinaryDataV6, denseIndex, disabled, 1, 100, stats, seq.DocsOrderDesc, sw) + tree, err := tryBuildBatchEvalTree(q.Root, config.BinaryDataV6, denseIndex, make(tidsCache), disabled, 1, 100, stats, seq.DocsOrderDesc, sw) require.ErrorIs(t, err, errBatchingUnsupported) assert.Nil(t, tree) assert.Equal(t, 0, stats.LeavesTotal) @@ -198,7 +198,7 @@ func TestTryBuildBatchEvalTree(t *testing.T) { q, err := parser.ParseSeqQL(`a:1 AND b:2`, nil) require.NoError(t, err) stats := &searchStats{} - tree, err := tryBuildBatchEvalTree(q.Root, config.BinaryDataV6, sparseIndex, queryOpt, 1, 100, stats, seq.DocsOrderDesc, sw) + tree, err := tryBuildBatchEvalTree(q.Root, config.BinaryDataV6, sparseIndex, make(tidsCache), queryOpt, 1, 100, stats, seq.DocsOrderDesc, sw) require.ErrorIs(t, err, errBatchingUnsupported) assert.Nil(t, tree) assert.Equal(t, 0, stats.LeavesTotal) @@ -213,7 +213,7 @@ func TestTryBuildBatchEvalTree(t *testing.T) { q, err := parser.ParseSeqQL(`a:1`, nil) require.NoError(t, err) stats := &searchStats{} - tree, err := tryBuildBatchEvalTree(q.Root, config.BinaryDataV6, index, queryOpt, 1, 100, stats, seq.DocsOrderDesc, sw) + tree, err := tryBuildBatchEvalTree(q.Root, config.BinaryDataV6, index, make(tidsCache), queryOpt, 1, 100, stats, seq.DocsOrderDesc, sw) require.ErrorIs(t, err, errBatchingUnsupported) assert.Nil(t, tree) assert.Equal(t, 0, stats.LeavesTotal) @@ -239,7 +239,7 @@ func TestTryBuildBatchEvalTree(t *testing.T) { q, err := parser.ParseSeqQL(`a:1 AND b:2`, nil) require.NoError(t, err) stats := &searchStats{} - tree, err := tryBuildBatchEvalTree(q.Root, config.BinaryDataV6, index, queryOpt, 1, 100, stats, seq.DocsOrderDesc, sw) + tree, err := tryBuildBatchEvalTree(q.Root, config.BinaryDataV6, index, make(tidsCache), queryOpt, 1, 100, stats, seq.DocsOrderDesc, sw) require.ErrorIs(t, err, errBatchingUnsupported) assert.Nil(t, tree) assert.Equal(t, 0, stats.LeavesTotal) @@ -250,7 +250,7 @@ func TestTryBuildBatchEvalTree(t *testing.T) { q, err := parser.ParseSeqQL(`NOT a:1`, nil) require.NoError(t, err) stats := &searchStats{} - tree, err := tryBuildBatchEvalTree(q.Root, config.BinaryDataV6, denseIndex, queryOpt, 1, 100, stats, seq.DocsOrderDesc, sw) + tree, err := tryBuildBatchEvalTree(q.Root, config.BinaryDataV6, denseIndex, make(tidsCache), queryOpt, 1, 100, stats, seq.DocsOrderDesc, sw) require.ErrorIs(t, err, errBatchingUnsupported) assert.Nil(t, tree) assert.Equal(t, 0, stats.LeavesTotal) diff --git a/frac/processor/eval_tree.go b/frac/processor/eval_tree.go index 5703abd00..efa682464 100644 --- a/frac/processor/eval_tree.go +++ b/frac/processor/eval_tree.go @@ -64,19 +64,18 @@ func buildEvalTree(root *parser.ASTNode, minVal, maxVal uint32, stats *searchSta func evalLeaf( ti tokenIndex, token parser.Token, + cache tidsCache, sw *stopwatch.Stopwatch, stats *searchStats, minLID, maxLID uint32, order seq.DocsOrder, ) (node.Node, error) { - m := sw.Start("get_tids_by_token_expr") - tids, err := ti.GetTIDsByTokenExpr(token) - m.Stop() + tids, err := cache.get(ti, token) if err != nil { return nil, err } - m = sw.Start("get_lids_from_tids") + m := sw.Start("get_lids_from_tids") lidsTids := ti.GetLIDsFromTIDs(tids, stats, minLID, maxLID, order) m.Stop() diff --git a/frac/processor/resolver.go b/frac/processor/resolver.go new file mode 100644 index 000000000..243333ae6 --- /dev/null +++ b/frac/processor/resolver.go @@ -0,0 +1,321 @@ +package processor + +import ( + "cmp" + "fmt" + "math" + "slices" + + "github.com/ozontech/seq-db/metric/stopwatch" + "github.com/ozontech/seq-db/parser" +) + +const ( + // NOTE(dkharms): These costs were picked empirically. + resolveCostExact = 1 // `foo:x` is resolved via binary search over a token range + resolveCostPrefix = 10 // `foo:x*` is resolved via binary search plus the matching token range + resolveCostScan = 1000 // `foo:*x*` is resolved by matching against every token block of the field +) + +type tidsCache map[parser.Token][]uint32 + +func (c tidsCache) get(ti tokenIndex, token parser.Token) ([]uint32, error) { + if tids, ok := c[token]; ok { + return tids, nil + } + + tids, err := ti.GetTIDsByTokenExpr(token) + if err != nil { + return nil, err + } + + c[token] = tids + return tids, nil +} + +// planNode is an intermediate representation of AST. +// Basically, all AND-clauses are flattened for convenience. +type planNode struct { + ast *parser.ASTNode + children []*planNode + + pos int + cost int + size uint64 +} + +func astResolve( + root *parser.ASTNode, ti tokenIndex, + cache tidsCache, sw *stopwatch.Stopwatch, +) (*parser.ASTNode, bool, error) { + plan := flatten(root) + sortByCost(plan) + + r := astResolver{ti: ti, cache: cache, sw: sw} + empty, err := r.estimateNodeSize(plan) + if err != nil || empty { + return nil, empty, err + } + + return rebuild(plan), false, nil +} + +func flatten(n *parser.ASTNode) *planNode { + p := &planNode{ast: n} + children := n.Children + + if isAnd(n) { + children = flattenAnd(n, nil) + } + + for i, c := range children { + child := flatten(c) + child.pos = i + p.children = append(p.children, child) + } + + return p +} + +func flattenAnd(n *parser.ASTNode, out []*parser.ASTNode) []*parser.ASTNode { + if !isAnd(n) { + return append(out, n) + } + + for _, c := range n.Children { + out = flattenAnd(c, out) + } + + return out +} + +func sortByCost(p *planNode) { + for _, c := range p.children { + sortByCost(c) + } + + p.cost = resolveCost(p) + if !isAnd(p.ast) { + return + } + + // TODO(dkharms): We need to put NOT at the end, + // because NOT-node almost never produces empty result. + slices.SortStableFunc(p.children, func(a, b *planNode) int { + return cmp.Compare(a.cost, b.cost) + }) +} + +func resolveCost(p *planNode) int { + switch t := p.ast.Value.(type) { + case *parser.Literal: + switch { + case len(t.Terms) == 1 && t.Terms[0].Kind == parser.TermText: + return resolveCostExact + case t.Terms[0].Kind == parser.TermText: + return resolveCostPrefix + } + return resolveCostScan + + case *parser.Re: + if len(t.Prefix.Value) > 0 && !t.Prefix.Foldable { + return resolveCostPrefix + } + return resolveCostScan + + case *parser.Logical: + switch t.Operator { + case parser.LogicalAnd: + return slices.MinFunc(p.children, func(a, b *planNode) int { + return cmp.Compare(a.cost, b.cost) + }).cost + case parser.LogicalOr: + return p.children[0].cost + p.children[1].cost + case parser.LogicalNAnd: + // TODO(dkharms): I need to transform NAND into X AND NOT Y + // to calculate resolve cost correctly. + // + // For example, consider query: `AND(NAND(neg=bar:*y*, reg=foo:x), baz:z*)`, + // resolve cost for NAND is one (since reg=foo:x), but we would have to resolve `bar` as well, + // while `baz` is cheaper to resolve and might return empty set. + // + // Once again, NOT-propagation, NAND-conjuction at parsing time was is not a good idea :) + return p.children[1].cost + case parser.LogicalNot: + return p.children[0].cost + } + } + + return resolveCostScan +} + +type astResolver struct { + ti tokenIndex + cache tidsCache + sw *stopwatch.Stopwatch +} + +func (r *astResolver) estimateNodeSize(p *planNode) (bool, error) { + logical, ok := p.ast.Value.(*parser.Logical) + if !ok { + return r.estimateLeafSize(p) + } + + switch logical.Operator { + case parser.LogicalAnd: + return r.estimateAndSize(p) + case parser.LogicalOr: + return r.estimateOrSize(p) + case parser.LogicalNAnd: + return r.estimateNAndSize(p) + case parser.LogicalNot: + // NOTE(dkharms): This is a pessimistic estimate since operator + // NOT may yield the whole LID range regardless of its child. + p.size = math.MaxUint32 + _, err := r.estimateNodeSize(p.children[0]) + return false, err + } + + return false, fmt.Errorf("unknown logical operator: %v", logical.Operator) +} + +func (r *astResolver) estimateLeafSize(p *planNode) (bool, error) { + m := r.sw.Start("get_tids_by_token_expr") + tids, err := r.cache.get(r.ti, p.ast.Value) + m.Stop() + + if err != nil || len(tids) == 0 { + return len(tids) == 0, err + } + + p.size = r.leafSize(p.ast.Value, tids) + return false, nil +} + +func (r *astResolver) leafSize(token parser.Token, tids []uint32) (size uint64) { + field := parser.GetField(token) + for _, freq := range r.ti.GetFreqsByTIDs(tids, field) { + size += uint64(freq) + } + return size +} + +func (r *astResolver) estimateAndSize(p *planNode) (bool, error) { + p.size = math.MaxUint64 + for _, c := range p.children { + empty, err := r.estimateNodeSize(c) + if err != nil { + return false, err + } + + if empty { + // NOTE(dkharms): This assignment is required for queries + // like `NOT (X AND Y)`. If Y is empty then we will build + // subtree which is equivalent to NOT(Y) (or NOT(EMPTY)). + p.children = []*planNode{c} + return true, nil + } + + p.size = min(p.size, c.size) + } + return false, nil +} + +func (r *astResolver) estimateOrSize(p *planNode) (bool, error) { + leftEmpty, err := r.estimateNodeSize(p.children[0]) + if err != nil { + return false, err + } + + rightEmpty, err := r.estimateNodeSize(p.children[1]) + if err != nil { + return false, err + } + + switch { + case leftEmpty && rightEmpty: + p.children = p.children[:1] + return true, nil + case leftEmpty: + p.children = p.children[1:] + case rightEmpty: + p.children = p.children[:1] + } + + for _, c := range p.children { + p.size += c.size + } + + return false, nil +} + +func (r *astResolver) estimateNAndSize(p *planNode) (bool, error) { + neg, reg := p.children[0], p.children[1] + regEmpty, err := r.estimateNodeSize(reg) + if err != nil { + return false, err + } + + if regEmpty { + p.children = p.children[1:] + return true, nil + } + + negEmpty, err := r.estimateNodeSize(neg) + if err != nil { + return false, err + } + + if negEmpty { + p.children = p.children[1:] + } + + p.size = reg.size + return false, nil +} + +func rebuild(p *planNode) *parser.ASTNode { + logical, ok := p.ast.Value.(*parser.Logical) + if !ok { + return p.ast + } + + if len(p.children) == 1 && logical.Operator != parser.LogicalNot { + return rebuild(p.children[0]) + } + + if logical.Operator == parser.LogicalAnd { + return rebuildAnd(p) + } + + children := make([]*parser.ASTNode, 0, len(p.children)) + for _, c := range p.children { + children = append(children, rebuild(c)) + } + + return &parser.ASTNode{Value: p.ast.Value, Children: children} +} + +func rebuildAnd(p *planNode) *parser.ASTNode { + slices.SortFunc(p.children, func(a, b *planNode) int { + return cmp.Or( + cmp.Compare(a.size, b.size), + cmp.Compare(a.pos, b.pos), + ) + }) + + root := rebuild(p.children[0]) + for _, c := range p.children[1:] { + root = &parser.ASTNode{ + Value: &parser.Logical{Operator: parser.LogicalAnd}, + Children: []*parser.ASTNode{root, rebuild(c)}, + } + } + + return root +} + +func isAnd(n *parser.ASTNode) bool { + l, ok := n.Value.(*parser.Logical) + return ok && l.Operator == parser.LogicalAnd +} diff --git a/frac/processor/resolver_test.go b/frac/processor/resolver_test.go new file mode 100644 index 000000000..d59aa66fd --- /dev/null +++ b/frac/processor/resolver_test.go @@ -0,0 +1,215 @@ +package processor + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/ozontech/seq-db/metric/stopwatch" + "github.com/ozontech/seq-db/parser" +) + +type recordingTokenIndex struct { + testTokenIndex + present map[string]bool + lookups []string +} + +func (d *recordingTokenIndex) GetTIDsByTokenExpr(token parser.Token) ([]uint32, error) { + key := astDump(&parser.ASTNode{Value: token}) + d.lookups = append(d.lookups, key) + if d.present[key] { + return []uint32{1}, nil + } + return nil, nil +} + +func TestASTResolver(t *testing.T) { + tests := []struct { + name string + query string + present []string + wantEmpty bool + wantAST string + wantLookups []string + }{ + { + name: "empty literal skips scan", + query: `a:x and b:*y*`, + wantEmpty: true, + wantLookups: []string{`a:x`}, + }, + { + name: "literal goes before scan regardless of position", + query: `b:*y* and a:x`, + present: []string{`b:*y*`}, + wantEmpty: true, + wantLookups: []string{`a:x`}, + }, + { + name: "literal goes before prefix", + query: `s:wms* and r:1`, + present: []string{`s:wms*`}, + wantEmpty: true, + wantLookups: []string{`r:1`}, + }, + { + name: "nested and chain is flattened", + query: `((s:wms* and r:1) and t:1)`, + present: []string{`s:wms*`, `r:1`}, + wantEmpty: true, + wantLookups: []string{`r:1`, `t:1`}, + }, + { + name: "or of literals goes after single literals", + query: `(a:1 or b:2) and c:3`, + wantEmpty: true, + wantLookups: []string{`c:3`}, + }, + { + name: "empty or branch is dropped", + query: `(a:1 or b:2) and c:3`, + present: []string{`a:1`, `c:3`}, + wantAST: `(a:1 and c:3)`, + wantLookups: []string{`c:3`, `a:1`, `b:2`}, + }, + { + name: "or of literals goes before scan", + query: `a:*x* and (b:1 or c:2)`, + present: []string{`a:*x*`}, + wantEmpty: true, + wantLookups: []string{`b:1`, `c:2`}, + }, + { + name: "empty and under not is pruned to its empty conjunct", + query: `not (b:*x* and c:2)`, + present: []string{`b:*x*`}, + wantAST: `(not c:2)`, + wantLookups: []string{`c:2`}, + }, + { + name: "or of empty branches is empty", + query: `a:1 or b:2`, + wantEmpty: true, + wantLookups: []string{`a:1`, `b:2`}, + }, + { + name: "empty negative part of nand is dropped", + query: `a:1 and not b:2`, + present: []string{`a:1`}, + wantAST: `a:1`, + wantLookups: []string{`a:1`, `b:2`}, + }, + { + name: "empty regular part of nand skips negative part", + query: `not a:1 and b:2`, + present: []string{`a:1`}, + wantEmpty: true, + wantLookups: []string{`b:2`}, + }, + { + name: "not of empty is not empty", + query: `not a:1`, + wantAST: `(not a:1)`, + wantLookups: []string{`a:1`}, + }, + { + name: "all present keeps the tree", + query: `a:1 and (b:2 or c:*3)`, + present: []string{`a:1`, `b:2`, `c:*3`}, + wantAST: `(a:1 and (b:2 or c:*3))`, + wantLookups: []string{`a:1`, `b:2`, `c:*3`}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + q, err := parser.ParseSeqQL(tt.query, nil) + require.NoError(t, err) + + before := astDump(q.Root) + index := &recordingTokenIndex{present: map[string]bool{}} + for _, p := range tt.present { + index.present[p] = true + } + + got, empty, err := astResolve(q.Root, index, make(tidsCache), stopwatch.New()) + require.NoError(t, err) + + assert.Equal(t, tt.wantEmpty, empty) + if !tt.wantEmpty { + assert.Equal(t, tt.wantAST, astDump(got)) + } + + assert.Equal(t, tt.wantLookups, index.lookups) + assert.Equal(t, before, astDump(q.Root), "shared AST must not be mutated") + }) + } +} + +func TestASTResolverOrder(t *testing.T) { + tests := []struct { + name string + query string + freqs map[uint32]uint32 + wantAST string + }{ + { + name: "rarest conjunct goes first", + query: `a:1 and b:2 and c:3`, + freqs: map[uint32]uint32{1: 100, 2: 10, 3: 0}, + wantAST: `((c:3 and b:2) and a:1)`, + }, + { + name: "equal sizes keep query order", + query: `c:3 and (b:2 and a:1)`, + wantAST: `((c:3 and b:2) and a:1)`, + }, + { + name: "or size is the sum of its branches", + query: `(a:1 or b:2) and c:3`, + freqs: map[uint32]uint32{1: 5, 2: 5, 3: 7}, + wantAST: `(c:3 and (a:1 or b:2))`, + }, + { + name: "smaller or stays first", + query: `(a:1 or b:2) and c:3`, + freqs: map[uint32]uint32{1: 5, 2: 5, 3: 20}, + wantAST: `((a:1 or b:2) and c:3)`, + }, + { + name: "regular part of nand is reordered", + query: `a:1 and b:2 and not c:3`, + freqs: map[uint32]uint32{1: 100, 2: 10, 3: 1}, + wantAST: `(not c:3 and (b:2 and a:1))`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + q, err := parser.ParseSeqQL(tt.query, nil) + require.NoError(t, err) + before := astDump(q.Root) + + index := &testTokenIndex{ + tids: map[string][]uint32{"a:1": {1}, "b:2": {2}, "c:3": {3}}, + freqs: tt.freqs, + } + + got, empty, err := astResolve(q.Root, index, make(tidsCache), stopwatch.New()) + require.NoError(t, err) + require.False(t, empty) + + assert.Equal(t, tt.wantAST, astDump(got)) + assert.Equal(t, before, astDump(q.Root), "shared AST must not be mutated") + }) + } +} + +func astDump(n *parser.ASTNode) string { + var b strings.Builder + n.DumpSeqQL(&b) + return b.String() +} diff --git a/frac/processor/search.go b/frac/processor/search.go index 97d1fb2d7..28efb2997 100644 --- a/frac/processor/search.go +++ b/frac/processor/search.go @@ -94,8 +94,22 @@ func IndexSearch( sw *stopwatch.Stopwatch, ) (qpr *seq.QPR, err error) { stats := &searchStats{} + cache := make(tidsCache) - m := sw.Start("get_lids_borders") + m := sw.Start("ast_resolve") + ast, empty, err := astResolve(params.AST, index, cache, sw) + m.Stop() + if err != nil { + return nil, err + } + + if empty { + stats.UpdateMetrics() + return emptyQPR(params), nil + } + + params.AST = ast + m = sw.Start("get_lids_borders") minLID, maxLID := getLIDsBorders(params, index) m.Stop() @@ -114,7 +128,7 @@ func IndexSearch( var evalTree node.BatchedNode if !hasSkipLIDs { evalTree, err = tryBuildBatchEvalTree( - params.AST, fracVer, index, queryOpt, minLID, maxLID, stats, params.Order, sw, + params.AST, fracVer, index, cache, queryOpt, minLID, maxLID, stats, params.Order, sw, ) } else { err = errBatchingUnsupported @@ -129,7 +143,7 @@ func IndexSearch( var nodeTree node.Node nodeTree, err = buildEvalTree(params.AST, minLID, maxLID, stats, params.Order.IsReverse(), func(token parser.Token) (node.Node, error) { - return evalLeaf(index, token, sw, stats, minLID, maxLID, params.Order) + return evalLeaf(index, token, cache, sw, stats, minLID, maxLID, params.Order) }, ) if err != nil { @@ -228,6 +242,15 @@ func IndexSearch( return qpr, nil } +// emptyQPR is the result IndexSearch produces when the eval tree yields no LIDs. +func emptyQPR(params SearchParams) *seq.QPR { + qpr := &seq.QPR{} + if params.HasHist() { + qpr.Histogram = map[seq.MID]uint64{} + } + return qpr +} + func iterateEvalTree( ctx context.Context, params SearchParams,