From c715c3d76176556d4cc78e15d29df281fd9fe999 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 19 Sep 2026 11:08:40 +0000 Subject: [PATCH] feat: EHR search table stakes (_has, uri, include wildcards, 2-hop chains) Implement the search features EHR apps actually send, rather than Lucene or _filter first: - Reverse chaining via _has:Type:ref:param (one nested _has) - uri SearchParameters with :below / :above, indexed as uri.* - _include / _revinclude wildcards (ResourceType:* and *:*) - Two-hop chained search (e.g. subject.organization.name) Recursive _include:iterate and chain depth > 2 stay unsupported. Co-authored-by: Adegoke Adewoye --- pkg/postgres/README.md | 1 + pkg/postgres/doc.go | 1 + pkg/postgres/search_store.go | 14 + pkg/runtime/README.md | 2 +- pkg/search/README.md | 12 +- pkg/search/doc.go | 8 +- pkg/search/ehr_search_test.go | 366 ++++++++++++++++++++++++ pkg/search/executor.go | 100 ++++++- pkg/search/integration_postgres_test.go | 116 +++++++- pkg/search/modifiers.go | 10 +- pkg/search/normalize.go | 11 + pkg/search/parser.go | 161 +++++++++-- pkg/search/parser_test.go | 11 +- pkg/search/planner.go | 273 +++++++++++++++++- pkg/search/query.go | 29 +- pkg/search/registry.go | 2 +- pkg/search/reindex_test.go | 111 ++++++- pkg/sqlite/README.md | 1 + pkg/sqlite/doc.go | 1 + pkg/sqlite/search_store.go | 2 + 20 files changed, 1169 insertions(+), 63 deletions(-) create mode 100644 pkg/search/ehr_search_test.go diff --git a/pkg/postgres/README.md b/pkg/postgres/README.md index 3d81f2a..71b0d2b 100644 --- a/pkg/postgres/README.md +++ b/pkg/postgres/README.md @@ -145,6 +145,7 @@ Search entries use typed table routing (same convention as sqlite): - `token.` → token index - `string.` → string index (default when no prefix) - `date.`, `number.`, `reference.` / `ref.` → typed tables +- `uri.` → string index (canonical/URI search parameters) FHIR search parsing lives outside this package; callers pass prepared `SearchIndexEntry` values on writes. diff --git a/pkg/postgres/doc.go b/pkg/postgres/doc.go index eef8845..2b9bc2b 100644 --- a/pkg/postgres/doc.go +++ b/pkg/postgres/doc.go @@ -147,6 +147,7 @@ // - date. → hai_search_date // - number. → hai_search_number // - reference. or ref. → hai_search_reference +// - uri. → hai_search_string // // Keys without a prefix default to hai_search_string. QueryPrepared supports the "by-field" // plan (args: key, value). AnalyticsStore QueryPrepared supports "by-name-since" diff --git a/pkg/postgres/search_store.go b/pkg/postgres/search_store.go index 1e0f64d..2bc7d46 100644 --- a/pkg/postgres/search_store.go +++ b/pkg/postgres/search_store.go @@ -56,6 +56,8 @@ func parseSearchFieldKey(key string) (searchTable, string, error) { return searchTableComposite, parts[1], nil case "text": return searchTableText, parts[1], nil + case "uri": + return searchTableString, parts[1], nil default: return searchTableString, key, nil } @@ -193,6 +195,18 @@ func (s *SearchStore) LookupMatch(ctx context.Context, match store.SearchMatch) AND (value LIKE $4 || '/%%' OR value LIKE $4 || '|%%') ORDER BY resource_id`, table) args = []any{s.tenantID, match.ResourceType, fieldKey, match.Value} + case (table == searchTableString) && op == "below": + query = fmt.Sprintf(` + SELECT resource_id FROM %s + WHERE tenant_id = $1 AND resource_type = $2 AND field_key = $3 AND value LIKE $4 || '%%' + ORDER BY resource_id`, table) + args = []any{s.tenantID, match.ResourceType, fieldKey, match.Value} + case (table == searchTableString) && op == "above": + query = fmt.Sprintf(` + SELECT resource_id FROM %s + WHERE tenant_id = $1 AND resource_type = $2 AND field_key = $3 AND $4 LIKE value || '%%' + ORDER BY resource_id`, table) + args = []any{s.tenantID, match.ResourceType, fieldKey, match.Value} case (table == searchTableDate || table == searchTableNumber) && isComparator(op): sqlOp, err := comparatorSQL(op) if err != nil { diff --git a/pkg/runtime/README.md b/pkg/runtime/README.md index 9d01412..29eb0f6 100644 --- a/pkg/runtime/README.md +++ b/pkg/runtime/README.md @@ -197,7 +197,7 @@ Build failures roll back partially opened resources before returning an error. | SQLite | Embedded/basic search via local `SearchStore` executor | | Postgres | Full search service + background reindex worker | -Advanced FHIR search features (`_include`, chained search, composites, FTS) remain **Postgres-first** per `pkg/search`. SQLite persists index rows and supports basic lookups. +Advanced FHIR search features (`_include` wildcards, `_has`, two-hop chains, uri, composites, FTS) remain **Postgres-first** per `pkg/search`. SQLite persists index rows and supports basic lookups. ## Sync by mode diff --git a/pkg/search/README.md b/pkg/search/README.md index d13b986..a5590d3 100644 --- a/pkg/search/README.md +++ b/pkg/search/README.md @@ -136,10 +136,11 @@ Postgres-first advanced FHIR search: | Registry-backed parameters | All installed SearchParameters for enabled resource types | | `_count` / `_offset` | Paging with max `_count` of 100 | | `_sort` | Registry-backed fields plus `_id` and `_lastUpdated` | -| Modifiers | `string:exact`, `string:contains`; token/reference modifiers per type | +| Modifiers | `string:exact`, `string:contains`; `uri:below`, `uri:above`; token/reference modifiers per type | | Prefixes | Date/number comparators: `eq`, `ne`, `gt`, `ge`, `lt`, `le`, `sa`, `eb`, `ap` | -| Chained search | Single-hop only (e.g. `subject.name`) | -| `_include` / `_revinclude` | Direct includes; wildcards deferred | +| Chained search | Up to two hops (e.g. `subject.name`, `subject.organization.name`) | +| Reverse chaining | `_has:Type:ref:param` (one extra nested `_has`) | +| `_include` / `_revinclude` | Direct includes plus `ResourceType:*` and `*:*` wildcards | | Composite search | Declared composite SearchParameters from registry | | `_summary` / `_elements` | Response projection at assembly time | | Full text | Postgres native FTS via indexed text documents | @@ -153,7 +154,7 @@ Unsupported semantics return explicit errors (`ErrUnsupportedFeature`, `ErrInval - Comma-separated values **OR** within one occurrence (`?name=Smith,Jones`) - `_count` and `_offset` apply to primary matches only (not included resources) - `_sort` uses registry metadata; tiebreak on resource id -- Chain depth limited to 1; wildcard includes and recursive includes are rejected +- Chain depth is limited to 2 hops; `_include:iterate` remains unsupported ## Index field keys @@ -165,6 +166,7 @@ Unsupported semantics return explicit errors (`ErrUnsupportedFeature`, `ErrInval | `string.*` | `string.family` | Normalized strings | | `date.*` | `date.birthdate` | Comparable date strings | | `reference.*` | `reference.patient` | Reference targets (typed/id/canonical forms) | +| `uri.*` | `uri.url` | Canonical/URI strings (stored with string indexes) | | `composite.*` | `composite.context-type-value` | Composite component values | | `text.*` | `text.document` | Postgres full-text document | @@ -209,7 +211,7 @@ url.Values → ParseQuery → ResolveQuery → BuildPlan → StoreExecut ## Current limits - Postgres is the primary complete execution backend; SQLite stores indexes and supports basic lookups but not advanced execution -- Chain depth is limited to 1; wildcard/recursive includes are deferred +- Chain depth is limited to 2; recursive `_include:iterate` is deferred - OpenSearch adapter seam is preserved via `SearchAdvancedExecutor`; not implemented yet - No HTTP `_search` endpoint in this package - Custom SearchParameters become searchable after snapshot rebuild and reindex completion diff --git a/pkg/search/doc.go b/pkg/search/doc.go index 65686f4..9a0723b 100644 --- a/pkg/search/doc.go +++ b/pkg/search/doc.go @@ -18,8 +18,10 @@ // // Postgres-first execution supports: // -// - _include and _revinclude (direct, non-wildcard) -// - single-hop chained search (e.g. subject.name) +// - _include and _revinclude, including ResourceType:* and *:* wildcards +// - chained search up to two hops (e.g. subject.name, subject.organization.name) +// - reverse chaining via _has:Type:ref:param +// - uri SearchParameters with :below / :above // - composite SearchParameters from the registry // - modifiers (:exact, :contains on string; date/number prefixes) // - _sort on registry-backed parameters plus _id / _lastUpdated @@ -33,7 +35,7 @@ // // RegistryIndexer evaluates each installed SearchParameter expression with pkg/fhirpath, // normalizes extracted values into typed field keys (token.*, string.*, date.*, -// reference.*, composite.*, text.*), and emits store.SearchIndexEntry rows consumed +// reference.*, uri.*, composite.*, text.*), and emits store.SearchIndexEntry rows consumed // by store.SearchStore. // // # Query semantics diff --git a/pkg/search/ehr_search_test.go b/pkg/search/ehr_search_test.go new file mode 100644 index 0000000..0cc9648 --- /dev/null +++ b/pkg/search/ehr_search_test.go @@ -0,0 +1,366 @@ +package search_test + +import ( + "context" + "testing" + "time" + + "github.com/degoke/health-ai-stack/pkg/search" + "github.com/degoke/health-ai-stack/pkg/store" + "github.com/degoke/health-ai-stack/pkg/types" +) + +func TestParseQueryHasAndTwoHopChain(t *testing.T) { + q, err := search.ParseQueryValues("Patient", map[string][]string{ + "_has:Observation:subject:code": {"8867-4"}, + "general-practitioner.name": {"Smith"}, + }) + if err != nil { + t.Fatalf("ParseQuery: %v", err) + } + if len(q.Has) != 1 || q.Has[0].SourceType != "Observation" || q.Has[0].RefCode != "subject" || q.Has[0].Param.Code != "code" { + t.Fatalf("has = %#v", q.Has) + } + if len(q.Has[0].Param.Values) != 1 || q.Has[0].Param.Values[0].Raw != "8867-4" { + t.Fatalf("has values = %#v", q.Has[0].Param.Values) + } + + twoHop, err := search.ParseQueryValues("Observation", map[string][]string{ + "subject.organization.name": {"Acme"}, + }) + if err != nil { + t.Fatalf("ParseQuery two-hop: %v", err) + } + if len(twoHop.Chains) != 1 || twoHop.Chains[0].RefCode != "subject" || twoHop.Chains[0].Nested == nil { + t.Fatalf("two-hop chain = %#v", twoHop.Chains) + } + if twoHop.Chains[0].Nested.RefCode != "organization" || twoHop.Chains[0].Nested.Param.Code != "name" { + t.Fatalf("nested chain = %#v", twoHop.Chains[0].Nested) + } +} + +func TestParseQueryIncludeWildcards(t *testing.T) { + q, err := search.ParseQueryValues("Observation", map[string][]string{ + "_include": {"Observation:*", "Observation:subject:Patient", "*:*"}, + "_revinclude": {"*:*", "Observation:*"}, + }) + if err != nil { + t.Fatalf("ParseQuery: %v", err) + } + if len(q.Includes) != 3 { + t.Fatalf("includes = %#v", q.Includes) + } + if q.Includes[0].ParamCode != "*" || q.Includes[1].TargetType != "Patient" || q.Includes[2].ParamCode != "*" { + t.Fatalf("includes = %#v", q.Includes) + } + if len(q.RevIncludes) != 2 || q.RevIncludes[0].SourceType != "*" || q.RevIncludes[1].ParamCode != "*" { + t.Fatalf("revincludes = %#v", q.RevIncludes) + } + + starParam, err := search.ParseQueryValues("Patient", map[string][]string{ + "_include": {"*:general-practitioner"}, + }) + if err != nil { + t.Fatalf("ParseQuery star source: %v", err) + } + if len(starParam.Includes) != 1 || starParam.Includes[0].SourceType != "Patient" || starParam.Includes[0].ParamCode != "general-practitioner" { + t.Fatalf("star source include = %#v", starParam.Includes) + } +} + +func TestResolveQueryHasUriTwoHopAndWildcards(t *testing.T) { + snapshot := testSnapshot(t, "Patient", "Observation", "Organization", "Questionnaire") + reg := search.NewSnapshotRegistry(snapshot) + + hasQuery, err := search.ParseQueryValues("Patient", map[string][]string{ + "_has:Observation:subject:code": {"8867-4"}, + }) + if err != nil { + t.Fatalf("ParseQuery: %v", err) + } + resolvedHas, err := search.ResolveQuery(reg, hasQuery) + if err != nil { + t.Fatalf("ResolveQuery _has: %v", err) + } + if len(resolvedHas.Has) != 1 || resolvedHas.Has[0].RefFieldKey != "reference.subject" || resolvedHas.Has[0].Param.FieldKey != "token.code" { + t.Fatalf("resolved has = %#v", resolvedHas.Has) + } + + chainQuery, err := search.ParseQueryValues("Observation", map[string][]string{ + "subject.organization.name": {"Acme"}, + }) + if err != nil { + t.Fatalf("ParseQuery: %v", err) + } + resolvedChain, err := search.ResolveQuery(reg, chainQuery) + if err != nil { + t.Fatalf("ResolveQuery two-hop: %v", err) + } + if len(resolvedChain.Chains) != 1 || resolvedChain.Chains[0].TargetType != "Patient" || resolvedChain.Chains[0].Nested == nil { + t.Fatalf("resolved chain = %#v", resolvedChain.Chains) + } + if resolvedChain.Chains[0].Nested.TargetType != "Organization" || resolvedChain.Chains[0].Nested.Param.FieldKey != "string.name" { + t.Fatalf("resolved nested chain = %#v", resolvedChain.Chains[0].Nested) + } + + uriQuery, err := search.ParseQueryValues("Questionnaire", map[string][]string{ + "url:below": {"http://example.org/fhir"}, + }) + if err != nil { + t.Fatalf("ParseQuery uri: %v", err) + } + resolvedURI, err := search.ResolveQuery(reg, uriQuery) + if err != nil { + t.Fatalf("ResolveQuery uri: %v", err) + } + if len(resolvedURI.Params) != 1 || resolvedURI.Params[0].ParamType != "uri" || resolvedURI.Params[0].FieldKey != "uri.url" { + t.Fatalf("resolved uri = %#v", resolvedURI.Params) + } + + incQuery, err := search.ParseQueryValues("Observation", map[string][]string{ + "_include": {"Observation:*"}, + }) + if err != nil { + t.Fatalf("ParseQuery include: %v", err) + } + resolvedInc, err := search.ResolveQuery(reg, incQuery) + if err != nil { + t.Fatalf("ResolveQuery include wildcard: %v", err) + } + if len(resolvedInc.Includes) < 2 { + t.Fatalf("wildcard includes = %#v", resolvedInc.Includes) + } + codes := map[string]bool{} + for _, inc := range resolvedInc.Includes { + if inc.ParamCode == "*" { + t.Fatalf("wildcard was not expanded: %#v", inc) + } + codes[inc.ParamCode] = true + } + if !codes["subject"] || !codes["encounter"] { + t.Fatalf("expanded includes missing subject/encounter: %#v", resolvedInc.Includes) + } + + revQuery, err := search.ParseQueryValues("Patient", map[string][]string{ + "_revinclude": {"Observation:*"}, + }) + if err != nil { + t.Fatalf("ParseQuery revinclude: %v", err) + } + resolvedRev, err := search.ResolveQuery(reg, revQuery) + if err != nil { + t.Fatalf("ResolveQuery revinclude wildcard: %v", err) + } + if len(resolvedRev.RevIncludes) == 0 { + t.Fatal("expected expanded revincludes") + } +} + +func TestStoreExecutorHasAndTwoHopChain(t *testing.T) { + ctx := context.Background() + backend := &memSearchBackend{ + entries: []store.SearchIndexEntry{ + {ResourceType: "Organization", ID: "org-1", Fields: map[string]string{"string.name": "Acme"}}, + {ResourceType: "Organization", ID: "org-2", Fields: map[string]string{"string.name": "Other"}}, + {ResourceType: "Patient", ID: "pat-1", Fields: map[string]string{"reference.organization": "Organization/org-1"}}, + {ResourceType: "Patient", ID: "pat-2", Fields: map[string]string{"reference.organization": "Organization/org-2"}}, + {ResourceType: "Observation", ID: "obs-1", Fields: map[string]string{"reference.subject": "Patient/pat-1", "token.code": "8867-4"}}, + {ResourceType: "Observation", ID: "obs-2", Fields: map[string]string{"reference.subject": "Patient/pat-2", "token.code": "1234-5"}}, + }, + } + resources := newMemResourceStore() + for _, id := range []string{"pat-1", "pat-2"} { + _ = resources.Create(ctx, &types.ResourceEnvelope{ResourceType: "Patient", ID: id}) + } + executor := search.NewStoreExecutor(backend, resources) + + hasPlan := &search.Plan{ + ResourceType: "Patient", + Count: 10, + HasPlans: []search.HasPlan{{ + SourceType: "Observation", + RefCode: "subject", + RefFieldKey: "reference.subject", + ParamPlan: search.ParamPlan{ + Code: "code", + FieldKey: "token.code", + ParamType: "token", + Predicates: []search.Predicate{{ + FieldKey: "token.code", + Value: "8867-4", + Operator: search.OpEqual, + }}, + }, + }}, + } + hasResult, err := executor.Execute(ctx, hasPlan) + if err != nil { + t.Fatalf("Execute _has: %v", err) + } + if len(hasResult.IDs) != 1 || hasResult.IDs[0] != "pat-1" { + t.Fatalf("_has ids = %v", hasResult.IDs) + } + + chainPlan := &search.Plan{ + ResourceType: "Observation", + Count: 10, + ChainPlans: []search.ChainPlan{{ + RefCode: "subject", + RefFieldKey: "reference.subject", + TargetType: "Patient", + Nested: &search.ChainPlan{ + RefCode: "organization", + RefFieldKey: "reference.organization", + TargetType: "Organization", + ParamPlan: search.ParamPlan{ + Code: "name", + FieldKey: "string.name", + ParamType: "string", + Predicates: []search.Predicate{{ + FieldKey: "string.name", + Value: "Acme", + Operator: search.OpEqual, + }}, + }, + }, + }}, + } + obsStore := newMemResourceStore() + _ = obsStore.Create(ctx, &types.ResourceEnvelope{ResourceType: "Observation", ID: "obs-1"}) + _ = obsStore.Create(ctx, &types.ResourceEnvelope{ResourceType: "Observation", ID: "obs-2"}) + chainExec := search.NewStoreExecutor(backend, obsStore) + chainResult, err := chainExec.Execute(ctx, chainPlan) + if err != nil { + t.Fatalf("Execute two-hop: %v", err) + } + if len(chainResult.IDs) != 1 || chainResult.IDs[0] != "obs-1" { + t.Fatalf("two-hop ids = %v", chainResult.IDs) + } +} + +func TestStoreExecutorUriBelowAndWildcardInclude(t *testing.T) { + ctx := context.Background() + backend := &memAdvancedSearchBackend{memSearchBackend: memSearchBackend{ + entries: []store.SearchIndexEntry{ + {ResourceType: "Questionnaire", ID: "q-1", Fields: map[string]string{"uri.url": "http://example.org/fhir/Questionnaire/q-1"}}, + {ResourceType: "Questionnaire", ID: "q-2", Fields: map[string]string{"uri.url": "http://other.org/fhir/Questionnaire/q-2"}}, + {ResourceType: "Observation", ID: "obs-1", Fields: map[string]string{"reference.subject": "Patient/pat-1", "reference.encounter": "Encounter/enc-1"}}, + }, + }} + resources := newMemResourceStore() + _ = resources.Create(ctx, &types.ResourceEnvelope{ResourceType: "Questionnaire", ID: "q-1"}) + _ = resources.Create(ctx, &types.ResourceEnvelope{ResourceType: "Questionnaire", ID: "q-2"}) + executor := search.NewStoreExecutor(backend, resources) + + uriPlan := &search.Plan{ + ResourceType: "Questionnaire", + Count: 10, + ParamPlans: []search.ParamPlan{{ + Code: "url", + FieldKey: "uri.url", + ParamType: "uri", + Predicates: []search.Predicate{{ + FieldKey: "uri.url", + Value: "http://example.org/fhir", + Operator: search.OpBelow, + }}, + }}, + } + uriResult, err := executor.Execute(ctx, uriPlan) + if err != nil { + t.Fatalf("Execute uri:below: %v", err) + } + if len(uriResult.IDs) != 1 || uriResult.IDs[0] != "q-1" { + t.Fatalf("uri:below ids = %v", uriResult.IDs) + } + + obsStore := newMemResourceStore() + _ = obsStore.Create(ctx, &types.ResourceEnvelope{ResourceType: "Observation", ID: "obs-1"}) + incExec := search.NewStoreExecutor(backend, obsStore) + incPlan := &search.Plan{ + ResourceType: "Observation", + Count: 10, + Includes: []search.IncludePlan{ + {SourceType: "Observation", ParamCode: "subject", RefFieldKey: "reference.subject"}, + {SourceType: "Observation", ParamCode: "encounter", RefFieldKey: "reference.encounter"}, + }, + } + incResult, err := incExec.Execute(ctx, incPlan) + if err != nil { + t.Fatalf("Execute wildcard includes: %v", err) + } + if len(incResult.Included) != 2 { + t.Fatalf("included = %#v", incResult.Included) + } +} + +func TestRegistryIndexerEmitsUriFields(t *testing.T) { + ctx := context.Background() + snapshot := testSnapshot(t, "Questionnaire") + reg := search.NewSnapshotRegistry(snapshot) + indexer, err := search.NewRegistryIndexer(search.RegistryIndexerConfig{ + Registry: reg, + Engine: testEngine(t), + }) + if err != nil { + t.Fatalf("NewRegistryIndexer: %v", err) + } + entries, err := indexer.Build(ctx, questionnaireResource(t, "q-1", "http://example.org/fhir/Questionnaire/q-1")) + if err != nil { + t.Fatalf("Build: %v", err) + } + values := fieldValues(entries) + if !containsValue(values, "uri.url", "http://example.org/fhir/Questionnaire/q-1") { + t.Fatalf("missing uri.url index: %#v", values) + } +} + +func questionnaireResource(t *testing.T, id, url string) *types.ResourceEnvelope { + t.Helper() + data := []byte(`{ + "resourceType":"Questionnaire", + "id":"` + id + `", + "status":"active", + "url":"` + url + `" + }`) + codec := types.NewJSONCodec() + env, err := codec.ParseJSON("Questionnaire", data) + if err != nil { + t.Fatalf("ParseJSON: %v", err) + } + env.LastUpdated = time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC) + return env +} + +func organizationResource(t *testing.T, id, name string) *types.ResourceEnvelope { + t.Helper() + data := []byte(`{ + "resourceType":"Organization", + "id":"` + id + `", + "name":"` + name + `" + }`) + codec := types.NewJSONCodec() + env, err := codec.ParseJSON("Organization", data) + if err != nil { + t.Fatalf("ParseJSON: %v", err) + } + env.LastUpdated = time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC) + return env +} + +func patientWithOrganization(t *testing.T, id, family, orgID string) *types.ResourceEnvelope { + t.Helper() + data := []byte(`{ + "resourceType":"Patient", + "id":"` + id + `", + "name":[{"family":"` + family + `","given":["Jane"]}], + "managingOrganization":{"reference":"Organization/` + orgID + `"} + }`) + codec := types.NewJSONCodec() + env, err := codec.ParseJSON("Patient", data) + if err != nil { + t.Fatalf("ParseJSON: %v", err) + } + env.LastUpdated = time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC) + return env +} diff --git a/pkg/search/executor.go b/pkg/search/executor.go index 0e1b0a6..a810ca1 100644 --- a/pkg/search/executor.go +++ b/pkg/search/executor.go @@ -67,6 +67,17 @@ func (e *StoreExecutor) Execute(ctx context.Context, plan *Plan) (*ExecuteResult candidateSets = append(candidateSets, ids) } + for _, hasPlan := range plan.HasPlans { + ids, err := e.executeHasPlan(ctx, plan.ResourceType, hasPlan) + if err != nil { + return nil, err + } + if len(ids) == 0 { + return &ExecuteResult{}, nil + } + candidateSets = append(candidateSets, ids) + } + var scores map[string]float64 if plan.FullText != "" { if e.Advanced == nil { @@ -272,7 +283,13 @@ func (e *StoreExecutor) executeParamPlan(ctx context.Context, resourceType strin } func (e *StoreExecutor) executeChainPlan(ctx context.Context, resourceType string, chain ChainPlan) ([]string, error) { - targetIDs, err := e.executeParamPlan(ctx, chain.TargetType, chain.ParamPlan) + var targetIDs []string + var err error + if chain.Nested != nil { + targetIDs, err = e.executeChainPlan(ctx, chain.TargetType, *chain.Nested) + } else { + targetIDs, err = e.executeParamPlan(ctx, chain.TargetType, chain.ParamPlan) + } if err != nil { return nil, err } @@ -306,6 +323,87 @@ func (e *StoreExecutor) executeChainPlan(ctx context.Context, resourceType strin return ids, nil } +func (e *StoreExecutor) executeHasPlan(ctx context.Context, searchType string, has HasPlan) ([]string, error) { + var sourceIDs []string + var err error + switch { + case has.Nested != nil: + sourceIDs, err = e.executeHasPlan(ctx, has.SourceType, *has.Nested) + case has.ChainPlan != nil: + sourceIDs, err = e.executeChainPlan(ctx, has.SourceType, *has.ChainPlan) + default: + sourceIDs, err = e.executeParamPlan(ctx, has.SourceType, has.ParamPlan) + } + if err != nil { + return nil, err + } + if len(sourceIDs) == 0 { + return nil, nil + } + + targetIDs, err := e.referencedTargetIDs(ctx, has.SourceType, has.RefFieldKey, searchType, sourceIDs) + if err != nil { + return nil, err + } + sort.Strings(targetIDs) + return targetIDs, nil +} + +func (e *StoreExecutor) referencedTargetIDs(ctx context.Context, sourceType, refFieldKey, targetType string, sourceIDs []string) ([]string, error) { + seen := make(map[string]struct{}) + var ids []string + add := func(typ, id string) { + if id == "" { + return + } + if targetType != "" && typ != "" && typ != targetType { + return + } + if _, ok := seen[id]; ok { + return + } + seen[id] = struct{}{} + ids = append(ids, id) + } + + if e.Advanced != nil { + refs, err := e.Advanced.LookupReferences(ctx, sourceType, refFieldKey, sourceIDs) + if err != nil { + return nil, err + } + for _, links := range refs { + for _, link := range links { + typ := link.TargetType + if typ == "" { + typ = parseIncludedReference(link.Literal) + } + add(typ, link.TargetID) + } + } + return ids, nil + } + + values, err := e.Backend.FieldValues(ctx, sourceType, refFieldKey, sourceIDs) + if err != nil { + return nil, err + } + for _, value := range values { + link := parseReferenceLink(value) + add(link.TargetType, link.TargetID) + } + return ids, nil +} + +func parseReferenceLink(value string) store.ReferenceLink { + if i := strings.Index(value, "|"); i > 0 && i < len(value)-1 { + return store.ReferenceLink{TargetType: value[:i], TargetID: value[i+1:], Literal: value} + } + if i := strings.Index(value, "/"); i > 0 && i < len(value)-1 { + return store.ReferenceLink{TargetType: value[:i], TargetID: value[i+1:], Literal: value} + } + return store.ReferenceLink{TargetID: value, Literal: value} +} + func referenceLookupValues(targetType, targetID string) []string { typed := targetType + "/" + targetID canonical := targetType + "|" + targetID diff --git a/pkg/search/integration_postgres_test.go b/pkg/search/integration_postgres_test.go index 1988b29..9bdf580 100644 --- a/pkg/search/integration_postgres_test.go +++ b/pkg/search/integration_postgres_test.go @@ -18,6 +18,10 @@ import ( ) func openPostgresSearchHarness(t *testing.T) (*core.ResourceService, *search.Service, *registry.Snapshot, *postgres.TenantDB, func()) { + return openPostgresSearchHarnessTypes(t, "Patient", "Observation") +} + +func openPostgresSearchHarnessTypes(t *testing.T, types ...string) (*core.ResourceService, *search.Service, *registry.Snapshot, *postgres.TenantDB, func()) { t.Helper() ctx := context.Background() db := postgrestest.SharedDB(t) @@ -34,7 +38,7 @@ func openPostgresSearchHarness(t *testing.T) (*core.ResourceService, *search.Ser if err := manager.SeedBundled(ctx); err != nil { t.Fatalf("SeedBundled: %v", err) } - for _, rt := range []string{"Patient", "Observation"} { + for _, rt := range types { if err := manager.EnableResource(ctx, rt); err != nil { t.Fatalf("EnableResource %s: %v", rt, err) } @@ -462,6 +466,116 @@ func TestPostgresIncludePatient(t *testing.T) { } } +func TestPostgresHasReverseChain(t *testing.T) { + if testing.Short() { + t.Skip("skipping postgres integration test in short mode") + } + svc, searchSvc, _, _, cleanup := openPostgresSearchHarness(t) + defer cleanup() + ctx := context.Background() + + if _, err := svc.Create(ctx, patientResource(t, "pat-1", "Doe", "555")); err != nil { + t.Fatalf("Create patient: %v", err) + } + if _, err := svc.Create(ctx, patientResource(t, "pat-2", "Smith", "556")); err != nil { + t.Fatalf("Create patient: %v", err) + } + if _, err := svc.Create(ctx, observationResource(t)); err != nil { + t.Fatalf("Create observation: %v", err) + } + + result, err := searchSvc.Search(ctx, "Patient", mustValues(t, map[string]string{ + "_has:Observation:subject:code": "8867-4", + })) + if err != nil { + t.Fatalf("_has search: %v", err) + } + if len(result.Resources) != 1 || result.Resources[0].ID != "pat-1" { + t.Fatalf("_has search = %#v", result.Resources) + } +} + +func TestPostgresTwoHopChain(t *testing.T) { + if testing.Short() { + t.Skip("skipping postgres integration test in short mode") + } + svc, searchSvc, _, _, cleanup := openPostgresSearchHarnessTypes(t, "Patient", "Observation", "Organization") + defer cleanup() + ctx := context.Background() + + if _, err := svc.Create(ctx, organizationResource(t, "org-1", "Acme")); err != nil { + t.Fatalf("Create organization: %v", err) + } + if _, err := svc.Create(ctx, patientWithOrganization(t, "pat-1", "Doe", "org-1")); err != nil { + t.Fatalf("Create patient: %v", err) + } + if _, err := svc.Create(ctx, observationResource(t)); err != nil { + t.Fatalf("Create observation: %v", err) + } + + result, err := searchSvc.Search(ctx, "Observation", mustValues(t, map[string]string{ + "subject.organization.name": "Acme", + })) + if err != nil { + t.Fatalf("two-hop search: %v", err) + } + if len(result.Resources) != 1 || result.Resources[0].ID != "obs-1" { + t.Fatalf("two-hop search = %#v", result.Resources) + } +} + +func TestPostgresUriBelowAndWildcardInclude(t *testing.T) { + if testing.Short() { + t.Skip("skipping postgres integration test in short mode") + } + svc, searchSvc, _, _, cleanup := openPostgresSearchHarnessTypes(t, "Patient", "Observation", "Questionnaire") + defer cleanup() + ctx := context.Background() + + if _, err := svc.Create(ctx, questionnaireResource(t, "q-1", "http://example.org/fhir/Questionnaire/q-1")); err != nil { + t.Fatalf("Create questionnaire: %v", err) + } + if _, err := svc.Create(ctx, questionnaireResource(t, "q-2", "http://other.org/fhir/Questionnaire/q-2")); err != nil { + t.Fatalf("Create questionnaire: %v", err) + } + + result, err := searchSvc.Search(ctx, "Questionnaire", mustValues(t, map[string]string{ + "url:below": "http://example.org/fhir", + })) + if err != nil { + t.Fatalf("uri:below search: %v", err) + } + if len(result.Resources) != 1 || result.Resources[0].ID != "q-1" { + t.Fatalf("uri:below search = %#v", result.Resources) + } + + if _, err := svc.Create(ctx, patientResource(t, "pat-1", "Doe", "555")); err != nil { + t.Fatalf("Create patient: %v", err) + } + if _, err := svc.Create(ctx, observationResource(t)); err != nil { + t.Fatalf("Create observation: %v", err) + } + included, err := searchSvc.Search(ctx, "Observation", mustValues(t, map[string]string{ + "_id": "obs-1", + "_include": "Observation:*", + })) + if err != nil { + t.Fatalf("wildcard include: %v", err) + } + if len(included.Included) == 0 { + t.Fatal("expected wildcard include to expand referenced resources") + } + foundPatient := false + for _, inc := range included.Included { + if inc.ResourceType == "Patient" && inc.ID == "pat-1" { + foundPatient = true + } + } + if !foundPatient { + t.Fatalf("wildcard include missing patient: %#v", included.Included) + } +} + func TestPostgresStringContainsModifier(t *testing.T) { if testing.Short() { t.Skip("skipping postgres integration test in short mode") diff --git a/pkg/search/modifiers.go b/pkg/search/modifiers.go index 8513fa6..195b42b 100644 --- a/pkg/search/modifiers.go +++ b/pkg/search/modifiers.go @@ -10,7 +10,6 @@ var deferredParams = map[string]struct{}{ "_containedType": {}, "_filter": {}, "_list": {}, - "_has": {}, "_type": {}, } @@ -29,6 +28,11 @@ var referenceModifiers = map[string]MatchOperator{ "type": OpType, } +var uriModifiers = map[string]MatchOperator{ + "below": OpBelow, + "above": OpAbove, +} + var dateNumberPrefixes = map[string]MatchOperator{ "eq": OpEqual, "ne": OpNotEqual, @@ -59,7 +63,9 @@ func validateModifier(paramType, modifier string) (MatchOperator, error) { return op, nil } case "uri": - return "", fmt.Errorf("%w: modifier %q on type %q", ErrUnsupportedFeature, modifier, paramType) + if op, ok := uriModifiers[modifier]; ok { + return op, nil + } } return "", fmt.Errorf("%w: modifier %q on type %q", ErrUnsupportedFeature, modifier, paramType) } diff --git a/pkg/search/normalize.go b/pkg/search/normalize.go index d7c46e8..fa66a5e 100644 --- a/pkg/search/normalize.go +++ b/pkg/search/normalize.go @@ -35,6 +35,8 @@ func fieldKeyForParam(code, paramType string) string { return "number." + code case "composite": return "composite." + code + case "uri": + return "uri." + code default: return "" } @@ -81,6 +83,8 @@ func normalizeValue(code, paramType string, v fhirpath.Value) []string { return normalizeReferenceValue(v) case "number", "quantity": return normalizeNumberValue(v) + case "uri": + return normalizeUriValue(v) default: return nil } @@ -260,6 +264,13 @@ func dateIndexVariants(value string) []string { return []string{value} } +func normalizeUriValue(v fhirpath.Value) []string { + if s, err := v.String(); err == nil && s != "" { + return []string{s} + } + return nil +} + func normalizeReferenceValue(v fhirpath.Value) []string { switch val := v.Raw().(type) { case *dtpb.Reference: diff --git a/pkg/search/parser.go b/pkg/search/parser.go index f112c11..60ad5c3 100644 --- a/pkg/search/parser.go +++ b/pkg/search/parser.go @@ -32,6 +32,16 @@ func ParseQuery(resourceType string, params url.Values) (*Query, error) { } switch baseKey { + case "_has": + if !hasModifier { + return nil, fmt.Errorf("%w: _has", ErrInvalidQuery) + } + hasClause, err := parseHasKey(modifier, values) + if err != nil { + return nil, err + } + q.Has = append(q.Has, hasClause) + continue case "_count": if err := parseCount(values, q); err != nil { return nil, err @@ -50,6 +60,9 @@ func ParseQuery(resourceType string, params url.Values) (*Query, error) { q.Sort = sortFields continue case "_include": + if hasModifier { + return nil, fmt.Errorf("%w: _include modifier %q", ErrUnsupportedFeature, modifier) + } for _, raw := range values { directive, err := parseIncludeValue(resourceType, raw) if err != nil { @@ -59,6 +72,9 @@ func ParseQuery(resourceType string, params url.Values) (*Query, error) { } continue case "_revinclude": + if hasModifier { + return nil, fmt.Errorf("%w: _revinclude modifier %q", ErrUnsupportedFeature, modifier) + } for _, raw := range values { directive, err := parseRevIncludeValue(resourceType, raw) if err != nil { @@ -138,8 +154,10 @@ func ParseQuery(resourceType string, params url.Values) (*Query, error) { } const ( - defaultCount = 20 - maxCount = 100 + defaultCount = 20 + maxCount = 100 + maxChainHops = 2 + maxHasNesting = 2 ) func parseCount(values []string, q *Query) error { @@ -204,12 +222,18 @@ func splitORValues(rawValue string) ([]ValueClause, error) { } func parseChainKey(key, modifier string, hasModifier bool, values []string) (ChainClause, error) { - parts := strings.SplitN(key, ".", 2) - if len(parts) != 2 { + parts := strings.Split(key, ".") + if len(parts) < 2 { return ChainClause{}, fmt.Errorf("%w: chained search %q", ErrUnsupportedFeature, key) } - if strings.Contains(parts[1], ".") { - return ChainClause{}, fmt.Errorf("%w: chain depth > 1 for %q", ErrUnsupportedFeature, key) + hops := len(parts) - 1 + if hops > maxChainHops { + return ChainClause{}, fmt.Errorf("%w: chain depth > %d for %q", ErrUnsupportedFeature, maxChainHops, key) + } + for _, part := range parts { + if strings.TrimSpace(part) == "" { + return ChainClause{}, fmt.Errorf("%w: chained search %q", ErrInvalidQuery, key) + } } if hasModifier && modifier == "missing" { return ChainClause{}, fmt.Errorf("%w: modifier %q on chain", ErrUnsupportedFeature, modifier) @@ -222,48 +246,137 @@ func parseChainKey(key, modifier string, hasModifier bool, values []string) (Cha } valueClauses = append(valueClauses, orValues...) } + return buildChainClause(parts, modifier, valueClauses), nil +} + +func buildChainClause(parts []string, modifier string, values []ValueClause) ChainClause { + if len(parts) == 2 { + return ChainClause{ + RefCode: parts[0], + Param: ParamClause{ + Code: parts[1], + Modifier: modifier, + Values: values, + }, + } + } + nested := buildChainClause(parts[1:], modifier, values) return ChainClause{ RefCode: parts[0], + Nested: &nested, + } +} + +func parseHasKey(rest string, values []string) (HasClause, error) { + return parseHasRest(rest, values, 1) +} + +func parseHasRest(rest string, values []string, depth int) (HasClause, error) { + if depth > maxHasNesting { + return HasClause{}, fmt.Errorf("%w: _has nesting > %d", ErrUnsupportedFeature, maxHasNesting) + } + parts := strings.SplitN(rest, ":", 3) + if len(parts) < 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" { + return HasClause{}, fmt.Errorf("%w: _has %q", ErrInvalidQuery, rest) + } + sourceType, refCode, remainder := parts[0], parts[1], parts[2] + if strings.HasPrefix(remainder, "_has:") { + nested, err := parseHasRest(strings.TrimPrefix(remainder, "_has:"), values, depth+1) + if err != nil { + return HasClause{}, err + } + return HasClause{ + SourceType: sourceType, + RefCode: refCode, + Nested: &nested, + }, nil + } + + paramKey, paramMod, hasMod := splitParamKey(remainder) + if hasMod && paramMod == "missing" { + return HasClause{}, fmt.Errorf("%w: modifier %q on _has", ErrUnsupportedFeature, paramMod) + } + if strings.Contains(paramKey, ".") { + chain, err := parseChainKey(paramKey, paramMod, hasMod, values) + if err != nil { + return HasClause{}, err + } + return HasClause{ + SourceType: sourceType, + RefCode: refCode, + Chain: &chain, + }, nil + } + + var valueClauses []ValueClause + for _, rawValue := range values { + orValues, err := splitORValues(rawValue) + if err != nil { + return HasClause{}, err + } + valueClauses = append(valueClauses, orValues...) + } + return HasClause{ + SourceType: sourceType, + RefCode: refCode, Param: ParamClause{ - Code: parts[1], - Modifier: modifier, + Code: paramKey, + Modifier: paramMod, Values: valueClauses, }, }, nil } func parseIncludeValue(sourceType, raw string) (IncludeDirective, error) { - parts := strings.Split(raw, ":") - if len(parts) != 2 { - return IncludeDirective{}, fmt.Errorf("%w: _include %q", ErrInvalidQuery, raw) + src, param, target, err := parseIncludeParts("_include", raw) + if err != nil { + return IncludeDirective{}, err } - if parts[0] == "*" || parts[1] == "*" { - return IncludeDirective{}, fmt.Errorf("%w: wildcard _include", ErrUnsupportedFeature) + if src == "*" { + src = sourceType } - if parts[0] != sourceType { - return IncludeDirective{}, fmt.Errorf("%w: _include source type %q does not match search type %q", ErrInvalidQuery, parts[0], sourceType) + if src != sourceType { + return IncludeDirective{}, fmt.Errorf("%w: _include source type %q does not match search type %q", ErrInvalidQuery, src, sourceType) } return IncludeDirective{ - SourceType: parts[0], - ParamCode: parts[1], + SourceType: src, + ParamCode: param, + TargetType: target, }, nil } func parseRevIncludeValue(searchType, raw string) (RevIncludeDirective, error) { - parts := strings.Split(raw, ":") - if len(parts) != 2 { - return RevIncludeDirective{}, fmt.Errorf("%w: _revinclude %q", ErrInvalidQuery, raw) + src, param, target, err := parseIncludeParts("_revinclude", raw) + if err != nil { + return RevIncludeDirective{}, err } - if parts[0] == "*" || parts[1] == "*" { - return RevIncludeDirective{}, fmt.Errorf("%w: wildcard _revinclude", ErrUnsupportedFeature) + if target != "" && target != searchType && target != "*" { + return RevIncludeDirective{}, fmt.Errorf("%w: _revinclude target type %q does not match search type %q", ErrInvalidQuery, target, searchType) } return RevIncludeDirective{ - SourceType: parts[0], - ParamCode: parts[1], + SourceType: src, + ParamCode: param, TargetType: searchType, }, nil } +func parseIncludeParts(kind, raw string) (source, param, target string, err error) { + parts := strings.Split(raw, ":") + if len(parts) < 2 || len(parts) > 3 { + return "", "", "", fmt.Errorf("%w: %s %q", ErrInvalidQuery, kind, raw) + } + for _, part := range parts { + if strings.TrimSpace(part) == "" { + return "", "", "", fmt.Errorf("%w: %s %q", ErrInvalidQuery, kind, raw) + } + } + source, param = parts[0], parts[1] + if len(parts) == 3 { + target = parts[2] + } + return source, param, target, nil +} + func parseSortValues(values []string) ([]SortField, error) { var out []SortField for _, raw := range values { diff --git a/pkg/search/parser_test.go b/pkg/search/parser_test.go index dece9ea..65875d9 100644 --- a/pkg/search/parser_test.go +++ b/pkg/search/parser_test.go @@ -79,7 +79,7 @@ func TestParseQueryAdvancedFeatures(t *testing.T) { } func TestParseQueryUnsupportedFeatures(t *testing.T) { - _, err := search.ParseQueryValues("Patient", map[string][]string{"subject.name.family": {"Smith"}}) + _, err := search.ParseQueryValues("Patient", map[string][]string{"subject.organization.partof.name": {"Acme"}}) if !errors.Is(err, search.ErrUnsupportedFeature) { t.Fatalf("expected chain depth error, got %v", err) } @@ -89,9 +89,14 @@ func TestParseQueryUnsupportedFeatures(t *testing.T) { t.Fatalf("expected modifier error, got %v", err) } - _, err = search.ParseQueryValues("Patient", map[string][]string{"_include": {"*:general-practitioner"}}) + _, err = search.ParseQueryValues("Patient", map[string][]string{"_include:iterate": {"Patient:general-practitioner"}}) if !errors.Is(err, search.ErrUnsupportedFeature) { - t.Fatalf("expected wildcard include error, got %v", err) + t.Fatalf("expected iterate include error, got %v", err) + } + + _, err = search.ParseQueryValues("Patient", map[string][]string{"_has:Observation:subject:_has:Encounter:diagnosis:_has:Condition:subject:code": {"x"}}) + if !errors.Is(err, search.ErrUnsupportedFeature) { + t.Fatalf("expected nested _has depth error, got %v", err) } } diff --git a/pkg/search/planner.go b/pkg/search/planner.go index a11201c..51d8611 100644 --- a/pkg/search/planner.go +++ b/pkg/search/planner.go @@ -20,6 +20,7 @@ func ResolveQuery(reg Registry, q *Query) (*Query, error) { resolved := *q resolved.Params = nil resolved.Chains = nil + resolved.Has = nil resolved.Includes = nil resolved.RevIncludes = nil resolved.Sort = nil @@ -40,20 +41,28 @@ func ResolveQuery(reg Registry, q *Query) (*Query, error) { resolved.Chains = append(resolved.Chains, resolvedChain) } + for _, has := range q.Has { + resolvedHas, err := resolveHasClause(reg, q.ResourceType, has) + if err != nil { + return nil, err + } + resolved.Has = append(resolved.Has, resolvedHas) + } + for _, inc := range q.Includes { - resolvedInc, err := resolveInclude(reg, q.ResourceType, inc) + expanded, err := resolveIncludes(reg, q.ResourceType, inc) if err != nil { return nil, err } - resolved.Includes = append(resolved.Includes, resolvedInc) + resolved.Includes = append(resolved.Includes, expanded...) } for _, rev := range q.RevIncludes { - resolvedRev, err := resolveRevInclude(reg, q.ResourceType, rev) + expanded, err := resolveRevIncludes(reg, q.ResourceType, rev) if err != nil { return nil, err } - resolved.RevIncludes = append(resolved.RevIncludes, resolvedRev) + resolved.RevIncludes = append(resolved.RevIncludes, expanded...) } for _, sortField := range q.Sort { @@ -150,6 +159,27 @@ func resolveChainClause(reg Registry, resourceType string, chain ChainClause) (C if refInfo.Type != "reference" { return ChainClause{}, fmt.Errorf("%w: chain left-hand %q is not a reference", ErrInvalidQuery, chain.RefCode) } + + if chain.Nested != nil { + targetType, err := inferChainTargetType(reg, refInfo, chain.Nested.RefCode) + if err != nil { + return ChainClause{}, err + } + if !reg.IsResourceEnabled(targetType) { + return ChainClause{}, ErrResourceTypeDisabled + } + nested, err := resolveChainClause(reg, targetType, *chain.Nested) + if err != nil { + return ChainClause{}, err + } + return ChainClause{ + RefCode: chain.RefCode, + RefFieldKey: fieldKeyForParam(chain.RefCode, "reference"), + TargetType: targetType, + Nested: &nested, + }, nil + } + targetType, err := inferChainTargetType(reg, refInfo, chain.Param.Code) if err != nil { return ChainClause{}, err @@ -171,6 +201,62 @@ func resolveChainClause(reg Registry, resourceType string, chain ChainClause) (C }, nil } +func resolveHasClause(reg Registry, searchType string, has HasClause) (HasClause, error) { + if !reg.IsResourceEnabled(has.SourceType) { + return HasClause{}, ErrResourceTypeDisabled + } + refInfo, err := lookupParam(reg, has.SourceType, has.RefCode) + if err != nil { + return HasClause{}, err + } + if refInfo.Type != "reference" { + return HasClause{}, fmt.Errorf("%w: _has reference %q is not a reference", ErrInvalidQuery, has.RefCode) + } + if err := validateHasTarget(refInfo, searchType); err != nil { + return HasClause{}, err + } + + resolved := HasClause{ + SourceType: has.SourceType, + RefCode: has.RefCode, + RefFieldKey: fieldKeyForParam(has.RefCode, "reference"), + } + + switch { + case has.Nested != nil: + nested, err := resolveHasClause(reg, has.SourceType, *has.Nested) + if err != nil { + return HasClause{}, err + } + resolved.Nested = &nested + case has.Chain != nil: + chain, err := resolveChainClause(reg, has.SourceType, *has.Chain) + if err != nil { + return HasClause{}, err + } + resolved.Chain = &chain + default: + param, err := resolveParamClause(reg, has.SourceType, has.Param) + if err != nil { + return HasClause{}, err + } + resolved.Param = param + } + return resolved, nil +} + +func validateHasTarget(refInfo ParameterInfo, searchType string) error { + if len(refInfo.Target) == 0 { + return nil + } + for _, target := range refInfo.Target { + if target == searchType { + return nil + } + } + return fmt.Errorf("%w: _has reference %q does not target %s", ErrInvalidQuery, refInfo.Code, searchType) +} + func inferChainTargetType(reg Registry, refInfo ParameterInfo, chainedCode string) (string, error) { if len(refInfo.Target) == 1 { return refInfo.Target[0], nil @@ -198,6 +284,99 @@ func inferChainTargetType(reg Registry, refInfo ParameterInfo, chainedCode strin return refInfo.Target[0], nil } +func resolveIncludes(reg Registry, resourceType string, inc IncludeDirective) ([]IncludeDirective, error) { + if inc.ParamCode != "*" { + resolved, err := resolveInclude(reg, resourceType, inc) + if err != nil { + return nil, err + } + return []IncludeDirective{resolved}, nil + } + var out []IncludeDirective + for _, info := range reg.SearchParametersFor(resourceType) { + if info.Type != "reference" { + continue + } + if inc.TargetType != "" && inc.TargetType != "*" && !paramTargets(info, inc.TargetType) { + continue + } + resolved, err := resolveInclude(reg, resourceType, IncludeDirective{ + SourceType: resourceType, + ParamCode: info.Code, + TargetType: inc.TargetType, + }) + if err != nil { + continue + } + out = append(out, resolved) + } + if len(out) == 0 { + return nil, fmt.Errorf("%w: _include %s:* matched no reference parameters", ErrInvalidQuery, resourceType) + } + return out, nil +} + +func resolveRevIncludes(reg Registry, targetType string, rev RevIncludeDirective) ([]RevIncludeDirective, error) { + if rev.SourceType != "*" && rev.ParamCode != "*" { + resolved, err := resolveRevInclude(reg, targetType, rev) + if err != nil { + return nil, err + } + return []RevIncludeDirective{resolved}, nil + } + + sourceTypes := []string{rev.SourceType} + if rev.SourceType == "*" { + sourceTypes = reg.EnabledResourceTypes() + } + + var out []RevIncludeDirective + for _, sourceType := range sourceTypes { + if !reg.IsResourceEnabled(sourceType) { + continue + } + for _, info := range reg.SearchParametersFor(sourceType) { + if info.Type != "reference" { + continue + } + if rev.ParamCode != "*" && info.Code != rev.ParamCode { + continue + } + if !paramTargets(info, targetType) && len(info.Target) > 0 { + continue + } + resolved, err := resolveRevInclude(reg, targetType, RevIncludeDirective{ + SourceType: sourceType, + ParamCode: info.Code, + TargetType: targetType, + }) + if err != nil { + continue + } + out = append(out, resolved) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("%w: _revinclude %s:%s matched no reference parameters", ErrInvalidQuery, rev.SourceType, rev.ParamCode) + } + return out, nil +} + +func paramTargets(info ParameterInfo, targetType string) bool { + if targetType == "" || targetType == "*" { + return true + } + if len(info.Target) == 0 { + return true + } + for _, t := range info.Target { + if t == targetType { + return true + } + } + return false +} + func resolveInclude(reg Registry, resourceType string, inc IncludeDirective) (IncludeDirective, error) { info, err := lookupParam(reg, resourceType, inc.ParamCode) if err != nil { @@ -207,10 +386,16 @@ func resolveInclude(reg Registry, resourceType string, inc IncludeDirective) (In return IncludeDirective{}, fmt.Errorf("%w: _include param %q is not a reference", ErrInvalidQuery, inc.ParamCode) } targetType := inc.TargetType - if targetType == "" && len(info.Target) == 1 { - targetType = info.Target[0] + if targetType == "*" { + targetType = "" } - if len(info.Target) > 1 { + if targetType != "" { + if !paramTargets(info, targetType) && len(info.Target) > 0 { + return IncludeDirective{}, fmt.Errorf("%w: _include param %q does not target %s", ErrInvalidQuery, inc.ParamCode, targetType) + } + } else if len(info.Target) == 1 { + targetType = info.Target[0] + } else if len(info.Target) > 1 { // Multi-target references are resolved at expansion time. targetType = "" } @@ -322,16 +507,19 @@ func BuildPlan(q *Query) (*Plan, error) { } for _, chain := range q.Chains { - pp, err := buildParamPlan(chain.TargetType, chain.Param) + cp, err := buildChainPlan(chain) if err != nil { return nil, err } - plan.ChainPlans = append(plan.ChainPlans, ChainPlan{ - RefCode: chain.RefCode, - RefFieldKey: chain.RefFieldKey, - TargetType: chain.TargetType, - ParamPlan: *pp, - }) + plan.ChainPlans = append(plan.ChainPlans, *cp) + } + + for _, has := range q.Has { + hp, err := buildHasPlan(has) + if err != nil { + return nil, err + } + plan.HasPlans = append(plan.HasPlans, *hp) } for _, inc := range q.Includes { @@ -355,6 +543,63 @@ func BuildPlan(q *Query) (*Plan, error) { return plan, nil } +func buildChainPlan(chain ChainClause) (*ChainPlan, error) { + plan := &ChainPlan{ + RefCode: chain.RefCode, + RefFieldKey: chain.RefFieldKey, + TargetType: chain.TargetType, + } + if chain.Nested != nil { + nested, err := buildChainPlan(*chain.Nested) + if err != nil { + return nil, err + } + plan.Nested = nested + return plan, nil + } + pp, err := buildParamPlan(chain.TargetType, chain.Param) + if err != nil { + return nil, err + } + if pp == nil { + return nil, fmt.Errorf("%w: chained search %q has no value", ErrInvalidQuery, chain.RefCode) + } + plan.ParamPlan = *pp + return plan, nil +} + +func buildHasPlan(has HasClause) (*HasPlan, error) { + plan := &HasPlan{ + SourceType: has.SourceType, + RefCode: has.RefCode, + RefFieldKey: has.RefFieldKey, + } + switch { + case has.Nested != nil: + nested, err := buildHasPlan(*has.Nested) + if err != nil { + return nil, err + } + plan.Nested = nested + case has.Chain != nil: + chain, err := buildChainPlan(*has.Chain) + if err != nil { + return nil, err + } + plan.ChainPlan = chain + default: + pp, err := buildParamPlan(has.SourceType, has.Param) + if err != nil { + return nil, err + } + if pp == nil { + return nil, fmt.Errorf("%w: _has %s:%s has no value", ErrInvalidQuery, has.SourceType, has.RefCode) + } + plan.ParamPlan = *pp + } + return plan, nil +} + func buildParamPlan(resourceType string, clause ParamClause) (*ParamPlan, error) { if len(clause.Values) == 0 { return nil, nil diff --git a/pkg/search/query.go b/pkg/search/query.go index 8120240..b9b2e5e 100644 --- a/pkg/search/query.go +++ b/pkg/search/query.go @@ -56,12 +56,24 @@ type ValueClause struct { Operator MatchOperator } -// ChainClause is a single-hop chained search parameter (e.g. subject.name). +// ChainClause is a chained search parameter (e.g. subject.name or subject.organization.name). +// Nested is set for multi-hop chains; Param is the terminal predicate on the last hop. type ChainClause struct { RefCode string RefFieldKey string TargetType string Param ParamClause + Nested *ChainClause +} + +// HasClause is a reverse-chained _has search (e.g. _has:Observation:subject:code=8867-4). +type HasClause struct { + SourceType string + RefCode string + RefFieldKey string + Param ParamClause + Chain *ChainClause + Nested *HasClause } // IncludeDirective requests direct include expansion for one reference parameter. @@ -95,6 +107,7 @@ type Query struct { ResourceType string Params []ParamClause Chains []ChainClause + Has []HasClause Includes []IncludeDirective RevIncludes []RevIncludeDirective Count int @@ -129,12 +142,23 @@ const ( combineAnd ) -// ChainPlan is a single-hop chained search execution stage. +// ChainPlan is a chained search execution stage. Nested walks one more hop. type ChainPlan struct { RefCode string RefFieldKey string TargetType string ParamPlan ParamPlan + Nested *ChainPlan +} + +// HasPlan is a reverse-chained _has execution stage. +type HasPlan struct { + SourceType string + RefCode string + RefFieldKey string + ParamPlan ParamPlan + ChainPlan *ChainPlan + Nested *HasPlan } // IncludePlan describes direct include expansion after primary search. @@ -158,6 +182,7 @@ type Plan struct { ResourceType string ParamPlans []ParamPlan ChainPlans []ChainPlan + HasPlans []HasPlan Includes []IncludePlan RevIncludes []RevIncludePlan Count int diff --git a/pkg/search/registry.go b/pkg/search/registry.go index da1d6d5..b41ac74 100644 --- a/pkg/search/registry.go +++ b/pkg/search/registry.go @@ -179,7 +179,7 @@ func lookupByCanonical(snapshot *registry.Snapshot, canonicalURL string) (regist func isSearchableType(paramType string) bool { switch paramType { - case "token", "string", "date", "reference", "number", "composite", "quantity": + case "token", "string", "date", "reference", "number", "composite", "quantity", "uri": return true default: return false diff --git a/pkg/search/reindex_test.go b/pkg/search/reindex_test.go index 08936af..758db18 100644 --- a/pkg/search/reindex_test.go +++ b/pkg/search/reindex_test.go @@ -2,6 +2,7 @@ package search_test import ( "context" + "strings" "testing" "github.com/degoke/health-ai-stack/pkg/search" @@ -39,6 +40,10 @@ func (m *memSearchBackend) QueryPrepared(context.Context, store.PreparedQuery, m } func (m *memSearchBackend) LookupMatch(_ context.Context, match store.SearchMatch) ([]string, error) { + op := match.Operator + if op == "" { + op = "eq" + } seen := make(map[string]struct{}) var ids []string for _, entry := range m.entries { @@ -46,18 +51,39 @@ func (m *memSearchBackend) LookupMatch(_ context.Context, match store.SearchMatc continue } for key, value := range entry.Fields { - if key == match.FieldKey && value == match.Value { - if _, ok := seen[entry.ID]; ok { - continue - } - seen[entry.ID] = struct{}{} - ids = append(ids, entry.ID) + if key != match.FieldKey { + continue + } + if !memMatchValue(op, value, match.Value) { + continue + } + if _, ok := seen[entry.ID]; ok { + continue } + seen[entry.ID] = struct{}{} + ids = append(ids, entry.ID) } } return ids, nil } +func memMatchValue(op, have, want string) bool { + switch op { + case "", "eq", "=": + return have == want + case "below": + return strings.HasPrefix(have, want) + case "above": + return strings.HasPrefix(want, have) + case "contains": + return strings.Contains(strings.ToLower(have), strings.ToLower(want)) + case "exact": + return have == want + default: + return have == want + } +} + func (m *memSearchBackend) FieldValues(_ context.Context, resourceType, fieldKey string, resourceIDs []string) (map[string]string, error) { out := make(map[string]string, len(resourceIDs)) for _, entry := range m.entries { @@ -80,6 +106,79 @@ func (m *memSearchBackend) FieldValues(_ context.Context, resourceType, fieldKey return out, nil } +type memAdvancedSearchBackend struct { + memSearchBackend +} + +func (m *memAdvancedSearchBackend) LookupReferences(_ context.Context, resourceType, fieldKey string, sourceIDs []string) (map[string][]store.ReferenceLink, error) { + wanted := make(map[string]struct{}, len(sourceIDs)) + for _, id := range sourceIDs { + wanted[id] = struct{}{} + } + out := make(map[string][]store.ReferenceLink) + for _, entry := range m.entries { + if resourceType != "" && entry.ResourceType != resourceType { + continue + } + if _, ok := wanted[entry.ID]; !ok { + continue + } + for key, value := range entry.Fields { + if key != fieldKey { + continue + } + link := store.ReferenceLink{Literal: value} + if i := strings.Index(value, "/"); i > 0 && i < len(value)-1 { + link.TargetType = value[:i] + link.TargetID = value[i+1:] + } else if i := strings.Index(value, "|"); i > 0 && i < len(value)-1 { + link.TargetType = value[:i] + link.TargetID = value[i+1:] + } else { + link.TargetID = value + } + out[entry.ID] = append(out[entry.ID], link) + } + } + return out, nil +} + +func (m *memAdvancedSearchBackend) LookupReferencing(_ context.Context, sourceType, fieldKey, targetType, targetID string) ([]string, error) { + values := []string{targetID} + if targetType != "" { + values = append(values, targetType+"/"+targetID, targetType+"|"+targetID) + } + wanted := make(map[string]struct{}, len(values)) + for _, v := range values { + wanted[v] = struct{}{} + } + seen := make(map[string]struct{}) + var ids []string + for _, entry := range m.entries { + if sourceType != "" && entry.ResourceType != sourceType { + continue + } + for key, value := range entry.Fields { + if key != fieldKey { + continue + } + if _, ok := wanted[value]; !ok { + continue + } + if _, ok := seen[entry.ID]; ok { + continue + } + seen[entry.ID] = struct{}{} + ids = append(ids, entry.ID) + } + } + return ids, nil +} + +func (m *memAdvancedSearchBackend) LookupFullText(context.Context, string, string) (store.FullTextMatch, error) { + return store.FullTextMatch{}, nil +} + type memResourceStore struct { byKey map[string]*types.ResourceEnvelope } diff --git a/pkg/sqlite/README.md b/pkg/sqlite/README.md index 7c135ea..180bdb6 100644 --- a/pkg/sqlite/README.md +++ b/pkg/sqlite/README.md @@ -124,6 +124,7 @@ err = session.Commit(ctx) | `date.` | `search_date` | | `number.` | `search_number` | | `reference.` or `ref.` | `search_reference` | +| `uri.` | `search_string` | Keys without a prefix (for example `family`) default to `search_string`. FHIR search parsing stays in `pkg/search`; this package only stores prepared entries. diff --git a/pkg/sqlite/doc.go b/pkg/sqlite/doc.go index 8e6c57f..d509302 100644 --- a/pkg/sqlite/doc.go +++ b/pkg/sqlite/doc.go @@ -127,6 +127,7 @@ // - date. → hai_search_date // - number. → hai_search_number // - reference. or ref. → hai_search_reference +// - uri. → hai_search_string // // Keys without a prefix (for example "family") default to hai_search_string. FHIR search // parsing and token extraction remain in pkg/search; this package stores prepared entries. diff --git a/pkg/sqlite/search_store.go b/pkg/sqlite/search_store.go index 5f6ffa3..2948841 100644 --- a/pkg/sqlite/search_store.go +++ b/pkg/sqlite/search_store.go @@ -53,6 +53,8 @@ func parseSearchFieldKey(key string) (searchTable, string, error) { return searchTableNumber, parts[1], nil case "reference", "ref": return searchTableReference, parts[1], nil + case "uri": + return searchTableString, parts[1], nil default: return searchTableString, key, nil }