From 2c047cfc0370d253e999e0ec1c71f6a3806e7eb1 Mon Sep 17 00:00:00 2001 From: orian Date: Mon, 7 Sep 2026 11:23:48 +0200 Subject: [PATCH] feat(locate): add table column selection --- README.md | 28 ++- cmd/hclexp/hclexp.go | 4 +- cmd/hclexp/locate.go | 263 ++++++++++++++++++++++++++--- cmd/hclexp/locate_test.go | 199 ++++++++++++++++++++-- docs/README.hcl.md | 22 +++ internal/loader/hcl/locate.go | 89 ++++++++++ internal/loader/hcl/locate_test.go | 39 +++++ 7 files changed, 597 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index e555bb5..911c54d 100644 --- a/README.md +++ b/README.md @@ -778,6 +778,11 @@ hclexp locate -manifest manifest.hcl -layer-root ./schema 'events*' person # Also report which per-node dump files declare it hclexp locate -manifest manifest.hcl -layer-root ./schema -dump prod/eu events +# Find selected columns within selected tables across every dump snapshot +hclexp locate -dump prod/eu \ + -tables 'flag_evaluations,sharded_flag_evaluations' \ + -columns 'person_properties,group*_properties' -format json + # Ad-hoc layer dirs or files, before a manifest exists (no placement info) hclexp locate -layer roles/shared,roles/ops/prod events @@ -805,6 +810,14 @@ or a `[raw ]` block. An object extended by others lists its children (`extended by: ...`), and each dump site names its node (from the dump's `node {}` block, else the filename). +Column lookup is a separate selector mode: provide both `-tables` and +`-columns` as comma-separated exact names or globs, with no positional object +patterns. It reports authored `column`, `patch_column`, and `modify_column` +blocks, grouped by database, table, and column; dump sites include the snapshot +node. It remains syntax-only, so raw SQL, `drop_columns`, and inherited columns +not written on the selected table are not invented. A valid query with no +matches succeeds and JSON includes `"columns": []`. + **Flags:** - `-manifest` — the same role manifest `plan`/`validate`/`load` consume. @@ -819,20 +832,25 @@ or a `[raw ]` block. An object extended by others lists its children - `-dump` — directory of per-node `.hcl` dumps (as written by `introspect`/`dump-cluster`); reports which node files also declare each matching object +- `-tables` / `-columns` — enter column-selector mode; both are required and + accept comma-separated exact names or globs. Table patterns match bare or + `database.table`; column patterns match bare, `table.column`, or + `database.table.column`. - `-format` — `text` (default) or `json` (a `{"patterns": [...], "objects": [...]}` / `{"duplicates": [...]}` document with per-site - file/line/layer/markers/placements) + file/line/layer/markers/placements, or a column-selector document containing + `table_patterns`, `column_patterns`, and `columns`) - `-duplicates` — takes no name argument and requires `-manifest` or - `-layer` (mutually exclusive with `-dump`); lists every object defined + `-layer` (mutually exclusive with `-dump` and column selectors); lists every object defined at more than one site and exits non-zero when any is found. Patch sites, `override = true` redeclarations, and declarations carrying `extend` are refinements rather than definitions and do not count. Abstract declarations do count: copying the same abstract schema is still duplication. Reported groups retain every site, including refinements. -Exit codes: `0` on success, `1` when any pattern matches nothing (a -scriptable existence check) or `-duplicates` finds any, `2` on usage -errors. +Exit codes: `0` on success (including an empty column query), `1` when any +positional object pattern matches nothing, `-duplicates` finds any, or an +input cannot be read/parsed; `2` on usage errors. ## Browse the schema in a web UI diff --git a/cmd/hclexp/hclexp.go b/cmd/hclexp/hclexp.go index 5517fac..5593d63 100644 --- a/cmd/hclexp/hclexp.go +++ b/cmd/hclexp/hclexp.go @@ -105,8 +105,8 @@ Commands: (-format json for structured output) decompose turn per-environment node dumps into shared and env HCL layers, with an exact compose-against-dump round-trip gate - locate find every declaration site of an object across manifest - layers and dump directories (-duplicates audits the once-only rule) + locate find object or table-column declaration sites across manifest + layers and dumps (-duplicates audits the object once-only rule) sql2hcl apply SQL DDL edits (CREATE/ALTER/DROP/RENAME) to an HCL schema load parse and resolve an HCL config, layer stack, or manifest role (default when flags are given) diff --git a/cmd/hclexp/locate.go b/cmd/hclexp/locate.go index 7056643..4f17eeb 100644 --- a/cmd/hclexp/locate.go +++ b/cmd/hclexp/locate.go @@ -49,6 +49,38 @@ type locateDump struct { Type string `json:"type"` } +type locateColumnSite struct { + File string `json:"file"` + Line int `json:"line"` + Layer string `json:"layer,omitempty"` + Type string `json:"type"` + Placements []locatePlacement `json:"placements,omitempty"` +} + +type locateColumnDump struct { + File string `json:"file"` + Line int `json:"line"` + Node string `json:"node"` + Type string `json:"type"` +} + +type locateColumn struct { + Database string `json:"database"` + Table string `json:"table"` + Name string `json:"column"` + Declarations []locateColumnSite `json:"declarations,omitempty"` + Dumps []locateColumnDump `json:"dumps,omitempty"` +} + +// locateColumnDoc is deliberately separate from the existing object-query +// document, keeping its JSON contract unchanged. Columns is always encoded, +// including as [] when a valid selector query finds nothing. +type locateColumnDoc struct { + TablePatterns []string `json:"table_patterns"` + ColumnPatterns []string `json:"column_patterns"` + Columns []locateColumn `json:"columns"` +} + // locateObject collects every declaration site of one object name. Objects // are keyed by (database, name) — the namespace ClickHouse object types // share — so a table and a raw block with the same name land in one entry, @@ -75,11 +107,15 @@ type locateDoc struct { // locateFlagsError reports the usage error in a locate invocation, if any. // Pure so the exit-2 paths are testable without a subprocess. -func locateFlagsError(manifest, layers, dump, format string, duplicates bool, patterns []string) error { +func locateFlagsError(manifest, layers, dump, format string, duplicates bool, patterns, tables, columns []string) error { if format != "text" && format != "json" { return fmt.Errorf("invalid -format %q (want text or json)", format) } + columnMode := len(tables) > 0 || len(columns) > 0 if duplicates { + if columnMode { + return fmt.Errorf("-duplicates cannot be combined with -tables or -columns") + } if manifest == "" && layers == "" { return fmt.Errorf("-duplicates requires -manifest or -layer (it audits authored layers)") } @@ -94,12 +130,34 @@ func locateFlagsError(manifest, layers, dump, format string, duplicates bool, pa if manifest == "" && layers == "" && dump == "" { return fmt.Errorf("at least one of -manifest, -layer, or -dump is required") } + if columnMode { + if len(patterns) != 0 { + return fmt.Errorf("-tables/-columns cannot be combined with name arguments") + } + if len(tables) == 0 { + return fmt.Errorf("-columns requires -tables") + } + if len(columns) == 0 { + return fmt.Errorf("-tables requires -columns") + } + if err := validateLocatePatterns("-tables", tables); err != nil { + return err + } + return validateLocatePatterns("-columns", columns) + } if len(patterns) == 0 { return fmt.Errorf("at least one argument is required") } + return validateLocatePatterns("name", patterns) +} + +func validateLocatePatterns(kind string, patterns []string) error { for _, pattern := range patterns { if _, err := filepath.Match(pattern, ""); err != nil { - return fmt.Errorf("invalid pattern %q: %w", pattern, err) + if kind == "name" { + return fmt.Errorf("invalid pattern %q: %w", pattern, err) + } + return fmt.Errorf("invalid %s pattern %q: %w", kind, pattern, err) } } return nil @@ -149,27 +207,7 @@ func parseManifestAllEnvs(path string) ([]locateStack, error) { func buildLocateDoc(stacks []locateStack, layerRoot string, extraLayers []string, dumpDir string, patterns []string, duplicates bool) (locateDoc, []string, error) { // Index which (role, env) stacks include each resolved layer, keeping // first-seen layer order so output is stable. - stacksByLayer := map[string][]locatePlacement{} - var layerOrder []string - for _, s := range stacks { - for _, l := range s.Layers { - resolved := filepath.Join(layerRoot, l) - if _, ok := stacksByLayer[resolved]; !ok { - layerOrder = append(layerOrder, resolved) - } - stacksByLayer[resolved] = appendUniquePlacement(stacksByLayer[resolved], locatePlacement{Role: s.Role, Env: s.Env}) - } - } - // Ad-hoc -layer entries scan after the manifest's layers. They resolve - // as given (not under -layer-root) and carry no placements. - for _, l := range extraLayers { - resolved := filepath.Clean(l) - if _, ok := stacksByLayer[resolved]; ok { - continue - } - stacksByLayer[resolved] = nil - layerOrder = append(layerOrder, resolved) - } + stacksByLayer, layerOrder := indexLocateLayers(stacks, layerRoot, extraLayers) // Scan each file once; a file reachable through several layers (e.g. a // dir layer and the same file listed directly) keeps its first @@ -282,6 +320,136 @@ func buildLocateDoc(stacks []locateStack, layerRoot string, extraLayers []string return doc, unmatched, nil } +func indexLocateLayers(stacks []locateStack, layerRoot string, extraLayers []string) (map[string][]locatePlacement, []string) { + stacksByLayer := map[string][]locatePlacement{} + var layerOrder []string + for _, s := range stacks { + for _, l := range s.Layers { + resolved := filepath.Join(layerRoot, l) + if _, ok := stacksByLayer[resolved]; !ok { + layerOrder = append(layerOrder, resolved) + } + stacksByLayer[resolved] = appendUniquePlacement(stacksByLayer[resolved], locatePlacement{Role: s.Role, Env: s.Env}) + } + } + // Ad-hoc -layer entries scan after the manifest's layers. They resolve + // as given (not under -layer-root) and carry no placements. + for _, l := range extraLayers { + resolved := filepath.Clean(l) + if _, ok := stacksByLayer[resolved]; ok { + continue + } + stacksByLayer[resolved] = nil + layerOrder = append(layerOrder, resolved) + } + return stacksByLayer, layerOrder +} + +func buildLocateColumnDoc(stacks []locateStack, layerRoot string, extraLayers []string, dumpDir string, tablePatterns, columnPatterns []string) (locateColumnDoc, error) { + doc := locateColumnDoc{ + TablePatterns: tablePatterns, + ColumnPatterns: columnPatterns, + Columns: []locateColumn{}, + } + type key struct{ db, table, column string } + index := map[key]int{} + upsert := func(d hclload.ColumnDeclaration) *locateColumn { + k := key{d.Database, d.Table, d.Name} + if i, ok := index[k]; ok { + return &doc.Columns[i] + } + index[k] = len(doc.Columns) + doc.Columns = append(doc.Columns, locateColumn{Database: d.Database, Table: d.Table, Name: d.Name}) + return &doc.Columns[len(doc.Columns)-1] + } + matches := func(d hclload.ColumnDeclaration) bool { + return matchesTablePatterns(tablePatterns, d.Database, d.Table) && matchesColumnPatterns(columnPatterns, d.Database, d.Table, d.Name) + } + + stacksByLayer, layerOrder := indexLocateLayers(stacks, layerRoot, extraLayers) + layerByFile := map[string]string{} + for _, layer := range layerOrder { + files, err := hclload.LayerFiles(layer) + if err != nil { + return locateColumnDoc{}, err + } + for _, file := range files { + if _, ok := layerByFile[file]; ok { + continue + } + layerByFile[file] = layer + decls, _, err := hclload.ScanFileColumnDeclarations(file) + if err != nil { + return locateColumnDoc{}, err + } + for _, d := range decls { + if !matches(d) { + continue + } + column := upsert(d) + column.Declarations = append(column.Declarations, locateColumnSite{ + File: d.File, Line: d.Line, Layer: layer, Type: d.Type, + Placements: stacksByLayer[layer], + }) + } + } + } + + if dumpDir != "" { + files, err := filepath.Glob(filepath.Join(dumpDir, "*.hcl")) + if err != nil { + return locateColumnDoc{}, fmt.Errorf("dump dir %q: %w", dumpDir, err) + } + sort.Strings(files) + for _, file := range files { + decls, node, err := hclload.ScanFileColumnDeclarations(file) + if err != nil { + return locateColumnDoc{}, err + } + if node == "" { + node = strings.TrimSuffix(filepath.Base(file), ".hcl") + } + for _, d := range decls { + if !matches(d) { + continue + } + column := upsert(d) + column.Dumps = append(column.Dumps, locateColumnDump{File: d.File, Line: d.Line, Node: node, Type: d.Type}) + } + } + } + + sort.SliceStable(doc.Columns, func(i, j int) bool { + left, right := doc.Columns[i], doc.Columns[j] + if left.Database != right.Database { + return left.Database < right.Database + } + if left.Table != right.Table { + return left.Table < right.Table + } + return left.Name < right.Name + }) + return doc, nil +} + +func matchesTablePatterns(patterns []string, database, table string) bool { + for _, pattern := range patterns { + if hclload.MatchesPattern(pattern, database, table) { + return true + } + } + return false +} + +func matchesColumnPatterns(patterns []string, database, table, column string) bool { + for _, pattern := range patterns { + if hclload.MatchesColumnPattern(pattern, database, table, column) { + return true + } + } + return false +} + // matchesAnyPattern reports whether any pattern matches the object and marks // every pattern that does in hits, so the caller can tell which patterns // found nothing across the whole scan. @@ -405,6 +573,25 @@ func renderLocateText(w io.Writer, doc locateDoc) { } } +func renderLocateColumnText(w io.Writer, doc locateColumnDoc) { + for _, column := range doc.Columns { + fmt.Fprintf(w, "column %s.%s.%s\n", column.Database, column.Table, column.Name) + for _, d := range column.Declarations { + marker := "" + if d.Type != "column" { + marker = " [" + d.Type + "]" + } + fmt.Fprintf(w, " %s:%d%s\n", d.File, d.Line, marker) + if len(d.Placements) > 0 { + fmt.Fprintf(w, " %s\n", formatPlacements(d.Placements)) + } + } + for _, d := range column.Dumps { + fmt.Fprintf(w, " dump: %s:%d (node %s)\n", d.File, d.Line, d.Node) + } + } +} + func renderDuplicatesText(w io.Writer, doc locateDoc) { if len(doc.Duplicates) == 0 { fmt.Fprintln(w, "no duplicate declarations") @@ -419,8 +606,9 @@ func renderDuplicatesText(w io.Writer, doc locateDoc) { // runLocate answers "where is object X declared?" across a manifest's layer // tree, ad-hoc -layer entries, and/or a dump directory, or (with // -duplicates) audits the layer tree for objects defined at more than one -// site. Read-only; exits 1 when any pattern matches nothing or duplicates -// exist, 2 on usage errors. +// site. Column selector mode returns success with an empty result. Object +// pattern mode exits 1 when any pattern matches nothing; duplicate mode exits +// 1 when duplicates exist. Usage errors exit 2. func runLocate(args []string) { fs := flag.NewFlagSet("hclexp locate", flag.ExitOnError) manifestFlag := fs.String("manifest", "", "HCL manifest: role blocks with env blocks; every (role, env) stack is searched") @@ -429,10 +617,14 @@ func runLocate(args []string) { dumpFlag := fs.String("dump", "", "directory of per-node .hcl dumps to search as well") formatFlag := fs.String("format", "text", "output format: text (default) or json") duplicatesFlag := fs.Bool("duplicates", false, "list every object defined at more than one site (patch/override/extend sites refine; abstracts define); takes no name argument") + tablesFlag := fs.String("tables", "", "comma-separated table names or globs for column lookup; requires -columns and no name argument") + columnsFlag := fs.String("columns", "", "comma-separated column names or globs to locate within -tables") _ = fs.Parse(args) patterns := fs.Args() - if err := locateFlagsError(*manifestFlag, *layersFlag, *dumpFlag, *formatFlag, *duplicatesFlag, patterns); err != nil { + tablePatterns := splitList(*tablesFlag) + columnPatterns := splitList(*columnsFlag) + if err := locateFlagsError(*manifestFlag, *layersFlag, *dumpFlag, *formatFlag, *duplicatesFlag, patterns, tablePatterns, columnPatterns); err != nil { slog.Error("invalid locate invocation", "err", err) os.Exit(2) } @@ -451,6 +643,25 @@ func runLocate(args []string) { } } + if len(tablePatterns) > 0 || len(columnPatterns) > 0 { + doc, err := buildLocateColumnDoc(stacks, *layerRootFlag, splitList(*layersFlag), *dumpFlag, tablePatterns, columnPatterns) + if err != nil { + slog.Error("locate failed", "err", err) + os.Exit(1) + } + if *formatFlag == "json" { + out, err := json.MarshalIndent(doc, "", " ") + if err != nil { + slog.Error("failed to render JSON", "err", err) + os.Exit(1) + } + fmt.Println(string(out)) + } else { + renderLocateColumnText(os.Stdout, doc) + } + return + } + doc, unmatched, err := buildLocateDoc(stacks, *layerRootFlag, splitList(*layersFlag), *dumpFlag, patterns, *duplicatesFlag) if err != nil { slog.Error("locate failed", "err", err) diff --git a/cmd/hclexp/locate_test.go b/cmd/hclexp/locate_test.go index cbc4531..f562a8d 100644 --- a/cmd/hclexp/locate_test.go +++ b/cmd/hclexp/locate_test.go @@ -2,7 +2,9 @@ package main import ( "bytes" + "encoding/json" "os" + "os/exec" "path/filepath" "testing" @@ -96,20 +98,27 @@ func TestLocateFlagsError(t *testing.T) { one := []string{"events"} several := []string{"events", "person_*"} - assert.NoError(t, locateFlagsError("m.hcl", "", "", "text", false, one)) - assert.NoError(t, locateFlagsError("", "", "dumps", "json", false, one)) - assert.NoError(t, locateFlagsError("", "a,b", "", "text", false, one), "-layer alone is a source") - assert.NoError(t, locateFlagsError("m.hcl", "", "", "text", false, several), "several patterns") - assert.NoError(t, locateFlagsError("m.hcl", "", "", "text", true, nil)) - assert.NoError(t, locateFlagsError("", "a", "", "text", true, nil), "-duplicates audits -layer too") - - assert.Error(t, locateFlagsError("", "", "", "text", false, one), "needs a source") - assert.Error(t, locateFlagsError("m.hcl", "", "", "yaml", false, one), "bad format") - assert.Error(t, locateFlagsError("m.hcl", "", "", "text", false, nil), "missing name") - assert.Error(t, locateFlagsError("m.hcl", "", "", "text", false, []string{"a", "[bad"}), "invalid glob among several") - assert.Error(t, locateFlagsError("", "", "dumps", "text", true, nil), "-duplicates without authored layers") - assert.Error(t, locateFlagsError("m.hcl", "", "dumps", "text", true, nil), "-duplicates with -dump") - assert.Error(t, locateFlagsError("m.hcl", "", "", "text", true, one), "-duplicates with name") + assert.NoError(t, locateFlagsError("m.hcl", "", "", "text", false, one, nil, nil)) + assert.NoError(t, locateFlagsError("", "", "dumps", "json", false, one, nil, nil)) + assert.NoError(t, locateFlagsError("", "a,b", "", "text", false, one, nil, nil), "-layer alone is a source") + assert.NoError(t, locateFlagsError("m.hcl", "", "", "text", false, several, nil, nil), "several patterns") + assert.NoError(t, locateFlagsError("m.hcl", "", "", "text", true, nil, nil, nil)) + assert.NoError(t, locateFlagsError("", "a", "", "text", true, nil, nil, nil), "-duplicates audits -layer too") + assert.NoError(t, locateFlagsError("", "", "dumps", "json", false, nil, []string{"events"}, []string{"id"}), "column mode") + + assert.Error(t, locateFlagsError("", "", "", "text", false, one, nil, nil), "needs a source") + assert.Error(t, locateFlagsError("m.hcl", "", "", "yaml", false, one, nil, nil), "bad format") + assert.Error(t, locateFlagsError("m.hcl", "", "", "text", false, nil, nil, nil), "missing name") + assert.Error(t, locateFlagsError("m.hcl", "", "", "text", false, []string{"a", "[bad"}, nil, nil), "invalid glob among several") + assert.Error(t, locateFlagsError("", "", "dumps", "text", true, nil, nil, nil), "-duplicates without authored layers") + assert.Error(t, locateFlagsError("m.hcl", "", "dumps", "text", true, nil, nil, nil), "-duplicates with -dump") + assert.Error(t, locateFlagsError("m.hcl", "", "", "text", true, one, nil, nil), "-duplicates with name") + assert.Error(t, locateFlagsError("m.hcl", "", "", "text", false, one, []string{"events"}, []string{"id"}), "columns with positional name") + assert.Error(t, locateFlagsError("m.hcl", "", "", "text", false, nil, nil, []string{"id"}), "columns require tables") + assert.Error(t, locateFlagsError("m.hcl", "", "", "text", false, nil, []string{"events"}, nil), "tables require columns") + assert.Error(t, locateFlagsError("m.hcl", "", "", "text", true, nil, []string{"events"}, []string{"id"}), "duplicates with columns") + assert.Error(t, locateFlagsError("m.hcl", "", "", "text", false, nil, []string{"[bad"}, []string{"id"}), "bad table glob") + assert.Error(t, locateFlagsError("m.hcl", "", "", "text", false, nil, []string{"events"}, []string{"[bad"}), "bad column glob") } func TestParseManifestAllEnvs(t *testing.T) { @@ -214,6 +223,54 @@ func TestBuildLocateDocMultiplePatterns(t *testing.T) { assert.Equal(t, []string{"person", "only_*", "nosuch*"}, doc.Patterns) } +func TestBuildLocateColumnDocFindsLayerAndDumpSites(t *testing.T) { + root := locateTree(t) + stacks, err := parseManifestAllEnvs(filepath.Join(root, "manifest.hcl")) + require.NoError(t, err) + + doc, err := buildLocateColumnDoc(stacks, root, nil, filepath.Join(root, "dumps"), + []string{"events*"}, []string{"uuid"}) + require.NoError(t, err) + assert.Equal(t, []string{"events*"}, doc.TablePatterns) + assert.Equal(t, []string{"uuid"}, doc.ColumnPatterns) + require.Len(t, doc.Columns, 2) + + events := doc.Columns[0] + assert.Equal(t, "posthog", events.Database) + assert.Equal(t, "events", events.Table) + assert.Equal(t, "uuid", events.Name) + assert.Empty(t, events.Declarations, "inherited columns are not invented by syntax-only locate") + assert.Equal(t, []locateColumnDump{{ + File: filepath.Join(root, "dumps", "node1.hcl"), Line: 7, Node: "node1", Type: "column", + }}, events.Dumps) + + base := doc.Columns[1] + assert.Equal(t, "events_base", base.Table) + require.Len(t, base.Declarations, 1) + assert.Equal(t, filepath.Join(root, "shared", "base.hcl"), base.Declarations[0].File) + assert.Equal(t, []locatePlacement{ + {Role: "ingestion", Env: "prod-us"}, + {Role: "ingestion", Env: "prod-eu"}, + {Role: "aux", Env: "prod-us"}, + }, base.Declarations[0].Placements) +} + +func TestBuildLocateColumnDocNoMatchIsEmpty(t *testing.T) { + root := locateTree(t) + doc, err := buildLocateColumnDoc(nil, root, nil, filepath.Join(root, "dumps"), + []string{"flag_evaluations"}, []string{"person_properties"}) + require.NoError(t, err) + assert.Empty(t, doc.Columns) + + body, err := json.Marshal(doc) + require.NoError(t, err) + assert.JSONEq(t, `{ + "table_patterns": ["flag_evaluations"], + "column_patterns": ["person_properties"], + "columns": [] +}`, string(body)) +} + // A dump file without a node{} block falls back to the filename stem, the // same identity drift uses. func TestBuildLocateDocDumpNodeFallback(t *testing.T) { @@ -330,3 +387,117 @@ func TestRenderLocateText(t *testing.T) { assert.Contains(t, dupBuf.String(), "duplicate table posthog.person") assert.Contains(t, dupBuf.String(), "base.hcl:7") } + +func TestRenderLocateColumnText(t *testing.T) { + doc := locateColumnDoc{Columns: []locateColumn{{ + Database: "posthog", + Table: "flag_evaluations", + Name: "person_properties", + Declarations: []locateColumnSite{{ + File: "schema/flags.hcl", Line: 12, Type: "modify_column", + Placements: []locatePlacement{{Role: "ingestion", Env: "prod-us"}}, + }}, + Dumps: []locateColumnDump{{File: "prod-us/node1.hcl", Line: 20, Node: "node1", Type: "column"}}, + }}} + + var buf bytes.Buffer + renderLocateColumnText(&buf, doc) + out := buf.String() + assert.Contains(t, out, "column posthog.flag_evaluations.person_properties") + assert.Contains(t, out, "schema/flags.hcl:12 [modify_column]") + assert.Contains(t, out, "(ingestion, prod-us)") + assert.Contains(t, out, "dump: prod-us/node1.hcl:20 (node node1)") +} + +func TestLocateColumnsCLIProcess(t *testing.T) { + if os.Getenv("HCLEXP_LOCATE_COLUMNS_HELPER") != "1" { + return + } + for i, arg := range os.Args { + if arg == "--" { + runLocate(os.Args[i+1:]) + os.Exit(0) + return + } + } + t.Fatal("missing locate CLI arguments") +} + +func TestLocateColumnsCLIEndToEnd(t *testing.T) { + root := t.TempDir() + dumps := filepath.Join(root, "prod-us") + require.NoError(t, os.MkdirAll(dumps, 0o755)) + write := func(name, body string) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(dumps, name), []byte(body), 0o600)) + } + write("node-a.hcl", ` +node "node-a" {} +database "posthog" { + table "flag_evaluations" { + column "person_properties" { type = "String" } + column "group0_properties" { type = "String" } + } + table "sharded_flag_evaluations" { + column "group4_properties" { type = "String" } + } + table "not_selected" { + column "person_properties" { type = "String" } + } +} +`) + write("node-b.hcl", ` +node "node-b" {} +database "posthog" { + table "flag_evaluations" { + column "person_properties" { type = "String" } + } + table "writable_flag_evaluations" { + column "group2_properties" { type = "String" } + } + table "kafka_flag_evaluations" { + column "group3_properties" { type = "String" } + column "event" { type = "String" } + } +} +`) + + tables := "flag_evaluations,sharded_flag_evaluations,writable_flag_evaluations,kafka_flag_evaluations" + columns := "person_properties,group0_properties,group1_properties,group2_properties,group3_properties,group4_properties" + output, err := runLocateColumnsCLI(t, "-dump", dumps, "-tables", tables, "-columns", columns, "-format", "json") + require.NoError(t, err, string(output)) + var doc locateColumnDoc + require.NoError(t, json.Unmarshal(output, &doc), string(output)) + require.Len(t, doc.Columns, 5) + assert.Equal(t, "flag_evaluations", doc.Columns[0].Table) + assert.Equal(t, "group0_properties", doc.Columns[0].Name) + assert.Equal(t, "person_properties", doc.Columns[1].Name) + assert.Equal(t, []string{"node-a", "node-b"}, []string{doc.Columns[1].Dumps[0].Node, doc.Columns[1].Dumps[1].Node}) + assert.Equal(t, "kafka_flag_evaluations", doc.Columns[2].Table) + assert.Equal(t, "sharded_flag_evaluations", doc.Columns[3].Table) + assert.Equal(t, "writable_flag_evaluations", doc.Columns[4].Table) + + empty, err := runLocateColumnsCLI(t, "-dump", dumps, "-tables", tables, "-columns", "removed_column", "-format", "json") + require.NoError(t, err, string(empty)) + var emptyDoc locateColumnDoc + require.NoError(t, json.Unmarshal(empty, &emptyDoc), string(empty)) + assert.Empty(t, emptyDoc.Columns) + assert.Contains(t, string(empty), `"columns": []`) + + broken := filepath.Join(root, "broken") + require.NoError(t, os.MkdirAll(broken, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(broken, "bad.hcl"), []byte(`database "posthog" {`), 0o600)) + failed, err := runLocateColumnsCLI(t, "-dump", broken, "-tables", tables, "-columns", columns, "-format", "json") + require.Error(t, err, string(failed)) + exitErr, ok := err.(*exec.ExitError) + require.True(t, ok) + assert.Equal(t, 1, exitErr.ExitCode(), string(failed)) +} + +func runLocateColumnsCLI(t *testing.T, args ...string) ([]byte, error) { + t.Helper() + commandArgs := append([]string{"-test.run=^TestLocateColumnsCLIProcess$", "--"}, args...) + cmd := exec.Command(os.Args[0], commandArgs...) + cmd.Env = append(os.Environ(), "HCLEXP_LOCATE_COLUMNS_HELPER=1") + return cmd.CombinedOutput() +} diff --git a/docs/README.hcl.md b/docs/README.hcl.md index 1f00759..4eaab15 100644 --- a/docs/README.hcl.md +++ b/docs/README.hcl.md @@ -1066,6 +1066,12 @@ hclexp locate -manifest manifest.hcl -layer-root ./schema 'posthog.person_*' eve # Also search per-node dumps (introspect / dump-cluster output) hclexp locate -manifest manifest.hcl -layer-root ./schema -dump ./dumps events +# Search several table/column names in every per-node snapshot at once +hclexp locate -dump ./prod-us \ + -tables 'flag_evaluations,sharded_flag_evaluations,writable_flag_evaluations,kafka_flag_evaluations' \ + -columns 'person_properties,group0_properties,group1_properties,group2_properties,group3_properties,group4_properties' \ + -format json + # Ad-hoc layer dirs or .hcl files, no manifest required (no placements) hclexp locate -layer ./schema/shared,./schema/ingestion events @@ -1090,6 +1096,22 @@ block, else the filename stem — the same identity `drift` uses). `-format json` emits the same document structurally, with the queried `patterns` echoed back. +Column lookup uses both `-tables` and `-columns`; each flag is a comma-separated +list of exact names or globs. Table selectors accept bare names and +`database.table`; column selectors accept bare names, `table.column`, and +`database.table.column`. Results are grouped by database/table/column and list +every authored layer site plus every matching dump file and node. Ordinary +`column`, child-local `patch_column`, and cross-layer `modify_column` blocks are +reported. The scan deliberately does not resolve inheritance or interpret raw +SQL/drop lists, preserving `locate`'s source-site semantics. + +Selector mode cannot be combined with positional object patterns or +`-duplicates`, and both selector flags are required. A valid query with no +matches is not an error: text output is empty, JSON returns +`{"table_patterns":[...],"column_patterns":[...],"columns":[]}`, and the exit +status is `0`. Missing/unparseable inputs still exit `1`; invalid flag +combinations exit `2`. + Objects are grouped by `(database, name)` — the namespace ClickHouse object types share — so a stray `view "events"` next to a `table "events"` shows up as one entry with both types. diff --git a/internal/loader/hcl/locate.go b/internal/loader/hcl/locate.go index d825285..38af667 100644 --- a/internal/loader/hcl/locate.go +++ b/internal/loader/hcl/locate.go @@ -29,6 +29,19 @@ type Declaration struct { RawKind string // raw blocks only: the kind label } +// ColumnDeclaration is one authored column site within a table or patch_table +// block. Type is column, patch_column, or modify_column. Drop lists are not +// declarations and raw SQL remains opaque, matching locate's syntax-only +// object discovery. +type ColumnDeclaration struct { + Database string + Table string + Name string + File string + Line int + Type string +} + // ScanDeclarations records every object declaration site in the given files, // in file order. Files must be native HCL syntax, same as the loader; a // parse error aborts the scan. @@ -91,6 +104,70 @@ func ScanFileDeclarations(path string) ([]Declaration, string, error) { return out, node, nil } +// ScanFileColumnDeclarations returns every authored table-column site in one +// native HCL file and the first node{} label for dump attribution. It does not +// resolve extend chains or patches: callers see the source sites as written. +func ScanFileColumnDeclarations(path string) ([]ColumnDeclaration, string, error) { + parser := hclparse.NewParser() + f, diags := parser.ParseHCLFile(path) + if diags.HasErrors() { + return nil, "", formatDiagnostics(parser, diags) + } + body, ok := f.Body.(*hclsyntax.Body) + if !ok { + return nil, "", fmt.Errorf("%s: not native HCL syntax", path) + } + + var out []ColumnDeclaration + node := "" + for _, blk := range body.Blocks { + switch blk.Type { + case "database": + if len(blk.Labels) != 1 { + continue + } + for _, table := range blk.Body.Blocks { + out = append(out, columnDeclarations(table, blk.Labels[0], path)...) + } + case "node": + if node == "" && len(blk.Labels) == 1 { + node = blk.Labels[0] + } + } + } + return out, node, nil +} + +func columnDeclarations(table *hclsyntax.Block, database, path string) []ColumnDeclaration { + if len(table.Labels) != 1 || (table.Type != "table" && table.Type != "patch_table") { + return nil + } + var out []ColumnDeclaration + for _, child := range table.Body.Blocks { + if len(child.Labels) != 1 { + continue + } + valid := child.Type == "column" + if table.Type == "table" { + valid = valid || child.Type == "patch_column" + } else { + valid = valid || child.Type == "modify_column" + } + if !valid { + continue + } + out = append(out, ColumnDeclaration{ + Database: database, + Table: table.Labels[0], + Name: child.Labels[0], + File: path, + Line: child.DefRange().Start.Line, + Type: child.Type, + }) + } + return out +} + // objectDeclaration converts one block nested in a database{} into a // Declaration. Blocks that are not object declarations (node, column, // unknown types) report ok = false. @@ -180,6 +257,18 @@ func MatchesPattern(pattern, database, name string) bool { return ok } +// MatchesColumnPattern matches a column by bare name, table.column, or +// database.table.column. It uses the same filepath.Match glob syntax as +// MatchesPattern. +func MatchesColumnPattern(pattern, database, table, name string) bool { + for _, candidate := range []string{name, table + "." + name, database + "." + table + "." + name} { + if ok, _ := filepath.Match(pattern, candidate); ok { + return true + } + } + return false +} + // DuplicateGroup is one object name defined at more than one site without the // composition system explaining the extras: patch sites are additive, // override = true is a deliberate replacement, and extend-carrying sites diff --git a/internal/loader/hcl/locate_test.go b/internal/loader/hcl/locate_test.go index 36454ed..efb2c7a 100644 --- a/internal/loader/hcl/locate_test.go +++ b/internal/loader/hcl/locate_test.go @@ -158,6 +158,37 @@ database "posthog" { assert.Empty(t, node) } +func TestScanFileColumnDeclarations(t *testing.T) { + dir := t.TempDir() + path := writeHCL(t, dir, "columns.hcl", ` +node "prod-ch-1a" {} +database "posthog" { + table "events" { + column "uuid" { type = "UUID" } + patch_column "properties" { codec = "ZSTD(1)" } + } + patch_table "events" { + modify_column "uuid" { type = "String" } + column "person_properties" { type = "String" } + drop_columns = ["obsolete"] + } + materialized_view "events_mv" { + column "ignored" { type = "UInt64" } + } +} +`) + + decls, node, err := ScanFileColumnDeclarations(path) + require.NoError(t, err) + assert.Equal(t, "prod-ch-1a", node) + assert.Equal(t, []ColumnDeclaration{ + {Database: "posthog", Table: "events", Name: "uuid", File: path, Line: 5, Type: "column"}, + {Database: "posthog", Table: "events", Name: "properties", File: path, Line: 6, Type: "patch_column"}, + {Database: "posthog", Table: "events", Name: "uuid", File: path, Line: 9, Type: "modify_column"}, + {Database: "posthog", Table: "events", Name: "person_properties", File: path, Line: 10, Type: "column"}, + }, decls) +} + func TestMatchesPattern(t *testing.T) { assert.True(t, MatchesPattern("events", "posthog", "events")) assert.True(t, MatchesPattern("events*", "posthog", "events_mv")) @@ -169,6 +200,14 @@ func TestMatchesPattern(t *testing.T) { assert.False(t, MatchesPattern("*.kafka_creds", "", "kafka_creds")) } +func TestMatchesColumnPattern(t *testing.T) { + assert.True(t, MatchesColumnPattern("person_properties", "posthog", "flag_evaluations", "person_properties")) + assert.True(t, MatchesColumnPattern("group*_properties", "posthog", "flag_evaluations", "group0_properties")) + assert.True(t, MatchesColumnPattern("flag_evaluations.person_properties", "posthog", "flag_evaluations", "person_properties")) + assert.True(t, MatchesColumnPattern("posthog.flag_*.group?_properties", "posthog", "flag_evaluations", "group4_properties")) + assert.False(t, MatchesColumnPattern("other.*", "posthog", "flag_evaluations", "person_properties")) +} + func TestFindDuplicates(t *testing.T) { decls := []Declaration{ // Plain duplicate: two plain declarations of posthog.person.