Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1522,9 +1522,10 @@ environment patches, but the desired schema is always reduced to one flat
table by the same rule:

Before resolving inheritance, hclexp merges the selected layers in their
declared order. A later `override = true` replaces the earlier declaration;
patch blocks are collected rather than applied during that merge. The resolver
then evaluates the table graph below.
declared order. A later `override = true` replaces the earlier declaration for
any managed object kind (`table`, `materialized_view`, `view`, `dictionary`,
`raw`, or `named_collection`); patch blocks are collected rather than applied
during that merge. The resolver then evaluates the table graph below.

```text
resolved(table) =
Expand Down Expand Up @@ -1665,8 +1666,9 @@ The resolver enforces several safety invariants around this model:
one **replaces it wholesale** (a one-key `settings` map loses every
inherited key). `primary_key`, `comment`, `cluster`, constraints, and
projections never flow through `extend`.
- `override = true` — required for a later layer to replace a table or
materialized view that an earlier layer already defined
- `override = true` — required for a later layer to replace a table,
materialized view, view, dictionary, raw object, or named collection that an
earlier layer already defined

Rule of thumb: `extend` is for *different tables sharing a shape*
(`events_local` / `events_distributed`); `patch_table` (below) is for *the
Expand Down
139 changes: 139 additions & 0 deletions cmd/hclexp/load_override_e2e_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
package main

import (
"os"
"os/exec"
"path/filepath"
"testing"

hclload "github.com/posthog/chschema/internal/loader/hcl"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestLoadOverrideCLIProcess(t *testing.T) {
if os.Getenv("HCLEXP_LOAD_OVERRIDE_HELPER") != "1" {
return
}
for i, arg := range os.Args {
if arg == "--" {
runLoad(os.Args[i+1:])
return
}
}
t.Fatal("missing load CLI arguments")
}

func TestLoadCLI_OverrideEveryManagedObjectKindEndToEnd(t *testing.T) {
root := t.TempDir()
base := writeLoadOverrideLayer(t, root, "base.hcl", `
named_collection "warehouse" {
param "host" { value = "production.internal" }
}

database "analytics" {
table "events" {
column "id" { type = "UInt64" }
engine "log" {}
}

materialized_view "events_mv" {
to_table = "analytics.events"
query = "SELECT id FROM production_source"
column "id" { type = "UInt64" }
}

view "environment" {
query = "SELECT 'production' AS name"
}

dictionary "labels" {
primary_key = ["id"]
attribute "id" { type = "UInt64" }
attribute "label" { type = "String" }
source "null" {}
layout "flat" {}
}

raw "view" "legacy_environment" {
sql = "CREATE VIEW analytics.legacy_environment AS SELECT 'production'"
}
}`)
override := writeLoadOverrideLayer(t, root, "override.hcl", `
named_collection "warehouse" {
override = true
param "host" { value = "development.internal" }
}

database "analytics" {
table "events" {
override = true
column "dev_id" { type = "UInt32" }
engine "log" {}
}

materialized_view "events_mv" {
override = true
to_table = "analytics.events"
query = "SELECT dev_id FROM development_source"
column "dev_id" { type = "UInt32" }
}

view "environment" {
override = true
query = "SELECT 'development' AS name"
}

dictionary "labels" {
override = true
primary_key = ["key"]
attribute "key" { type = "String" }
attribute "label" { type = "String" }
source "null" {}
layout "flat" {}
}

raw "view" "legacy_environment" {
override = true
sql = "CREATE VIEW analytics.legacy_environment AS SELECT 'development'"
}
}`)
out := filepath.Join(root, "resolved.hcl")

cmd := exec.Command(os.Args[0], "-test.run=^TestLoadOverrideCLIProcess$", "--",
"-layer", base+","+override,
"-out", out,
)
cmd.Env = append(os.Environ(), "HCLEXP_LOAD_OVERRIDE_HELPER=1")
output, err := cmd.CombinedOutput()
require.NoError(t, err, string(output))

schema, err := hclload.ParseFile(out)
require.NoError(t, err)
require.NoError(t, hclload.Resolve(schema))
require.Len(t, schema.NamedCollections, 1)
require.Len(t, schema.Databases, 1)
db := schema.Databases[0]
require.Len(t, db.Tables, 1)
require.Len(t, db.MaterializedViews, 1)
require.Len(t, db.Views, 1)
require.Len(t, db.Dictionaries, 1)
require.Len(t, db.Raws, 1)
assert.Equal(t, "development.internal", schema.NamedCollections[0].Params[0].Value)
assert.Equal(t, "dev_id", db.Tables[0].Columns[0].Name)
assert.Contains(t, db.MaterializedViews[0].Query, "development_source")
assert.Contains(t, db.Views[0].Query, "development")
assert.Equal(t, []string{"key"}, db.Dictionaries[0].PrimaryKey)
assert.Contains(t, db.Raws[0].SQL, "development")

resolved, err := os.ReadFile(out)
require.NoError(t, err)
assert.NotContains(t, string(resolved), "override = true", "layer control metadata must not leak into resolved output")
}

func writeLoadOverrideLayer(t *testing.T, root, name, body string) string {
t.Helper()
path := filepath.Join(root, name)
require.NoError(t, os.WriteFile(path, []byte(body), 0o600))
return path
}
9 changes: 6 additions & 3 deletions docs/FAQ.md
Original file line number Diff line number Diff line change
Expand Up @@ -239,9 +239,11 @@ it, put a `patch_table` in each. If the column is present in every
environment, just add it to the base `table`. Don't try to be clever — the
local-knowledge wins over global indirection.

## How do I replace a table entirely in one environment?
## How do I replace an object entirely in one environment?

Declare the table in the environment layer with `override = true`.
Redeclare it in the environment layer with `override = true`. The same rule
applies to tables, materialized views, views, dictionaries, raw objects, and
named collections.

```hcl
# envs/dev/events.hcl
Expand All @@ -255,7 +257,8 @@ database "posthog" {
```

Without `override = true`, the cross-layer collision is an error. With it,
the dev-layer definition wins.
the dev-layer definition wins. Raw objects collide by `(kind, name)`; the
other database objects and named collections collide by name.

## What's the difference between `extend` and `patch_table`?

Expand Down
22 changes: 15 additions & 7 deletions docs/README.hcl.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,10 @@ below. `patch_column` is valid only when the table sets `extend`.
- `abstract = true` — declares this table as inheritable-only; it is not
emitted as a real ClickHouse table.
- `override = true` — declares that this block replaces an earlier-layer
`table` of the same name. Without it, a cross-layer name collision is an
error.
object of the same identity. It is supported uniformly by `table`,
`materialized_view`, `view`, `dictionary`, `raw`, and `named_collection`;
without it, a cross-layer collision is an error. A raw object's identity is
its `(kind, name)` label pair.

## `column`

Expand Down Expand Up @@ -636,9 +638,10 @@ A later layer may fully replace an MV by redeclaring it with
When the loader processes a layered set, this pipeline runs:

1. **Parse** every `.hcl` file in every layer (ordered).
2. **Merge** databases by name. Tables and materialized views collide on name
unless the later declaration sets `override = true`. Patch blocks
accumulate.
2. **Merge** databases by name. Every managed object (`table`,
`materialized_view`, `view`, `dictionary`, `raw`, and `named_collection`)
collides on identity unless the later declaration sets `override = true`.
Patch blocks accumulate.
3. **Apply view/dictionary patches.** Table patches remain attached to their
named targets while the inheritance graph resolves.
4. **Resolve each table parent-first**: inherit the resolved parent, apply
Expand All @@ -661,8 +664,9 @@ which question you are asking:
- **`patch_table`**: *"this is the **same table**, and one layer wants to
adjust it."* The table stays declared exactly **once**; the env layer
contributes a modification, not a declaration.
- **`override = true`**: *"in this environment the table is genuinely
different."* A full replacement declaration, sanctioned by the flag.
- **`override = true`**: *"in this environment the object is genuinely
different."* A full replacement declaration, sanctioned by the flag. It
follows the same rule for every managed object kind.

The declaration count is not cosmetic: `hclexp locate -duplicates` (the
once-only CI guard) counts definition sites, while treating patch,
Expand Down Expand Up @@ -690,6 +694,8 @@ existing table, which stays authoritative except where patched.
| A table differing beyond the patchable fields (`primary_key`, constraints, …) | `override = true` |
| Replace a table entirely in one environment | `override = true` |
| Replace a materialized view entirely in one environment | `override = true` |
| Replace a view or dictionary entirely in one environment | `override = true` |
| Replace a raw object or named collection entirely in one environment | `override = true` |

## Dependency validation — `hclexp validate`

Expand Down Expand Up @@ -1334,6 +1340,7 @@ database "posthog" {

| Attribute | Required | Meaning |
|------------------|----------|---------|
| `override` | no | Set `true` to replace the same view from an earlier layer. |
| `query` | yes | the `AS SELECT ...` body (verbatim text) |
| `column_aliases` | no | `CREATE VIEW v (a, b, ...) AS ...` |
| `sql_security` | no | `SQL SECURITY` clause: `definer`, `invoker`, or `none` (canonical lowercase; case-insensitive on parse) |
Expand Down Expand Up @@ -1375,6 +1382,7 @@ SQL
|--------|---------|
| `kind` (1st label) | `table`, `materialized_view`, `view`, or `dictionary`. Drives the `DROP` form on a recreate. |
| `name` (2nd label) | The object name. |
| `override` | Set `true` to replace an earlier raw block with the same `(kind, name)`. |
| `sql` | The original `CREATE` statement, emitted verbatim on apply. |

**Semantics.** Raw objects are opaque:
Expand Down
3 changes: 3 additions & 0 deletions internal/loader/hcl/dictionary_dump.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ import (
// writeDictionary emits a `dictionary` block body. Sub-blocks (lifetime,
// range, attribute, source, layout) are emitted in a consistent order.
func writeDictionary(body *hclwrite.Body, d DictionarySpec) {
if d.Override {
body.SetAttributeValue("override", cty.True)
}
body.SetAttributeValue("primary_key", stringList(d.PrimaryKey))
if len(d.Settings) > 0 {
body.SetAttributeValue("settings", stringMap(d.Settings))
Expand Down
12 changes: 12 additions & 0 deletions internal/loader/hcl/dump.go
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,9 @@ func writeDatabase(body *hclwrite.Body, db DatabaseSpec) {
// as a quoted string otherwise. Heredoc emission is exact: the re-parsed value
// equals the stored SQL.
func writeRaw(body *hclwrite.Body, r RawSpec) {
if r.Override {
body.SetAttributeValue("override", cty.True)
}
setSQLAttribute(body, "sql", r.SQL)
}

Expand Down Expand Up @@ -333,6 +336,9 @@ func setQueryAttribute(body *hclwrite.Body, query string) {
}

func writeView(body *hclwrite.Body, v ViewSpec) {
if v.Override {
body.SetAttributeValue("override", cty.True)
}
setQueryAttribute(body, v.Query)
if len(v.ColumnAliases) > 0 {
body.SetAttributeValue("column_aliases", stringList(v.ColumnAliases))
Expand All @@ -352,6 +358,9 @@ func writeView(body *hclwrite.Body, v ViewSpec) {
}

func writeMaterializedView(body *hclwrite.Body, mv MaterializedViewSpec) {
if mv.Override {
body.SetAttributeValue("override", cty.True)
}
body.SetAttributeValue("to_table", cty.StringVal(mv.ToTable))
setQueryAttribute(body, mv.Query)
if mv.Cluster != nil {
Expand All @@ -366,6 +375,9 @@ func writeMaterializedView(body *hclwrite.Body, mv MaterializedViewSpec) {
}

func writeTable(body *hclwrite.Body, t TableSpec) {
if t.Override {
body.SetAttributeValue("override", cty.True)
}
if t.Comment != nil {
body.SetAttributeValue("comment", cty.StringVal(*t.Comment))
}
Expand Down
19 changes: 19 additions & 0 deletions internal/loader/hcl/dump_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"os"
"path/filepath"
"sort"
"strings"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -178,6 +179,24 @@ func TestDumpSchema_RendersNodeBlock(t *testing.T) {
assert.Equal(t, want, reparsed.Nodes[0])
}

func TestWriteLayerPreservesOverrideForEveryManagedObjectKind(t *testing.T) {
schema := &Schema{
NamedCollections: []NamedCollectionSpec{{Name: "credentials", Override: true}},
Databases: []DatabaseSpec{{
Name: "posthog",
Tables: []TableSpec{{Name: "events", Override: true}},
MaterializedViews: []MaterializedViewSpec{{Name: "events_mv", Override: true}},
Views: []ViewSpec{{Name: "events_view", Override: true, Query: "SELECT 1"}},
Dictionaries: []DictionarySpec{{Name: "geo", Override: true}},
Raws: []RawSpec{{Kind: "view", Name: "legacy", Override: true, SQL: "CREATE VIEW posthog.legacy AS SELECT 1\n"}},
}},
}

var buf bytes.Buffer
require.NoError(t, WriteLayer(&buf, schema))
assert.Equal(t, 6, strings.Count(buf.String(), "= true"), buf.String())
}

func TestDumpSchema_RendersViewBlocks(t *testing.T) {
want := ViewSpec{
Name: "metrics",
Expand Down
Loading