Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1759,7 +1759,7 @@ columns/indexes. Any descendant inherits that completed result, while siblings
remain on their own inheritance path. Abstract and concrete tables follow the
same rule.

`patch_materialized_view` does the same for MV queries and output columns;
`patch_materialized_view` does the same for MV destinations (`to_table`), queries, and output columns;
its column operations run after `extend`, so inherited columns are patchable.
`patch_view` and `patch_dictionary` cover views (`query`, `comment`) and
dictionaries (`source`/`layout`/`lifetime` replace, `settings` merge). See the
Expand Down
9 changes: 5 additions & 4 deletions cmd/hclexp/decompose.go
Original file line number Diff line number Diff line change
Expand Up @@ -571,9 +571,6 @@ func tablePatch(from, to *hclload.Schema, object decomposeObject) (hclload.Patch
func materializedViewPatch(from, to *hclload.Schema, object decomposeObject) (hclload.PatchMaterializedViewSpec, error) {
f, t := onlyMaterializedView(from), onlyMaterializedView(to)
var unsupported []string
if f.ToTable != t.ToTable {
unsupported = append(unsupported, "to_table")
}
if !reflect.DeepEqual(f.Cluster, t.Cluster) {
unsupported = append(unsupported, "cluster")
}
Expand All @@ -588,6 +585,10 @@ func materializedViewPatch(from, to *hclload.Schema, object decomposeObject) (hc
return hclload.PatchMaterializedViewSpec{}, fmt.Errorf("%s cannot use patch_materialized_view without loss: %s", object.key(), strings.Join(unsupported, ", "))
}
patch := hclload.PatchMaterializedViewSpec{Name: object.Name, Columns: adds, ModifyColumns: modifies, DropColumns: drops}
if f.ToTable != t.ToTable {
toTable := t.ToTable
patch.ToTable = &toTable
}
if f.Query != t.Query {
query := t.Query
patch.Query = &query
Expand Down Expand Up @@ -747,7 +748,7 @@ func onlyMaterializedView(schema *hclload.Schema) *hclload.MaterializedViewSpec
}

func patchMaterializedViewEmpty(p hclload.PatchMaterializedViewSpec) bool {
return len(p.Columns)+len(p.ModifyColumns)+len(p.DropColumns) == 0 && p.Query == nil
return len(p.Columns)+len(p.ModifyColumns)+len(p.DropColumns) == 0 && p.Query == nil && p.ToTable == nil
}

func patchDictionaryEmpty(p hclload.PatchDictionarySpec) bool {
Expand Down
19 changes: 19 additions & 0 deletions cmd/hclexp/decompose_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,25 @@ func TestBuildDecomposition_PatchesEveryPatchableObjectKind(t *testing.T) {
assert.Contains(t, patch, "first = true")
})

t.Run("materialized view destination only", func(t *testing.T) {
makeSchema := func(destination string) *hclload.Schema {
return &hclload.Schema{Databases: []hclload.DatabaseSpec{{Name: "analytics", MaterializedViews: []hclload.MaterializedViewSpec{{
Name: "events_mv", ToTable: destination, Query: "SELECT 1",
Columns: []hclload.ColumnSpec{{Name: "id", Type: "UInt64"}},
}}}}}
}
generated, err := buildDecomposition([]decomposeSnapshot{
{Env: "eu", Role: "events", Schema: makeSchema("analytics.events")},
{Env: "us", Role: "events", Schema: makeSchema("analytics.writable_events")},
}, []string{"eu", "us"}, decomposeAssignment{Version: 1, Objects: map[string]decomposeObjectAssignment{}})
require.NoError(t, err)
patch := string(generated.Files[envLayerPath("us", "events")])
assert.Contains(t, patch, `patch_materialized_view "events_mv"`)
assert.Contains(t, patch, `to_table = "analytics.writable_events"`)
assert.NotContains(t, patch, "query")
assert.NotContains(t, patch, "column")
})

t.Run("view", func(t *testing.T) {
commentA, commentB := "old", "new"
from := &hclload.Schema{Databases: []hclload.DatabaseSpec{{Name: "analytics", Views: []hclload.ViewSpec{{Name: "events", Query: "SELECT 1", Comment: &commentA}}}}}
Expand Down
8 changes: 5 additions & 3 deletions docs/README.hcl.md
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,7 @@ declared once, the env layer replaces just the fields it sets.
```hcl
database "posthog" {
patch_materialized_view "events_mv" {
to_table = "posthog.writable_events"
query = file("sql/events_mv_dev.sql")
modify_column "team_id" { type = "UInt64" }
column "region" { type = "LowCardinality(String)" }
Expand All @@ -480,11 +481,12 @@ database "posthog" {
}
```

- `patch_materialized_view` fields: `query` (replace), plus `column`,
- `patch_materialized_view` fields: `to_table` and `query` (replace when set), plus `column`,
`modify_column`, and `drop_columns` with the same semantics and placement
options as `patch_table`. MV column patches apply after `extend`, so they
may target columns inherited from an abstract table. The query is
normalized like a declared MV query.
normalized like a declared MV query. A patch setting only `to_table` preserves
the declared query and columns.
- `patch_view` fields: `query`, `comment` — each replaces the target's
value when set. The patched query normalizes to the canonical beautified
form, so a heredoc patch and a one-liner declaration of the same SQL
Expand Down Expand Up @@ -688,7 +690,7 @@ existing table, which stays authoritative except where patched.
| Add / modify / drop a column on the same table in one environment | `patch_table` |
| Change a setting, index, `order_by`/`partition_by`/`ttl`, or the engine on the same table in one environment | `patch_table` |
| A Distributed table whose target moves with the env's topology | `patch_table` with `engine` |
| An MV's `query` or columns differing per environment | `patch_materialized_view` |
| An MV's `to_table`, `query`, or columns differing per environment | `patch_materialized_view` |
| A view's `query` or a dictionary's `source` differing per environment | `patch_view` / `patch_dictionary` |
| Add a projection to the same table in one environment | `patch_table` with `projection` |
| A table differing beyond the patchable fields (`primary_key`, constraints, …) | `override = true` |
Expand Down
3 changes: 3 additions & 0 deletions internal/loader/hcl/dump.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,9 @@ func writeLayerDatabase(body *hclwrite.Body, db DatabaseSpec) {
for _, patch := range mvPatches {
body.AppendNewline()
pb := body.AppendNewBlock("patch_materialized_view", []string{patch.Name}).Body()
if patch.ToTable != nil {
pb.SetAttributeValue("to_table", cty.StringVal(*patch.ToTable))
}
if patch.Query != nil {
setQueryAttribute(pb, *patch.Query)
}
Expand Down
47 changes: 47 additions & 0 deletions internal/loader/hcl/patch_vocabulary_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,13 +247,60 @@ database "posthog" {
require.Len(t, db.MaterializedViews, 1)
mv := db.MaterializedViews[0]
assert.Contains(t, mv.Query, "FROM source_dev")
assert.Equal(t, "posthog.events", mv.ToTable)
assert.Equal(t, []ColumnSpec{
{Name: "id", Type: "UInt64"},
{Name: "extra", Type: "String"},
}, mv.Columns)
assert.Empty(t, db.MaterializedViewPatches, "patches are consumed during resolution")
}

func TestPatchMaterializedView_ToTable(t *testing.T) {
root := t.TempDir()
base := writePatchLayer(t, root, "base/mv.hcl", `
database "posthog" {
materialized_view "events_mv" {
to_table = "posthog.events"
query = "SELECT id FROM source"
column "id" { type = "UInt64" }
}
}`)
for _, tc := range []struct {
name string
patch string
want string
}{
{"destination only", `to_table = "posthog.writable_events"`, "posthog.writable_events"},
{"last patch wins", `to_table = "posthog.writable_events"
}
patch_materialized_view "events_mv" {
to_table = "other.events"`, "other.events"},
{"empty destination rejected", `to_table = ""`, ""},
} {
t.Run(tc.name, func(t *testing.T) {
patch := writePatchLayer(t, root, "role/patch.hcl", `database "posthog" {
patch_materialized_view "events_mv" {
`+tc.patch+`
}
}`)
schema, err := LoadLayers([]string{base, patch})
require.NoError(t, err)
err = Resolve(schema)
if tc.want == "" {
require.ErrorContains(t, err, "materialized_view requires to_table")
return
}
require.NoError(t, err)
mv := schema.Databases[0].MaterializedViews[0]
assert.Equal(t, tc.want, mv.ToTable)
assert.Contains(t, mv.Query, "SELECT id")
assert.Contains(t, mv.Query, "FROM source")
assert.Equal(t, []ColumnSpec{{Name: "id", Type: "UInt64"}}, mv.Columns)
assert.Empty(t, schema.Databases[0].MaterializedViewPatches)
})
}
}

func TestPatchMaterializedView_UnknownAndInvalidColumnTargets(t *testing.T) {
unknown := DatabaseSpec{
Name: "posthog",
Expand Down
3 changes: 3 additions & 0 deletions internal/loader/hcl/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -786,6 +786,9 @@ func applyMaterializedViewPatches(db *DatabaseSpec) error {
copy(target.Columns[pos+1:], target.Columns[pos:])
target.Columns[pos] = column
}
if patch.ToTable != nil {
target.ToTable = *patch.ToTable
}
if patch.Query != nil {
target.Query = *patch.Query
}
Expand Down
3 changes: 2 additions & 1 deletion internal/loader/hcl/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,11 +98,12 @@ type MaterializedViewSpec struct {
}

// PatchMaterializedViewSpec is the materialized-view counterpart of
// patch_table. Query replaces when set; columns follow patch_table's
// patch_table. ToTable and Query replace when set; columns follow patch_table's
// modify -> drop -> add semantics. It is consumed after MV extend resolution,
// so inherited columns may be modified or dropped too.
type PatchMaterializedViewSpec struct {
Name string `hcl:"name,label"`
ToTable *string `hcl:"to_table,optional"`
Columns []ColumnSpec `hcl:"column,block"`
ModifyColumns []ColumnSpec `hcl:"modify_column,block"`
DropColumns []string `hcl:"drop_columns,optional"`
Expand Down