From 1acaecd8dc7decd95b98f1b6c66fc2b9af0e5375 Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Wed, 7 Oct 2026 22:35:33 +0200 Subject: [PATCH 01/14] Number integer primary keys 1..N and list rows in key order A table's single-column integer primary key was a shuffled sample starting at 0, so the first customers came out as 4251, 4557, 1941. It now numbers the rows 1..N like an identity column, and the rows come out in that order. A key drawn in an explicit min/max range or with a distribution, and an integer key that is also a foreign key (a one-to-one child keyed by its parent), keeps its draw and is put in ascending order: the key is drawn before any other column of the row, so sorting it sorts the rows. Text, uuid and composite keys are unchanged. Later days already continue from the largest key; an empty table now starts at 1 rather than 0. This is a deliberate output change under the per-version determinism promise: the key no longer draws from the table's stream, so every column after it moves. The pinned frames of earlier releases are pinned again. Co-Authored-By: Claude Opus 5.5 --- model2data/generate/days.py | 2 +- model2data/generate/faker.py | 16 +++++++++ tests/test_column_time_hints.py | 3 +- tests/test_distributions.py | 14 ++++---- tests/test_shaping.py | 54 +++++++++++++++--------------- tests/test_table_seeds.py | 16 ++++++--- tests/test_timeline.py | 59 +++++++++++++++++---------------- 7 files changed, 97 insertions(+), 67 deletions(-) diff --git a/model2data/generate/days.py b/model2data/generate/days.py index fa19104..bd6313c 100644 --- a/model2data/generate/days.py +++ b/model2data/generate/days.py @@ -620,7 +620,7 @@ def _draw( and kinds.is_integer_type(column.data_type) ): held = state[column.name].dropna() - start = int(held.max()) + 1 if len(held) else 0 + start = int(held.max()) + 1 if len(held) else 1 return list(range(start, start + count)) values = generate_column_values( column=column, diff --git a/model2data/generate/faker.py b/model2data/generate/faker.py index dd42394..aa3b4ef 100644 --- a/model2data/generate/faker.py +++ b/model2data/generate/faker.py @@ -897,6 +897,11 @@ def generator() -> int: values = [generator() for _ in range(row_count)] if ensure_unique: values = _deduplicate(values, generator, column_name=unique_label) + elif ensure_unique and "pk" in column.settings and not had_explicit_range: + # A table's own integer primary key numbers its rows 1..N, as an + # identity column does: the first row is 1 and the last is N. + # Before 1.15 it was a shuffled sample starting at 0. + values = list(range(1, row_count + 1)) elif ensure_unique: if not had_explicit_range: # No user-specified range: widen the default so there's @@ -1017,6 +1022,17 @@ def generator() -> float: _stats_state["unmapped"].append((column.name, column.data_type)) values = [_sentence(3) for _ in range(row_count)] + # A table's rows come out in the order of an integer primary key: its + # values are put in ascending order here, before any other column of the + # row is tied to them (they are drawn in the first pass, independently of + # each other), so sorting the key is sorting the rows. Covers a key drawn + # in an explicit `min`/`max` range or with a `distribution`, and a key that + # is also a foreign key (a one-to-one child keyed by its parent). Only a + # single-column `pk` carries the setting; text and composite keys keep the + # order they are drawn in. + if "pk" in column.settings and is_integer_type(base_type) and None not in values: + values.sort() + # ----------------------------------------------------- # Nullability # ----------------------------------------------------- diff --git a/tests/test_column_time_hints.py b/tests/test_column_time_hints.py index eb433fa..3029de0 100644 --- a/tests/test_column_time_hints.py +++ b/tests/test_column_time_hints.py @@ -196,7 +196,8 @@ def test_no_hint_and_uniform_profile_matches_the_pre_feature_pinned_frame(): } df = generate_data_from_dbml(tables, [], base_rows=8, seed=2024, as_of=ANCHOR_DATE)["events"] - assert list(df["created_at"])[:2] == ["2026-01-24 23:52:50", "2025-08-07 01:00:18"] + # As re-pinned in 1.15.0 (integer keys numbered 1..N). + assert list(df["created_at"])[:2] == ["2025-12-07 15:29:52", "2025-05-20 12:00:46"] # --------------------------------------------------------- diff --git a/tests/test_distributions.py b/tests/test_distributions.py index 1ea3b48..2b649fa 100644 --- a/tests/test_distributions.py +++ b/tests/test_distributions.py @@ -38,17 +38,19 @@ def test_no_distribution_hint_reproduces_pre_1_7_frames(): ) } data = generate_data_from_dbml(tables, [], base_rows=8, seed=777) + # Re-pinned in 1.15.0: the primary key numbers the rows 1..N rather than + # drawing a shuffled sample, so the columns after it start elsewhere in the stream. assert data["t"]["amount"].tolist() == [ + 7784.79, + 37.17, + 9617.56, + 4384.36, + 6564.75, 4169.09, 3848.52, 5163.13, - 3006.66, - 25.14, - 4363.61, - 685.34, - 8357.29, ] - assert data["t"]["qty"].tolist() == [7, 89, 51, 52, 82, 100, 97, 9] + assert data["t"]["qty"].tolist() == [38, 80, 0, 33, 55, 8, 50, 57] # --------------------------------------------------------- diff --git a/tests/test_shaping.py b/tests/test_shaping.py index f67b679..122b393 100644 --- a/tests/test_shaping.py +++ b/tests/test_shaping.py @@ -83,62 +83,64 @@ def test_a_nullable_enum_column_takes_the_default_null_rate(): def test_uniform_generation_reproduces_pre_1_5_frames(): """A run that passes no shaping option at all must still produce exactly what 1.4.0 produced with the same seed -- this is the guard rail every - later change in this module works against.""" + later change in this module works against. Re-pinned in 1.15.0, whose + primary keys number the rows 1..N instead of drawing a shuffled sample, so + every column after the key draws from a different point of the stream.""" data = generate_data_from_dbml( _users_orders_schema(), _USERS_ORDERS_REFS, base_rows=12, seed=2024 ) orders = data["orders"] - assert orders["id"].tolist() == [67, 7, 12, 23, 83, 48, 85, 42, 74, 72, 13, 87] - assert orders["user_id"].tolist() == [41, 82, 31, 21, 48, 39, 99, 82, 21, 0, 48, 41] + assert orders["id"].tolist() == list(range(1, 13)) + assert orders["user_id"].tolist() == [9, 1, 2, 3, 11, 7, 11, 6, 10, 10, 2, 11] assert orders["status"].tolist() == [ - "delivered", + "cancelled", "cancelled", "pending", - "delivered", + "pending", "cancelled", "delivered", "pending", "cancelled", + "delivered", + "cancelled", "pending", - "pending", - "pending", - "shipped", + "delivered", ] assert orders["is_paid"].tolist() == [ False, - True, - None, False, + True, False, + True, + True, + None, + True, None, - False, True, True, - False, - False, False, ] - assert pd.isna(orders["total"].tolist()[1]) - assert pd.isna(orders["total"].tolist()[9]) + assert pd.isna(orders["total"].tolist()[0]) + assert pd.isna(orders["total"].tolist()[2]) non_null_totals = [v for v in orders["total"].tolist() if not pd.isna(v)] assert non_null_totals == [ - 2435.11, - 1746.91, - 370.85, - 1610.47, - 6398.42, - 1936.29, - 2677.74, - 4708.56, - 3946.17, - 1678.0, + 2530.43, + 6586.53, + 8365.73, + 9516.01, + 3703.4, + 3187.15, + 3350.27, + 3432.97, + 1887.32, + 2337.57, ] assert orders["note"].tolist() == [ "Argue we pretty.", "One big indicate.", + "Range billion.", None, - "Raise west hotel.", "Agreement pattern yet.", "Radio trip administration.", "Machine tell big.", diff --git a/tests/test_table_seeds.py b/tests/test_table_seeds.py index 18545a1..81aca76 100644 --- a/tests/test_table_seeds.py +++ b/tests/test_table_seeds.py @@ -115,14 +115,20 @@ def test_referential_integrity_holds_against_the_new_parent(self): assert set(rolled["orders"]["customer_id"]) <= set(rolled["customers"]["id"]) assert set(rolled["order_items"]["order_id"]) <= set(rolled["orders"]["id"]) - def test_the_old_parent_rows_are_genuinely_gone(self): - """A child that still pointed at the pre-roll ids would be dangling.""" + def test_the_re_rolled_parent_holds_new_rows_under_the_same_numbering(self): + """An integer key numbers the rows 1..N (1.15.0), so a re-rolled parent + keeps its ids and every child still points at a row that exists; the rows + behind those ids are new.""" baseline = _generate() rolled = _generate(table_seeds={"orders": 7}) - stale = set(baseline["orders"]["id"]) - set(rolled["orders"]["id"]) - assert stale, "the re-rolled parent has to hand out at least some new ids" - assert not set(rolled["order_items"]["order_id"]) & stale + assert rolled["orders"]["id"].tolist() == baseline["orders"]["id"].tolist() + assert ( + not rolled["orders"] + .drop(columns=["id"]) + .equals(baseline["orders"].drop(columns=["id"])) + ) + assert set(rolled["order_items"]["order_id"]) <= set(rolled["orders"]["id"]) def test_two_tables_can_be_re_rolled_at_once(self): baseline = _generate() diff --git a/tests/test_timeline.py b/tests/test_timeline.py index b0f4970..1f11d28 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -54,26 +54,29 @@ def _created_updated_schema() -> dict[str, TableDef]: def test_uniform_profile_reproduces_pre_1_5_frames(): """A table with no ordered pair is untouched by the ordering pass, so its - frame under the uniform profile is byte-identical to what 1.4.0 generated.""" + frame under the uniform profile is byte-identical to what 1.4.0 generated -- + but for 1.15.0's primary key, which numbers the rows 1..N instead of drawing + a shuffled sample, so every column after it draws from a different point of + the table's stream (pinned again in 1.15.0).""" df = generate_data_from_dbml( _single_timestamp_and_date_schema(), [], base_rows=8, seed=2024, as_of=ANCHOR )["events"] - assert list(df["id"]) == [88, 21, 43, 40, 41, 10, 22, 79] + assert list(df["id"]) == [1, 2, 3, 4, 5, 6, 7, 8] assert list(df["created_at"]) == [ - "2026-01-24 23:52:50", - "2025-08-07 01:00:18", - "2025-10-11 16:45:31", - "2025-03-20 05:56:47", - "2025-11-02 12:14:24", - "2025-05-06 07:44:23", - "2025-11-12 09:20:05", - "2025-11-23 20:22:21", + "2025-12-07 15:29:52", + "2025-05-20 12:00:46", + "2026-01-20 17:46:51", + "2026-01-31 06:10:57", + "2025-07-25 17:30:00", + "2025-07-15 05:24:33", + "2025-07-18 08:26:12", + "2026-01-28 22:28:20", ] assert list(df["event_date"]) == [ date(2025, 7, 31), - date(2025, 10, 21), None, + date(2026, 3, 8), date(2024, 11, 19), date(2024, 11, 5), date(2024, 5, 16), @@ -90,26 +93,26 @@ def test_ordering_changes_created_updated_pairs_under_the_same_seed(): _created_updated_schema(), [], base_rows=8, seed=2024, as_of=ANCHOR )["events"] - assert list(df["id"]) == [88, 21, 43, 40, 41, 10, 22, 79] + assert list(df["id"]) == [1, 2, 3, 4, 5, 6, 7, 8] assert list(df["created_at"]) == [ - "2026-01-24 23:52:50", - "2025-08-07 01:00:18", - "2025-10-11 16:45:31", - "2025-03-20 05:56:47", - "2025-11-02 12:14:24", - "2025-05-06 07:44:23", - "2025-11-12 09:20:05", - "2025-11-23 20:22:21", + "2025-12-07 15:29:52", + "2025-05-20 12:00:46", + "2026-01-20 17:46:51", + "2026-01-31 06:10:57", + "2025-07-25 17:30:00", + "2025-07-15 05:24:33", + "2025-07-18 08:26:12", + "2026-01-28 22:28:20", ] assert list(df["updated_at"]) == [ - "2026-01-25 02:30:36", - "2025-08-10 22:27:25", - "2025-10-19 02:05:52", - "2025-03-24 01:14:16", - "2025-11-07 20:42:41", + "2025-12-10 13:57:04", + "2025-05-23 16:21:10", None, - "2025-11-16 11:53:12", - "2025-11-24 20:45:24", + "2026-02-07 13:00:47", + "2025-07-29 19:39:57", + "2025-07-15 15:51:09", + "2025-07-18 17:37:38", + "2026-01-30 01:59:33", ] # Every non-null updated_at is at or after its own row's created_at. for created, updated in zip(df["created_at"], df["updated_at"], strict=True): @@ -425,7 +428,7 @@ def test_single_timestamp_table_is_byte_identical_to_the_pinned_frame(): _single_timestamp_and_date_schema(), [], base_rows=8, seed=2024, as_of=ANCHOR )["events"] - assert list(df["created_at"])[0] == "2026-01-24 23:52:50" + assert list(df["created_at"])[0] == "2025-12-07 15:29:52" def test_cyclic_after_hints_raise_naming_table_and_columns(): From 7febffddf663996716bea90f3c328f0e3f5065d8 Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Wed, 7 Oct 2026 22:35:52 +0200 Subject: [PATCH 02/14] Keep a child row's creation date on or after its parents' Half the orders of the ecommerce example were placed before their customer signed up. A table's creation column -- its first date or timestamp without `after` named for the created stage (created_at, signup_date, start_date), else its first named for no stage (order_date) -- is now kept on or after the creation column of every parent row its foreign keys point at, through a foreign key onto the parent's primary or unique key; with several parents, after the latest. A date and a timestamp compare by day. Self references, null foreign keys, parents without a date and foreign keys that break a cycle are not followed. The child keeps its own shape: a row dated before its parent takes another parent created by then, drawn from the parents the column already points at, so its date keeps the growth, seasonality and business hours it was drawn with. Only a row with no such parent (or a one-to-one, which cannot change parent) moves its date, drawn again from the column's day weights and hours cut to the part of the window on or after the parent's, never past as_of. Moving dates alone would have flattened the coffee webshop's gift-season peak (3.1x May before, 0.6x after); taking older parents keeps it (3.0-3.5x) at the cost of a steeper skew (top fifth of customers: 66% of orders before, about 80% after). A composite-key repair or a `when` fill that dates a row before its parent afterwards moves only that row's date and places its later dates again. On later days a new row whose parent was inserted the same day lands later that day. `after_parent: false` on the creation column turns the rule off for the table, and the generator rejects it anywhere else. The late_arriving defect prefers rows that stay clear of their parents once moved back, so it keeps breaking only the incremental model; a review test moves to a seed whose late rows still show what it checks. Co-Authored-By: Claude Opus 5.5 --- model2data/defects/apply.py | 40 ++- model2data/generate/core.py | 64 ++++- model2data/generate/days.py | 30 +++ model2data/generate/hints.py | 24 +- model2data/generate/parents.py | 420 ++++++++++++++++++++++++++++++++ model2data/generate/timeline.py | 52 +++- tests/test_defects_review.py | 6 +- 7 files changed, 626 insertions(+), 10 deletions(-) create mode 100644 model2data/generate/parents.py diff --git a/model2data/defects/apply.py b/model2data/defects/apply.py index 746601b..f4cfed6 100644 --- a/model2data/defects/apply.py +++ b/model2data/defects/apply.py @@ -67,8 +67,10 @@ from model2data.defects.checks import Columns, as_seeded, failing, fails from model2data.defects.report import AppliedDefect, DefectsReport, ExpectedFailure from model2data.generate import kinds +from model2data.generate.core import parent_rules_for from model2data.generate.days import DayResult, TableDay from model2data.generate.history import IS_CURRENT, VALID_FROM, VALID_TO, history_key, primary_key +from model2data.generate.parents import ParentRule, floor_on_day, moments from model2data.model.engine import EngineInputs, to_engine from model2data.model.types import Defect, Model from model2data.parse.dbml import ColumnDef, TableDef @@ -111,8 +113,11 @@ def __init__( refs: list[dict], results: Optional[list[DayResult]], incremental: Any, + rule: Optional[ParentRule] = None, ): self.key = key + # The parents the table's creation date follows (see generate.parents), if any. + self.rule = rule self.table = table self.frames = frames self.frame = frames[key] @@ -533,7 +538,7 @@ def _late_arriving(table: _Table, defect: Defect, rng: random.Random, count: int ] temporal = [c for c in table.columns if table.is_temporal(c)] candidates = table.free(candidates, temporal) - chosen = _choose(rng, candidates, count, set()) + chosen = _choose(rng, candidates, count, _before_parents_once_late(table, candidates, name)) patches: list[_Patch] = [] # Earliest day first: a row moved back lowers what the loader has seen on later days. for position in sorted(chosen, key=lambda p: (table.inserted_day[p], p)): @@ -634,6 +639,37 @@ def _event_column(table: _Table) -> Optional[str]: return next((c for c in table.columns if table.is_temporal(c)), None) +def _before_parents_once_late(table: _Table, candidates: list[int], name: str) -> set[int]: + """The candidates moving back would date before a parent row (see generate.parents). + + A late-arriving row is moved back past the previous load's cutoff, every date of + it; when the table's creation date follows its parents', that can put it before + one, which the dbt test `model2data_not_before_parent` then reports. Rows that + stay clear of their parents are taken first, so the defect breaks the + incremental model it is meant to and no test besides. + """ + rule = table.rule + if rule is None or not candidates: + return set() + floor = floor_on_day(table.frame, rule, table.frames).tolist() + created = moments(table.frame[rule.column], rule.kind).tolist() + values = table.frame[name].tolist() + cutoffs: dict[int, Any] = {} + risky = set() + for position in candidates: + if pd.isna(floor[position]) or pd.isna(created[position]): + continue + day = table.inserted_day[position] + if day not in cutoffs: + cutoffs[day] = table.loaded_max(day - 1, name) + cutoff, moment, back = cutoffs[day], _moment(values[position]), 1 + while cutoff is not None and moment - timedelta(days=back) >= cutoff: + back += 1 + if created[position] - pd.Timedelta(days=back) < floor[position]: + risky.add(position) + return risky + + def _moment(value: Any) -> datetime: return pd.Timestamp(value).to_pydatetime() @@ -818,6 +854,7 @@ def break_tables( applied: Applied = [] tables: dict[str, _Table] = {} + rules = parent_rules_for(inputs.tables, inputs.refs) for key, entries in defects.items(): if key not in inputs.tables: raise ValueError(f"defects name the table {key!r}, which is not in the model") @@ -840,6 +877,7 @@ def break_tables( inputs.refs, results, inputs.incremental.get(target), + rules.get(target), ) table = tables[target] rng = random.Random(defect_stream_seed(base_seed, key, defect)) diff --git a/model2data/generate/core.py b/model2data/generate/core.py index 1830958..8de2e4a 100644 --- a/model2data/generate/core.py +++ b/model2data/generate/core.py @@ -22,6 +22,7 @@ from model2data.generate.hints import validate_hints from model2data.generate.kinds import is_integer_type from model2data.generate.options import UNIFORM, TimeProfile, validate_skew +from model2data.generate.parents import ParentRule, follow_parents, parent_rules from model2data.generate.relationships import ( build_fk_lookup, classify_refs, @@ -157,6 +158,8 @@ def generate_data_from_dbml( # Generate tables in dependency order # --------------------------------------------------------- ordered_tables, deferred_edges = _table_order(tables, fk_refs) + # Which tables' creation dates follow their parents' (see generate.parents). + rules = parent_rules(tables, fk_refs, ordered_tables) generated: dict[str, pd.DataFrame] = {} for table_name in ordered_tables: @@ -244,6 +247,13 @@ def generate_data_from_dbml( ) df = pd.DataFrame(data) + # A row comes into being on or after the parent rows it points at: + # before the row's own later dates are ordered after its creation date, + # so they follow it wherever it ends up (see generate.parents). + rule = rules.get(table_name) + if rule is not None: + creation = next(c for c in table_def.columns if c.name == rule.column) + follow_parents(df, rule, creation, generated, as_of=as_of, time_profile=profile) # Ordering runs before FK resolution/dedup so a self-ref repair or a # composite-key retry regenerates a temporal column's value into a # frame that already respects created/updated/closed ordering, rather @@ -285,6 +295,8 @@ def generate_data_from_dbml( as_of=as_of, time_profile=profile, ) + if rule is not None: + df = _hold_parent_rule(df, table_def, rule, generated, as_of, profile) df = _coerce_integer_dtypes(df, table_def) generated[table_name] = df @@ -370,9 +382,46 @@ def _mirror_attributes( return df +def parent_rules_for(tables: dict[str, TableDef], refs: list[dict]) -> dict[str, ParentRule]: + """The tables whose creation date `generate_data_from_dbml` keeps after their parents'. + + By table key, each with the foreign keys it follows (see + `generate.parents`). What the dbt export tests, so it is worked out exactly + as generation works it out. + """ + fk_refs, _ = classify_refs(tables, refs) + order, _broken, _leftover = _plan_table_order(tables, fk_refs) + return parent_rules(tables, fk_refs, order) + + # --------------------------------------------------------- # Internal helpers # --------------------------------------------------------- +def _hold_parent_rule( + df: pd.DataFrame, + table_def: TableDef, + rule: ParentRule, + generated: dict[str, pd.DataFrame], + as_of: AsOf, + profile: TimeProfile, +) -> pd.DataFrame: + """Keep the parent rule after the passes that run once it was applied. + + A composite key's repair can give a row another parent, and `when` can fill + a creation date the null pass had left empty; either can leave a row dated + before its parent. Such a row's date moves (its parents stay: the + attributes mirrored through them are final by now), and the row's later + dates are placed after it again. A table neither touched comes back as it was. + """ + creation = next(c for c in table_def.columns if c.name == rule.column) + moved = follow_parents( + df, rule, creation, generated, as_of=as_of, time_profile=profile, repick=False + ) + if moved: + df = order_row_times(df, table_def, as_of=as_of, time_profile=profile, rows=moved) + return df + + def _midnight(as_of: AsOf) -> datetime: """Midnight of the run's anchor day: the latest moment a first-day timestamp takes.""" anchor = _resolve_anchor(as_of) @@ -680,6 +729,17 @@ def _table_order( columns are drawn once their parents exist (see `_deferred_fk_columns`). A schema without a cycle breaks nothing, so its order is what it always was. """ + order, broken, leftover = _plan_table_order(tables, fk_refs) + if leftover: + _cycle_state["cyclic_tables"] = leftover + return order, broken + + +def _plan_table_order( + tables: dict[str, TableDef], + fk_refs: list[dict], +) -> tuple[list[str], set[tuple[str, str]], list[str]]: + """`_table_order`, plus the tables left in a cycle, without recording them.""" graph: dict[str, set[str]] = defaultdict(set) indegree: dict[str, int] = dict.fromkeys(tables.keys(), 0) # An edge is breakable when every reference along it is a nullable FK. @@ -742,11 +802,9 @@ def _table_order( # unusual-but-not-invalid schema), but record it so the CLI can warn the # user their generated FK data may not respect every relationship. leftover = sorted(name for name in tables if name not in order) - if leftover: - _cycle_state["cyclic_tables"] = leftover order.extend(leftover) - return order, broken + return order, broken, leftover def _deferred_fk_columns( diff --git a/model2data/generate/days.py b/model2data/generate/days.py index bd6313c..52f49ab 100644 --- a/model2data/generate/days.py +++ b/model2data/generate/days.py @@ -89,6 +89,7 @@ ) from model2data.generate.hints import validate_hints from model2data.generate.options import UNIFORM, TimeProfile, validate_skew +from model2data.generate.parents import ParentRule, floor_on_day, parent_rules from model2data.generate.relationships import build_fk_lookup, classify_refs from model2data.generate.timeline import HOUR_WEIGHTS, _build_dependencies, _topological_order from model2data.generate.when import SeedFor, apply_when, end_of_day, update_when @@ -253,6 +254,7 @@ def iter_days( anchor=anchor, seed=seed, table_seeds=seeds, + rules=parent_rules(tables, fk_refs, order), ) state = dict(frames) for day in range(1, days + 1): @@ -332,6 +334,7 @@ class _Engine: day_date: date = field(default_factory=date.today) warnings: list[str] = field(default_factory=list) states: dict[str, pd.DataFrame] = field(default_factory=dict) + rules: dict[str, ParentRule] = field(default_factory=dict) # -- one table, one day --------------------------------------------- def advance( @@ -715,6 +718,7 @@ def _place_in_day( moments = self._timestamps(len(frame), profile) stamps[name] = pd.Series(moments, index=frame.index).where(present, pd.NaT) if not only_updated_at: + self._after_parents(table, frame, stamps) self._order(table, frame, by_name, stamps) for name, series in stamps.items(): if name == inc.updated_at: @@ -726,6 +730,32 @@ def _place_in_day( latest = pd.DataFrame(stamps).max(axis=1) frame[inc.updated_at] = _format(latest) + def _after_parents( + self, table: TableDef, frame: pd.DataFrame, stamps: dict[str, pd.Series] + ) -> None: + """A new row's creation timestamp is not before its parents' (see generate.parents). + + Only a parent inserted the same day can be later: every other one is + from an earlier day. Such a row is moved to a time between its latest + parent and the end of the day, evenly. A date column falls on the day, + which is never before a parent's. + """ + rule = self.rules.get(table.name) + if rule is None or rule.column not in stamps: + return + floor = floor_on_day(frame, rule, self.states) + current = stamps[rule.column] + late = current.notna() & floor.notna() & (current < floor) + if not bool(late.any()): + return + end = pd.Timestamp(self._day_start()) + pd.Timedelta(seconds=86399) + moved = current.copy() + for label in frame.index[late.to_numpy()]: + lower = floor.at[label] + room = max(0, int((end - lower).total_seconds())) + moved.at[label] = lower + pd.Timedelta(seconds=random.randint(0, room)) + stamps[rule.column] = moved + def _timestamps(self, count: int, profile: TimeProfile) -> pd.DatetimeIndex: if profile.business_hours: hours = random.choices(range(24), weights=HOUR_WEIGHTS, k=count) diff --git a/model2data/generate/hints.py b/model2data/generate/hints.py index cf810d6..13fd0ab 100644 --- a/model2data/generate/hints.py +++ b/model2data/generate/hints.py @@ -3,7 +3,7 @@ A note has always been either plain text (a comment, ignored by generation) or a JSON object read for `min`/`max`. This module documents the rest of that object's vocabulary -- `null_rate`, `weights`, `true_rate`, `distinct`, `skew`, -`after`, `business_hours`, `growth`, `seasonality`, `distribution` and its +`after`, `business_hours`, `growth`, `seasonality`, `after_parent`, `distribution` and its parameters -- and checks it once, before a single row is generated, so a typo'd enum value or a hint on the wrong kind of column fails with a message naming the table and column rather than surfacing as a wrong-looking dataset @@ -21,6 +21,7 @@ | `business_hours` | date/timestamp columns | overrides the run-level `TimeProfile.business_hours` for this column | | `growth` | date/timestamp columns | overrides the run-level `TimeProfile.growth` for this column | | `seasonality` | date/timestamp columns | overrides the run-level `TimeProfile.seasonality` for this column | +| `after_parent` | a table's creation column (see generate.timeline.creation_column) | `false`: not kept on or after its parents' creation dates (see generate.parents) | | `distribution` | numeric columns | draw shape: `uniform` (default), `normal`, `lognormal`, `exponential` | | `mean` | numeric columns, with `distribution: normal` or `exponential` | normal: the centre; exponential: the average (scale) | | `stddev` | numeric columns, with `distribution: normal` | normal: the spread | @@ -35,6 +36,7 @@ from model2data.generate import kinds from model2data.generate.relationships import build_fk_lookup, classify_refs +from model2data.generate.timeline import creation_column from model2data.parse.dbml import ColumnDef, TableDef @@ -82,6 +84,8 @@ def validate_hints(tables: Mapping[str, TableDef], refs: list[dict]) -> None: is_unique="unique" in column.settings or column.name in composite_unique_columns, columns_by_name=columns_by_name, ) + if "after_parent" in note: + _check_after_parent(table_def, column, note["after_parent"]) # --------------------------------------------------------- @@ -304,6 +308,24 @@ def _check_after( ) +def _check_after_parent(table_def: TableDef, column: ColumnDef, value: object) -> None: + label = f"{table_def.name}.{column.name}" + if column.enum_values or not _is_temporal_type(_base_type(column.data_type)): + raise ValueError(f'{label}: "after_parent" only applies to date/timestamp columns.') + _check_bool(label, "after_parent", value) + creation = creation_column(table_def) + if creation is None or creation.name != column.name: + where = ( + f"which in {table_def.name} is {creation.name}" + if creation is not None + else f"and {table_def.name} has none (every date follows another column)" + ) + raise ValueError( + f'{label}: "after_parent" sits on the date a row comes into being with, {where}: ' + "only that column is kept on or after the parent rows' own." + ) + + def _check_when( label: str, table_name: str, diff --git a/model2data/generate/parents.py b/model2data/generate/parents.py new file mode 100644 index 0000000..a9e0ef3 --- /dev/null +++ b/model2data/generate/parents.py @@ -0,0 +1,420 @@ +"""A child row comes into being on or after the parent rows it points at. + +An order placed before its customer signed up, a review written before the +product existed: each column is plausible on its own and the pair is not. So a +table's **creation column** (see `generate.timeline.creation_column`: its +`created_at`, `signup_date`, or else its first date that follows nothing, such +as `order_date`) is kept on or after the creation column of every parent row +its foreign keys point at, on every row, by the rule `parent_rules` works out: + +- The rule follows a foreign key onto a parent's primary or unique key (one + parent row per value), to a parent that has a creation column and is + generated before the child. A self-reference, a foreign key that breaks a + cycle (its parent is generated after the child), a parent without a date + and a reference onto a column that is no key are not followed; a null + foreign key constrains nothing. +- `after_parent: false` on the child's creation column turns the rule off for + the table. +- A date compared with a timestamp compares by day: a date is not before the + timestamp's day, and a timestamp is not before the date's midnight. + +How a row is made to hold (`follow_parents`), keeping the child's own shape: + +1. A row that already holds keeps every value: the column was drawn exactly + as it is without the rule. +2. A row whose date is before a parent's takes another parent through that + foreign key, one created on or before its date, drawn from the parents the + column already points at (so a popular customer stays popular, and one + created earlier has had longer to order). The child's date keeps its draw, + so its growth, seasonality and business hours stay exactly as asked. A + foreign key that must be unique (a one-to-one) keeps its parent. +3. A row with no such parent to take (it is dated before every parent the + column points at, or its foreign key is unique) moves its date instead, + drawn again from the column's own shape cut to the part of the window on or + after the latest of its parents: the same day weights (growth, seasonality, + weekdays) and, for a timestamp, the same hours. A parent created at the + very end of the window gives a child at the very end of it, never past it. + +Because a kept row is a draw from the column's shape and a moved one is a draw +from that shape cut at the parent, step 3 alone gives each row exactly the +shape cut at its parent; step 2 keeps the column's shape whole wherever a +parent allows it. + +All draws come from the table's own stream (seeded by `generate.core`), so the +rule is as reproducible as the rest of the table. +""" + +from __future__ import annotations + +import random +from bisect import bisect_right +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timedelta +from itertools import accumulate +from typing import Optional + +import numpy as np +import pandas as pd + +from model2data.generate import kinds +from model2data.generate.faker import _DATE_WINDOW_YEARS, _column_time_profile, _years_before +from model2data.generate.options import UNIFORM, TimeProfile +from model2data.generate.relationships import _key_kind +from model2data.generate.timeline import ( + _TIMESTAMP_WINDOW_DAYS, + HOUR_WEIGHTS, + AsOf, + _day_weight, + _resolve_anchor, + _window_days, + column_business_hours, + creation_column, +) +from model2data.parse.dbml import ColumnDef, TableDef + +_TS_FORMAT = "%Y-%m-%d %H:%M:%S" + + +@dataclass(frozen=True) +class ParentLink: + """One foreign key of a child along which its creation column follows a parent's. + + `fk` is the child's foreign-key column, `parent_key` the parent column it + references, `parent_column` the parent's creation column, of `parent_kind` + (`"date"` or `"timestamp"`). `unique` is true + when the foreign key takes each parent at most once (a one-to-one), so a + row cannot simply take another parent. `by_day` is true when the two + columns compare by day (one is a date, the other a timestamp). + """ + + fk: str + parent_table: str + parent_key: str + parent_column: str + parent_kind: str + unique: bool + by_day: bool + + +@dataclass(frozen=True) +class ParentRule: + """A table's creation column (`column`, of `kind`) and the parents it follows.""" + + table: str + column: str + kind: str + links: tuple[ParentLink, ...] + + +def after_parent_enabled(column: ColumnDef) -> bool: + """False when the column opts out with `after_parent: false`.""" + return (column.note or {}).get("after_parent") is not False + + +def parent_rules( + tables: Mapping[str, TableDef], fk_refs: list[dict], order: list[str] +) -> dict[str, ParentRule]: + """The rule of every table it applies to, by table key. + + `fk_refs` are the foreign keys as `generate.relationships.classify_refs` + finds them, `order` the order tables are generated in: a parent counts + only when it comes before the child, which leaves out a self-reference and + a foreign key that breaks a cycle (its parent is drawn after the child). + """ + position = {name: index for index, name in enumerate(order)} + rules: dict[str, ParentRule] = {} + for table_name, table_def in tables.items(): + column = creation_column(table_def) + if column is None or not after_parent_enabled(column): + continue + kind = kinds.temporal_kind(column.data_type) + assert kind is not None + by_name = {c.name: c for c in table_def.columns} + links: list[ParentLink] = [] + for ref in fk_refs: + if ref["source_table"] != table_name: + continue + parent = ref["target_table"] + parent_def = tables.get(parent) + fk_column = by_name.get(ref["source_column"]) + if ( + parent_def is None + or fk_column is None + or parent == table_name + or position.get(parent, len(order)) >= position.get(table_name, -1) + or _key_kind(dict(tables), ref) not in ("pk", "unique") + ): + continue + parent_column = creation_column(parent_def) + if parent_column is None: + continue + parent_kind = kinds.temporal_kind(parent_column.data_type) + assert parent_kind is not None + link = ParentLink( + fk=fk_column.name, + parent_table=parent, + parent_key=ref["target_column"], + parent_column=parent_column.name, + parent_kind=parent_kind, + unique="pk" in fk_column.settings or "unique" in fk_column.settings, + by_day=parent_kind != kind, + ) + if link not in links: + links.append(link) + if links: + rules[table_name] = ParentRule(table_name, column.name, kind, tuple(links)) + return rules + + +# --------------------------------------------------------- +# Reading moments +# --------------------------------------------------------- +def moments(series: pd.Series, kind: str) -> pd.Series: + """A column's values as `datetime64` (NaT for a null or a value that is no moment). + + A timestamp column holds `"YYYY-MM-DD HH:MM:SS"` text, a date column `date`s. + """ + if pd.api.types.is_datetime64_any_dtype(series): + return series + if kind == "timestamp": + return pd.to_datetime(series, format=_TS_FORMAT, errors="coerce") + return pd.to_datetime(series, errors="coerce") + + +def parent_floor( + frame: pd.DataFrame, link: ParentLink, parents: Mapping[str, pd.DataFrame] +) -> pd.Series: + """Per row of `frame`: the creation moment of the parent its `link` points at. + + NaT where the foreign key is null, points at no row, or the parent's date + is null. By day (midnight) when `link.by_day`. + """ + parent = parents.get(link.parent_table) + if parent is None or link.fk not in frame.columns: + return pd.Series(pd.NaT, index=frame.index, dtype="datetime64[ns]") + created = moments(parent[link.parent_column], link.parent_kind) + if link.by_day: + created = created.dt.normalize() + keys = parent[link.parent_key] + lookup = pd.Series(created.to_numpy(), index=_key_index(keys)) + lookup = lookup[~lookup.index.duplicated(keep="first")] + values = frame[link.fk] + mapped = _key_values(values).map(lookup) + return pd.Series(pd.to_datetime(mapped.to_numpy()), index=frame.index) + + +def _key_index(keys: pd.Series) -> pd.Index: + return pd.Index(_key_values(keys).tolist(), dtype=object) + + +def _key_values(values: pd.Series) -> pd.Series: + """Keys as plain Python values, so an `Int64` parent and an int child match.""" + return pd.Series( + [None if _is_na(value) else _plain(value) for value in values.tolist()], + index=values.index, + dtype=object, + ) + + +def _is_na(value: object) -> bool: + try: + return bool(pd.isna(value)) + except (TypeError, ValueError): + return False + + +def _plain(value: object) -> object: + if isinstance(value, np.generic): + return value.item() + return value + + +# --------------------------------------------------------- +# Day 0: making a table's rows hold +# --------------------------------------------------------- +def follow_parents( + df: pd.DataFrame, + rule: ParentRule, + column: ColumnDef, + parents: Mapping[str, pd.DataFrame], + *, + as_of: AsOf = None, + time_profile: Optional[TimeProfile] = None, + repick: bool = True, +) -> list: + """Make every row of `df` hold `rule`, in place; return the index labels whose date moved. + + See the module docstring for how. `parents` holds the generated parent + tables. `repick` false skips step 2 (taking another parent), for a pass + that must leave the foreign keys as they are. + """ + if rule.column not in df.columns or not len(df): + return [] + own = moments(df[rule.column], rule.kind) + if not bool(own.notna().any()): + return [] + + floors: list[pd.Series] = [] + for link in rule.links: + floor = parent_floor(df, link, parents) + late = own.notna() & floor.notna() & (own < floor) + if repick and not link.unique and bool(late.any()): + floor = _repick(df, link, own, floor, late) + floors.append(floor) + + lowest = pd.concat(floors, axis=1).max(axis=1) if len(floors) > 1 else floors[0] + late = own.notna() & lowest.notna() & (own < lowest) + if not bool(late.any()): + return [] + + window = _Window(rule.kind, column, as_of, time_profile) + positions = np.flatnonzero(late.to_numpy()) + floor_values = lowest.to_numpy() + values = df[rule.column].tolist() + for position in positions: + lower = pd.Timestamp(floor_values[position]).to_pydatetime() + values[position] = _stored(window.draw(lower), rule.kind) + df[rule.column] = pd.Series(values, index=df.index, dtype=object) + return df.index[positions].tolist() + + +def _repick( + df: pd.DataFrame, link: ParentLink, own: pd.Series, floor: pd.Series, late: pd.Series +) -> pd.Series: + """Give each late row a parent created by its date, drawn from those the column holds. + + Returns the rows' parent moments after the change. A row no parent the + column holds was created by keeps its parent (its date moves instead). + """ + floor_ns = floor.to_numpy(dtype="datetime64[ns]").astype("int64") + held = (floor.notna() & df[link.fk].notna()).to_numpy() + keys = df[link.fk].tolist() + held_positions = np.flatnonzero(held) + order = held_positions[np.argsort(floor_ns[held_positions], kind="stable")] + stamps = floor_ns[order].tolist() + own_ns = own.to_numpy(dtype="datetime64[ns]").astype("int64") + new_floor = floor_ns.copy() + changed = False + for position in np.flatnonzero(late.to_numpy()).tolist(): + count = bisect_right(stamps, int(own_ns[position])) + if count == 0: + continue + source = int(order[random.randrange(count)]) + keys[position] = keys[source] + new_floor[position] = floor_ns[source] + changed = True + if not changed: + return floor + df[link.fk] = pd.Series(keys, index=df.index, dtype=df[link.fk].dtype) + # NaT went through int64 as its sentinel and comes back as NaT. + return pd.Series(new_floor.view("datetime64[ns]"), index=floor.index) + + +def _stored(moment: datetime, kind: str) -> object: + """A moment as the column stores it: `"YYYY-MM-DD HH:MM:SS"` text, or a `date`.""" + if kind == "timestamp": + return moment.strftime(_TS_FORMAT) + return moment.date() + + +class _Window: + """A column's own window and shape, drawn from on or after a given moment.""" + + def __init__( + self, + kind: str, + column: ColumnDef, + as_of: AsOf, + time_profile: Optional[TimeProfile], + ) -> None: + anchor = _resolve_anchor(as_of) + self.kind = kind + self.business_hours = column_business_hours(column, time_profile) + profile = _column_time_profile(column, time_profile) or UNIFORM + if kind == "date": + start = _years_before(anchor, _DATE_WINDOW_YEARS) + end = anchor + else: + start = anchor - timedelta(days=_TIMESTAMP_WINDOW_DAYS) + end = anchor - timedelta(days=1) + self.start = start + self.days = _window_days(start, end) + weights = [_day_weight(day, start, end, profile) for day in self.days] + self.added_up = list(accumulate(weights)) + self.weights = weights + # A timestamp's window ends at midnight of the anchor, a date's on the anchor. + self.cap = datetime(anchor.year, anchor.month, anchor.day) + + def draw(self, lower: datetime) -> datetime: + """A moment on or after `lower` in the window, in the column's shape.""" + if self.kind == "date": + return self._date(lower) + return self._timestamp(lower) + + def _date(self, lower: datetime) -> datetime: + first = max(0, (lower.date() - self.start).days) + if first >= len(self.days): + return lower + before = self.added_up[first - 1] if first else 0.0 + point = before + random.random() * (self.added_up[-1] - before) + index = min(bisect_right(self.added_up, point), len(self.days) - 1) + index = max(index, first) + day = self.days[index] + return datetime(day.year, day.month, day.day) + + def _timestamp(self, lower: datetime) -> datetime: + if lower >= self.cap: + # A parent created at the very end of the window (a date on the + # anchor day): the child is created at the end too, never past it. + return lower + first = (lower.date() - self.start).days + if first < 0: + first, lower = 0, datetime.combine(self.start, datetime.min.time()) + offset = lower.hour * 3600 + lower.minute * 60 + lower.second + first_share = self.weights[first] * self._share_after(offset) + rest = self.added_up[-1] - self.added_up[first] + point = random.random() * (first_share + rest) + if point < first_share or rest <= 0: + day = self.days[first] + seconds = self._time_after(offset) + else: + target = self.added_up[first] + (point - first_share) + index = min(bisect_right(self.added_up, target), len(self.days) - 1) + day = self.days[max(index, first + 1)] + seconds = self._time_after(0) + moment = datetime(day.year, day.month, day.day) + timedelta(seconds=seconds) + return max(moment, lower) + + def _share_after(self, offset: int) -> float: + """The share of a day's draws that fall at or after `offset` seconds into it.""" + if not self.business_hours: + return (86400 - offset) / 86400 + hour, into = divmod(offset, 3600) + remaining = HOUR_WEIGHTS[hour] * (3600 - into) / 3600 + sum(HOUR_WEIGHTS[hour + 1 :]) + return remaining / sum(HOUR_WEIGHTS) + + def _time_after(self, offset: int) -> int: + """Seconds into a day, at or after `offset`, drawn as the column draws a time of day.""" + if not self.business_hours: + return random.randint(offset, 86399) + hour, into = divmod(offset, 3600) + hours = list(range(hour, 24)) + weights = [HOUR_WEIGHTS[hour] * (3600 - into) / 3600] + [HOUR_WEIGHTS[h] for h in hours[1:]] + chosen = random.choices(hours, weights=weights)[0] + if chosen == hour: + return hour * 3600 + random.randint(into, 3599) + return chosen * 3600 + random.randrange(3600) + + +# --------------------------------------------------------- +# Later days: a new row's timestamp on its day +# --------------------------------------------------------- +def floor_on_day( + frame: pd.DataFrame, rule: ParentRule, parents: Mapping[str, pd.DataFrame] +) -> pd.Series: + """Per new row: the latest creation moment among the parents it points at (NaT for none).""" + floors = [parent_floor(frame, link, parents) for link in rule.links] + if not floors: + return pd.Series(pd.NaT, index=frame.index, dtype="datetime64[ns]") + return pd.concat(floors, axis=1).max(axis=1) if len(floors) > 1 else floors[0] diff --git a/model2data/generate/timeline.py b/model2data/generate/timeline.py index c6b63d2..798dd7c 100644 --- a/model2data/generate/timeline.py +++ b/model2data/generate/timeline.py @@ -340,6 +340,46 @@ def _infer_stage(column_name: str) -> Optional[int]: return None +def creation_column_name(columns: list[tuple[str, bool]]) -> Optional[str]: + """Which of a table's date/timestamp columns the row comes into being with. + + `columns` are the table's date and timestamp columns in table order, each + as `(name, follows)`, `follows` being whether it has an `after` hint. A + column with `after` follows another column of its row, so it is never the + one the row starts with. Of the rest, the first whose name puts it at the + created stage (`created_at`, `signup_date`, `start_date`: see + `_STAGE_WORDS`) is the creation column; failing that, the first whose name + puts it at no stage at all (`order_date`, `hire_date`), a birth date + excepted. A table whose dates all follow something, or are all updated- or + closed-stage (`updated_at`, `deleted_at`), has none. + """ + leading = [name for name, follows in columns if not follows] + for name in leading: + if _infer_stage(name) == 0: + return name + for name in leading: + if _infer_stage(name) is None and not _EXCLUDED_FROM_CHAIN.intersection(_tokenize(name)): + return name + return None + + +def creation_column(table_def: TableDef) -> Optional[ColumnDef]: + """The date/timestamp column a row of `table_def` comes into being with, or None. + + See `creation_column_name`. It is the column a child row's own date is + kept on or after its parent's by (see `generate.parents`). + """ + temporal = [ + column + for column in table_def.columns + if not column.enum_values and _column_kind(column.data_type) is not None + ] + name = creation_column_name( + [(column.name, bool((column.note or {}).get("after"))) for column in temporal] + ) + return next((column for column in temporal if column.name == name), None) + + def _column_kind(data_type: str) -> Optional[str]: """`"timestamp"`, `"date"`, or None for anything else -- including plain `time`. @@ -476,7 +516,11 @@ def _topological_order(table_def: TableDef, deps: dict[str, set[str]]) -> list[s def order_row_times( - df, table_def: TableDef, as_of: AsOf = None, time_profile: Optional[TimeProfile] = None + df, + table_def: TableDef, + as_of: AsOf = None, + time_profile: Optional[TimeProfile] = None, + rows: Optional[list] = None, ): """Fix up temporal columns so an earlier-stage column never lands later. @@ -492,7 +536,9 @@ def order_row_times( Leaves `df` untouched when the table holds no `after` hint and no recognisable created/updated/closed-style name pair, and never modifies a - column that isn't a date/timestamp/datetime type. + column that isn't a date/timestamp/datetime type. `rows`, when given, are + the only index labels it places again (a row whose creation date moved + after the table was ordered); every row otherwise. """ columns_by_name = {column.name: column for column in table_def.columns} temporal_columns = [ @@ -525,7 +571,7 @@ def order_row_times( required_kinds[name] = other_kind business_hours = column_business_hours(columns_by_name[column_name], time_profile) - for idx in df.index: + for idx in df.index if rows is None else rows: if _parse_value(df.at[idx, column_name], kind) is None: # Already null for this row (or holds a non-temporal default): # nothing to reorder, and forcing a value in would undo the diff --git a/tests/test_defects_review.py b/tests/test_defects_review.py index 9e9c4ff..3874af2 100644 --- a/tests/test_defects_review.py +++ b/tests/test_defects_review.py @@ -287,7 +287,9 @@ def test_an_undone_row_without_days(): def test_late_rows_a_later_defect_makes_on_time_are_not_counted(): model = load(SHOP) - days = generate_days(model, 3, seed=11, as_of=AS_OF) + # A seed whose late rows include one the nulls make on time (engine 1.15.0's + # parent dates changed which rows seed 11 picks). + days = generate_days(model, 3, seed=2, as_of=AS_OF) plan = { "orders": [ Defect("late_arriving", count=3), @@ -295,7 +297,7 @@ def test_late_rows_a_later_defect_makes_on_time_are_not_counted(): Defect("nulls", column="updated_at", count=10_000), ] } - _, report = apply_defects(model, days, plan, seed=11) + _, report = apply_defects(model, days, plan, seed=2) late = report.defects[0] assert late.applied < 3 assert "no longer before the cutoff once the table's other defects are in" in late.note From 0d4c46d357f7afdd5e3a36dad019e3dd3ece721e Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Wed, 7 Oct 2026 22:36:09 +0200 Subject: [PATCH 03/14] Test parent dates in the generated dbt project With hint tests on, each foreign key a creation date follows writes `model2data_not_before_parent` on the child's creation column: it joins the child to the parent model through the foreign key and returns the rows dated before the parent row, by day for a date against a timestamp. A key several parent rows share counts its earliest, so a duplicate_keys defect stays the unique test's to report. The macro ships in macros/model2data_parent_tests.sql, written only when a project has such a test, so other projects are what they were. `hint_tests_for` takes the model's refs beside bare tables, and the defects report checks the test the way its SQL reads, so a defect that dates a row before its parent lists it in EXPECTED_FAILURES.md. A dbt run confirms the test passes on clean data and fails on a customer created after its order. Co-Authored-By: Claude Opus 5.5 --- model2data/dbt/hint_tests.py | 67 +++++++++++++++++-- .../hint_macros/model2data_parent_tests.sql | 37 ++++++++++ model2data/dbt/tests.py | 20 ++++-- model2data/defects/checks.py | 38 +++++++++++ tests/test_defects_dbt.py | 29 ++++++++ 5 files changed, 178 insertions(+), 13 deletions(-) create mode 100644 model2data/dbt/templates/hint_macros/model2data_parent_tests.sql diff --git a/model2data/dbt/hint_tests.py b/model2data/dbt/hint_tests.py index d5afbba..a1b3033 100644 --- a/model2data/dbt/hint_tests.py +++ b/model2data/dbt/hint_tests.py @@ -16,6 +16,13 @@ | `generate.distinct: n` | `model2data_max_distinct` | none | | `generate.when` | `model2data_when` | none | | table `grain` | `model2data_unique_combination` | none | +| a creation date after its parents' | `model2data_not_before_parent` | none | + +The last one comes from no hint: it is what generation keeps on every table +whose creation date follows a parent's (see `model2data.generate.parents`), one +test per foreign key it follows, on the child's creation column. It needs the +model's references, so a caller passing bare tables passes `refs` too; a column +with `after_parent: false` gets none. A column with `when` and `null_rate` has its null share tested among the rows `when` matches (`model2data_when_max_null_share`), since only those rows can @@ -28,8 +35,9 @@ The generic tests live in `macros/model2data_hint_tests.sql` of the generated project (see `write_hint_macros`), and the `when` ones in -`macros/model2data_when_tests.sql`, written only for a model that has one, so -a project without `when` is what it was: plain SQL, no dbt package. +`macros/model2data_when_tests.sql` and the parent one in +`macros/model2data_parent_tests.sql`, each written only for a model that has +one, so a project without either is what it was: plain SQL, no dbt package. """ from __future__ import annotations @@ -41,6 +49,8 @@ from typing import Any, Literal, Optional, Union from model2data.generate import kinds +from model2data.generate.core import parent_rules_for +from model2data.generate.parents import ParentRule from model2data.model.engine import EngineInputs, to_engine from model2data.model.types import Model from model2data.parse.dbml import ColumnDef, TableDef @@ -53,6 +63,8 @@ _MACROS_SOURCE = Path(__file__).parent / "templates" / "hint_macros" / MACROS_FILE WHEN_MACROS_FILE = "model2data_when_tests.sql" _WHEN_MACROS_SOURCE = Path(__file__).parent / "templates" / "hint_macros" / WHEN_MACROS_FILE +PARENT_MACROS_FILE = "model2data_parent_tests.sql" +_PARENT_MACROS_SOURCE = Path(__file__).parent / "templates" / "hint_macros" / PARENT_MACROS_FILE @dataclass(frozen=True) @@ -63,7 +75,10 @@ class HintTest: `column` is None for a table-level test (grain). `test` is the generic test's name, `arguments` its parameters as dbt's `arguments:` block, and `hint` the hint it comes from (`min`, `max`, `after`, `null_rate`, - `distinct`, `when`, `grain`). `min` and `max` of one column share one test. + `distinct`, `when`, `grain`, or `after_parent` for a creation date that + follows a parent's). `min` and `max` of one column share one test. + `parent` is the `(table, column)` a test looks rows up in: the parent's key + for `after_parent`, None for every other test. """ table: str @@ -71,6 +86,7 @@ class HintTest: test: str arguments: dict[str, Any] = field(default_factory=dict) hint: str = "" + parent: Optional[tuple[str, str]] = None def to_dbt(self, severity: str = "warn") -> dict[str, dict[str, Any]]: """The entry for a `tests:` list: `{name: {arguments: ..., config: {severity: ...}}}`.""" @@ -83,12 +99,16 @@ def hint_tests_for( model_or_tables: Union[Model, EngineInputs, Mapping[str, TableDef]], *, tolerance: float = DEFAULT_TOLERANCE, + refs: Optional[list[dict]] = None, ) -> list[HintTest]: """The tests implied by the hints of a model, in table then column order. Takes a `Model`, the `EngineInputs` of one, or the `tables` dict (as `to_engine(model).tables`, or the renamed tables the CLI hands `generate_dbt_yml`). Table-level tests follow the table's column tests. + With `tables`, `refs` are the references as `to_engine` gives them; without + them no `after_parent` test is written (a `Model` or `EngineInputs` brings + its own). `tolerance` is the absolute slack of the statistical test (the null share may be `null_rate + tolerance`); hard constraints take none. Severity is @@ -97,18 +117,25 @@ def hint_tests_for( if tolerance < 0: raise ValueError("tolerance must not be negative") if isinstance(model_or_tables, Model): - tables: Mapping[str, TableDef] = to_engine(model_or_tables).tables + inputs = to_engine(model_or_tables) + tables: Mapping[str, TableDef] = inputs.tables + refs = inputs.refs elif isinstance(model_or_tables, EngineInputs): tables = model_or_tables.tables + refs = model_or_tables.refs else: tables = model_or_tables + rules = parent_rules_for(dict(tables), refs) if refs else {} found: list[HintTest] = [] for key, table in tables.items(): by_name = {column.name: column for column in table.columns} names = set(by_name) + rule = rules.get(key) for column in table.columns: found.extend(_column_tests(key, column, by_name, tolerance)) + if rule is not None and rule.column == column.name: + found.extend(_parent_tests(key, rule)) grain = (table.note or {}).get("grain") if grain and all(name in names for name in grain): found.append( @@ -202,6 +229,31 @@ def _column_tests( return tests +def _parent_tests(table: str, rule: ParentRule) -> list[HintTest]: + """One `model2data_not_before_parent` test per foreign key the creation date follows.""" + tests = [] + for link in rule.links: + arguments: dict[str, Any] = { + "foreign_key": link.fk, + "to": f"ref('stg_{link.parent_table}')", + "field": link.parent_key, + "parent_column": link.parent_column, + } + if link.by_day: + arguments["granularity"] = "day" + tests.append( + HintTest( + table, + rule.column, + "model2data_not_before_parent", + arguments, + "after_parent", + parent=(link.parent_table, link.parent_key), + ) + ) + return tests + + def _when_conditions(when: Any, kinds_of: dict[str, ColumnDef]) -> Optional[dict[str, list]]: """A `when` hint as the test's `conditions`: an enum's values as the member text it holds.""" if not isinstance(when, dict) or not when or not all(name in kinds_of for name in when): @@ -212,13 +264,14 @@ def _when_conditions(when: Any, kinds_of: dict[str, ColumnDef]) -> Optional[dict } -def write_hint_macros(dest: Path, *, when: bool = False) -> Path: +def write_hint_macros(dest: Path, *, when: bool = False, parents: bool = False) -> Path: """Write the generic tests into `dest/macros/`, and return the file's path. A project whose YAML holds `hint_tests_for` tests needs this file next to it; `generate_dbt_yml` writes it itself when it emits any. With `when`, the file of the `when` tests is written beside it (`WHEN_MACROS_FILE`), which a - project holding a `model2data_when` test needs too. + project holding a `model2data_when` test needs too; with `parents`, the + file of `model2data_not_before_parent` (`PARENT_MACROS_FILE`). """ macros = dest / "macros" macros.mkdir(parents=True, exist_ok=True) @@ -226,4 +279,6 @@ def write_hint_macros(dest: Path, *, when: bool = False) -> Path: target.write_text(_MACROS_SOURCE.read_text()) if when: (macros / WHEN_MACROS_FILE).write_text(_WHEN_MACROS_SOURCE.read_text()) + if parents: + (macros / PARENT_MACROS_FILE).write_text(_PARENT_MACROS_SOURCE.read_text()) return target diff --git a/model2data/dbt/templates/hint_macros/model2data_parent_tests.sql b/model2data/dbt/templates/hint_macros/model2data_parent_tests.sql new file mode 100644 index 0000000..ada9bbe --- /dev/null +++ b/model2data/dbt/templates/hint_macros/model2data_parent_tests.sql @@ -0,0 +1,37 @@ +{#- + The generic test model2data writes where a table's creation date follows its + parents': a row is not dated before the parent row its foreign key points at + (an order not before its customer signed up). Written beside + model2data_hint_tests.sql, and only by a model that has such a table. Plain + ANSI SQL, runs on DuckDB and Postgres. Returns the rows that break it. + `column_name` arrives already quoted by the schema YAML; `foreign_key`, + `field` (the parent's key) and `parent_column` (the parent's creation date) + are raw names and are quoted here. A row whose foreign key is null, or points + at no parent row, is left to the not_null and relationships tests, and a key + several parent rows share (left to the unique test) counts its earliest one. + `granularity='day'` compares a date with a timestamp by day. +-#} + +{% test model2data_not_before_parent(model, column_name, foreign_key, to, field, parent_column, granularity=none) %} +{%- set mine = 'child.' ~ column_name -%} +{%- set theirs = 'parent.parent_value' -%} +{%- if granularity == 'day' -%} + {%- set mine = 'cast(' ~ mine ~ ' as date)' -%} + {%- set theirs = 'cast(' ~ theirs ~ ' as date)' -%} +{%- endif %} +select + child.{{ column_name }} as value, + parent.parent_value +from {{ model }} as child +join ( + select + {{ adapter.quote(field) }} as parent_key, + min({{ adapter.quote(parent_column) }}) as parent_value + from {{ to }} + where {{ adapter.quote(parent_column) }} is not null + group by {{ adapter.quote(field) }} +) as parent + on child.{{ adapter.quote(foreign_key) }} = parent.parent_key +where child.{{ column_name }} is not null + and {{ mine }} < {{ theirs }} +{% endtest %} diff --git a/model2data/dbt/tests.py b/model2data/dbt/tests.py index 9722b95..53cff78 100644 --- a/model2data/dbt/tests.py +++ b/model2data/dbt/tests.py @@ -66,10 +66,10 @@ def generate_dbt_yml( staging_path = dest / "models" / "staging" staging_path.mkdir(parents=True, exist_ok=True) - hinted = _hinted(tables, hint_tests, test_tolerance) + hinted = _hinted(tables, hint_tests, test_tolerance, refs) if hinted: - when = any(test.hint == "when" for tests in hinted.values() for test in tests) - write_hint_macros(dest, when=when) + hints = {test.hint for tests in hinted.values() for test in tests} + write_hint_macros(dest, when="when" in hints, parents="after_parent" in hints) if any(_history_entries(table) for table in tables.values()): write_history_macros(dest) fk_map = _fk_map(tables, refs) @@ -114,14 +114,14 @@ def generate_dbt_yml( def _hinted( - tables: dict, hint_tests: str, test_tolerance: float + tables: dict, hint_tests: str, test_tolerance: float, refs: list[dict] ) -> dict[tuple[str, Optional[str]], list[HintTest]]: """The hint tests by (table, column), column None for a table-level one; none for `off`.""" if hint_tests not in ("error", "warn", "off"): raise ValueError(f"hint_tests must be error, warn or off, not {hint_tests!r}") hinted: dict[tuple[str, Optional[str]], list[HintTest]] = defaultdict(list) if hint_tests != "off": - for hint_test in hint_tests_for(tables, tolerance=test_tolerance): + for hint_test in hint_tests_for(tables, tolerance=test_tolerance, refs=refs): hinted[(hint_test.table, hint_test.column)].append(hint_test) return hinted @@ -193,7 +193,8 @@ class DbtTest: `relationships`, a `model2data_*` hint test) or `unique_combination` for the singular test of a composite key. `arguments` are the test's parameters, `severity` is `error`, or the hint tests' severity, and `parent` is the - `(table, column)` a `relationships` test looks values up in. + `(table, column)` a `relationships` test looks values up in, or the parent + key a `model2data_not_before_parent` test joins on. """ name: str @@ -218,7 +219,7 @@ def dbt_tests( name dbt gives the test in the generated project. Unit tests are not data tests and are not listed. """ - hinted = _hinted(tables, hint_tests, test_tolerance) + hinted = _hinted(tables, hint_tests, test_tolerance, refs) fk_map = _fk_map(tables, refs) found: list[DbtTest] = [] for table in tables.values(): @@ -226,6 +227,9 @@ def dbt_tests( for col in table.columns: column_ref = _dbt_column_ref(col.name) parents = iter(fk_map.get((table.name, col.name), [])) + hint_parents = iter( + t.parent for t in hinted.get((table.name, col.name), []) if t.parent is not None + ) for entry in _column_entries(table, col, fk_map, hinted, hint_tests): test_type, body = _entry_parts(entry) arguments = dict(body.get("arguments") or {}) @@ -233,6 +237,8 @@ def dbt_tests( if test_type == "relationships": ref = next(parents) parent = (ref["target_table"], ref["target_column"]) + elif test_type == "model2data_not_before_parent": + parent = next(hint_parents) found.append( DbtTest( name=generic_test_name( diff --git a/model2data/defects/checks.py b/model2data/defects/checks.py index 852d095..9fffb79 100644 --- a/model2data/defects/checks.py +++ b/model2data/defects/checks.py @@ -14,6 +14,9 @@ and `model2data_max_distinct`: as their macros in `dbt/templates/hint_macros/model2data_hint_tests.sql`, nulls skipped where the SQL skips them. +- `model2data_not_before_parent`: as its macro in + `dbt/templates/hint_macros/model2data_parent_tests.sql`: a row dated before + the earliest parent row its foreign key matches, by day with `granularity: day`. - `model2data_when` and `model2data_when_max_null_share`: as their macros in `dbt/templates/hint_macros/model2data_when_tests.sql`, a row matching when each named column's text is one of its values (a boolean as `true`/`false` @@ -338,6 +341,40 @@ def _not_before(test: DbtTest, frame: pd.DataFrame, seeded: Mapping, cache: Colu return bool((mine < theirs).any()) +def _not_before_parent(test: DbtTest, frame: pd.DataFrame, seeded: Mapping, cache: Columns) -> bool: + if test.parent is None: + return False + parent = seeded.get(test.parent[0]) + if parent is None: + return False + arguments = test.arguments + mine = pd.DataFrame( + { + "key": cache.numbers(frame, arguments["foreign_key"]), + "moment": cache.values(frame, str(test.column)), + } + ).dropna() + theirs = pd.DataFrame( + { + "key": cache.numbers(parent, arguments["field"]), + "parent_moment": cache.values(parent, arguments["parent_column"]), + } + ).dropna() + if mine.empty or theirs.empty: + return False + # A key several parent rows share counts its earliest, as the SQL's `min` does. + theirs["parent_moment"] = pd.to_datetime(theirs["parent_moment"], format="mixed") + earliest = theirs.groupby("key", sort=False)["parent_moment"].min().reset_index() + pairs = mine.merge(earliest, on="key") + if pairs.empty: + return False + child = pd.to_datetime(pairs["moment"], format="mixed") + parents = pairs["parent_moment"] + if arguments.get("granularity") == "day": + child, parents = child.dt.normalize(), parents.dt.normalize() + return bool((child < parents).any()) + + def _max_null_share(test: DbtTest, frame: pd.DataFrame, seeded: Mapping, cache: Columns) -> bool: if not len(frame): return False @@ -433,6 +470,7 @@ def _no_overlapping_ranges( "relationships": _relationships, "model2data_between": _between, "model2data_not_before": _not_before, + "model2data_not_before_parent": _not_before_parent, "model2data_max_null_share": _max_null_share, "model2data_max_distinct": _max_distinct, "model2data_when": _when, diff --git a/tests/test_defects_dbt.py b/tests/test_defects_dbt.py index 698b817..250cb7f 100644 --- a/tests/test_defects_dbt.py +++ b/tests/test_defects_dbt.py @@ -182,6 +182,35 @@ def test_a_preset_fails_exactly_the_tests_the_report_names( assert late == {"late_arriving", "late_updates"} +PARENT_TEST = ( + "model2data_not_before_parent_stg_orders_ordered_at" + "__id__customer_id__created_at__ref_stg_customers_" +) + + +def test_the_parent_dates_test_passes_clean_and_fails_a_customer_created_late( + tmp_path, monkeypatch +): + """Orders follow their customers' `created_at`: dbt agrees, and catches one that does not.""" + project = _generate(tmp_path, monkeypatch, SHOP) + assert (project / "macros" / "model2data_parent_tests.sql").exists() + assert _build(project) == set() + + seeds = project / "seeds" / "raw" + orders = (seeds / "orders.csv").read_text().splitlines() + customer = orders[1].split(",")[1] + lines = (seeds / "customers.csv").read_text().splitlines() + header = lines[0].split(",") + at, key = header.index("created_at"), header.index("id") + for number, line in enumerate(lines[1:], start=1): + cells = line.split(",") + if cells[key] == customer: + cells[at] = "2099-01-01 00:00:00" + lines[number] = ",".join(cells) + (seeds / "customers.csv").write_text("\n".join(lines) + "\n") + assert _build(project) == {PARENT_TEST} + + def test_a_history_fails_only_its_overlap_test(tmp_path, monkeypatch): model = load(HISTORY) model.tables["orders"].defects = [Defect("overlapping_history", count=3)] From 8d713149603dfb319edeab379df9e27947fe90ae Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Wed, 7 Oct 2026 22:36:16 +0200 Subject: [PATCH 04/14] Cut spec 0.5.0 for `after_parent` `after_parent` is new optional vocabulary, so by the spec's own rule it is a minor version: allowed only in a document that says `model2data: 0.5.0`, and an error naming the version to write in a 0.2, 0.3 or 0.4 document, which otherwise read exactly as before. It is a boolean on the table's creation column; anywhere else the reader names the column that is. A writer keeps the version a document was read with and writes 0.5.0 once it uses the hint; DBML with an `after_parent` note converts to 0.5.0. The spec README says what 0.5.0 also says of every document's data, as a minor release may before 1.0.0: a child's creation date is not before its parents' (References), and a one-column integer primary key numbers the rows 1..n in order (Keys). The schema's $id moves to /spec/0.5.0/, PUBLISHING.md lists the 0.5.0 files of engine 1.15.0 beside the older ones, and the guide pages name 0.5.0. Co-Authored-By: Claude Opus 5.5 --- model2data/cli.py | 17 +- model2data/guide/setup.md | 4 +- model2data/guide/tune.md | 28 +- model2data/model/__init__.py | 2 +- model2data/model/dbml.py | 14 +- model2data/model/document.py | 15 +- model2data/model/types.py | 2 +- model2data/model/validate.py | 64 ++++- model2data/spec/PUBLISHING.md | 26 +- model2data/spec/README.md | 36 ++- model2data/spec/model.schema.json | 17 +- tests/test_defects.py | 3 +- tests/test_model_messages.py | 8 +- tests/test_parent_dates.py | 440 ++++++++++++++++++++++++++++++ 14 files changed, 617 insertions(+), 59 deletions(-) create mode 100644 tests/test_parent_dates.py diff --git a/model2data/cli.py b/model2data/cli.py index 96f3bca..9962dd5 100644 --- a/model2data/cli.py +++ b/model2data/cli.py @@ -184,7 +184,7 @@ def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: cls=_GenerateByDefault, help=( "model2data: Generate analytics-ready datasets from a data model.\n\n" - "Given a model -- a .model2data.yml document (spec 0.4.0), the same as JSON,\n" + "Given a model -- a .model2data.yml document (spec 0.5.0), the same as JSON,\n" "or a DBML file -- this tool produces:\n" "• Synthetic but realistic data\n" "• A runnable dbt project scaffold\n" @@ -837,12 +837,13 @@ def pick(option: Any, from_run: Any, default: Any) -> Any: def _spec_of(file: Path) -> str: - """The spec a conforming model is written against: 0.4.0, 0.3.0, or 0.2.0 (DBML without - `when` included).""" + """The spec a conforming model is written against: 0.5.0, 0.4.0, 0.3.0, or 0.2.0 (DBML + without `when` or `after_parent` included).""" version = str(load(file).version) - return ( - "0.4.0" if version.startswith("0.4") else "0.3.0" if version.startswith("0.3") else "0.2.0" - ) + for minor in ("0.5", "0.4", "0.3"): + if version.startswith(minor): + return f"{minor}.0" + return "0.2.0" def _print_defects(report: DefectsReport) -> None: @@ -934,7 +935,7 @@ def validate_command( ), ), ): - """Check that models conform to spec 0.4.0 (or 0.2.x, 0.3.x), printing every issue with its path. + """Check that models conform to spec 0.5.0 (or 0.2.x-0.4.x), printing every issue with its path. Takes one or more files and/or --glob patterns. A metrics file (*.metrics.yml) is checked against metrics spec 0.1.0 and against its model. Exits 1 when any file has an error; @@ -1029,7 +1030,7 @@ def convert_command( force: bool = typer.Option(False, "--force", help="Overwrite the output file if it exists."), ): """Convert a model -- DBML, typically -- to a .model2data.yml document (spec 0.2.0, or - 0.3.0 when it has defects, 0.4.0 when it has `when`).""" + 0.3.0 when it has defects, 0.4.0 when it has `when`, 0.5.0 when it has `after_parent`).""" try: model = load(file) except ModelError as error: diff --git a/model2data/guide/setup.md b/model2data/guide/setup.md index 8f74485..9c14152 100644 --- a/model2data/guide/setup.md +++ b/model2data/guide/setup.md @@ -15,7 +15,7 @@ runnable dbt project. You write the model; it writes everything else. Create `.model2data.yml` in this directory. Start from this and replace the tables: ```yaml -model2data: 0.4.0 +model2data: 0.5.0 name: shop enums: @@ -61,7 +61,7 @@ model2data --file .model2data.yml cd dbt_ && dbt build ``` -Done looks like: `✅ .model2data.yml conforms to spec 0.4.0.`, a summary with +Done looks like: `✅ .model2data.yml conforms to spec 0.5.0.`, a summary with `Columns using generic fallback text: 0` and no ⚠️ lines, and `dbt build` ending in `ERROR=0`. Anything else: `model2data guide triage`. diff --git a/model2data/guide/tune.md b/model2data/guide/tune.md index ad2dd39..e0cdab0 100644 --- a/model2data/guide/tune.md +++ b/model2data/guide/tune.md @@ -5,7 +5,7 @@ | Command | Does | Exit | |---|---|---| | `model2data --file M` | same as `model2data generate --file M` | see below | -| `model2data validate M...` | check models against spec 0.4.0, and `*.metrics.yml` files against the metrics spec and their model (`--model M`, else the model beside it with the same stem) | 0 conforms (warnings allowed), 1 any error | +| `model2data validate M...` | check models against spec 0.5.0, and `*.metrics.yml` files against the metrics spec and their model (`--model M`, else the model beside it with the same stem) | 0 conforms (warnings allowed), 1 any error | | `model2data convert M` | print M as `.model2data.yml`; `-o FILE` writes it, `--force` overwrites | 0 ok, 1 unreadable model or output exists | | `model2data metrics list -f M [--metrics F]` | every metric: the file's, then the inferred ones | 0, 1 on a model or metrics error | | `model2data metrics export -f M [--metrics F] --to ossie\|lightdash` | the model and its metrics as Apache Ossie 0.1.1, or as Lightdash meta in one dbt properties file for the staging models; `-o FILE` writes it, `--force` overwrites | 0, 1 on an error or an existing output, 2 on another `--to` | @@ -62,7 +62,7 @@ columns. Add `not_null` to every column that must always have a value. `references`, `measure`, `generate`. - `generate`: `min`, `max`, `distribution`, `null_rate`, `weights`, `true_rate`, `distinct`, `skew`, `after`, `business_hours`, `growth`, `seasonality`, `transitions`, - `when`. + `when`, `after_parent`. - Table: `columns`, `description`, `color`, `role`, `grain`, `keys`, `foreign_keys`, `incremental`, `defects`. - Top level: `model2data` (spec version), `name`, `description`, `enums`, `tables`, @@ -71,7 +71,7 @@ columns. Add `not_null` to every column that must always have a value. Each key in use (this model validates): ```yaml -model2data: 0.4.0 +model2data: 0.5.0 name: saas enums: plan_tier: [free, pro, enterprise] @@ -84,6 +84,7 @@ tables: tier: {type: plan_tier, generate: {weights: {free: 5, pro: 3, enterprise: 1}}} is_verified: {type: boolean, generate: {true_rate: 0.8}} region: {type: city, generate: {distinct: 4}} + created_at: {type: timestamp, not_null: true} subscriptions: incremental: {new_per_day: 5, update_rate: 0.1, changes: [status], updated_at: updated_at} keys: [{unique: [account_id, started_at]}] @@ -99,6 +100,11 @@ tables: updated_at: {type: timestamp, generate: {after: started_at}} defects: - {type: nulls, column: account_id, count: 2} + audit_events: + columns: + id: {type: bigint, pk: true} + account_id: {type: bigint, references: accounts.id} + logged_at: {type: timestamp, not_null: true, generate: {after_parent: false}} run: rows: 100 rows_per_table: {accounts: 20} @@ -110,7 +116,7 @@ run: `transitions` act only on days after the first: it needs `incremental` and `--days`. -`when` (spec 0.4.0, so the model says `model2data: 0.4.0`) ties a column to another column's value: `cancelled_at` above holds a timestamp on +`when` (added in spec version 0.4.0, which the model then names) ties a column to another column's value: `cancelled_at` above holds a timestamp on every `cancelled` row and is null on every other one, so no `active` subscription has a `cancelled_at`. Name another column of the table (an enum, boolean, number or text column) and list the values it must hold; several columns must all match. Use it for every @@ -120,6 +126,20 @@ without a `default`, and not a key, unique or a foreign key. A `null_rate` besid only the matching rows. On later days a row whose status moves into a listed value gets a value that day (its `updated_at` when the table has one), and one moving out loses it. It writes the dbt test `model2data_when`. + +A row is never created before the parent rows it points at: `started_at` above (the +subscription's creation column) is on or after its account's `created_at`, on every row, +with no hint. The creation column is a table's first date or timestamp without `after` +whose name says created (`created_at`, `signup_date`, `start_date`), else its first that +names no later stage (`order_date`); a child follows each parent that has one, through a +foreign key onto the parent's key. A row dated too early takes another parent created by +then, so the child's growth and seasonality stay as asked; with no such parent its date +moves. Self-references, nullable foreign keys left null, parents without a date and +foreign keys that break a cycle are not followed. `after_parent: false` on the creation +column (spec 0.5.0) turns it off for that table: `audit_events` above copies old events, so +its `logged_at` may predate the account. With `--hint-tests` each followed foreign key +writes the dbt test `model2data_not_before_parent`. Integer primary keys number the rows +1..N, and every output lists the rows in key order. The full spec and JSON Schema ship in the package: `model2data/spec/README.md`, `model2data/spec/model.schema.json`. diff --git a/model2data/model/__init__.py b/model2data/model/__init__.py index 3513491..940d8d5 100644 --- a/model2data/model/__init__.py +++ b/model2data/model/__init__.py @@ -1,4 +1,4 @@ -"""The model2data model: spec 0.4.0 (and 0.2.x, 0.3.x), read, checked, written, and generated from. +"""The model2data model: spec 0.5.0 (and 0.2.x-0.4.x), read, checked, written, and generated from. A model is one document -- `.model2data.yml`, or the same in JSON -- as `model2data/spec/README.md` and `model2data/spec/model.schema.json` define it. diff --git a/model2data/model/dbml.py b/model2data/model/dbml.py index bf3e380..1b2b9fd 100644 --- a/model2data/model/dbml.py +++ b/model2data/model/dbml.py @@ -197,12 +197,18 @@ def document(self) -> dict[str, Any]: if self._named("the table", table.name, f"tables.{key}"): self.tables[key] = self._table(key, table) document["tables"] = self.tables - # A note hint `when` is spec 0.4.0: the document says so, and any other stays 0.2.0. - if any( - isinstance(column, dict) and "when" in (column.get("generate") or {}) + # A note hint `when` is spec 0.4.0 and `after_parent` 0.5.0: the document says so, + # and any other stays 0.2.0. + hints = { + hint for table in self.tables.values() for column in (table.get("columns") or {}).values() - ): + if isinstance(column, dict) + for hint in (column.get("generate") or {}) + } + if "after_parent" in hints: + document["model2data"] = "0.5.0" + elif "when" in hints: document["model2data"] = "0.4.0" relationships = self._refs() diff --git a/model2data/model/document.py b/model2data/model/document.py index 2ea5dd7..3822c08 100644 --- a/model2data/model/document.py +++ b/model2data/model/document.py @@ -246,16 +246,27 @@ def uses_0_4(model: Model) -> bool: ) +def uses_0_5(model: Model) -> bool: + """Whether the model uses what spec 0.5.0 added: a column's `after_parent`.""" + return any( + "after_parent" in column.generate + for table in model.tables.values() + for column in table.columns.values() + ) + + def document_version(model: Model) -> Any: """The version the model's document is written against. The version the model was read with, unless the model uses what that - version does not have: 0.4.0 when it uses `when` and was read as 0.2 or - 0.3, 0.3.0 when it uses `defects` or `incremental.history` and was read as + version does not have: 0.5.0 when it uses `after_parent` and was read as + 0.2-0.4, 0.4.0 when it uses `when` and was read as 0.2 or 0.3, 0.3.0 when it uses `defects` or `incremental.history` and was read as 0.2. A document without them is written as the version it was, as always. """ version = model.version minor = str(version).split(".")[:2] + if uses_0_5(model) and minor in (["0", "2"], ["0", "3"], ["0", "4"]): + return "0.5.0" if uses_0_4(model) and minor in (["0", "2"], ["0", "3"]): return "0.4.0" if uses_0_3(model) and minor == ["0", "2"]: diff --git a/model2data/model/types.py b/model2data/model/types.py index f773ad6..f5e6673 100644 --- a/model2data/model/types.py +++ b/model2data/model/types.py @@ -1,4 +1,4 @@ -"""The model as typed values: what a spec 0.4.0 (or 0.2.x, 0.3.x) document says, read. +"""The model as typed values: what a spec 0.5.0 (or 0.2.x-0.4.x) document says, read. Every class is a plain, comparable dataclass, so `load(dump(m)) == m` means what it says. A value left out of the document is the class default (`None`, diff --git a/model2data/model/validate.py b/model2data/model/validate.py index bdc3d91..ed4fef8 100644 --- a/model2data/model/validate.py +++ b/model2data/model/validate.py @@ -1,4 +1,4 @@ -"""Does a document conform to spec 0.4.0 (or 0.2.x, 0.3.x): the schema, then the checks beyond it. +"""Does a document conform to spec 0.5.0 (or 0.2.x-0.4.x): the schema, then the checks beyond it. `check(document)` returns every issue it finds, each with the document path of the value at fault and a severity; it never stops at the first. The schema is @@ -25,29 +25,30 @@ from model2data.dbt.naming import dbt_identifier from model2data.generate import kinds +from model2data.generate.timeline import creation_column_name from model2data.model.errors import Issue, PathPart, format_path -SPEC_VERSION = "0.4.0" +SPEC_VERSION = "0.5.0" # The columns `incremental.history` adds to `_history`. HISTORY_COLUMNS = ("valid_from", "valid_to", "is_current") # The minor versions this reader implements. 0.3.0 only adds `defects` and -# `incremental.history`, 0.4.0 only `when`, so an older document reads exactly as it -# did; it just cannot use them. -READS = ("0.2", "0.3", "0.4") +# `incremental.history`, 0.4.0 only `when`, 0.5.0 only `after_parent`, so an older +# document reads exactly as it did; it just cannot use them. +READS = ("0.2", "0.3", "0.4", "0.5") _URL = "https://www.jbanalytica.com/model2data/spec/{}/model.schema.json" SCHEMA_URL = _URL.format(SPEC_VERSION) def schema_url(version: Any) -> str: """The schema URL a document of `version` points editors at: 0.2.0's for a 0.2 one, - 0.3.0's for a 0.3 one, the current one otherwise.""" - older = {"0.2": "0.2.0", "0.3": "0.3.0"}.get(_minor(version) or "") + 0.3.0's for a 0.3 one, 0.4.0's for a 0.4 one, the current one otherwise.""" + older = {"0.2": "0.2.0", "0.3": "0.3.0", "0.4": "0.4.0"}.get(_minor(version) or "") return _URL.format(older) if older else SCHEMA_URL def _minor(version: Any) -> Optional[str]: if isinstance(version, float): - return {0.2: "0.2", 0.3: "0.3", 0.4: "0.4"}.get(version) + return {0.2: "0.2", 0.3: "0.3", 0.4: "0.4", 0.5: "0.5"}.get(version) if isinstance(version, str): return ".".join(version.split(".")[:2]) return None @@ -55,7 +56,7 @@ def _minor(version: Any) -> Optional[str]: @lru_cache(maxsize=1) def schema() -> dict[str, Any]: - """The packaged, normative JSON Schema of spec 0.4.0, which also reads 0.2 and 0.3 documents.""" + """The packaged, normative JSON Schema of spec 0.5.0, which also reads 0.2-0.4 documents.""" text = resources.files("model2data").joinpath("spec/model.schema.json").read_text("utf-8") return json.loads(text) @@ -99,7 +100,7 @@ def _version_issues(document: Mapping) -> list[Issue]: version = document.get("model2data") if version is None: return [] # reported by the schema as missing - if version in (0.2, 0.3, 0.4) and not isinstance(version, bool): + if version in (0.2, 0.3, 0.4, 0.5) and not isinstance(version, bool): return [] if isinstance(version, str): parts = version.split(".") @@ -110,7 +111,7 @@ def _version_issues(document: Mapping) -> list[Issue]: Issue( "model2data", f"the document is written against spec {version}, and this reader " - f"implements spec {SPEC_VERSION} (0.2.x, 0.3.x and 0.4.x). " + f"implements spec {SPEC_VERSION} (0.2.x, 0.3.x, 0.4.x and 0.5.x). " + ( "Convert a 0.1 model, which is DBML, with `model2data convert`." if parts[:2] == ["0", "1"] @@ -428,6 +429,18 @@ def add(self, path: list[PathPart], message: str, *, warning: bool = False) -> N self.issues.append(Issue(format_path(path), message, severity=severity)) # -- enums --------------------------------------------------------- + def _creation_column(self, key: str) -> Optional[str]: + """The date a row of `key` comes into being with (`generate.timeline.creation_column`).""" + temporal = [] + for name, column in self.columns[key].items(): + type_text = column.get("type") + if not isinstance(type_text, str) or self.enum_named(type_text): + continue + if kinds.is_temporal_type(type_text): + follows = isinstance(_mapping(column.get("generate")).get("after"), str) + temporal.append((name, follows)) + return creation_column_name(temporal) + def enum_named(self, type_name: Any) -> Optional[tuple[str, list]]: if not isinstance(type_name, str): return None @@ -652,6 +665,17 @@ def _needs_0_4(self, path: list[PathPart], what: str = "when") -> bool: ) return True + def _needs_0_5(self, path: list[PathPart], what: str = "after_parent") -> bool: + """Report what 0.5.0 added (`what`) in a 0.2-0.4 document; True when it was reported.""" + if self.minor not in ("0.2", "0.3", "0.4"): + return False + self.add( + path, + f"`{what}` is spec 0.5.0, and the document is written against {self.minor}: " + "write `model2data: 0.5.0`", + ) + return True + def table_rows(self, key: str) -> Optional[int]: """The rows the document's `run` gives a table, or None when it leaves them to the reader.""" run = _mapping(self.document.get("run")) @@ -1074,6 +1098,24 @@ def _hints( f"names {after}, which is not a date or timestamp column", ) + if ( + "after_parent" in generate + and temporal + and not self._needs_0_5([*gen_path, "after_parent"]) + ): + creation = self._creation_column(key) + if creation != name: + where = ( + f"which in {key} is {creation}" + if creation is not None + else f"and every date of {key} follows another column" + ) + self.add( + [*gen_path, "after_parent"], + f"sits on the date a row comes into being with, {where}: only that " + "column is kept on or after its parent rows' own", + ) + if integer: bounds = {} for bound in ("min", "max"): diff --git a/model2data/spec/PUBLISHING.md b/model2data/spec/PUBLISHING.md index 3f2f76e..650636f 100644 --- a/model2data/spec/PUBLISHING.md +++ b/model2data/spec/PUBLISHING.md @@ -3,11 +3,11 @@ Every model file's first line points editors at the schema: ```yaml -# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.4.0/model.schema.json +# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json ``` -(a 0.2.0 or 0.3.0 document keeps pointing at `/spec/0.2.0/` or `/spec/0.3.0/`, so every version -is served). +(a 0.2.0, 0.3.0 or 0.4.0 document keeps pointing at `/spec/0.2.0/`, `/spec/0.3.0/` or +`/spec/0.4.0/`, so every version is served). and `model.schema.json` carries the same URL as its `$id`. Until that URL answers, VS Code, JetBrains and every other YAML-aware editor fail to fetch the schema and offer nothing: no @@ -24,6 +24,8 @@ On the website (`JB-Analytica/jba-website`), serve this directory's files at the | --- | --- | | `https://www.jbanalytica.com/model2data/spec/metrics/0.1.0/metrics.schema.json` | `model2data/spec/metrics/metrics.schema.json` of engine 1.13.0 | | `https://www.jbanalytica.com/model2data/spec/metrics/0.1.0/README.md` (optional, or render it as a page) | `model2data/spec/metrics/README.md` of engine 1.13.0 | +| `https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json` | `model2data/spec/model.schema.json` of engine 1.15.0 | +| `https://www.jbanalytica.com/model2data/spec/0.5.0/README.md` (optional, or render it as a page) | `model2data/spec/README.md` of engine 1.15.0 | | `https://www.jbanalytica.com/model2data/spec/0.4.0/model.schema.json` | `model2data/spec/model.schema.json` of engine 1.11.0 | | `https://www.jbanalytica.com/model2data/spec/0.4.0/README.md` (optional, or render it as a page) | `model2data/spec/README.md` of engine 1.11.0 | | `https://www.jbanalytica.com/model2data/spec/0.3.0/model.schema.json` | `model2data/spec/model.schema.json` of engine 1.9.0 | @@ -33,14 +35,14 @@ On the website (`JB-Analytica/jba-website`), serve this directory's files at the Requirements: -- **Exactly this path, one directory per version.** A version is never overwritten; 0.4.0 gets - `/spec/0.4.0/` beside the others. The `$id` inside the file must equal the URL it is served at. +- **Exactly this path, one directory per version.** A version is never overwritten; 0.5.0 gets + `/spec/0.5.0/` beside the others. The `$id` inside the file must equal the URL it is served at. - **`Access-Control-Allow-Origin: *`**, so browser-based editors (the studio, vscode.dev, the Monaco playgrounds) can fetch it. - **`Content-Type: application/schema+json`** (or `application/json`). - Copy the file from the released engine version, not from a branch: the tag that ships each - version (0.2.0: engine 1.8.0; 0.3.0: engine 1.9.0; 0.4.0: engine 1.11.0; metrics 0.1.0: - engine 1.13.0). + version (0.2.0: engine 1.8.0; 0.3.0: engine 1.9.0; 0.4.0: engine 1.11.0; 0.5.0: engine 1.15.0; + metrics 0.1.0: engine 1.13.0). - The metrics spec is versioned on its own, under `/spec/metrics//`, beside the model spec's versions; a metrics file's first line points at it the same way: `# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/metrics/0.1.0/metrics.schema.json`. @@ -48,8 +50,8 @@ Requirements: Check it: ```bash -curl -sI https://www.jbanalytica.com/model2data/spec/0.4.0/model.schema.json | grep -i -e content-type -e access-control -curl -s https://www.jbanalytica.com/model2data/spec/0.4.0/model.schema.json | python -c "import json,sys; print(json.load(sys.stdin)['\$id'])" +curl -sI https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json | grep -i -e content-type -e access-control +curl -s https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json | python -c "import json,sys; print(json.load(sys.stdin)['\$id'])" ``` The second command must print the URL itself. Check the metrics schema the same way, at @@ -67,9 +69,9 @@ adding this entry to `src/api/json/catalog.json` (the list is kept alphabetical ```json { "name": "model2data model", - "description": "A data model for model2data: tables, keys, relationships and how each column's values are generated (spec 0.4.0)", + "description": "A data model for model2data: tables, keys, relationships and how each column's values are generated (spec 0.5.0)", "fileMatch": ["*.model2data.yml", "*.model2data.yaml", "*.model2data.json"], - "url": "https://www.jbanalytica.com/model2data/spec/0.4.0/model.schema.json" + "url": "https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json" } ``` @@ -89,7 +91,7 @@ without `fileMatch`, so a file that points at the schema on its first line still The schema stays hosted on our site (SchemaStore links to it, it does not copy it), so a new spec version only needs the `url` updated in a follow-up PR (an entry already registered -with the 0.3.0 URL moves to 0.4.0, which also reads 0.2 and 0.3 documents), or a `versions` map +with an older URL moves to 0.5.0, which also reads 0.2, 0.3 and 0.4 documents), or a `versions` map added to offer the older ones too. ## After both diff --git a/model2data/spec/README.md b/model2data/spec/README.md index 4736936..8cbbca3 100644 --- a/model2data/spec/README.md +++ b/model2data/spec/README.md @@ -1,4 +1,4 @@ -# The model2data model — spec 0.4.0 +# The model2data model — spec 0.5.0 A model2data model is one document: the tables of a data model, their columns and keys, the relationships between them, the enums a column can be typed as, and how every column's values @@ -77,7 +77,7 @@ The spec is versioned on its own, apart from the engine and the studio, with [semantic versioning](https://semver.org). The version is in the schema's `$id`: ``` -https://www.jbanalytica.com/model2data/spec/0.4.0/model.schema.json +https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json ``` A patch release changes wording only. A minor release adds something optional. A major release @@ -97,6 +97,15 @@ document keeps saying the version it says and pointing at that version's schema; uses `when` says `model2data: 0.4.0`, and `when` in a document that says 0.2 or 0.3 is an error. A writer writes the version the document was read with, and 0.4.0 once it uses `when`. +0.5.0 adds a column's [`after_parent`](#references), and nothing else in the document: a +0.2, 0.3 or 0.4 document is a 0.5.0 document that does not use it. A document that uses it says +`model2data: 0.5.0`, and `after_parent` in a document that says an earlier version is an +error. A writer writes the version the document was read with, and 0.5.0 once it uses +`after_parent`. 0.5.0 also says two things about the generated data of every document, as +until 1.0.0 a minor release may: a child row's creation date is not before its parents' (see +[References](#references)), and an integer primary key numbers the rows 1 to n, in order (see +[Keys](#keys)). + 0.2.0 replaced 0.1.0, which carried hints as JSON inside DBML notes. See [From 0.1](#from-01). @@ -158,6 +167,11 @@ case-insensitively. member of a `pk` key is not null. A table has at most one primary key, whichever way it is written. +A primary key of one integer column numbers a table's rows: the first day's n rows hold 1 to +n, and a reader lists them in that order (spec 0.5.0). With a `min`/`max` or `distribution`, +or as a foreign key (a child keyed by its parent), the values are drawn and the rows listed +in ascending key order. Any other key leaves the values and the row order to the engine. + ### References A foreign key of one column is written on it: `references: customers.id`, or @@ -171,6 +185,21 @@ reference is generated like any foreign key -- every non-null child value is dra values the parent column actually holds, so each one exists in the parent -- but it does not imply one parent row per value, as a reference onto a key does. +A child row comes into being on or after the parent rows it points at (spec 0.5.0): an order +is not placed before its customer signed up. A table's **creation column** is its first date +or timestamp column without `after` whose name puts it at the created stage (`created_at`, +`signup_date`, `start_date`: the engine's stage words), or, when none does, its first without +`after` whose name puts it at no stage (`order_date`), a birth date excepted; a table whose +dates all follow something, or are all updated- or closed-stage, has none. On every row, the +child's creation column is not before the creation column of the parent row each foreign key +points at, when the key references the parent's primary key or a unique column and the parent +has a creation column. A date compared with a timestamp compares by day. A null foreign key, +a self-reference, a foreign key that breaks a cycle of references and a parent without a +creation column constrain nothing. The child's later dates follow its creation date as they +always do. `after_parent: false` on the creation column leaves it free of its parents. How +the engine keeps the child's own shape while doing so is the engine's (see +`model2data.generate.parents`). + A foreign key over several columns is written on its table under `foreign_keys`, pairing `columns` with `to_columns` in order. A many-to-many relationship, which no table holds the key of, is written under the top-level `relationships`; it is drawn and documented but generates @@ -269,7 +298,8 @@ orders: - A day's new rows continue the existing ones: an integer key continues after the largest value held, any other unique value differs from every value held, and a foreign key points at a row that exists by then, one inserted the same day included (parents are inserted before - children). A foreign key that must be unique takes each parent once, so a table whose parents + children), and a new row's creation column is not before that parent's. A foreign key that + must be unique takes each parent once, so a table whose parents run out inserts fewer rows that day. Every date and timestamp column of a new row falls on the day (a timestamp at any time of it, weighted toward working hours when the run shape has `business_hours`), a nullable one staying null where an ordinary draw would be, and a column diff --git a/model2data/spec/model.schema.json b/model2data/spec/model.schema.json index 9ae0a25..05e208a 100644 --- a/model2data/spec/model.schema.json +++ b/model2data/spec/model.schema.json @@ -1,18 +1,18 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://www.jbanalytica.com/model2data/spec/0.4.0/model.schema.json", + "$id": "https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json", "title": "model2data model", "description": "A data model: its tables, their columns and keys, the relationships between them, the enums a column can be typed as, and how each column's values are generated. Written as YAML (the `.model2data.yml` file) or as JSON; both are the same document. This schema is normative together with model2data/spec/README.md, which defines the YAML profile a reader must apply and the checks that depend on more than one part of the document.", "type": "object", "required": ["model2data", "tables"], "properties": { "model2data": { - "description": "The spec version the document is written against, `MAJOR.MINOR` or `MAJOR.MINOR.PATCH`. Write it as `0.4.0`. A 0.2 or 0.3 document is still read, and reads as it did: 0.3.0 only adds `defects` and `incremental.history`, which a 0.2 document cannot use, and 0.4.0 only a column's `when`, which a 0.2 or 0.3 document cannot use. A reader that met the YAML number 0.4 reads it as `0.4`.", + "description": "The spec version the document is written against, `MAJOR.MINOR` or `MAJOR.MINOR.PATCH`. Write it as `0.5.0`. A 0.2, 0.3 or 0.4 document is still read, and reads as it did: 0.3.0 only adds `defects` and `incremental.history`, which a 0.2 document cannot use, 0.4.0 only a column's `when`, which a 0.2 or 0.3 document cannot use, and 0.5.0 only a column's `after_parent`, which a 0.2, 0.3 or 0.4 document cannot use. A reader that met the YAML number 0.5 reads it as `0.5`.", "oneOf": [ - { "type": "string", "pattern": "^0\\.[234](\\.(0|[1-9][0-9]*))?$" }, - { "enum": [0.2, 0.3, 0.4] } + { "type": "string", "pattern": "^0\\.[2345](\\.(0|[1-9][0-9]*))?$" }, + { "enum": [0.2, 0.3, 0.4, 0.5] } ], - "examples": ["0.4.0"] + "examples": ["0.5.0"] }, "name": { "description": "The model's name. Names the generated dbt project, normalised to `[a-z0-9_]`. Defaults to the file's name without `.model2data.yml`.", @@ -486,7 +486,7 @@ "x-model2data-applies-to": ["non-key"] }, "skew": { - "description": "How unevenly the foreign key's values spread over the parent rows, replacing the run's skew for this column. 0: every parent equally likely. Above 0: the parents, in a seeded random order, are weighted exp(−8 · skew² · i / n) by rank i of n; at 0.8 the top fifth of parents hold roughly 65–75% of the rows, at 1.0 roughly 80%.", + "description": "How unevenly the foreign key's values spread over the parent rows, replacing the run's skew for this column. 0: every parent equally likely. Above 0: the parents, in a seeded random order, are weighted exp(−8 · skew² · i / n) by rank i of n; at 0.8 the top fifth of parents hold roughly 65–75% of the rows, at 1.0 roughly 80%. Where the child's creation date follows its parent's (see `after_parent`), the parents created earliest also take the rows dated before later ones existed, so the spread is steeper.", "$ref": "#/$defs/fraction", "x-model2data-applies-to": ["foreign-key"] }, @@ -511,6 +511,11 @@ "$ref": "#/$defs/fraction", "x-model2data-applies-to": ["temporal"] }, + "after_parent": { + "description": "Whether this column, the date a row comes into being with, is kept on or after the creation date of every parent row the row's foreign keys point at: an order is not placed before its customer signed up (see References in the spec README). `true` is the default; `false` draws the column on its own, its parents' dates aside. It sits only on its table's creation column: the first date or timestamp column without `after` whose name puts it at the created stage (`created_at`, `signup_date`, `start_date`), or, when none does, the first without `after` whose name puts it at no stage (`order_date`), a birth date excepted. Spec 0.5.0.", + "type": "boolean", + "x-model2data-applies-to": ["temporal"] + }, "transitions": { "description": "The states each enum member may move to when the row is updated on a later generated day (see a table's `incremental`): a mapping of member to the members that may follow it. A member with no entry, or an empty list, does not change. Every key and every target is a member of the column's enum.", "type": "object", diff --git a/tests/test_defects.py b/tests/test_defects.py index 656f4b5..c221cde 100644 --- a/tests/test_defects.py +++ b/tests/test_defects.py @@ -161,7 +161,8 @@ def test_the_version_as_a_yaml_number_0_3_is_read(): assert schema_url(0.3).endswith("/0.3.0/model.schema.json") assert schema_url(0.2).endswith("/0.2.0/model.schema.json") # A version it does not name points at the current spec. - assert schema_url(None).endswith("/0.4.0/model.schema.json") + assert schema_url(0.4).endswith("/0.4.0/model.schema.json") + assert schema_url(None).endswith("/0.5.0/model.schema.json") assert schema_url(0.4).endswith("/0.4.0/model.schema.json") diff --git a/tests/test_model_messages.py b/tests/test_model_messages.py index 975cda4..ab08677 100644 --- a/tests/test_model_messages.py +++ b/tests/test_model_messages.py @@ -98,10 +98,10 @@ def test_a_key_with_both_pk_and_unique_says_to_write_two_keys(): def test_a_newer_spec_says_to_upgrade(): - issues = _issues({"model2data": "0.5.0", "tables": {"t": {"columns": COLUMNS}}}) + issues = _issues({"model2data": "0.6.0", "tables": {"t": {"columns": COLUMNS}}}) assert issues == [ - "model2data: the document is written against spec 0.5.0, and this reader implements " - "spec 0.4.0 (0.2.x, 0.3.x and 0.4.x). Upgrade model2data to read it." + "model2data: the document is written against spec 0.6.0, and this reader implements " + "spec 0.5.0 (0.2.x, 0.3.x, 0.4.x and 0.5.x). Upgrade model2data to read it." ] @@ -229,7 +229,7 @@ def _table(**table) -> dict: ), pytest.param( {"model2data": "abc", "tables": {"t": {"columns": COLUMNS}}}, - 'model2data: must match the pattern ^0\\.[234](\\.(0|[1-9][0-9]*))?$ (got "abc")', + 'model2data: must match the pattern ^0\\.[2345](\\.(0|[1-9][0-9]*))?$ (got "abc")', id="version-pattern", ), pytest.param( diff --git a/tests/test_parent_dates.py b/tests/test_parent_dates.py new file mode 100644 index 0000000..aad1e5b --- /dev/null +++ b/tests/test_parent_dates.py @@ -0,0 +1,440 @@ +"""A child row is created on or after its parents, and integer keys number the rows 1..N.""" + +from __future__ import annotations + +import copy +from datetime import date, datetime +from typing import Any + +import pandas as pd +import pytest + +from model2data.dbt.hint_tests import PARENT_MACROS_FILE, hint_tests_for +from model2data.dbt.tests import dbt_tests, generate_dbt_yml +from model2data.defects.checks import as_seeded, fails +from model2data.generate.core import generate_data_from_dbml, parent_rules_for +from model2data.generate.days import generate_days +from model2data.generate.hints import validate_hints +from model2data.generate.options import TimeProfile +from model2data.generate.parents import _Window +from model2data.generate.timeline import creation_column +from model2data.model import from_dict, to_engine +from model2data.model.errors import ModelError +from model2data.parse.dbml import ColumnDef, TableDef + +AS_OF = date(2026, 10, 7) +MIDNIGHT = datetime(2026, 10, 7) + +SHOP: dict[str, Any] = { + "model2data": "0.5.0", + "tables": { + "customers": { + "incremental": {"new_per_day": 8}, + "columns": { + "id": {"type": "bigint", "pk": True}, + "email": {"type": "email", "unique": True, "not_null": True}, + "created_at": { + "type": "timestamp", + "not_null": True, + "generate": {"growth": 0.5}, + }, + }, + }, + "products": { + "columns": { + "id": {"type": "bigint", "pk": True}, + "launched_on": {"type": "date", "not_null": True}, + } + }, + "orders": { + "incremental": {"new_per_day": 30}, + "columns": { + "id": {"type": "bigint", "pk": True}, + "customer_id": { + "type": "bigint", + "not_null": True, + "references": "customers.id", + "generate": {"skew": 0.8}, + }, + "order_date": { + "type": "timestamp", + "not_null": True, + "generate": {"business_hours": True, "growth": 0.5, "seasonality": 0.6}, + }, + "shipped_at": {"type": "timestamp", "generate": {"after": "order_date"}}, + }, + }, + "reviews": { + "columns": { + "id": {"type": "bigint", "pk": True}, + "product_id": {"type": "bigint", "not_null": True, "references": "products.id"}, + "customer_id": {"type": "bigint", "references": "customers.id"}, + "review_date": {"type": "date", "not_null": True}, + } + }, + }, +} + +ROWS = {"customers": 300, "products": 40, "orders": 3000, "reviews": 800} + + +def _generate(document: dict, seed: int = 3, **kwargs) -> dict[str, pd.DataFrame]: + inputs = to_engine(from_dict(document)) + return generate_data_from_dbml( + inputs.tables, + inputs.refs, + base_rows=100, + row_overrides=kwargs.pop("rows", ROWS), + seed=seed, + as_of=AS_OF, + **kwargs, + ) + + +def _joined( + frames: dict[str, pd.DataFrame], child: str, fk: str, column: str, parent: str, created: str +) -> pd.DataFrame: + parent_frame = frames[parent][["id", created]].rename( + columns={"id": "_key", created: "_parent"} + ) + rows = frames[child][[fk, column]].dropna().rename(columns={fk: "_key"}) + rows["_key"] = rows["_key"].astype("int64") + parent_frame["_key"] = parent_frame["_key"].astype("int64") + joined = rows.merge(parent_frame, on="_key") + joined["_mine"] = pd.to_datetime(joined[column]) + joined["_theirs"] = pd.to_datetime(joined["_parent"]) + return joined + + +# --------------------------------------------------------------------------- +# Which column a row comes into being with +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + ("columns", "expected"), + [ + ([("updated_at", "timestamp"), ("created_at", "timestamp")], "created_at"), + ([("order_date", "timestamp"), ("shipped_at", "timestamp")], "order_date"), + ([("birth_date", "date"), ("hire_date", "date")], "hire_date"), + ([("updated_at", "timestamp"), ("deleted_at", "timestamp")], None), + ([("name", "varchar")], None), + ], +) +def test_the_creation_column_is_the_created_stage_else_the_first_unstaged_date(columns, expected): + table = TableDef("t", [ColumnDef(name, kind) for name, kind in columns]) + found = creation_column(table) + assert (found.name if found else None) == expected + + +def test_a_column_with_after_is_never_the_creation_column(): + table = TableDef( + "t", + [ + ColumnDef("event_date", "date", note={"after": "logged_at"}), + ColumnDef("logged_at", "timestamp"), + ], + ) + found = creation_column(table) + assert found is not None and found.name == "logged_at" + + +# --------------------------------------------------------------------------- +# The first day +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("seed", [1, 2, 3]) +def test_no_order_is_placed_before_its_customer_signed_up(seed): + frames = _generate(SHOP, seed=seed) + joined = _joined(frames, "orders", "customer_id", "order_date", "customers", "created_at") + assert len(joined) == len(frames["orders"]) + assert (joined["_mine"] >= joined["_theirs"]).all() + shipped = frames["orders"].dropna(subset=["shipped_at"]) + assert (pd.to_datetime(shipped["shipped_at"]) >= pd.to_datetime(shipped["order_date"])).all() + + +def test_a_date_follows_every_parent_and_compares_with_a_timestamp_by_day(): + frames = _generate(SHOP) + by_product = _joined(frames, "reviews", "product_id", "review_date", "products", "launched_on") + assert (by_product["_mine"] >= by_product["_theirs"]).all() + by_customer = _joined( + frames, "reviews", "customer_id", "review_date", "customers", "created_at" + ) + assert (by_customer["_mine"] >= by_customer["_theirs"].dt.normalize()).all() + # A nullable foreign key left null constrains nothing, and stays null. + assert frames["reviews"]["customer_id"].isna().any() + + +def test_the_orders_keep_their_own_shape(): + """Late rows take an older parent rather than moving their date: the order + dates' seasonal peak and growth are what the column asks for.""" + frames = _generate(SHOP, rows={**ROWS, "orders": 20000}) + moments = pd.to_datetime(frames["orders"]["order_date"]) + gift = ((moments >= "2025-11-28") & (moments <= "2025-12-24")).sum() / 27 + may = ((moments >= "2026-05-01") & (moments <= "2026-05-31")).sum() / 31 + assert gift / may > 2.5 + hours = moments.dt.hour + assert ((hours >= 8) & (hours < 18)).mean() > 0.7 + + +def test_after_parent_false_draws_the_column_on_its_own(): + document = copy.deepcopy(SHOP) + document["tables"]["orders"]["columns"]["order_date"]["generate"]["after_parent"] = False + frames = _generate(document) + joined = _joined(frames, "orders", "customer_id", "order_date", "customers", "created_at") + assert (joined["_mine"] < joined["_theirs"]).any() + assert "orders" not in parent_rules_for(*_engine(document)) + + +def _engine(document: dict) -> tuple[dict, list]: + inputs = to_engine(from_dict(document)) + return inputs.tables, inputs.refs + + +def test_a_one_to_one_child_keeps_its_parent_and_moves_its_date(): + document: dict[str, Any] = { + "model2data": "0.5.0", + "tables": { + "users": { + "columns": { + "id": {"type": "int", "pk": True}, + "created_at": {"type": "timestamp", "not_null": True}, + } + }, + "profiles": { + "columns": { + "id": {"type": "int", "pk": True}, + "user_id": { + "type": "int", + "not_null": True, + "references": {"to": "users.id", "one_to_one": True}, + }, + "created_at": {"type": "timestamp", "not_null": True}, + } + }, + }, + } + frames = _generate(document, rows={"users": 200, "profiles": 150}) + profiles = frames["profiles"] + assert profiles["user_id"].is_unique + joined = _joined(frames, "profiles", "user_id", "created_at", "users", "created_at") + assert (joined["_mine"] >= joined["_theirs"]).all() + assert (joined["_mine"] < MIDNIGHT + pd.Timedelta(seconds=1)).all() + + +def test_self_references_and_parents_without_a_date_are_not_followed(): + document: dict[str, Any] = { + "model2data": "0.5.0", + "tables": { + "teams": {"columns": {"id": {"type": "int", "pk": True}, "name": "word"}}, + "employees": { + "columns": { + "id": {"type": "int", "pk": True}, + "team_id": {"type": "int", "not_null": True, "references": "teams.id"}, + "manager_id": {"type": "int", "references": "employees.id"}, + "hired_on": {"type": "date", "not_null": True}, + } + }, + }, + } + assert parent_rules_for(*_engine(document)) == {} + frames = _generate(document, rows={"teams": 5, "employees": 50}) + assert set(frames["employees"]["team_id"]) <= set(frames["teams"]["id"]) + + +@pytest.mark.parametrize("kind", ["date", "timestamp"]) +def test_a_parent_created_on_the_last_day_gives_a_child_on_it_not_past_it(kind): + column = ColumnDef("created_at", kind, {"not null"}) + window = _Window(kind, column, AS_OF, TimeProfile(business_hours=True, growth=0.5)) + last = datetime(2026, 10, 7) if kind == "date" else datetime(2026, 10, 6, 23, 59, 59) + for _ in range(50): + moment = window.draw(last) + assert last <= moment <= MIDNIGHT + if kind == "timestamp": + assert window.draw(MIDNIGHT) == MIDNIGHT + + +def test_a_moved_timestamp_keeps_business_hours(): + column = ColumnDef("created_at", "timestamp", {"not null"}) + window = _Window("timestamp", column, AS_OF, TimeProfile(business_hours=True)) + lower = datetime(2026, 3, 2, 6, 0, 0) + moments = [window.draw(lower) for _ in range(2000)] + assert all(lower <= m < MIDNIGHT for m in moments) + assert sum(8 <= m.hour < 18 for m in moments) / len(moments) > 0.7 + + +def test_the_rule_is_deterministic(): + first = _generate(SHOP, seed=11) + second = _generate(SHOP, seed=11) + for key, frame in first.items(): + pd.testing.assert_frame_equal(frame, second[key]) + + +# --------------------------------------------------------------------------- +# The days after the first +# --------------------------------------------------------------------------- +def test_every_day_keeps_new_orders_after_their_customers(): + results = generate_days( + from_dict(SHOP), 6, base_rows=100, row_overrides=ROWS, seed=4, as_of=AS_OF + ) + for result in results: + joined = _joined( + result.state, "orders", "customer_id", "order_date", "customers", "created_at" + ) + assert (joined["_mine"] >= joined["_theirs"]).all(), result.day + last = results[-1].state + assert last["orders"]["id"].tolist() == list(range(1, len(last["orders"]) + 1)) + + +# --------------------------------------------------------------------------- +# Validation of the hint +# --------------------------------------------------------------------------- +def _issues(document: dict) -> list[str]: + try: + from_dict(document) + except ModelError as error: + return [str(issue) for issue in error.issues] + return [] + + +def test_after_parent_on_another_date_names_the_creation_column(): + document = copy.deepcopy(SHOP) + document["tables"]["orders"]["columns"]["shipped_at"]["generate"]["after_parent"] = False + (issue,) = _issues(document) + assert "tables.orders.columns.shipped_at.generate.after_parent" in issue + assert "which in orders is order_date" in issue + + +def test_after_parent_on_a_non_date_and_a_non_boolean_are_errors(): + document = copy.deepcopy(SHOP) + document["tables"]["orders"]["columns"]["customer_id"]["generate"]["after_parent"] = False + document["tables"]["orders"]["columns"]["order_date"]["generate"]["after_parent"] = "no" + issues = _issues(document) + assert any( + "customer_id.generate.after_parent" in i and "date or timestamp" in i for i in issues + ) + assert any("order_date.generate.after_parent" in i for i in issues) + + +def test_after_parent_needs_spec_0_5(): + document = copy.deepcopy(SHOP) + document["model2data"] = "0.4.0" + document["tables"]["orders"]["columns"]["order_date"]["generate"]["after_parent"] = False + (issue,) = _issues(document) + assert "write `model2data: 0.5.0`" in issue + + +def test_the_generator_checks_after_parent_on_bare_tables(): + tables = { + "orders": TableDef( + "orders", + [ + ColumnDef("id", "int", {"pk"}), + ColumnDef("order_date", "date"), + ColumnDef("total", "int", note={"after_parent": False}), + ], + ) + } + with pytest.raises(ValueError, match="only applies to date/timestamp columns"): + validate_hints(tables, []) + tables["orders"].columns[2] = ColumnDef("shipped_on", "date", note={"after_parent": False}) + with pytest.raises(ValueError, match="which in orders is order_date"): + validate_hints(tables, []) + + +# --------------------------------------------------------------------------- +# dbt +# --------------------------------------------------------------------------- +def test_dbt_gets_one_parent_test_per_followed_foreign_key(tmp_path): + tables, refs = _engine(SHOP) + tests = [t for t in hint_tests_for(from_dict(SHOP)) if t.hint == "after_parent"] + assert [(t.table, t.column, t.arguments["foreign_key"]) for t in tests] == [ + ("orders", "order_date", "customer_id"), + ("reviews", "review_date", "product_id"), + ("reviews", "review_date", "customer_id"), + ] + by_key = {t.arguments["foreign_key"]: t for t in tests if t.table == "reviews"} + assert "granularity" not in by_key["product_id"].arguments + assert by_key["customer_id"].arguments["granularity"] == "day" + assert tests[0].arguments == { + "foreign_key": "customer_id", + "to": "ref('stg_customers')", + "field": "id", + "parent_column": "created_at", + } + # Bare tables need their refs for it. + assert not [t for t in hint_tests_for(tables) if t.hint == "after_parent"] + + generate_dbt_yml(tmp_path, tables, refs, hint_tests="warn") + assert (tmp_path / "macros" / PARENT_MACROS_FILE).exists() + assert ( + "model2data_not_before_parent" in (tmp_path / "models/staging/stg_orders.yml").read_text() + ) + plain = tmp_path / "plain" + generate_dbt_yml(plain, tables, refs, hint_tests="off") + assert not (plain / "macros" / PARENT_MACROS_FILE).exists() + + +def test_the_defects_check_reads_the_parent_test_as_its_sql_does(): + tables, refs = _engine(SHOP) + frames = _generate(SHOP) + tests = [ + t + for t in dbt_tests(tables, refs, hint_tests="warn") + if t.type == "model2data_not_before_parent" + ] + assert tests and all(t.parent is not None for t in tests) + seeded = {key: as_seeded(frame) for key, frame in frames.items()} + assert not any(fails(t, seeded) for t in tests) + + broken = frames["orders"].copy() + broken.loc[0, "order_date"] = "2000-01-01 00:00:00" + seeded["orders"] = as_seeded(broken) + orders_test = next(t for t in tests if t.table == "orders") + assert fails(orders_test, seeded) + + +# --------------------------------------------------------------------------- +# Integer keys +# --------------------------------------------------------------------------- +def test_integer_primary_keys_number_the_rows_in_order(): + frames = _generate(SHOP) + for key, frame in frames.items(): + assert frame["id"].tolist() == list(range(1, len(frame) + 1)), key + assert set(frames["orders"]["customer_id"]) <= set(frames["customers"]["id"]) + + +def test_a_drawn_or_borrowed_integer_key_is_listed_in_order(): + document: dict[str, Any] = { + "model2data": "0.5.0", + "tables": { + "accounts": { + "columns": { + "id": {"type": "int", "pk": True, "generate": {"min": 1000, "max": 9999}}, + } + }, + "settings": { + "columns": { + "account_id": { + "type": "int", + "pk": True, + "references": {"to": "accounts.id", "one_to_one": True}, + }, + "theme": "word", + } + }, + "tags": { + "columns": {"code": {"type": "varchar", "pk": True}, "label": "word"}, + }, + }, + } + frames = _generate(document, rows={"accounts": 60, "settings": 40, "tags": 20}) + accounts = frames["accounts"]["id"].tolist() + assert accounts == sorted(accounts) and accounts[0] >= 1000 and len(set(accounts)) == 60 + settings = frames["settings"]["account_id"].tolist() + assert settings == sorted(settings) and set(settings) <= set(accounts) + + +def test_later_days_continue_the_numbering(): + results = generate_days( + from_dict(SHOP), 2, base_rows=100, row_overrides=ROWS, seed=4, as_of=AS_OF + ) + day1 = results[1].tables["customers"].inserted + assert day1["id"].tolist() == list(range(301, 309)) From 1b08dd478e45a54c7f23501523ca00e27d6dc480 Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Wed, 7 Oct 2026 22:36:34 +0200 Subject: [PATCH 05/14] Document parent dates and numbered keys, and re-record determinism The changelog's Unreleased section says what 1.15.0 changes and which models generate different bytes for the same seed (every model with an integer primary key; where the parent rule applies, its children too). The README, LLMS.md and the time-shapes, distributions and dbt-project pages describe the rule, the opt-out and the new dbt test. The determinism digests are recorded again for these deliberate changes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 60 +++ LLMS.md | 14 + README.md | 12 +- README_PYPI.md | 12 +- docs/dbt-project.md | 1 + docs/distributions.md | 6 +- docs/time-shapes.md | 27 ++ tests/fixtures/determinism.json | 654 +++++++++++++++++--------------- 8 files changed, 462 insertions(+), 324 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00b1a00..8d28be1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,66 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Added +- **A child row is created on or after its parents.** An order is no longer placed before its + customer signed up (in `examples/ecommerce.model2data.yml` half the orders were). A table's + **creation column** is its first date or timestamp column without `after` whose name says the + row is created (`created_at`, `signup_date`, `start_date`, ...), or, when none does, its first + that names no later stage (`order_date`, `hire_date`); a birth date never is. On every row, + a child's creation column is on or after the creation column of each parent row its foreign + keys point at (a foreign key onto the parent's primary key or a unique column, to a parent + that has a creation column); with several parents, after the latest. A date compared with a + timestamp compares by day, and the child's later dates (`after`, updated/closed names) follow + its creation date as before. Not followed, and documented as such: self-references, nullable + foreign keys left null, parents without a creation column, and foreign keys that break a + cycle of references. + - **The child keeps its own shape.** A row whose date falls before its parent's takes another + parent, one created by then, drawn from the parents the column already points at; its date + keeps its draw, so growth, seasonality, business hours and the `as_of` window hold exactly. + Only a row with no parent created early enough (or a one-to-one, which cannot change + parent) moves its date instead, drawn again from the column's own shape cut to the part of + the window on or after the parent's; a parent created on the window's last day gives a child + on that day, never past `as_of`. Parents created early collect more children, so a skewed + foreign key is steeper than before: on a coffee-webshop model with `skew: 0.8` and + customers growing over the year, the top fifth of customers hold about 80% of the orders + rather than 65%, while the orders' gift-season peak and growth are unchanged. + - **Days after the first** keep it too: a new row whose parent was inserted the same day lands + later that day than its parent. + - **`after_parent: false`** on the creation column draws it on its own, its parents aside, for + the table whose rows really can predate them (imported history, say). It is validated like + any hint: on a column that is not a date or timestamp, or a date that is not the table's + creation column, it is an error naming the column that is. It is new vocabulary, so it + needs **spec 0.5.0** (`model2data: 0.5.0`); a 0.2, 0.3 or 0.4 document using it is an error + naming the version to write, and reads exactly as before otherwise. DBML with an + `after_parent` note converts to 0.5.0. The schema's `$id` moves to `/spec/0.5.0/`. + - **dbt: `model2data_not_before_parent`.** With hint tests on (`--hint-tests`), each foreign key + a creation date follows writes a relationship-aware test on the child's creation column: no + row dated before the parent row it joins to (by day for a date against a timestamp; a key + several parent rows share counts its earliest, leaving duplicates to the `unique` test). + The macro ships in `macros/model2data_parent_tests.sql`, written only when a project has + such a test. `hint_tests_for(tables, refs=...)` takes the references for it when given bare + tables. The defects report checks it like every other test, so a defect that dates a row + before its parent lists it in `EXPECTED_FAILURES.md`; `late_arriving` takes rows that stay + clear of their parents first, so it still breaks no test where it can. + +### Changed +- **Integer primary keys number the rows 1..N, and the rows come out in key order.** A table's + single-column integer primary key was a shuffled sample starting at 0 (the first customers + were 4251, 4557, 1941, ...); it is now 1, 2, 3, ... N, like an identity column, and every + output (seeds, the days' batches and change log, the defects' files) lists the rows in that + order. A key with a `min`/`max` or a `distribution`, and an integer key that is also a foreign + key (a one-to-one child keyed by its parent), is drawn as before and listed in ascending + order. Text, uuid and composite keys keep their values and order. Foreign keys, one-to-one + foreign keys and `--table-seed` re-rolls work as before (a re-rolled table keeps its ids 1..N + with new rows behind them), and a table with `incremental` continues from the largest key on + later days, as before (from 1 when it starts empty). +- **Output changes for the same seed (the determinism promise is per version).** Every model + with an integer primary key generates different bytes than 1.14.0: the key no longer draws + from the table's random stream, so every column after it draws from a different point of it. + A model whose child tables have a creation date and a dated parent also changes where the + rule applies. The same model, seed, `--as-of` and options still give the same bytes on any + machine and any day with 1.15.0. Pin `model2data==1.14.0` to keep fixtures of the old shape. + ## [1.14.0] - 2026-10-07 ### Added diff --git a/LLMS.md b/LLMS.md index d5fd084..f9868ab 100644 --- a/LLMS.md +++ b/LLMS.md @@ -161,6 +161,20 @@ shipped_at timestamp [note: '{"after": "ordered_at"}'] A birth-date-style column (`birth_date`, `date_of_birth`, `dob`) is never folded into this chain — it describes the person, not the record. +**A child row is never created before its parents.** A table's creation column (its first +date/timestamp without `after` named like `created_at`/`signup_date`/`start_date`, else its first +named for no stage, like `order_date`) is on or after the creation column of every parent row its +foreign keys point at (onto a pk or unique key): no order before its customer's `created_at`. +Rows drawn too early take another, older parent, so the child's own growth/seasonality/business +hours hold; a row with none (or a one-to-one) moves its date. Not followed: self-references, null +foreign keys, parents without a date, cycle-breaking foreign keys. Opt out per table on the +creation column, spec 0.5.0 (`model2data: 0.5.0` in YAML): +```dbml +logged_at timestamp [note: '{"after_parent": false}'] +``` +With `--hint-tests`, each followed foreign key writes the dbt test `model2data_not_before_parent`. +Integer primary keys are 1..N and every output lists rows in key order. + **Tie lifecycle columns to the status they belong to with `when`.** Without it, every nullable column is filled or nulled independently of the others, so a `todo` task gets a `completed_at` and a `cancelled` subscription has no `cancelled_at`. `when` maps another column of the same diff --git a/README.md b/README.md index c880923..1f4dab2 100644 --- a/README.md +++ b/README.md @@ -42,9 +42,11 @@ emails, not `Lorem ipsum`. Type a column with any Faker provider (`billing_count come from one country. → [The model file](docs/model-file.md) **Foreign keys that resolve.** Tables are generated in dependency order and every foreign key -points at a parent row that exists — onto a primary key or a unique one. Composite primary and -unique keys hold, and a column that repeats a parent's value through a foreign key -(`orders.customer_email`) holds the email of the order's own customer. +points at a parent row that exists — onto a primary key or a unique one — created no later than +the child: no order is placed before its customer signed up. Integer primary keys number the +rows 1..N, in order. Composite primary and unique keys hold, and a column that repeats a +parent's value through a foreign key (`orders.customer_email`) holds the email of the order's +own customer. → [The model file](docs/model-file.md) **A dbt project that runs.** Seeds, staging models that `ref()` them, `not_null`, `unique`, @@ -68,8 +70,8 @@ byte-identical. → [The determinism promise](docs/generating.md#the-determinism **When things happen.** `--business-hours`, `--growth` and `--seasonality` shape every timestamp (weekdays and working hours, a trend, a Q4 peak), or one column at a time. `updated_at` never -lands before `created_at`, `after` orders any two columns, and `when` gives `shipped_at` a value -on shipped and delivered orders only. → [When things happen](docs/time-shapes.md) +lands before `created_at`, a child's `created_at` never before its parent's, `after` orders any +two columns, and `when` gives `shipped_at` a value on shipped and delivered orders only. → [When things happen](docs/time-shapes.md) **How the data is spread.** `--skew` lets a few customers place most of the orders; `weights`, `true_rate`, `null_rate` and `distinct` shape a column; `distribution` draws a number from a diff --git a/README_PYPI.md b/README_PYPI.md index b03d4a9..421fed2 100644 --- a/README_PYPI.md +++ b/README_PYPI.md @@ -42,9 +42,11 @@ emails, not `Lorem ipsum`. Type a column with any Faker provider (`billing_count come from one country. → [The model file](https://github.com/JB-Analytica/model2data/blob/main/docs/model-file.md) **Foreign keys that resolve.** Tables are generated in dependency order and every foreign key -points at a parent row that exists — onto a primary key or a unique one. Composite primary and -unique keys hold, and a column that repeats a parent's value through a foreign key -(`orders.customer_email`) holds the email of the order's own customer. +points at a parent row that exists — onto a primary key or a unique one — created no later than +the child: no order is placed before its customer signed up. Integer primary keys number the +rows 1..N, in order. Composite primary and unique keys hold, and a column that repeats a +parent's value through a foreign key (`orders.customer_email`) holds the email of the order's +own customer. → [The model file](https://github.com/JB-Analytica/model2data/blob/main/docs/model-file.md) **A dbt project that runs.** Seeds, staging models that `ref()` them, `not_null`, `unique`, @@ -68,8 +70,8 @@ byte-identical. → [The determinism promise](https://github.com/JB-Analytica/mo **When things happen.** `--business-hours`, `--growth` and `--seasonality` shape every timestamp (weekdays and working hours, a trend, a Q4 peak), or one column at a time. `updated_at` never -lands before `created_at`, `after` orders any two columns, and `when` gives `shipped_at` a value -on shipped and delivered orders only. → [When things happen](https://github.com/JB-Analytica/model2data/blob/main/docs/time-shapes.md) +lands before `created_at`, a child's `created_at` never before its parent's, `after` orders any +two columns, and `when` gives `shipped_at` a value on shipped and delivered orders only. → [When things happen](https://github.com/JB-Analytica/model2data/blob/main/docs/time-shapes.md) **How the data is spread.** `--skew` lets a few customers place most of the orders; `weights`, `true_rate`, `null_rate` and `distinct` shape a column; `distribution` draws a number from a diff --git a/docs/dbt-project.md b/docs/dbt-project.md index 61f9ff8..52e42c5 100644 --- a/docs/dbt-project.md +++ b/docs/dbt-project.md @@ -52,6 +52,7 @@ dbt package, so `dbt build` still works offline): | `generate: {distinct: n}` | `model2data_max_distinct`: at most `n` distinct values | none | | `generate: {when}` | `model2data_when`: null exactly where the condition does not hold (with `null_rate`, only there; its null share is then tested among the matching rows) | none | | `grain` on a table | `model2data_unique_combination` | none | +| a creation date that follows its parents' (no hint: see [When things happen](time-shapes.md)) | `model2data_not_before_parent`, one per foreign key it follows, in `macros/model2data_parent_tests.sql`: no row dated before the parent row it joins to | none | | an enum-typed column | `accepted_values` (always written) | none | `true_rate`, `weights`, `skew`, `distribution` and the temporal shape hints (`business_hours`, diff --git a/docs/distributions.md b/docs/distributions.md index 73f3d3b..1e5d99f 100644 --- a/docs/distributions.md +++ b/docs/distributions.md @@ -34,7 +34,11 @@ orders: generate: {distinct: 12} ``` -`skew` on a foreign key overrides `--skew` for just that column. `weights` biases an enum column +`skew` on a foreign key overrides `--skew` for just that column. When the child's creation date +follows its parent's (see [When things happen](time-shapes.md)), the parents created early also +pick up the children dated before the later ones existed, so the spread is steeper than `skew` +alone: on the coffee-webshop example at `skew: 0.8`, the top fifth of customers hold about 80% +of the orders rather than 65%. `weights` biases an enum column toward the values named (unnamed values still appear, at weight 1). `true_rate` is the fraction of non-null rows a boolean column comes back `true`. `null_rate` replaces the column's default null fraction outright. `distinct` draws the column's values from a fixed-size pool instead of a fresh diff --git a/docs/time-shapes.md b/docs/time-shapes.md index 4d0f3c5..0579c0d 100644 --- a/docs/time-shapes.md +++ b/docs/time-shapes.md @@ -19,6 +19,33 @@ shipped_at: generate: {after: ordered_at} ``` +A row is never created before the parent rows it points at: an order's `order_date` is on or +after its customer's `created_at`, a review's `review_date` on or after the product's launch. +It needs no hint. A table's **creation column** is its first date or timestamp without `after` +whose name says created (`created_at`, `signup_date`, `start_date`, ...), else its first that +names no later stage (`order_date`, `hire_date`); a child's creation column follows the creation +column of each parent its foreign keys point at, through a foreign key onto the parent's +primary or unique key, and its own later dates (`shipped_at`, `updated_at`) follow it as +always. A date compared with a timestamp compares by day. + +The child keeps its own shape. A row drawn before its parent takes another parent, one created +by then, from the parents the column already points at, so the order dates keep their growth, +seasonality and working hours exactly, and the customers who signed up early end up with more +orders, as they would. Only a row no parent was created early enough for (or a one-to-one, +which cannot change parent) moves its date, drawn again from the column's own shape on or +after the parent's, and never past `as_of`. On days after the first, a new row whose parent was +inserted the same day lands later that day than the parent. + +Self-references, nullable foreign keys left null, parents without a creation column and +foreign keys that break a cycle are not followed. To draw a creation column on its own, as +before 1.15, say so on it (spec 0.5.0: the model says `model2data: 0.5.0`): + +```yaml +logged_at: + type: timestamp + generate: {after_parent: false} +``` + A column that only has a value in some states says which with `when` (spec 0.4.0: the model says `model2data: 0.4.0`): it holds a value on exactly the rows whose named column holds one of the listed values, and is null on every other diff --git a/tests/fixtures/determinism.json b/tests/fixtures/determinism.json index 08866f6..0e79fe2 100644 --- a/tests/fixtures/determinism.json +++ b/tests/fixtures/determinism.json @@ -11,58 +11,60 @@ "dbt_det/seeds/raw/monthly_metrics.csv": "7bc73af68b0a73d9c368f12af5c353b7607fbbe38f6be8f425e854dc40dc8625" }, "days-2": { - "dbt_det/days/customers/day_000.csv": "0ae988dbb32bcba86047756c3fa4cb22cb74d4b5e6dcc604c18cb56e0507ccf8", - "dbt_det/days/customers/day_001.csv": "e5472ddc4da3d0eb647832e1c3bbc64d85c5b26bfad783e03b37c8c328ecae6e", - "dbt_det/days/customers/day_002.csv": "2772ecdd0256bd955a1e11aa9b5164397d9f1ceee2f9e7dc7a30ed38f22867df", - "dbt_det/days/order_items/day_000.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/days/order_items/day_001.csv": "c745a9e7052352d861d82b57b5a58371f3d026ecb739125deb60a00237e03b18", - "dbt_det/days/order_items/day_002.csv": "5a39f76aa79e2af21fbbb25b1fbe60f1bfe0e2b7a4714c09d12a4639484e928d", - "dbt_det/days/orders/day_000.csv": "969ed7ce4f3e94e1ea5f70eb05f3e79c9cdd9ec199acf3d5f4917fb01c6fdf24", - "dbt_det/days/orders/day_001.csv": "f98eb9b2adbd4881b1e523b3021add667086bb410d2a529ba7e89696b97db77e", - "dbt_det/days/orders/day_002.csv": "425efd6753c4156f87650a4b61a743ff262e2ab40c5664373cf9643cfc864673", - "dbt_det/days/product_reviews/day_000.csv": "e7bd78f152f82b650b2392a38e2bac3f6837a8f3b1cb7818a3f834d88a897768", - "dbt_det/days/products/day_000.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3", + "dbt_det/days/customers/day_000.csv": "2b34b876f615650d684d7ba23e0cf0f63d8f6addc7f9798c2aa16735c9b6a45d", + "dbt_det/days/customers/day_001.csv": "cc77e19974d959378cb83527952efe5bb16a684b0a1eb1532b82d4dcd02ccdbe", + "dbt_det/days/customers/day_002.csv": "c04d968bdf4c4c4f453435a18ff0caefc93fd2733c07d18975d2cf964bb59888", + "dbt_det/days/order_items/day_000.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/days/order_items/day_001.csv": "9329a9f8a2c87ade72747cb20955c84cf21feb7042ff0c272c3610cac7918332", + "dbt_det/days/order_items/day_002.csv": "914e65e5be574ffb7ffe9e80066334c9fb5d00b50037636dc683d4112a4191f2", + "dbt_det/days/orders/day_000.csv": "2c1bbab44be2fb1a46c78878945c319f2035a13792d1cc2d6d3ec54741dfd8d5", + "dbt_det/days/orders/day_001.csv": "df0ac1c8a20cea535ae074ff64d513270a0e5b45956e1cd55809a29431a2d1ea", + "dbt_det/days/orders/day_002.csv": "13627253fff83faaa8d42f666a00370f316e032735795f420f700273387809c3", + "dbt_det/days/product_reviews/day_000.csv": "95f9a92f499d61ebc0d9747dd27474e1a5d96eaf3cec91b3d36d1a9b2ceb3b07", + "dbt_det/days/products/day_000.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/ecommerce_daily.model2data.yml": "45723a5c83cf7058915b01c881ad18720de7d9c795a2e21abe1debde60141053", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "fe63c9f2fea411ec373b19a500e90c4e0c0b7352ba2d4591a33cbcf4a138ed7c", + "dbt_det/models/staging/stg_orders.yml": "2a2ace25b7164723120ba0b2b7def4839dd286a17ccfb5990f304070bbe05d77", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", "dbt_det/models/staging/stg_product_reviews.yml": "e7692c4d586af9ea57c9afbb421d5424be907fb28f4464bf82f75033b402ef48", "dbt_det/models/staging/stg_products.sql": "7455128b10e0357213f26c209913a73df3f417be28ddd9d4dd8681390e7dcdab", "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "f10588f38f838c163af5a58349670e4abc2e753619f1ae623b98ef2758d8be08", - "dbt_det/seeds/raw/customers.csv": "0c8da1a709ceddee645e3759ead7375b171b0bd633269eabde81825ef0940ad9", - "dbt_det/seeds/raw/order_items.csv": "96cc81037b430ecdfbbfa37028ea4187e7f8d0b3397a9fe104248202e854d3aa", - "dbt_det/seeds/raw/orders.csv": "fb1b2eba4662933b3b28b249ca073c0c383a0c97ad01b60f8d93082e88fae64c", - "dbt_det/seeds/raw/product_reviews.csv": "e7bd78f152f82b650b2392a38e2bac3f6837a8f3b1cb7818a3f834d88a897768", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "6defda7aa81c3356c00810db9ad97ddaa86414d0918d22d448881062e12af7b0", + "dbt_det/seeds/raw/order_items.csv": "b69a6502281d767a73f378d0dcdb5f3e27bbf8444559981a7cde59a14e5c0687", + "dbt_det/seeds/raw/orders.csv": "7ef9ce9e0a541918da6c5368bd570622a4b22f786a2b72ba8c54c27627e380c4", + "dbt_det/seeds/raw/product_reviews.csv": "95f9a92f499d61ebc0d9747dd27474e1a5d96eaf3cec91b3d36d1a9b2ceb3b07", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "days-changelog": { - "dbt_det/EXPECTED_FAILURES.md": "233eff9d9cd993bd7a2b6621870f8ef315785230eda5e23dca07286b6f68c192", - "dbt_det/changelog/customers.csv": "34a3935407e9d0a735afcdc4228a0af00852697d6a855b1b875c62a4bc890b5d", - "dbt_det/changelog/order_items.csv": "d10b706ab493adcd4b756111414566c762d88151e7d5be65b85ff2442e11f369", - "dbt_det/changelog/orders.csv": "13166256bec10becc62c05f4abd81da33e699d66e6654bd5b1b1ed7ea34b6c07", - "dbt_det/changelog/product_reviews.csv": "c3ee5d22c9bf6d9539ca5c9f9ad2a45b56b7550d2c4768973dc9efcfde200f03", - "dbt_det/changelog/products.csv": "e3b6302dbe42bb9eaa200527a813d7803f108d1b475321871f28b54ca2e244d0", - "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/defects_report.json": "02983affaff02f2248b52acc957c9c5f9a1dd3c071d72708b008d7707e39355c", + "dbt_det/EXPECTED_FAILURES.md": "c82f6c9408675f08f87220b5743eab42835193beedffcee66eb9f97a57270de6", + "dbt_det/changelog/customers.csv": "64723172134833e4a38fe38f9719ef138b9e963c811c9c2d1343875e8ed9793e", + "dbt_det/changelog/order_items.csv": "eaffbcfb70ef3fe221dd69797f5e153fc59ad6439ff9e3543a1a54f6f29cafba", + "dbt_det/changelog/orders.csv": "bec8cc0f7aa5b88ab98ee61a7c56cac129dbc3abf1c728449462b3407d26d526", + "dbt_det/changelog/product_reviews.csv": "27158a359cf7fb5be1ef4afc29fe6e9e6bc79b90c62d8992b65e860f7d1e25fb", + "dbt_det/changelog/products.csv": "5f9013e0977d99598a2228ef69f0cecd153ae2ad8461739f2ae88ca6ec7a63f0", + "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", + "dbt_det/defects_report.json": "b4213c8e2027d566e6d266551c9bf28e27e07f01584a9f83c069c9df5137b3cb", "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "fe63c9f2fea411ec373b19a500e90c4e0c0b7352ba2d4591a33cbcf4a138ed7c", + "dbt_det/models/staging/stg_orders.yml": "2a2ace25b7164723120ba0b2b7def4839dd286a17ccfb5990f304070bbe05d77", "dbt_det/models/staging/stg_orders_history.sql": "937bea699b5df5da6808580fe3963f432163f3741afc0f71275e5261d81f655b", "dbt_det/models/staging/stg_orders_history.yml": "b6bc4e53dc5078994b19f94b38371ea27308a452cccf351bfd43b6c489e2152a", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", @@ -71,12 +73,12 @@ "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "03fbba2c4026678a9eef7cfe48aeaf3391a2919b05b1f67b2744ee0891ff199f", - "dbt_det/seeds/raw/customers.csv": "3443d991581aefaabc9648fd96302c948db616761e0b4137401beae0f7add2db", - "dbt_det/seeds/raw/order_items.csv": "81bcd163cc42c3474dfcf5201634164189bd3ba19c7cb05fce833c80b2d81a55", - "dbt_det/seeds/raw/orders.csv": "0c7f5319bb652bd73a6b89649825377e875c8c166086c299b971ee8d760e375f", - "dbt_det/seeds/raw/orders_history.csv": "755df44c63bbdfa922276db2382e5b9cfb9603e6fc02f5c72552b9aaa14e32e8", - "dbt_det/seeds/raw/product_reviews.csv": "f9f33d24a151c6ff680ce5c4521f87418284a4de045dcb0eacb11fba048b7324", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "70e43d36b97a9b805aba04eb1d8a57dfca6618c7f652f1f8883aca1a8c366854", + "dbt_det/seeds/raw/order_items.csv": "1f40a7f749ff9f692b3b641d1519b3a59d77a75c09c500820ee6a5836cf3e02e", + "dbt_det/seeds/raw/orders.csv": "60cba64af43039d501e98115dcb93d977566f5aa7502f49f27464ab6d5e4f494", + "dbt_det/seeds/raw/orders_history.csv": "a3a359b615b4c84c2917e602eeeb319af9e3d7bc5d4333e57a0fb04a3e689e84", + "dbt_det/seeds/raw/product_reviews.csv": "49ba5a52aa90dd18f3a12a15afd1d1f954a10e0572b084f8c94c89255c553e2b", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "example-advanced_features.dbml": { "dbt_det/advanced_features.dbml": "3e4a90555639ce29adf33441aab4c4a6aefcfe2ecebd79b1aa65f575312c7379", @@ -84,6 +86,7 @@ "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_departments.sql": "78a2fdafd3f86532cf31320c9a0159b01598f07b3eeeadfc9ec4f59548617946", "dbt_det/models/staging/stg_departments.yml": "75b04766a3b24c5af493b7d4c604657263efdd2de62474b72e8b4d0fd5a7bc6e", "dbt_det/models/staging/stg_employees.sql": "c63a2b2b4ed86d0b3c2cce77aa5d4dfb032c229a23013c6e65dd396588b53109", @@ -97,16 +100,16 @@ "dbt_det/models/staging/stg_projects.sql": "6cec86f0e361aafc8af230dc62156fad1acdc1b53873efc59bd94142542fba01", "dbt_det/models/staging/stg_projects.yml": "3ffec29ac7abd60b8a78f6c9d4393a6c855fcaf3ca4f048091f2dd2342ec42ef", "dbt_det/models/staging/stg_time_entries.sql": "7327696835b984aa9c8570f5ab3ced16980d3971a44df99607daa35c6b07833a", - "dbt_det/models/staging/stg_time_entries.yml": "7ae8fcdb4d5868d67d9b24c213df8bb0736f0d47ad1174e6e93eea6758ee2756", + "dbt_det/models/staging/stg_time_entries.yml": "40635a89998b8169d6403f6a7473e120c05935ca72c0c125bbbff2535baa9ba9", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "8f515972d5a5d2603da2698c02f5b2da487fd708296eb8f1c6ab656eff95e22a", - "dbt_det/seeds/raw/departments.csv": "da3ef29c7e92e330a29828fa1572a31f48e9a3e3c21fa272c46266767d9d1b6a", - "dbt_det/seeds/raw/employees.csv": "6d00e81fbf8e33e330e7edb43b8589f466592e8ad82a5420f1d4529033e1142b", - "dbt_det/seeds/raw/expenses.csv": "ab71ba261a5f744e178932a94309649f9138ec8d2981d55c30245299bf0e2880", - "dbt_det/seeds/raw/performance_ratings.csv": "2608e33bf028aafe1256c31f308a9ce5085c7b5d966b5b1b54c856656103bb0c", - "dbt_det/seeds/raw/project_assignments.csv": "e3bbff7299f66b1307f268150a022b60ba76734ce6e626e61ab0494d904a5a1a", - "dbt_det/seeds/raw/projects.csv": "f823c5dda3ee570755a8b84817fb11d74921081c0c6df44db0f13b139748174f", - "dbt_det/seeds/raw/time_entries.csv": "17c98a76800db5399c8602b54031883d20dca2badbe319108cf830dee49d0c9a" + "dbt_det/seeds/raw/departments.csv": "7318193a19e6eab8947550adcab1fd308e639c0eb357f8b4b66db1d21a9d0d4b", + "dbt_det/seeds/raw/employees.csv": "2249a9edf2f07d7425d66d73a0cfb38d4ce9d274d99155ef6b2b252069228fef", + "dbt_det/seeds/raw/expenses.csv": "f8bbfa02eed973dfce68f8ab927f3c16e9cf263a71913b00867422e9c690142e", + "dbt_det/seeds/raw/performance_ratings.csv": "a74dfcf15466ed15a8f54c8fbefb74b886eab474993a5002bed1c5ef9344fc2a", + "dbt_det/seeds/raw/project_assignments.csv": "6e1c2ffb0c7b1419ead3e3709006d92ef150b3410bf9f5cf9b5dcda2a19be404", + "dbt_det/seeds/raw/projects.csv": "1841df28ba42d29cacd54cc1c6babb56ebb64f7fdda81d1048a6d2280f62b2f9", + "dbt_det/seeds/raw/time_entries.csv": "20c8eda681fb6df63b2155d3143567ef14353163a389e52698cb555005ff82f2" }, "example-advanced_features.model2data.yml": { "dbt_det/advanced_features.model2data.yml": "fc0727267e3b7d123a5666b759ed245a40e120562fa63aad5f79dcca28191d71", @@ -114,6 +117,7 @@ "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_departments.sql": "78a2fdafd3f86532cf31320c9a0159b01598f07b3eeeadfc9ec4f59548617946", "dbt_det/models/staging/stg_departments.yml": "75b04766a3b24c5af493b7d4c604657263efdd2de62474b72e8b4d0fd5a7bc6e", "dbt_det/models/staging/stg_employees.sql": "c63a2b2b4ed86d0b3c2cce77aa5d4dfb032c229a23013c6e65dd396588b53109", @@ -127,100 +131,104 @@ "dbt_det/models/staging/stg_projects.sql": "6cec86f0e361aafc8af230dc62156fad1acdc1b53873efc59bd94142542fba01", "dbt_det/models/staging/stg_projects.yml": "3ffec29ac7abd60b8a78f6c9d4393a6c855fcaf3ca4f048091f2dd2342ec42ef", "dbt_det/models/staging/stg_time_entries.sql": "7327696835b984aa9c8570f5ab3ced16980d3971a44df99607daa35c6b07833a", - "dbt_det/models/staging/stg_time_entries.yml": "7ae8fcdb4d5868d67d9b24c213df8bb0736f0d47ad1174e6e93eea6758ee2756", + "dbt_det/models/staging/stg_time_entries.yml": "40635a89998b8169d6403f6a7473e120c05935ca72c0c125bbbff2535baa9ba9", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "8f515972d5a5d2603da2698c02f5b2da487fd708296eb8f1c6ab656eff95e22a", - "dbt_det/seeds/raw/departments.csv": "da3ef29c7e92e330a29828fa1572a31f48e9a3e3c21fa272c46266767d9d1b6a", - "dbt_det/seeds/raw/employees.csv": "6d00e81fbf8e33e330e7edb43b8589f466592e8ad82a5420f1d4529033e1142b", - "dbt_det/seeds/raw/expenses.csv": "ab71ba261a5f744e178932a94309649f9138ec8d2981d55c30245299bf0e2880", - "dbt_det/seeds/raw/performance_ratings.csv": "2608e33bf028aafe1256c31f308a9ce5085c7b5d966b5b1b54c856656103bb0c", - "dbt_det/seeds/raw/project_assignments.csv": "e3bbff7299f66b1307f268150a022b60ba76734ce6e626e61ab0494d904a5a1a", - "dbt_det/seeds/raw/projects.csv": "f823c5dda3ee570755a8b84817fb11d74921081c0c6df44db0f13b139748174f", - "dbt_det/seeds/raw/time_entries.csv": "17c98a76800db5399c8602b54031883d20dca2badbe319108cf830dee49d0c9a" + "dbt_det/seeds/raw/departments.csv": "7318193a19e6eab8947550adcab1fd308e639c0eb357f8b4b66db1d21a9d0d4b", + "dbt_det/seeds/raw/employees.csv": "2249a9edf2f07d7425d66d73a0cfb38d4ce9d274d99155ef6b2b252069228fef", + "dbt_det/seeds/raw/expenses.csv": "f8bbfa02eed973dfce68f8ab927f3c16e9cf263a71913b00867422e9c690142e", + "dbt_det/seeds/raw/performance_ratings.csv": "a74dfcf15466ed15a8f54c8fbefb74b886eab474993a5002bed1c5ef9344fc2a", + "dbt_det/seeds/raw/project_assignments.csv": "6e1c2ffb0c7b1419ead3e3709006d92ef150b3410bf9f5cf9b5dcda2a19be404", + "dbt_det/seeds/raw/projects.csv": "1841df28ba42d29cacd54cc1c6babb56ebb64f7fdda81d1048a6d2280f62b2f9", + "dbt_det/seeds/raw/time_entries.csv": "20c8eda681fb6df63b2155d3143567ef14353163a389e52698cb555005ff82f2" }, "example-ecommerce.dbml": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/ecommerce.dbml": "b72b2b7be5459c597f8cc9c6787ee24066e1ff12ef5265e2e13c7320cb5deab9", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "88ed2f885637e38367e119250b09f764e4570e8ef244ae4bdfa55e0d832d466a", + "dbt_det/models/staging/stg_orders.yml": "cba055fdb3f2d6145cddcf8696242fbfb34463577e681284eca78e6f11b96ede", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", "dbt_det/models/staging/stg_product_reviews.yml": "e7692c4d586af9ea57c9afbb421d5424be907fb28f4464bf82f75033b402ef48", "dbt_det/models/staging/stg_products.sql": "7455128b10e0357213f26c209913a73df3f417be28ddd9d4dd8681390e7dcdab", "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "f10588f38f838c163af5a58349670e4abc2e753619f1ae623b98ef2758d8be08", - "dbt_det/seeds/raw/customers.csv": "0ae988dbb32bcba86047756c3fa4cb22cb74d4b5e6dcc604c18cb56e0507ccf8", - "dbt_det/seeds/raw/order_items.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/seeds/raw/orders.csv": "a9f5c94298c8d5d7a90a9a5d8e95f6a5adcaac3fc56badf830c615a0987e0d23", - "dbt_det/seeds/raw/product_reviews.csv": "e7bd78f152f82b650b2392a38e2bac3f6837a8f3b1cb7818a3f834d88a897768", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "2b34b876f615650d684d7ba23e0cf0f63d8f6addc7f9798c2aa16735c9b6a45d", + "dbt_det/seeds/raw/order_items.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/seeds/raw/orders.csv": "3124312b95043b7592f149a40fd51ab19a03276313bef80973716e6759376fdb", + "dbt_det/seeds/raw/product_reviews.csv": "95f9a92f499d61ebc0d9747dd27474e1a5d96eaf3cec91b3d36d1a9b2ceb3b07", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "example-ecommerce.model2data.yml": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "88ed2f885637e38367e119250b09f764e4570e8ef244ae4bdfa55e0d832d466a", + "dbt_det/models/staging/stg_orders.yml": "cba055fdb3f2d6145cddcf8696242fbfb34463577e681284eca78e6f11b96ede", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", "dbt_det/models/staging/stg_product_reviews.yml": "e7692c4d586af9ea57c9afbb421d5424be907fb28f4464bf82f75033b402ef48", "dbt_det/models/staging/stg_products.sql": "7455128b10e0357213f26c209913a73df3f417be28ddd9d4dd8681390e7dcdab", "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "f10588f38f838c163af5a58349670e4abc2e753619f1ae623b98ef2758d8be08", - "dbt_det/seeds/raw/customers.csv": "0ae988dbb32bcba86047756c3fa4cb22cb74d4b5e6dcc604c18cb56e0507ccf8", - "dbt_det/seeds/raw/order_items.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/seeds/raw/orders.csv": "a9f5c94298c8d5d7a90a9a5d8e95f6a5adcaac3fc56badf830c615a0987e0d23", - "dbt_det/seeds/raw/product_reviews.csv": "e7bd78f152f82b650b2392a38e2bac3f6837a8f3b1cb7818a3f834d88a897768", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "2b34b876f615650d684d7ba23e0cf0f63d8f6addc7f9798c2aa16735c9b6a45d", + "dbt_det/seeds/raw/order_items.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/seeds/raw/orders.csv": "3124312b95043b7592f149a40fd51ab19a03276313bef80973716e6759376fdb", + "dbt_det/seeds/raw/product_reviews.csv": "95f9a92f499d61ebc0d9747dd27474e1a5d96eaf3cec91b3d36d1a9b2ceb3b07", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "example-ecommerce_daily.model2data.yml": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/ecommerce_daily.model2data.yml": "45723a5c83cf7058915b01c881ad18720de7d9c795a2e21abe1debde60141053", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "fe63c9f2fea411ec373b19a500e90c4e0c0b7352ba2d4591a33cbcf4a138ed7c", + "dbt_det/models/staging/stg_orders.yml": "2a2ace25b7164723120ba0b2b7def4839dd286a17ccfb5990f304070bbe05d77", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", "dbt_det/models/staging/stg_product_reviews.yml": "e7692c4d586af9ea57c9afbb421d5424be907fb28f4464bf82f75033b402ef48", "dbt_det/models/staging/stg_products.sql": "7455128b10e0357213f26c209913a73df3f417be28ddd9d4dd8681390e7dcdab", "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "f10588f38f838c163af5a58349670e4abc2e753619f1ae623b98ef2758d8be08", - "dbt_det/seeds/raw/customers.csv": "0ae988dbb32bcba86047756c3fa4cb22cb74d4b5e6dcc604c18cb56e0507ccf8", - "dbt_det/seeds/raw/order_items.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/seeds/raw/orders.csv": "969ed7ce4f3e94e1ea5f70eb05f3e79c9cdd9ec199acf3d5f4917fb01c6fdf24", - "dbt_det/seeds/raw/product_reviews.csv": "e7bd78f152f82b650b2392a38e2bac3f6837a8f3b1cb7818a3f834d88a897768", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "2b34b876f615650d684d7ba23e0cf0f63d8f6addc7f9798c2aa16735c9b6a45d", + "dbt_det/seeds/raw/order_items.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/seeds/raw/orders.csv": "2c1bbab44be2fb1a46c78878945c319f2035a13792d1cc2d6d3ec54741dfd8d5", + "dbt_det/seeds/raw/product_reviews.csv": "95f9a92f499d61ebc0d9747dd27474e1a5d96eaf3cec91b3d36d1a9b2ceb3b07", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "example-ecommerce_training.model2data.yml": { - "dbt_det/EXPECTED_FAILURES.md": "008dfb57695e6faceb51cc4db8b76d22d3350b0cbbcaa24928d839ccf0a5f2f1", + "dbt_det/EXPECTED_FAILURES.md": "d786b65ca362828c22ce6b6cad9f8c189dc98286a6b2c898dd6a9a90c0648187", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/defects_report.json": "119e7e868ed1ffeb25c35108c64548eeb3b5137d7f48181265ff7d5fea1f622b", + "dbt_det/defects_report.json": "4f4f15d9a12bf2b8c2926b74db6df1824e8a277a231535b296390ed24f9c6d1e", "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "fe63c9f2fea411ec373b19a500e90c4e0c0b7352ba2d4591a33cbcf4a138ed7c", + "dbt_det/models/staging/stg_orders.yml": "2a2ace25b7164723120ba0b2b7def4839dd286a17ccfb5990f304070bbe05d77", "dbt_det/models/staging/stg_orders_history.sql": "937bea699b5df5da6808580fe3963f432163f3741afc0f71275e5261d81f655b", "dbt_det/models/staging/stg_orders_history.yml": "b6bc4e53dc5078994b19f94b38371ea27308a452cccf351bfd43b6c489e2152a", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", @@ -229,22 +237,24 @@ "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "03fbba2c4026678a9eef7cfe48aeaf3391a2919b05b1f67b2744ee0891ff199f", - "dbt_det/seeds/raw/customers.csv": "e4fe1e951c25ed14b11404bc86d45ecff7138f7ddcb127a7e47c0e54b977c9bb", - "dbt_det/seeds/raw/order_items.csv": "b0f59df407bcf8f889a05909397d12476a0666ddef5a8904de26fc73a12a58a4", - "dbt_det/seeds/raw/orders.csv": "50f07fb17956f72f0f35c3d261666f3626cc73defbb72647fa23828e1dd234a5", - "dbt_det/seeds/raw/orders_history.csv": "b32e87c9b25a091f29b394f662e367a9fb56b469026171f551243d5268caf1e8", - "dbt_det/seeds/raw/product_reviews.csv": "f9f33d24a151c6ff680ce5c4521f87418284a4de045dcb0eacb11fba048b7324", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "9fd2c3ca7fef5aa02f50cdf3427c84b7691f6decd97efa72a6f5f49059056d16", + "dbt_det/seeds/raw/order_items.csv": "67ae17ad8e9912c8c174546dda3b739a695f7feeda9994c974715e1adc91ba7b", + "dbt_det/seeds/raw/orders.csv": "3db35c196bc5175703bac0f6811277c0e05c23d8fa3d00ef5d17dd7be6eb9f8c", + "dbt_det/seeds/raw/orders_history.csv": "749263196575dcb373edb2f06a5d96c423d5f551a10faffaf3539a4151ef16a6", + "dbt_det/seeds/raw/product_reviews.csv": "49ba5a52aa90dd18f3a12a15afd1d1f954a10e0572b084f8c94c89255c553e2b", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "example-hackernews.dbml": { "dbt_det/data-tests/unique_combination_stg_stories_id__dlt_id.sql": "9d7be27ee3a57189665be95228cbb4bba3ceba2ba547465c35a5bae9fb4fc466", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/hackernews.dbml": "06ec4f08ac1fdb4617e82ebfbd6ca53f546530cac938438edcff152215f357a8", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", + "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg__dlt_loads.sql": "bf6c0705877e236e8feb73268a9fd8cba50b5ba50330eda30e7a1b77f1a780cb", "dbt_det/models/staging/stg__dlt_loads.yml": "69bf37a12589b0b10bbfea9a6d5aab046b16234313306a698235ab83b816e9c5", "dbt_det/models/staging/stg__dlt_pipeline_state.sql": "921824090c1985a61a063350b9c87a296c28d5dafe4d26a0fd9c3a1a13e98624", - "dbt_det/models/staging/stg__dlt_pipeline_state.yml": "aeaa684b4339d83bebcd708fff826ec413ed608ed53d8200320a853cb8f7dd72", + "dbt_det/models/staging/stg__dlt_pipeline_state.yml": "01d85053a17d1aa27f2a17537fdcc5affb0f394212d579d7a05217576a7237a1", "dbt_det/models/staging/stg__dlt_version.sql": "5d279d508581eae585acdbbeaa36622b1faab61996b3956c6e2373c0d1294a8d", "dbt_det/models/staging/stg__dlt_version.yml": "dec6894006cce86c5950e66e5939ea8d16ebacf5722509ff8235e7fe9710b1a9", "dbt_det/models/staging/stg_stories.sql": "1bef38284308be0963e9933f19f53339e110020d4d2993766d48a7c452930397", @@ -254,7 +264,7 @@ "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "2c712c4932785690fc4d9d9117d964a0e784989b1bc65cb8e2afaff4a18bcc16", "dbt_det/seeds/raw/_dlt_loads.csv": "4bc16cd04e968225f6fb788f9f48ea51282e3d089e40dc369b71db1f9b433107", - "dbt_det/seeds/raw/_dlt_pipeline_state.csv": "67f4f6828c4340bb33744fa1784bca9a1c2ff9093f415ad1aa16cae27edb67e6", + "dbt_det/seeds/raw/_dlt_pipeline_state.csv": "9387c8e111defbf0cf0cd7f36093c205ebf53997e398b3151b08f077e9837ca4", "dbt_det/seeds/raw/_dlt_version.csv": "bddf097235105ad5ced6c56380c70d2b9bc915e1a0e4a1368be3d48e534b7d3f", "dbt_det/seeds/raw/stories.csv": "2869db5e096a04b0092bf2a97266cf7969ec40bef9fd45efc0540a1db178a362", "dbt_det/seeds/raw/stories__kids.csv": "c65b11d84a910e24ee31f22eb44df742663e65a175b1b8c2083fa1e8e95f9005" @@ -264,10 +274,12 @@ "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/hackernews.model2data.yml": "f74d4df703b909d58d082b42e29623b8ca0c326c46c5d37fb144e8eb11992c3b", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", + "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg__dlt_loads.sql": "bf6c0705877e236e8feb73268a9fd8cba50b5ba50330eda30e7a1b77f1a780cb", "dbt_det/models/staging/stg__dlt_loads.yml": "69bf37a12589b0b10bbfea9a6d5aab046b16234313306a698235ab83b816e9c5", "dbt_det/models/staging/stg__dlt_pipeline_state.sql": "921824090c1985a61a063350b9c87a296c28d5dafe4d26a0fd9c3a1a13e98624", - "dbt_det/models/staging/stg__dlt_pipeline_state.yml": "aeaa684b4339d83bebcd708fff826ec413ed608ed53d8200320a853cb8f7dd72", + "dbt_det/models/staging/stg__dlt_pipeline_state.yml": "01d85053a17d1aa27f2a17537fdcc5affb0f394212d579d7a05217576a7237a1", "dbt_det/models/staging/stg__dlt_version.sql": "5d279d508581eae585acdbbeaa36622b1faab61996b3956c6e2373c0d1294a8d", "dbt_det/models/staging/stg__dlt_version.yml": "dec6894006cce86c5950e66e5939ea8d16ebacf5722509ff8235e7fe9710b1a9", "dbt_det/models/staging/stg_stories.sql": "1bef38284308be0963e9933f19f53339e110020d4d2993766d48a7c452930397", @@ -277,7 +289,7 @@ "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "2c712c4932785690fc4d9d9117d964a0e784989b1bc65cb8e2afaff4a18bcc16", "dbt_det/seeds/raw/_dlt_loads.csv": "4bc16cd04e968225f6fb788f9f48ea51282e3d089e40dc369b71db1f9b433107", - "dbt_det/seeds/raw/_dlt_pipeline_state.csv": "67f4f6828c4340bb33744fa1784bca9a1c2ff9093f415ad1aa16cae27edb67e6", + "dbt_det/seeds/raw/_dlt_pipeline_state.csv": "9387c8e111defbf0cf0cd7f36093c205ebf53997e398b3151b08f077e9837ca4", "dbt_det/seeds/raw/_dlt_version.csv": "bddf097235105ad5ced6c56380c70d2b9bc915e1a0e4a1368be3d48e534b7d3f", "dbt_det/seeds/raw/stories.csv": "2869db5e096a04b0092bf2a97266cf7969ec40bef9fd45efc0540a1db178a362", "dbt_det/seeds/raw/stories__kids.csv": "c65b11d84a910e24ee31f22eb44df742663e65a175b1b8c2083fa1e8e95f9005" @@ -292,8 +304,8 @@ "dbt_det/models/staging/stg_user_accounts.yml": "043f05233912b1a27bdb6fd18f5d4ed2f10c65b0fa2dd1f41d9257843894d645", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "3a77e5dc79f8b376b8cfc3460d19490965ffccb7ff1cc900d5f28ceaabc6e059", - "dbt_det/seeds/raw/orders.csv": "9e7ceef611c6133a0b0c3b7eee48de20ad4747656201a0349fd90a8ea9aa3271", - "dbt_det/seeds/raw/user_accounts.csv": "a43e5ec8034c02d8242f4398ed69f1f570c85cb34ae1674a44a306543586e16a" + "dbt_det/seeds/raw/orders.csv": "76cc3d3a91fab1a1c33f48f9d612b00d67b7e02e87b31a2ed0e2416a957a9e3a", + "dbt_det/seeds/raw/user_accounts.csv": "225d20a294eb1c7805c2e355f11904e143f515d2dd6e4708df0a0957f44bcfb6" }, "example-mixed_quotes_crlf.model2data.yml": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", @@ -305,66 +317,68 @@ "dbt_det/models/staging/stg_user_accounts.yml": "043f05233912b1a27bdb6fd18f5d4ed2f10c65b0fa2dd1f41d9257843894d645", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "3a77e5dc79f8b376b8cfc3460d19490965ffccb7ff1cc900d5f28ceaabc6e059", - "dbt_det/seeds/raw/orders.csv": "9e7ceef611c6133a0b0c3b7eee48de20ad4747656201a0349fd90a8ea9aa3271", - "dbt_det/seeds/raw/user_accounts.csv": "a43e5ec8034c02d8242f4398ed69f1f570c85cb34ae1674a44a306543586e16a" + "dbt_det/seeds/raw/orders.csv": "76cc3d3a91fab1a1c33f48f9d612b00d67b7e02e87b31a2ed0e2416a957a9e3a", + "dbt_det/seeds/raw/user_accounts.csv": "225d20a294eb1c7805c2e355f11904e143f515d2dd6e4708df0a0957f44bcfb6" }, "example-saas_platform.dbml": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_api_usage.sql": "3a453410ffeba21c7c299fe9144f762aab2565cd9f02b32ed3fc6a25c4a7cd4b", "dbt_det/models/staging/stg_api_usage.yml": "67de79479deb6266fc70d5e38651b730a19c2c2ae4641c72ad2a1e2e685d1ccd", "dbt_det/models/staging/stg_audit_events.sql": "062020dbc347feeb7b7b7e9c47e8112691cd434d839d783d147fbbd811c5f040", - "dbt_det/models/staging/stg_audit_events.yml": "cb9f16b3ec30544c0166905dde97589404cd5f653b75d73895971de56762f9c8", + "dbt_det/models/staging/stg_audit_events.yml": "6c6a5d9d56bf277782a723e9e0df2577d3afb46a0370524abb6321bbc4f11320", "dbt_det/models/staging/stg_feature_limits.sql": "8dd6ccb9ee9da75f8102460879f2da89f541e5fc71336b29b9e2372ee917de1d", "dbt_det/models/staging/stg_feature_limits.yml": "381963d063dca0966d5aef605a04b1bb9ca39b6c36fc35e9c53f931ec11dce90", "dbt_det/models/staging/stg_invoices.sql": "6a282cd534a83387c28062ec83924bca249e8466659ed4b7614652a4853e609d", - "dbt_det/models/staging/stg_invoices.yml": "7b72a9cf5ecd110f8a998558f76707cef23fc5a53220cc0b76a1796220dd772e", + "dbt_det/models/staging/stg_invoices.yml": "bde339657aae0e12e3dc820b1f08989ca31719511625ead8189cf7edad5c1089", "dbt_det/models/staging/stg_organizations.sql": "7f29bdc034068d313c826a71bff56d49bfae6039108d376f6034de8c97c9bb19", "dbt_det/models/staging/stg_organizations.yml": "3bcf56cc79a94657ba0c6132ee1eaff87f89b7d3f099c839ade87146e24c4ee5", "dbt_det/models/staging/stg_subscriptions.sql": "2a2d9ee7ba0ae44b610935af21bd4cdd1a38986e1f4fca772fb6775296b0377e", - "dbt_det/models/staging/stg_subscriptions.yml": "30320554d1087f4b2b905d2a8307966d04e1b8cd078b439d52deea368c40d8b1", + "dbt_det/models/staging/stg_subscriptions.yml": "308fc79cb2f277a28ebffd69aab4c9e578ace1512e0c37e7f503b05baf394e63", "dbt_det/models/staging/stg_users.sql": "0327dec706116d8aeb53093adad11614d9b456bc574276a9f44fbaa906be4e65", "dbt_det/models/staging/stg_users.yml": "b162ae9b22d7b026e9f546b58ad06f8bc9313351e2fe3d1fa4fd55eb235e3060", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/saas_platform.dbml": "a444d3d626b2a104920a5a2af9512ec311ac02e7260e18cb11750fc09adf8735", "dbt_det/seeds/raw/__seed_config.yml": "b87c6d38dae29df61e3c7bd66560b0ec505833a04a7e784e3b22142c78f3c4b7", - "dbt_det/seeds/raw/api_usage.csv": "e44c13b2daa250b3737767fc60b3466e34744c0e03e69ebc37fff7da0a270342", - "dbt_det/seeds/raw/audit_events.csv": "d0dbbd7e2225a5a8ae15c9c01fcc2839c49a0fd8f1ea1a4698a202537e94dae2", - "dbt_det/seeds/raw/feature_limits.csv": "49da3c9213cc8579f864d122b0dd43febdbaaa7b7385b29a9d19560c2b12dcfb", - "dbt_det/seeds/raw/invoices.csv": "274f79168492a7e5cc89512ba029ff923030dbbca0252636346432372f60745b", - "dbt_det/seeds/raw/organizations.csv": "bb9019c41083da011f0512d4a5ea4225fa766a8c9d3feaabdfb6713d19a412e0", - "dbt_det/seeds/raw/subscriptions.csv": "a556d928e9c22eeedb2c01302b049f723ef4e862dd04d528b90dc07ce67b8ec6", - "dbt_det/seeds/raw/users.csv": "3c861f291abb7d7dfb832aec2c3ab6ff7a4332cee37831440bf0a305cd087fd6" + "dbt_det/seeds/raw/api_usage.csv": "5494440e04f34c2e9c7b672c1d9993ff4042b0bd88963e5307e22e9acb27811b", + "dbt_det/seeds/raw/audit_events.csv": "ae4f86727f8fc3cbff817811812f2e529cd072ceef427118ee5f8bc30fac06c2", + "dbt_det/seeds/raw/feature_limits.csv": "e57be8667393acd8745eba627d0fb3f2b218df515d1c472c8a011eeee3a71926", + "dbt_det/seeds/raw/invoices.csv": "0be42f028ba7337159c335b8f53cbc49ea5c4458dabcf206dac2ad2bc827e31d", + "dbt_det/seeds/raw/organizations.csv": "b12a3a33bd8558d15c4e79f3c1284af6f2101e80bab540ea3457540d5a7abb8e", + "dbt_det/seeds/raw/subscriptions.csv": "e5af6aefaf597fefd16cb1276e388146c54741f2483e6de7deb6e5527b73ddae", + "dbt_det/seeds/raw/users.csv": "f6f7dac78e0631991ef382e0fc9331df6d2f137b497f8ccb9a22a22c8d5376df" }, "example-saas_platform.model2data.yml": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_api_usage.sql": "3a453410ffeba21c7c299fe9144f762aab2565cd9f02b32ed3fc6a25c4a7cd4b", "dbt_det/models/staging/stg_api_usage.yml": "67de79479deb6266fc70d5e38651b730a19c2c2ae4641c72ad2a1e2e685d1ccd", "dbt_det/models/staging/stg_audit_events.sql": "062020dbc347feeb7b7b7e9c47e8112691cd434d839d783d147fbbd811c5f040", - "dbt_det/models/staging/stg_audit_events.yml": "cb9f16b3ec30544c0166905dde97589404cd5f653b75d73895971de56762f9c8", + "dbt_det/models/staging/stg_audit_events.yml": "6c6a5d9d56bf277782a723e9e0df2577d3afb46a0370524abb6321bbc4f11320", "dbt_det/models/staging/stg_feature_limits.sql": "8dd6ccb9ee9da75f8102460879f2da89f541e5fc71336b29b9e2372ee917de1d", "dbt_det/models/staging/stg_feature_limits.yml": "381963d063dca0966d5aef605a04b1bb9ca39b6c36fc35e9c53f931ec11dce90", "dbt_det/models/staging/stg_invoices.sql": "6a282cd534a83387c28062ec83924bca249e8466659ed4b7614652a4853e609d", - "dbt_det/models/staging/stg_invoices.yml": "7b72a9cf5ecd110f8a998558f76707cef23fc5a53220cc0b76a1796220dd772e", + "dbt_det/models/staging/stg_invoices.yml": "bde339657aae0e12e3dc820b1f08989ca31719511625ead8189cf7edad5c1089", "dbt_det/models/staging/stg_organizations.sql": "7f29bdc034068d313c826a71bff56d49bfae6039108d376f6034de8c97c9bb19", "dbt_det/models/staging/stg_organizations.yml": "3bcf56cc79a94657ba0c6132ee1eaff87f89b7d3f099c839ade87146e24c4ee5", "dbt_det/models/staging/stg_subscriptions.sql": "2a2d9ee7ba0ae44b610935af21bd4cdd1a38986e1f4fca772fb6775296b0377e", - "dbt_det/models/staging/stg_subscriptions.yml": "30320554d1087f4b2b905d2a8307966d04e1b8cd078b439d52deea368c40d8b1", + "dbt_det/models/staging/stg_subscriptions.yml": "308fc79cb2f277a28ebffd69aab4c9e578ace1512e0c37e7f503b05baf394e63", "dbt_det/models/staging/stg_users.sql": "0327dec706116d8aeb53093adad11614d9b456bc574276a9f44fbaa906be4e65", "dbt_det/models/staging/stg_users.yml": "b162ae9b22d7b026e9f546b58ad06f8bc9313351e2fe3d1fa4fd55eb235e3060", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/saas_platform.model2data.yml": "23a8b7391c75e50ecc975ae77ee8abe0507093492b73d65b990b2c7c82b4e42c", "dbt_det/seeds/raw/__seed_config.yml": "b87c6d38dae29df61e3c7bd66560b0ec505833a04a7e784e3b22142c78f3c4b7", - "dbt_det/seeds/raw/api_usage.csv": "e44c13b2daa250b3737767fc60b3466e34744c0e03e69ebc37fff7da0a270342", - "dbt_det/seeds/raw/audit_events.csv": "d0dbbd7e2225a5a8ae15c9c01fcc2839c49a0fd8f1ea1a4698a202537e94dae2", - "dbt_det/seeds/raw/feature_limits.csv": "49da3c9213cc8579f864d122b0dd43febdbaaa7b7385b29a9d19560c2b12dcfb", - "dbt_det/seeds/raw/invoices.csv": "274f79168492a7e5cc89512ba029ff923030dbbca0252636346432372f60745b", - "dbt_det/seeds/raw/organizations.csv": "bb9019c41083da011f0512d4a5ea4225fa766a8c9d3feaabdfb6713d19a412e0", - "dbt_det/seeds/raw/subscriptions.csv": "a556d928e9c22eeedb2c01302b049f723ef4e862dd04d528b90dc07ce67b8ec6", - "dbt_det/seeds/raw/users.csv": "3c861f291abb7d7dfb832aec2c3ab6ff7a4332cee37831440bf0a305cd087fd6" + "dbt_det/seeds/raw/api_usage.csv": "5494440e04f34c2e9c7b672c1d9993ff4042b0bd88963e5307e22e9acb27811b", + "dbt_det/seeds/raw/audit_events.csv": "ae4f86727f8fc3cbff817811812f2e529cd072ceef427118ee5f8bc30fac06c2", + "dbt_det/seeds/raw/feature_limits.csv": "e57be8667393acd8745eba627d0fb3f2b218df515d1c472c8a011eeee3a71926", + "dbt_det/seeds/raw/invoices.csv": "0be42f028ba7337159c335b8f53cbc49ea5c4458dabcf206dac2ad2bc827e31d", + "dbt_det/seeds/raw/organizations.csv": "b12a3a33bd8558d15c4e79f3c1284af6f2101e80bab540ea3457540d5a7abb8e", + "dbt_det/seeds/raw/subscriptions.csv": "e5af6aefaf597fefd16cb1276e388146c54741f2483e6de7deb6e5527b73ddae", + "dbt_det/seeds/raw/users.csv": "f6f7dac78e0631991ef382e0fc9331df6d2f137b497f8ccb9a22a22c8d5376df" }, "example-tagging_m2m.dbml": { "dbt_det/data-tests/unique_combination_stg_post_tags_post_id_tag_id.sql": "2d667449d4fbddd50af1ed68b1441ff5a75c497dc69d822a4876490f6fab0900", @@ -378,9 +392,9 @@ "dbt_det/models/staging/stg_tags.yml": "e5d01f122d4dacb1e1e1b081865f9eaa8fbd13ae3facbf5d641d4d7860606965", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "a07ae262b9b64458de087c002c1e53ce7c48ef9b5cc4b1b7509839995b224734", - "dbt_det/seeds/raw/post_tags.csv": "84272958ecbc1825a3c8d6f63449c5fa264e59523cfea5a2e20a8ed8e05b915c", - "dbt_det/seeds/raw/posts.csv": "9ff3974029b8abe6963df969ca678c2a065f8727bfc89dd0781ad031e305feba", - "dbt_det/seeds/raw/tags.csv": "2e62bd75d25d6a696402d2044463bfd886eb42a415ce18ec833c723bb16d480a", + "dbt_det/seeds/raw/post_tags.csv": "a3294ef6c7ccf6008b7a618b09d4b4db8ef8ecb3928f6e5581cce5110e9bbefe", + "dbt_det/seeds/raw/posts.csv": "dbb25c2e25dbb31f33db2feac19471b074c11cfc3489bd0bee5a8d6814289f25", + "dbt_det/seeds/raw/tags.csv": "f733d56677d66899c25d6663c0e4c118533bba8446a3de807d188748d478ff99", "dbt_det/tagging_m2m.dbml": "451c4fc137124480f9efd53ed2d3807647ad9dcf679c5125be035c7f62751a82" }, "example-tagging_m2m.model2data.yml": { @@ -395,16 +409,16 @@ "dbt_det/models/staging/stg_tags.yml": "e5d01f122d4dacb1e1e1b081865f9eaa8fbd13ae3facbf5d641d4d7860606965", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "a07ae262b9b64458de087c002c1e53ce7c48ef9b5cc4b1b7509839995b224734", - "dbt_det/seeds/raw/post_tags.csv": "84272958ecbc1825a3c8d6f63449c5fa264e59523cfea5a2e20a8ed8e05b915c", - "dbt_det/seeds/raw/posts.csv": "9ff3974029b8abe6963df969ca678c2a065f8727bfc89dd0781ad031e305feba", - "dbt_det/seeds/raw/tags.csv": "2e62bd75d25d6a696402d2044463bfd886eb42a415ce18ec833c723bb16d480a", + "dbt_det/seeds/raw/post_tags.csv": "a3294ef6c7ccf6008b7a618b09d4b4db8ef8ecb3928f6e5581cce5110e9bbefe", + "dbt_det/seeds/raw/posts.csv": "dbb25c2e25dbb31f33db2feac19471b074c11cfc3489bd0bee5a8d6814289f25", + "dbt_det/seeds/raw/tags.csv": "f733d56677d66899c25d6663c0e4c118533bba8446a3de807d188748d478ff99", "dbt_det/tagging_m2m.model2data.yml": "7d76749e97cb0900c31e26c58acaabd361a35e8f312f3ef0d477805fb7a191ed" }, "incremental-history": { - "dbt_det/days/orders/day_000.csv": "934ddbe4f90f2d763742529e6e92a92ae8a69c566d77a6e57994d0995e6e1411", - "dbt_det/days/orders/day_001.csv": "975205fca907bf8f1df26e777d17c81fd4f7de1eda68bc92ff3a69f215458e8e", - "dbt_det/days/orders/day_002.csv": "3e9fd7566951b42ba15f2515371893a3bdb177617d47a70da20a54efed2c425e", - "dbt_det/days/orders/day_003.csv": "02cf9efaad0bcf0d9a0b7e681c43cd95ff715d226c2f26a0e10371096e00aeef", + "dbt_det/days/orders/day_000.csv": "f79b34dad5d3fbd2e7eb092d8a6ce6974c2e4592aae7266094c3a27f5bd732bf", + "dbt_det/days/orders/day_001.csv": "92241b7669c124e0880ea1870a1d109e0f90458da7b710b38528d37013a2742e", + "dbt_det/days/orders/day_002.csv": "4f46d5cf388e6e53266519f66ac2796542a0d60e4d10ac7b23f1994f3d9bb8b1", + "dbt_det/days/orders/day_003.csv": "35599442b7ee4b790d589d8fbf408c7ecb118fa577e6a7bc4ef0cd9c03e57b21", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/hcol.model2data.yml": "22d9b56cf0716e52f9d16fd01f3e153b6196091b73f3a0a55acb52a7dee442c4", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", @@ -415,148 +429,154 @@ "dbt_det/models/staging/stg_orders_history.yml": "178b9497122d0df06fea9a131ac625eba99ba2ef1a1a9a0248d0f86054110970", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "95e7df9627831b628bf1aebef60603b9bc7a0ad099758b76356f5639c2bc7341", - "dbt_det/seeds/raw/orders.csv": "9f58996eb8945d78e874f0d033c7cfb679467af190991590c5916638676e8d8b", - "dbt_det/seeds/raw/orders_history.csv": "496e2aa4a39c6e52bc0c7ea39441a2c369cc340c38dff05f11642ebe80291351" + "dbt_det/seeds/raw/orders.csv": "fc7fa8ea3924e22cddb3203fb041f5cb8eb8ab5b8f96074362cd5a034ea9d31b", + "dbt_det/seeds/raw/orders_history.csv": "48a354c01767342b65b0349cfefac219e60a538301175090f27688a73c198326" }, "locale-de_DE": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "88ed2f885637e38367e119250b09f764e4570e8ef244ae4bdfa55e0d832d466a", + "dbt_det/models/staging/stg_orders.yml": "cba055fdb3f2d6145cddcf8696242fbfb34463577e681284eca78e6f11b96ede", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", "dbt_det/models/staging/stg_product_reviews.yml": "e7692c4d586af9ea57c9afbb421d5424be907fb28f4464bf82f75033b402ef48", "dbt_det/models/staging/stg_products.sql": "7455128b10e0357213f26c209913a73df3f417be28ddd9d4dd8681390e7dcdab", "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "f10588f38f838c163af5a58349670e4abc2e753619f1ae623b98ef2758d8be08", - "dbt_det/seeds/raw/customers.csv": "9161ffba48a265b67adc38b5e5042afb1881fe482d58f0a71c48205280a7c35e", - "dbt_det/seeds/raw/order_items.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/seeds/raw/orders.csv": "a9f5c94298c8d5d7a90a9a5d8e95f6a5adcaac3fc56badf830c615a0987e0d23", - "dbt_det/seeds/raw/product_reviews.csv": "536ea01ea72793ae0e00ff2fc75484a77ece1c56a4045a3b49b8b5101cd2b933", - "dbt_det/seeds/raw/products.csv": "b9dc9c64bcb63d4fad1563c61de752ca3c77c59ffc22ba28be618aad77babed6" + "dbt_det/seeds/raw/customers.csv": "5003aa63dc9cbad185fc950c4a61cd094fecd4800518c2908b3f8f8d1fc64284", + "dbt_det/seeds/raw/order_items.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/seeds/raw/orders.csv": "3124312b95043b7592f149a40fd51ab19a03276313bef80973716e6759376fdb", + "dbt_det/seeds/raw/product_reviews.csv": "2cb46501bc7b6237ae5c6be2416b86df22120fda878baaa903e8b202a88acbf4", + "dbt_det/seeds/raw/products.csv": "f3d659818873ffd3e484e840d252e3c4d9940e335714b1b97d838e75fe9ecd28" }, "locale-en_US": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "88ed2f885637e38367e119250b09f764e4570e8ef244ae4bdfa55e0d832d466a", + "dbt_det/models/staging/stg_orders.yml": "cba055fdb3f2d6145cddcf8696242fbfb34463577e681284eca78e6f11b96ede", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", "dbt_det/models/staging/stg_product_reviews.yml": "e7692c4d586af9ea57c9afbb421d5424be907fb28f4464bf82f75033b402ef48", "dbt_det/models/staging/stg_products.sql": "7455128b10e0357213f26c209913a73df3f417be28ddd9d4dd8681390e7dcdab", "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "f10588f38f838c163af5a58349670e4abc2e753619f1ae623b98ef2758d8be08", - "dbt_det/seeds/raw/customers.csv": "0ae988dbb32bcba86047756c3fa4cb22cb74d4b5e6dcc604c18cb56e0507ccf8", - "dbt_det/seeds/raw/order_items.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/seeds/raw/orders.csv": "a9f5c94298c8d5d7a90a9a5d8e95f6a5adcaac3fc56badf830c615a0987e0d23", - "dbt_det/seeds/raw/product_reviews.csv": "e7bd78f152f82b650b2392a38e2bac3f6837a8f3b1cb7818a3f834d88a897768", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "2b34b876f615650d684d7ba23e0cf0f63d8f6addc7f9798c2aa16735c9b6a45d", + "dbt_det/seeds/raw/order_items.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/seeds/raw/orders.csv": "3124312b95043b7592f149a40fd51ab19a03276313bef80973716e6759376fdb", + "dbt_det/seeds/raw/product_reviews.csv": "95f9a92f499d61ebc0d9747dd27474e1a5d96eaf3cec91b3d36d1a9b2ceb3b07", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "locale-fr_FR": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "88ed2f885637e38367e119250b09f764e4570e8ef244ae4bdfa55e0d832d466a", + "dbt_det/models/staging/stg_orders.yml": "cba055fdb3f2d6145cddcf8696242fbfb34463577e681284eca78e6f11b96ede", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", "dbt_det/models/staging/stg_product_reviews.yml": "e7692c4d586af9ea57c9afbb421d5424be907fb28f4464bf82f75033b402ef48", "dbt_det/models/staging/stg_products.sql": "7455128b10e0357213f26c209913a73df3f417be28ddd9d4dd8681390e7dcdab", "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "f10588f38f838c163af5a58349670e4abc2e753619f1ae623b98ef2758d8be08", - "dbt_det/seeds/raw/customers.csv": "b41033e4473094c142f43a7ba035c8c8326dd7106b4df984af8bb12ffd8daabd", - "dbt_det/seeds/raw/order_items.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/seeds/raw/orders.csv": "a9f5c94298c8d5d7a90a9a5d8e95f6a5adcaac3fc56badf830c615a0987e0d23", - "dbt_det/seeds/raw/product_reviews.csv": "42cf619cc654895ed435e81fea30935dd5bd20ee54912c55e5f52bb4b480ea89", - "dbt_det/seeds/raw/products.csv": "52b9ace99163dc819e6a2841021146a723855911495ae1aba6c41a63436ef25c" + "dbt_det/seeds/raw/customers.csv": "7d7cbb839226413e32cfce693b34bc50e8392b179a3380366dd1fd9301c43f8f", + "dbt_det/seeds/raw/order_items.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/seeds/raw/orders.csv": "3124312b95043b7592f149a40fd51ab19a03276313bef80973716e6759376fdb", + "dbt_det/seeds/raw/product_reviews.csv": "5f05a33a0cdc56cc9df2184230b4b698002f427cc31aa140fb99961d235314f6", + "dbt_det/seeds/raw/products.csv": "80277e873fce9458f88e0079e23d52757d5346c6d1e3dc5901a5d7f72e7b6153" }, "locale-ja_JP": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "88ed2f885637e38367e119250b09f764e4570e8ef244ae4bdfa55e0d832d466a", + "dbt_det/models/staging/stg_orders.yml": "cba055fdb3f2d6145cddcf8696242fbfb34463577e681284eca78e6f11b96ede", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", "dbt_det/models/staging/stg_product_reviews.yml": "e7692c4d586af9ea57c9afbb421d5424be907fb28f4464bf82f75033b402ef48", "dbt_det/models/staging/stg_products.sql": "7455128b10e0357213f26c209913a73df3f417be28ddd9d4dd8681390e7dcdab", "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "f10588f38f838c163af5a58349670e4abc2e753619f1ae623b98ef2758d8be08", - "dbt_det/seeds/raw/customers.csv": "eb360f952c0914d77c69b953ca20eca5653e05c44f15fe9c65352512bbfeb02e", - "dbt_det/seeds/raw/order_items.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/seeds/raw/orders.csv": "a9f5c94298c8d5d7a90a9a5d8e95f6a5adcaac3fc56badf830c615a0987e0d23", - "dbt_det/seeds/raw/product_reviews.csv": "11f366c6c209690847b2d9b891cbc4a27f1aa57bfb5d717d92af09ebafd48f91", - "dbt_det/seeds/raw/products.csv": "ccdc1fb1525f8fd3c8886f0a1e6683fe854672e47fb0513e0f1d98e1fbb7a8f3" + "dbt_det/seeds/raw/customers.csv": "26ad1982472060d653afc29c0ca7d4c517b9ee48a2c65eec55789be508b0619a", + "dbt_det/seeds/raw/order_items.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/seeds/raw/orders.csv": "3124312b95043b7592f149a40fd51ab19a03276313bef80973716e6759376fdb", + "dbt_det/seeds/raw/product_reviews.csv": "8aefd4e4db8b8346d7e04a03b4486bc52fc44559c149633b56e13ace70f7ddf3", + "dbt_det/seeds/raw/products.csv": "7d46c1d40b524cecca1534c1458979cf4b0641a6e6e0d25c3a130b799acda7c6" }, "locale-nl_BE": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "88ed2f885637e38367e119250b09f764e4570e8ef244ae4bdfa55e0d832d466a", + "dbt_det/models/staging/stg_orders.yml": "cba055fdb3f2d6145cddcf8696242fbfb34463577e681284eca78e6f11b96ede", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", "dbt_det/models/staging/stg_product_reviews.yml": "e7692c4d586af9ea57c9afbb421d5424be907fb28f4464bf82f75033b402ef48", "dbt_det/models/staging/stg_products.sql": "7455128b10e0357213f26c209913a73df3f417be28ddd9d4dd8681390e7dcdab", "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "f10588f38f838c163af5a58349670e4abc2e753619f1ae623b98ef2758d8be08", - "dbt_det/seeds/raw/customers.csv": "dd19936a279b4be579eaec5e79f0bdb442608c1f095ec296fa6da447e852a6bf", - "dbt_det/seeds/raw/order_items.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/seeds/raw/orders.csv": "a9f5c94298c8d5d7a90a9a5d8e95f6a5adcaac3fc56badf830c615a0987e0d23", - "dbt_det/seeds/raw/product_reviews.csv": "bcc6a29dba1224d32aca2bcdb1e0d83d618dfa8e8168d6c1381b5bb549f6b1d6", - "dbt_det/seeds/raw/products.csv": "89a4ff01ab5a53712fc8b47728dd463104a008eb3d31b3fee1b6a95e0d8e1188" + "dbt_det/seeds/raw/customers.csv": "605855d6a63790f55d225bc53b250df8738be3b1aa5d6fd0b74f89cfa3e3101e", + "dbt_det/seeds/raw/order_items.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/seeds/raw/orders.csv": "3124312b95043b7592f149a40fd51ab19a03276313bef80973716e6759376fdb", + "dbt_det/seeds/raw/product_reviews.csv": "5596ca33f906f048e366921b26adf5991a66cd4ad3c5d73b027cd3d8d95f1fdc", + "dbt_det/seeds/raw/products.csv": "501382a8f893aa8dd7473652c420baccf189ff0a5f544d16e47a6a52f6e12b39" }, "metrics": { "dbt_det/coffee_webshop.metrics.yml": "16981620dc46a34c01f2c6cfc36644522bf9c5a80defb49206cc2a82eca67c32", "dbt_det/coffee_webshop.model2data.yml": "ae20f85480b4e5503e2f54ea5a71d24673b8bb5990644c850e51f91c5eddbc4a", - "dbt_det/data-tests/metrics/metric_average_order_value.sql": "d47204215d92b41b5216e61559e5593745d3ab13e0549045a6f9a816a0c1171b", - "dbt_det/data-tests/metrics/metric_coffee_units_sold.sql": "e8f15c7726dcbce43d0e72caca13d537fba8eb93a60e19e1ebbed780ce41ee7b", + "dbt_det/data-tests/metrics/metric_average_order_value.sql": "9292cf214a82e05011817057157da7eaa467baf5fda2443a80a826b2302c0bd6", + "dbt_det/data-tests/metrics/metric_coffee_units_sold.sql": "6e9d8b6182348163d5311790f71c5b41875c26b8cd7e96aa064075ba1849d19f", "dbt_det/data-tests/metrics/metric_order_items_count.sql": "3d0e8d87ceb9c396106908800b18872c499f9927b634780868271d003df3a415", - "dbt_det/data-tests/metrics/metric_order_items_quantity.sql": "918e178f40d3deb4ccb9bce56adb09ded838e7bdeebe75c219474a1f50037799", + "dbt_det/data-tests/metrics/metric_order_items_quantity.sql": "babb93e95a292808bbb365c81930820093535c8212139398d6d0e89b1b93f7f9", "dbt_det/data-tests/metrics/metric_order_lines.sql": "20bd6bf0823be4a1e62cec088f29ad84273ff9854599125862b7d1783deefc5d", "dbt_det/data-tests/metrics/metric_orders.sql": "8f0ec4c078bfe0d28a5efb93c9e1763a19f88c2314f0cb6ffb3ab7709d7a1c49", "dbt_det/data-tests/metrics/metric_orders_count.sql": "628a2ac01a89dbf22c8b83618704b2be1d674b440e23af6aceb4246398b0944d", - "dbt_det/data-tests/metrics/metric_orders_total_amount.sql": "072dcd484960eddac5609721af41e0efca0e6ba1ae2cf1cab3500bf7a212ccd4", - "dbt_det/data-tests/metrics/metric_revenue.sql": "23235c3ffb49ed5a7ee427628ce64921d49034bedbc0da360f1afcde26a4639b", - "dbt_det/data-tests/metrics/metric_web_revenue.sql": "c13c50a47a735df3a275d45b4bcaa7e97f028bbd7dd1451116bf02c83e01eddb", - "dbt_det/data-tests/metrics/metric_web_share.sql": "e6b2d720239105a41b1a75164c2f07f0051aa9e2f21ce2b078526af377abb58a", + "dbt_det/data-tests/metrics/metric_orders_total_amount.sql": "0d403fb45b83ed1b9e855df1b5abbecec9c24502ef53a2e4d25d206f73cbaf97", + "dbt_det/data-tests/metrics/metric_revenue.sql": "e14def79e2147d816377953fc18a91d4b11a8a2a78d92a6982c5a53c045a3803", + "dbt_det/data-tests/metrics/metric_web_revenue.sql": "36790666a4a8986f359d872ecb33b5e73a1a98ee7a4582377e54603972ad282c", + "dbt_det/data-tests/metrics/metric_web_share.sql": "8a65d805521923c8a214b2428294d0d6bd063b56680595770dcabfe6ac1687d5", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/metric_values.json": "37a66e08fbbe5d227a141d6be3efd31be6925b835e67cc95cbfd6b3922ec1316", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/metric_values.json": "36848fdba7e43423ea8ab71d2d03eae08c6e20742c1378038caf3871625feadc", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "02f18d31e87a4bfb8314588180ef3382b8db5e5b5460e4222f6e094e4006582c", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "5733941fb5c1419091efa5a9044030c4f205be95a2b68413df81fdd220eaa8c7", + "dbt_det/models/staging/stg_orders.yml": "18528079a15322d70b54cec2f6d6c6d039d375f5b9d69805e62ff614d5aa2174", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", "dbt_det/models/staging/stg_product_reviews.yml": "62bab917381e2a43ee1067a5cd56119082cb7f7dff10f23a687c2a9345b2a07c", "dbt_det/models/staging/stg_products.sql": "7455128b10e0357213f26c209913a73df3f417be28ddd9d4dd8681390e7dcdab", @@ -564,54 +584,55 @@ "dbt_det/osi/det.yml": "5d005ea6c0d5af8887e3c43bb9dea9a1be080e94cc6b913253c95e86d7b64dc7", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "8e3df67807677a8ed3e8949f8dd7c7dba6a088d4f5221236a770638802d62105", - "dbt_det/seeds/raw/customers.csv": "f97be9bf40f3f5f3c97eb9697a866733d5118518ed1e48475e3d4363b3febbb4", - "dbt_det/seeds/raw/order_items.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/seeds/raw/orders.csv": "1b4b057db7bfd59f49182f3effd13ecafa5ff3dba1de0ef766162e2b90b53e6b", - "dbt_det/seeds/raw/product_reviews.csv": "dd0b9a9087b147836358609c315cf2f15a83c9770c225bfee2f26742eab8e48a", - "dbt_det/seeds/raw/products.csv": "980c75d598d2bbb80b768df98c0958789f6720627c94bc542a40e1acac9c8ee5" + "dbt_det/seeds/raw/customers.csv": "96c9215cc0a7f8e718996f072547cc480dcbbe28c2bc0abd9fc86d3ef8e27b10", + "dbt_det/seeds/raw/order_items.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/seeds/raw/orders.csv": "e5231e80c3233300af22b9d54ca67048abce2b9fe56fa2bc93fb39fb3da1bb4e", + "dbt_det/seeds/raw/product_reviews.csv": "fa9537361e10ceabbaaa9e95ca0b43123a959c3243285835e3cd7c57f00b4909", + "dbt_det/seeds/raw/products.csv": "bde1e60cc1dd87d8dc8d3c78c6dfa73714b8eef3426ee87bda363340fbc473ff" }, "metrics-defects-days": { - "dbt_det/EXPECTED_FAILURES.md": "233eff9d9cd993bd7a2b6621870f8ef315785230eda5e23dca07286b6f68c192", - "dbt_det/data-tests/metrics/metric_average_order.sql": "d80e42fddda755c509ba6b8234fc5a7148439200562b615f7ad672b2f559a9d8", - "dbt_det/data-tests/metrics/metric_customers_ordering.sql": "59e9c3257ec14d3f88ef1f8e12abd01988d46caccccd5357fbac943542e6ee41", - "dbt_det/data-tests/metrics/metric_delivered_revenue.sql": "ba26ce0181f17bc1875d476c6aba9481cb242b1c7d13cf332ade14d05f0cdedf", - "dbt_det/data-tests/metrics/metric_item_value.sql": "a20af11fe63a2be7aa8fd678903fd1b112e47dc7580c4290e204c94c4f49daba", - "dbt_det/data-tests/metrics/metric_items_of_recent_orders.sql": "8278a4996aedd85c53d3872ce84fffafb4dad8dd1f366b1cb5add01803c5b898", + "dbt_det/EXPECTED_FAILURES.md": "c82f6c9408675f08f87220b5743eab42835193beedffcee66eb9f97a57270de6", + "dbt_det/data-tests/metrics/metric_average_order.sql": "cea708ea4d7b0bcfeaa9c53f1fd51fb90abbf5348fc4a942e82d3465ddf9f022", + "dbt_det/data-tests/metrics/metric_customers_ordering.sql": "6f062ed136db77eb616b85c78366d37909fd791717eb5ee5bdb7693ce395611e", + "dbt_det/data-tests/metrics/metric_delivered_revenue.sql": "3e50f1fd2a8d305930e2b86d521c52a900ca693f8a46eeb37292de25412dc3eb", + "dbt_det/data-tests/metrics/metric_item_value.sql": "6be75aa04a4d177ef1351b4f81720369c2909be0aea88345f076109bbea85e55", + "dbt_det/data-tests/metrics/metric_items_of_recent_orders.sql": "c5876b2b5b9d687869216385ccaae56da958adb555e1e74273742eadff6de3e2", "dbt_det/data-tests/metrics/metric_largest_quantity.sql": "c4bacbc70b08ccacaac62a0e2b88229d10ac57674c40f45c89945bf976ced4e5", - "dbt_det/data-tests/metrics/metric_median_order.sql": "c31f160119f37f0b9da2e1e28fc6e7cfded949c1c9fc52e318ec88cf4d8134bc", + "dbt_det/data-tests/metrics/metric_median_order.sql": "9813d18b2b29eaeb2847b406cb8eaf6fcf86681b9add80affc8c8a5be506ab78", "dbt_det/data-tests/metrics/metric_odd_mix.sql": "1cfaae567a6832ea2ae0991f18ecb3fded0a8fe5896da9f5c0d6478eedebfcab", - "dbt_det/data-tests/metrics/metric_open_orders.sql": "2e17d92acec3359b59999af6038b7b61dfc5f65cf7a5abf97b2c2a235426b13f", - "dbt_det/data-tests/metrics/metric_order_items_quantity.sql": "56203c4e6fb6484b4edd93c71a4b65112988acefd88ced83920523675ed4bb40", + "dbt_det/data-tests/metrics/metric_open_orders.sql": "31cce9eefafc42221bf3704ce71a02d498a44db20e7a5b2f0f46b61ab38afa34", + "dbt_det/data-tests/metrics/metric_order_items_quantity.sql": "2d3881048ca2457cb4d00080dafcca851d11fb0d158b35d0a5019f64c72d495b", "dbt_det/data-tests/metrics/metric_orders_count.sql": "592476a2c421c58056ad647ca7ad0a777a6642bbedc31bbb853aba9dc34ff396", - "dbt_det/data-tests/metrics/metric_per_order.sql": "441cc6ad7c7c41652bd90be2571caf22c54100b5a16bc133b8ac1b1b5182e530", - "dbt_det/data-tests/metrics/metric_rows_with_a_status.sql": "eaa990b17ba39de81eab1b85412944a609da72da59df8a6cc02f1e348a29d874", - "dbt_det/data-tests/metrics/metric_smallest_order.sql": "19a15f9618e90b2fa00c9fe61e5ba94335ee94b4e43fdb31bef5222db037de57", - "dbt_det/data-tests/metrics/metric_units_per_order.sql": "a2fd7b54c5c331564cf9912c2c6cb331c4249817c0cc3681a7ce8168ceb361db", - "dbt_det/days/customers/day_000.csv": "b0f8a38e0fb85d2a5ebd8ca80d56d9439b5938cbce6b4013bf91c4fb24502552", - "dbt_det/days/customers/day_001.csv": "220e7214482b5ac03b117c6d5c0fa2007f8ce17e1dcc9fc4c253a41df6acaa28", - "dbt_det/days/customers/day_002.csv": "f23b8f5fdc0e9abdb5efd01f6090d5c4f10d99fd47b07e228cd882bffa4c9738", - "dbt_det/days/order_items/day_000.csv": "68fda4e887148c46a19e39d5cc4ce3df20adbc5e846e2406b7fec80ac85d7594", - "dbt_det/days/order_items/day_001.csv": "c745a9e7052352d861d82b57b5a58371f3d026ecb739125deb60a00237e03b18", - "dbt_det/days/order_items/day_002.csv": "d22872e01590564174fbaa5f0f399a67b71a7f365cb3171e58259d8676c5cbf4", - "dbt_det/days/orders/day_000.csv": "a640c05ab85ad9125a030b24cb81932845ea56f0f0561229849c82208ee6a1b4", - "dbt_det/days/orders/day_001.csv": "4340b37b948e5b999c412d9d0542cdacef598bee46cd8f82c407a1759c744d74", - "dbt_det/days/orders/day_002.csv": "5de8d8e169904130aec139b1b65221ecd4c994d640f4e1dee72c332811d7941e", - "dbt_det/days/product_reviews/day_000.csv": "f9f33d24a151c6ff680ce5c4521f87418284a4de045dcb0eacb11fba048b7324", - "dbt_det/days/products/day_000.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3", - "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/defects_report.json": "02983affaff02f2248b52acc957c9c5f9a1dd3c071d72708b008d7707e39355c", + "dbt_det/data-tests/metrics/metric_per_order.sql": "c8ab4c16ac0037b398c8df9f911bcc6bb4ecb10eb84991e9af757a9b15f17d23", + "dbt_det/data-tests/metrics/metric_rows_with_a_status.sql": "4d4cae52a3c1841738b97aad069e14e3da9bc2f6744eeec919a528bbfdb4a0d3", + "dbt_det/data-tests/metrics/metric_smallest_order.sql": "09885768ee2f3cb0a0e14e8dbb30355c9e29b8936bcdb25599a3a678f8f7bbfb", + "dbt_det/data-tests/metrics/metric_units_per_order.sql": "88a2bf334072c0711bf1a9abf1517d86a638ff6b7c632555a58f0f0ed61ad97f", + "dbt_det/days/customers/day_000.csv": "e22a98c316139176e90be9fe56f147b5721b493588037382bf2cd619f140eec4", + "dbt_det/days/customers/day_001.csv": "ccee1dbf6f47a98326b8665e1d2654cbd68fb52761055bf4352e3acc0a37982d", + "dbt_det/days/customers/day_002.csv": "09e1eb40e21651e6080f55b58a0b5e328be5d82649d29a0b59c9d71174abcf19", + "dbt_det/days/order_items/day_000.csv": "3c0b1aaf744626559d92bab15abde9f1d306fc799fb100ec634d95427a6ba2fb", + "dbt_det/days/order_items/day_001.csv": "9329a9f8a2c87ade72747cb20955c84cf21feb7042ff0c272c3610cac7918332", + "dbt_det/days/order_items/day_002.csv": "0988e21aa8b3605f8301079a360a406cc79fc54584832405b4ef802a629c584c", + "dbt_det/days/orders/day_000.csv": "0b929251b1f9e50681ec1ad5dd724af6c4eb3e7a6f4dbd71eafd906d500511e5", + "dbt_det/days/orders/day_001.csv": "df0ac1c8a20cea535ae074ff64d513270a0e5b45956e1cd55809a29431a2d1ea", + "dbt_det/days/orders/day_002.csv": "411d703b8421d04b7f00ced1211630e3aa937dbe38163087c2dca4a6a16883b4", + "dbt_det/days/product_reviews/day_000.csv": "49ba5a52aa90dd18f3a12a15afd1d1f954a10e0572b084f8c94c89255c553e2b", + "dbt_det/days/products/day_000.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747", + "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", + "dbt_det/defects_report.json": "b4213c8e2027d566e6d266551c9bf28e27e07f01584a9f83c069c9df5137b3cb", "dbt_det/ecommerce_training.metrics.yml": "01324c4ec6103dee37f66d254c82d85515e894ccabd13262368814ab2fc9c6c1", "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", - "dbt_det/metric_values.json": "1f52439dfaa03f459e00e3fb6d7c24e6e5cd672b5005353596ac33acc2e25a1f", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/metric_values.json": "f47fa930aa4e3bb4ff42a8cb8f2505355dd7047c96a6833f7102fbb9c75b85a2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "fe63c9f2fea411ec373b19a500e90c4e0c0b7352ba2d4591a33cbcf4a138ed7c", + "dbt_det/models/staging/stg_orders.yml": "2a2ace25b7164723120ba0b2b7def4839dd286a17ccfb5990f304070bbe05d77", "dbt_det/models/staging/stg_orders_history.sql": "937bea699b5df5da6808580fe3963f432163f3741afc0f71275e5261d81f655b", "dbt_det/models/staging/stg_orders_history.yml": "b6bc4e53dc5078994b19f94b38371ea27308a452cccf351bfd43b6c489e2152a", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", @@ -621,41 +642,42 @@ "dbt_det/osi/det.yml": "638f0692719b23c6fb6d7e55a6cc09661bd3fda8c77eecd1a73c008058c58a31", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "03fbba2c4026678a9eef7cfe48aeaf3391a2919b05b1f67b2744ee0891ff199f", - "dbt_det/seeds/raw/customers.csv": "3443d991581aefaabc9648fd96302c948db616761e0b4137401beae0f7add2db", - "dbt_det/seeds/raw/order_items.csv": "81bcd163cc42c3474dfcf5201634164189bd3ba19c7cb05fce833c80b2d81a55", - "dbt_det/seeds/raw/orders.csv": "0c7f5319bb652bd73a6b89649825377e875c8c166086c299b971ee8d760e375f", - "dbt_det/seeds/raw/orders_history.csv": "755df44c63bbdfa922276db2382e5b9cfb9603e6fc02f5c72552b9aaa14e32e8", - "dbt_det/seeds/raw/product_reviews.csv": "f9f33d24a151c6ff680ce5c4521f87418284a4de045dcb0eacb11fba048b7324", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "70e43d36b97a9b805aba04eb1d8a57dfca6618c7f652f1f8883aca1a8c366854", + "dbt_det/seeds/raw/order_items.csv": "1f40a7f749ff9f692b3b641d1519b3a59d77a75c09c500820ee6a5836cf3e02e", + "dbt_det/seeds/raw/orders.csv": "60cba64af43039d501e98115dcb93d977566f5aa7502f49f27464ab6d5e4f494", + "dbt_det/seeds/raw/orders_history.csv": "a3a359b615b4c84c2917e602eeeb319af9e3d7bc5d4333e57a0fb04a3e689e84", + "dbt_det/seeds/raw/product_reviews.csv": "49ba5a52aa90dd18f3a12a15afd1d1f954a10e0572b084f8c94c89255c553e2b", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "preset-clean": { - "dbt_det/EXPECTED_FAILURES.md": "14e2890fcde05b3d20274fb5a531e0f57b92864ccbd54c78ffbe1cea52c68942", - "dbt_det/days/customers/day_000.csv": "69f9b44ab0cfdf2f6bca58f63f2be044b749c9f1cc80501a8355c4d3dc915104", - "dbt_det/days/customers/day_001.csv": "220e7214482b5ac03b117c6d5c0fa2007f8ce17e1dcc9fc4c253a41df6acaa28", - "dbt_det/days/customers/day_002.csv": "516bfb259fb8031023b68ef02dc8118a68952b63d1f5e2cdfc1822cde21c8987", - "dbt_det/days/customers/day_003.csv": "f7c6ad82ef5708829656a8617ebb9d926a1bead8b3f593bce09278ae66a0e2db", - "dbt_det/days/order_items/day_000.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/days/order_items/day_001.csv": "c745a9e7052352d861d82b57b5a58371f3d026ecb739125deb60a00237e03b18", - "dbt_det/days/order_items/day_002.csv": "5a39f76aa79e2af21fbbb25b1fbe60f1bfe0e2b7a4714c09d12a4639484e928d", - "dbt_det/days/order_items/day_003.csv": "375ceb438e8e165920f17c22852d47cdab21602ad98741d8ff1d6cdd1a451be4", - "dbt_det/days/orders/day_000.csv": "969ed7ce4f3e94e1ea5f70eb05f3e79c9cdd9ec199acf3d5f4917fb01c6fdf24", - "dbt_det/days/orders/day_001.csv": "f98eb9b2adbd4881b1e523b3021add667086bb410d2a529ba7e89696b97db77e", - "dbt_det/days/orders/day_002.csv": "425efd6753c4156f87650a4b61a743ff262e2ab40c5664373cf9643cfc864673", - "dbt_det/days/orders/day_003.csv": "7c3d1f62995b18e8d962af6be3cbddf75c63e0726f27f7c42b096e995cc79fdc", - "dbt_det/days/product_reviews/day_000.csv": "f9f33d24a151c6ff680ce5c4521f87418284a4de045dcb0eacb11fba048b7324", - "dbt_det/days/products/day_000.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3", - "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/defects_report.json": "12c2a1f0372d2518164fb1d870ab4273f836715fa8bf65e42dac461d9f8162e8", + "dbt_det/EXPECTED_FAILURES.md": "8463898366a9a499301153abfc47c62d642344bc16150bf2bbde2382ac87e675", + "dbt_det/days/customers/day_000.csv": "f046d54585e9a33afc602a26253d2143962de3162bbfa4a7897ca2052bf9f04c", + "dbt_det/days/customers/day_001.csv": "ccee1dbf6f47a98326b8665e1d2654cbd68fb52761055bf4352e3acc0a37982d", + "dbt_det/days/customers/day_002.csv": "ffe4d6e2022d07c96acea4f43a80eff1e1054bf8582d86a838fee5d8ce0ab57b", + "dbt_det/days/customers/day_003.csv": "6309b52d27191ef0228fc6aa9e4fef63bc020dde3433b622db0587a2e34e6bc6", + "dbt_det/days/order_items/day_000.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/days/order_items/day_001.csv": "9329a9f8a2c87ade72747cb20955c84cf21feb7042ff0c272c3610cac7918332", + "dbt_det/days/order_items/day_002.csv": "914e65e5be574ffb7ffe9e80066334c9fb5d00b50037636dc683d4112a4191f2", + "dbt_det/days/order_items/day_003.csv": "06ec522cbee32a75cc9a03b1774e78f970f307b8b98f0ab0c10c94e9b123d212", + "dbt_det/days/orders/day_000.csv": "2c1bbab44be2fb1a46c78878945c319f2035a13792d1cc2d6d3ec54741dfd8d5", + "dbt_det/days/orders/day_001.csv": "df0ac1c8a20cea535ae074ff64d513270a0e5b45956e1cd55809a29431a2d1ea", + "dbt_det/days/orders/day_002.csv": "13627253fff83faaa8d42f666a00370f316e032735795f420f700273387809c3", + "dbt_det/days/orders/day_003.csv": "e1bdcfb55f762a300b0f50177c2e8b7460520e340f5fe4c5ae4a0ae47e34d227", + "dbt_det/days/product_reviews/day_000.csv": "49ba5a52aa90dd18f3a12a15afd1d1f954a10e0572b084f8c94c89255c553e2b", + "dbt_det/days/products/day_000.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747", + "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", + "dbt_det/defects_report.json": "1d1826d7a4de48153a39ca9723ddf7328f7497c023225bee1a77c20581182858", "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "fe63c9f2fea411ec373b19a500e90c4e0c0b7352ba2d4591a33cbcf4a138ed7c", + "dbt_det/models/staging/stg_orders.yml": "2a2ace25b7164723120ba0b2b7def4839dd286a17ccfb5990f304070bbe05d77", "dbt_det/models/staging/stg_orders_history.sql": "937bea699b5df5da6808580fe3963f432163f3741afc0f71275e5261d81f655b", "dbt_det/models/staging/stg_orders_history.yml": "b6bc4e53dc5078994b19f94b38371ea27308a452cccf351bfd43b6c489e2152a", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", @@ -664,41 +686,42 @@ "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "03fbba2c4026678a9eef7cfe48aeaf3391a2919b05b1f67b2744ee0891ff199f", - "dbt_det/seeds/raw/customers.csv": "b2d24583a298151383a0763a5eb2091cbca6ff8568d53c840334e8706e1b8c06", - "dbt_det/seeds/raw/order_items.csv": "e5cf5db6cdf6130d8664f347d0b681186c5c14892ea41b35305dd86e395aec90", - "dbt_det/seeds/raw/orders.csv": "6f28818d91185565349ef60ce47a8403b6fc7a0996f51b2185c1360d428d7d52", - "dbt_det/seeds/raw/orders_history.csv": "47814c72399664c3b382c976ab785c6727d419b21024ab2c63f6134d3635e0e9", - "dbt_det/seeds/raw/product_reviews.csv": "f9f33d24a151c6ff680ce5c4521f87418284a4de045dcb0eacb11fba048b7324", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "2814de2606aa15ef87e0c26e20d01792acf8174bfa638f4efaa4400e5447dd58", + "dbt_det/seeds/raw/order_items.csv": "4d443f4acd43433a879d8062d6446a6adfdc1ad03baa0db0aede26402bcacea2", + "dbt_det/seeds/raw/orders.csv": "773279d4bb00b4982293166fa270b490daaa089977e387a0afb09448560c99a3", + "dbt_det/seeds/raw/orders_history.csv": "4f7978148b95474b75fa122ea4cfefb51394e76f1f8fc6763d999c1385630a0b", + "dbt_det/seeds/raw/product_reviews.csv": "49ba5a52aa90dd18f3a12a15afd1d1f954a10e0572b084f8c94c89255c553e2b", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "preset-messy": { - "dbt_det/EXPECTED_FAILURES.md": "e5a52342918e110d2dea0bb3b9ab17289cfa682829ecb7732d52c43122511971", - "dbt_det/days/customers/day_000.csv": "f2a401d9f6813080208ca9986e3d894b52af7b0c127dd995dde132c63aa37b79", - "dbt_det/days/customers/day_001.csv": "220e7214482b5ac03b117c6d5c0fa2007f8ce17e1dcc9fc4c253a41df6acaa28", - "dbt_det/days/customers/day_002.csv": "d0cf754369be62c463f9618c147bfcbeeac06bf65d71b8d6029e00fe008eee47", - "dbt_det/days/customers/day_003.csv": "88f92d5f433bb039bfe7c86e51aab71aa9ece09a51b187790f8c78ea931afa18", - "dbt_det/days/order_items/day_000.csv": "2193db1c2e1dc4f610a5099647c9a76bb567ab33c97b05d892852d0dc1f2f27b", - "dbt_det/days/order_items/day_001.csv": "e7c51b2fcc79539a9b092c9d321fb6c76b230e1909239f52c068987d829d4bef", - "dbt_det/days/order_items/day_002.csv": "3cf040fdf8010b43ccfb820eba4ded0020658ec6c2570891c798a2fb14c5a59c", - "dbt_det/days/order_items/day_003.csv": "ba8b60a2ee747afaefc95eebf6795960678df3145939900aa3136baeb38e354c", - "dbt_det/days/orders/day_000.csv": "61724e464bd9883c7e91e46d05791be8ee8225ea4106bbc6f6ab43492c386ed3", - "dbt_det/days/orders/day_001.csv": "f98eb9b2adbd4881b1e523b3021add667086bb410d2a529ba7e89696b97db77e", - "dbt_det/days/orders/day_002.csv": "0609bc6d4cd596bc7c6a66ab659b2726f157f24d24170b3f27aea2292f0ad1c3", - "dbt_det/days/orders/day_003.csv": "804de3678aa932f52a2e66ee7eb93d03772cdf1ec8e6a61966c205ae9a45f80c", - "dbt_det/days/product_reviews/day_000.csv": "8f5134e01eda3a72d3a751342b3efc861f505163c85e06aa54e2f0dc888bd22b", - "dbt_det/days/products/day_000.csv": "b9fe660dacb738033adddff7f8b6861933899991687d1fe0e93bfaa53d49d5f4", - "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/defects_report.json": "fd73fc8cc81e795e0f09d23071af20f19293f3205d6a2548472559853dcd902a", + "dbt_det/EXPECTED_FAILURES.md": "f7e999862942a6b3ea41096fe27391d8363698c4b1aa14a0dac2673b06d496fd", + "dbt_det/days/customers/day_000.csv": "15ddf8182eb3fbaeea26da57e8eebce9e68b8eb4989726b9dc28d6c22a9c554a", + "dbt_det/days/customers/day_001.csv": "4addd39e087139349719ca3b40b29a46d98c25eaeb0022983ba87dc4ab6d5973", + "dbt_det/days/customers/day_002.csv": "9924635ab7f99db005710c2c9810af2799b3ad2f76e760946bf03c59cdee6dbe", + "dbt_det/days/customers/day_003.csv": "6309b52d27191ef0228fc6aa9e4fef63bc020dde3433b622db0587a2e34e6bc6", + "dbt_det/days/order_items/day_000.csv": "c786f85922da9e68311974b80c92800ee6e9a12d40ddf2d811e028d8369511f7", + "dbt_det/days/order_items/day_001.csv": "f5d9a3e3ff4df2ccc176a12e116066d5679ef5fd784e7cdce54e0cd5c5d31c28", + "dbt_det/days/order_items/day_002.csv": "32e34229c8c5d45527faa085667853ec3f781fc55e7053086ee0d7fd7ba6bd62", + "dbt_det/days/order_items/day_003.csv": "5eaf31664c70b4c2ceaa90a5972411d2fc98b9958315b4b7ac0289421e91e7f6", + "dbt_det/days/orders/day_000.csv": "bc379db11fccdc30ca214fa6a9391b86769bda8750f6a7be2f339cdbcce25561", + "dbt_det/days/orders/day_001.csv": "df0ac1c8a20cea535ae074ff64d513270a0e5b45956e1cd55809a29431a2d1ea", + "dbt_det/days/orders/day_002.csv": "19d6473bc44d8d7f35cbf5a7d05f87833e215250eacba725806e8d68572bed1e", + "dbt_det/days/orders/day_003.csv": "2502136f1bda3fc977f79f5c23a727875d1e5d9ae4542c5263af853ee1027636", + "dbt_det/days/product_reviews/day_000.csv": "bcfe616185bd1a980da65f26b02ec6e14331c358a1ec1d8fb038609c432e06b6", + "dbt_det/days/products/day_000.csv": "5a494e7fbd0e86dca2ed3d22d57e000f4b774206cfa2bcfeab0e3aae6713b589", + "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", + "dbt_det/defects_report.json": "34b8c5d248468829a7e69654b43c45d42b16c5c4c1c7f185a175533790ba096e", "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "fe63c9f2fea411ec373b19a500e90c4e0c0b7352ba2d4591a33cbcf4a138ed7c", + "dbt_det/models/staging/stg_orders.yml": "2a2ace25b7164723120ba0b2b7def4839dd286a17ccfb5990f304070bbe05d77", "dbt_det/models/staging/stg_orders_history.sql": "937bea699b5df5da6808580fe3963f432163f3741afc0f71275e5261d81f655b", "dbt_det/models/staging/stg_orders_history.yml": "b6bc4e53dc5078994b19f94b38371ea27308a452cccf351bfd43b6c489e2152a", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", @@ -707,39 +730,40 @@ "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "03fbba2c4026678a9eef7cfe48aeaf3391a2919b05b1f67b2744ee0891ff199f", - "dbt_det/seeds/raw/customers.csv": "a9a807951dc51e6ac05b4d12bd59f4e6a7d83dffcf93f696db819e7f01d53caa", - "dbt_det/seeds/raw/order_items.csv": "39793c3a378fe846d23b09d9ce8b7a96dcc8d3a954b3de16a18bc1dcddf17adf", - "dbt_det/seeds/raw/orders.csv": "580c9fdf10fd47299a94ac79c472e59c65167634da9e6e0444f0b82878e0466a", - "dbt_det/seeds/raw/orders_history.csv": "22f6a4408c9fa2f18aa9218dcdb5d3fad95bd1e4736cdd2c595286a2b0e63e63", - "dbt_det/seeds/raw/product_reviews.csv": "8f5134e01eda3a72d3a751342b3efc861f505163c85e06aa54e2f0dc888bd22b", - "dbt_det/seeds/raw/products.csv": "b9fe660dacb738033adddff7f8b6861933899991687d1fe0e93bfaa53d49d5f4" + "dbt_det/seeds/raw/customers.csv": "f160037fab572e68f2f5eb5539e1e159f6d7593d19aadda8ed4e6a89ca9b0a02", + "dbt_det/seeds/raw/order_items.csv": "1f661f73c051fe8dc47180a6ef571467f06fd4b8fd8004e1348108054cd3804f", + "dbt_det/seeds/raw/orders.csv": "82c447a2cfad4fa6df60d4e9073d4fc8fc9efd98a8efaf8ccd2c8b540318b394", + "dbt_det/seeds/raw/orders_history.csv": "8f75aa3bb9d562b29a6c13646ec8c87957ddb56778c58b84c5045771caccbf14", + "dbt_det/seeds/raw/product_reviews.csv": "bcfe616185bd1a980da65f26b02ec6e14331c358a1ec1d8fb038609c432e06b6", + "dbt_det/seeds/raw/products.csv": "5a494e7fbd0e86dca2ed3d22d57e000f4b774206cfa2bcfeab0e3aae6713b589" }, "preset-none": { - "dbt_det/days/customers/day_000.csv": "0ae988dbb32bcba86047756c3fa4cb22cb74d4b5e6dcc604c18cb56e0507ccf8", - "dbt_det/days/customers/day_001.csv": "e5472ddc4da3d0eb647832e1c3bbc64d85c5b26bfad783e03b37c8c328ecae6e", - "dbt_det/days/customers/day_002.csv": "2772ecdd0256bd955a1e11aa9b5164397d9f1ceee2f9e7dc7a30ed38f22867df", - "dbt_det/days/customers/day_003.csv": "f7c6ad82ef5708829656a8617ebb9d926a1bead8b3f593bce09278ae66a0e2db", - "dbt_det/days/order_items/day_000.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/days/order_items/day_001.csv": "c745a9e7052352d861d82b57b5a58371f3d026ecb739125deb60a00237e03b18", - "dbt_det/days/order_items/day_002.csv": "5a39f76aa79e2af21fbbb25b1fbe60f1bfe0e2b7a4714c09d12a4639484e928d", - "dbt_det/days/order_items/day_003.csv": "375ceb438e8e165920f17c22852d47cdab21602ad98741d8ff1d6cdd1a451be4", - "dbt_det/days/orders/day_000.csv": "969ed7ce4f3e94e1ea5f70eb05f3e79c9cdd9ec199acf3d5f4917fb01c6fdf24", - "dbt_det/days/orders/day_001.csv": "f98eb9b2adbd4881b1e523b3021add667086bb410d2a529ba7e89696b97db77e", - "dbt_det/days/orders/day_002.csv": "425efd6753c4156f87650a4b61a743ff262e2ab40c5664373cf9643cfc864673", - "dbt_det/days/orders/day_003.csv": "7c14789029323a747f9eedb545351a47a5f6f6d032376278020ee8dc24f642f0", - "dbt_det/days/product_reviews/day_000.csv": "e7bd78f152f82b650b2392a38e2bac3f6837a8f3b1cb7818a3f834d88a897768", - "dbt_det/days/products/day_000.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3", + "dbt_det/days/customers/day_000.csv": "2b34b876f615650d684d7ba23e0cf0f63d8f6addc7f9798c2aa16735c9b6a45d", + "dbt_det/days/customers/day_001.csv": "cc77e19974d959378cb83527952efe5bb16a684b0a1eb1532b82d4dcd02ccdbe", + "dbt_det/days/customers/day_002.csv": "c04d968bdf4c4c4f453435a18ff0caefc93fd2733c07d18975d2cf964bb59888", + "dbt_det/days/customers/day_003.csv": "6309b52d27191ef0228fc6aa9e4fef63bc020dde3433b622db0587a2e34e6bc6", + "dbt_det/days/order_items/day_000.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/days/order_items/day_001.csv": "9329a9f8a2c87ade72747cb20955c84cf21feb7042ff0c272c3610cac7918332", + "dbt_det/days/order_items/day_002.csv": "914e65e5be574ffb7ffe9e80066334c9fb5d00b50037636dc683d4112a4191f2", + "dbt_det/days/order_items/day_003.csv": "06ec522cbee32a75cc9a03b1774e78f970f307b8b98f0ab0c10c94e9b123d212", + "dbt_det/days/orders/day_000.csv": "2c1bbab44be2fb1a46c78878945c319f2035a13792d1cc2d6d3ec54741dfd8d5", + "dbt_det/days/orders/day_001.csv": "df0ac1c8a20cea535ae074ff64d513270a0e5b45956e1cd55809a29431a2d1ea", + "dbt_det/days/orders/day_002.csv": "13627253fff83faaa8d42f666a00370f316e032735795f420f700273387809c3", + "dbt_det/days/orders/day_003.csv": "a28ec5c14974ddf2acc6d725c362af49fce437eeef670f1093152c20e8e12503", + "dbt_det/days/product_reviews/day_000.csv": "95f9a92f499d61ebc0d9747dd27474e1a5d96eaf3cec91b3d36d1a9b2ceb3b07", + "dbt_det/days/products/day_000.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "fe63c9f2fea411ec373b19a500e90c4e0c0b7352ba2d4591a33cbcf4a138ed7c", + "dbt_det/models/staging/stg_orders.yml": "2a2ace25b7164723120ba0b2b7def4839dd286a17ccfb5990f304070bbe05d77", "dbt_det/models/staging/stg_orders_history.sql": "937bea699b5df5da6808580fe3963f432163f3741afc0f71275e5261d81f655b", "dbt_det/models/staging/stg_orders_history.yml": "b6bc4e53dc5078994b19f94b38371ea27308a452cccf351bfd43b6c489e2152a", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", @@ -748,41 +772,42 @@ "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "03fbba2c4026678a9eef7cfe48aeaf3391a2919b05b1f67b2744ee0891ff199f", - "dbt_det/seeds/raw/customers.csv": "4e8efb8a7297acf6108fed71f3775d33944e32db1ee770bfc4b872c742a8af33", - "dbt_det/seeds/raw/order_items.csv": "e5cf5db6cdf6130d8664f347d0b681186c5c14892ea41b35305dd86e395aec90", - "dbt_det/seeds/raw/orders.csv": "17fd9439472001b34f73a644d97c9296aa0cfed71a56c3832388a246bc9c9c4f", - "dbt_det/seeds/raw/orders_history.csv": "47814c72399664c3b382c976ab785c6727d419b21024ab2c63f6134d3635e0e9", - "dbt_det/seeds/raw/product_reviews.csv": "e7bd78f152f82b650b2392a38e2bac3f6837a8f3b1cb7818a3f834d88a897768", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "65d4e3f90d055d1942d3bd037e8a5c6a398c928331c697d19e5082ec56674265", + "dbt_det/seeds/raw/order_items.csv": "4d443f4acd43433a879d8062d6446a6adfdc1ad03baa0db0aede26402bcacea2", + "dbt_det/seeds/raw/orders.csv": "259622a00b29620dcd7f308941aa394918f71c76ff67ad71ca35c30cbe9cc653", + "dbt_det/seeds/raw/orders_history.csv": "4f7978148b95474b75fa122ea4cfefb51394e76f1f8fc6763d999c1385630a0b", + "dbt_det/seeds/raw/product_reviews.csv": "95f9a92f499d61ebc0d9747dd27474e1a5d96eaf3cec91b3d36d1a9b2ceb3b07", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "preset-training": { - "dbt_det/EXPECTED_FAILURES.md": "12decde04d0cf12381076e1d9b6690b266dfc3612ddef08896757c4d8c6222a8", - "dbt_det/days/customers/day_000.csv": "b0f8a38e0fb85d2a5ebd8ca80d56d9439b5938cbce6b4013bf91c4fb24502552", - "dbt_det/days/customers/day_001.csv": "220e7214482b5ac03b117c6d5c0fa2007f8ce17e1dcc9fc4c253a41df6acaa28", - "dbt_det/days/customers/day_002.csv": "f23b8f5fdc0e9abdb5efd01f6090d5c4f10d99fd47b07e228cd882bffa4c9738", - "dbt_det/days/customers/day_003.csv": "f7c6ad82ef5708829656a8617ebb9d926a1bead8b3f593bce09278ae66a0e2db", - "dbt_det/days/order_items/day_000.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/days/order_items/day_001.csv": "c745a9e7052352d861d82b57b5a58371f3d026ecb739125deb60a00237e03b18", - "dbt_det/days/order_items/day_002.csv": "be071432489a9a0b706e7752798b202bad31799e6c5b65398135a45af2bb0e01", - "dbt_det/days/order_items/day_003.csv": "a5dba62edd0d374bd10e3b5f6819e0d6ee2ea8ccc9e5a4d2ee18d487068d470f", - "dbt_det/days/orders/day_000.csv": "fea5d7ff9d1b95bbe3f2b671ac1e4aba638bf46ccb93f6ee525adfad6045c9c7", - "dbt_det/days/orders/day_001.csv": "f98eb9b2adbd4881b1e523b3021add667086bb410d2a529ba7e89696b97db77e", - "dbt_det/days/orders/day_002.csv": "425efd6753c4156f87650a4b61a743ff262e2ab40c5664373cf9643cfc864673", - "dbt_det/days/orders/day_003.csv": "804de3678aa932f52a2e66ee7eb93d03772cdf1ec8e6a61966c205ae9a45f80c", - "dbt_det/days/product_reviews/day_000.csv": "f9f33d24a151c6ff680ce5c4521f87418284a4de045dcb0eacb11fba048b7324", - "dbt_det/days/products/day_000.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3", - "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/defects_report.json": "e3455249e86dd1526d8ce9bd63b1f128221a92fac7e9ed471742a39f567ba425", + "dbt_det/EXPECTED_FAILURES.md": "836eaa2dd21c37b3a580191a63c753b816520379c2d24091a91aa1e314f5ebfa", + "dbt_det/days/customers/day_000.csv": "e22a98c316139176e90be9fe56f147b5721b493588037382bf2cd619f140eec4", + "dbt_det/days/customers/day_001.csv": "ccee1dbf6f47a98326b8665e1d2654cbd68fb52761055bf4352e3acc0a37982d", + "dbt_det/days/customers/day_002.csv": "09e1eb40e21651e6080f55b58a0b5e328be5d82649d29a0b59c9d71174abcf19", + "dbt_det/days/customers/day_003.csv": "6309b52d27191ef0228fc6aa9e4fef63bc020dde3433b622db0587a2e34e6bc6", + "dbt_det/days/order_items/day_000.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/days/order_items/day_001.csv": "9329a9f8a2c87ade72747cb20955c84cf21feb7042ff0c272c3610cac7918332", + "dbt_det/days/order_items/day_002.csv": "c6845c6c213ab09d005a750e3b83816f71f623fc4144ff0ea83983618b7c7d02", + "dbt_det/days/order_items/day_003.csv": "6eb1f4727a7fea03eaf9454ab82f12d4d9224a8695a0a725883642e8f8b2fd28", + "dbt_det/days/orders/day_000.csv": "f920c175a6151d78b2a33cf5ccaf7c10c32108d4d7a09aedbcf2729b7841dba7", + "dbt_det/days/orders/day_001.csv": "df0ac1c8a20cea535ae074ff64d513270a0e5b45956e1cd55809a29431a2d1ea", + "dbt_det/days/orders/day_002.csv": "13627253fff83faaa8d42f666a00370f316e032735795f420f700273387809c3", + "dbt_det/days/orders/day_003.csv": "bc6b4d98aedd12a0f296bdca6fcf4128d4845d2b58f4e1d78d1578cd90388d43", + "dbt_det/days/product_reviews/day_000.csv": "49ba5a52aa90dd18f3a12a15afd1d1f954a10e0572b084f8c94c89255c553e2b", + "dbt_det/days/products/day_000.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747", + "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", + "dbt_det/defects_report.json": "a044d1215bd9a47ee1c430f6cbf0344ee8aa3ac202bab5e7e2afd79094c54cc3", "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "fe63c9f2fea411ec373b19a500e90c4e0c0b7352ba2d4591a33cbcf4a138ed7c", + "dbt_det/models/staging/stg_orders.yml": "2a2ace25b7164723120ba0b2b7def4839dd286a17ccfb5990f304070bbe05d77", "dbt_det/models/staging/stg_orders_history.sql": "937bea699b5df5da6808580fe3963f432163f3741afc0f71275e5261d81f655b", "dbt_det/models/staging/stg_orders_history.yml": "b6bc4e53dc5078994b19f94b38371ea27308a452cccf351bfd43b6c489e2152a", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", @@ -791,91 +816,94 @@ "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "03fbba2c4026678a9eef7cfe48aeaf3391a2919b05b1f67b2744ee0891ff199f", - "dbt_det/seeds/raw/customers.csv": "a60c623aa039f7e29411366b11ec2cea274bf22fd2c43c5bea32222eef2c7754", - "dbt_det/seeds/raw/order_items.csv": "8603705932635a30cc681472ce98a12c285301285faf387ad8d06f8d182922f5", - "dbt_det/seeds/raw/orders.csv": "840e5997eed6a9aad2d239f0adeca14956de06d01d30d08a5ef94498a1258a50", - "dbt_det/seeds/raw/orders_history.csv": "22f6a4408c9fa2f18aa9218dcdb5d3fad95bd1e4736cdd2c595286a2b0e63e63", - "dbt_det/seeds/raw/product_reviews.csv": "f9f33d24a151c6ff680ce5c4521f87418284a4de045dcb0eacb11fba048b7324", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "f7e6e58f0584d300d7beb309a65a7bc7322f0768dd8463f0af194e59de3de24c", + "dbt_det/seeds/raw/order_items.csv": "3f7810b2156a3261bcdbf50ff201b12e7690ef2bcec0051c1743e84deeb87113", + "dbt_det/seeds/raw/orders.csv": "d34df7da47e80a0aa67f644a13f23274f655849cc2fbd01efca37ea561e58f34", + "dbt_det/seeds/raw/orders_history.csv": "8f75aa3bb9d562b29a6c13646ec8c87957ddb56778c58b84c5045771caccbf14", + "dbt_det/seeds/raw/product_reviews.csv": "49ba5a52aa90dd18f3a12a15afd1d1f954a10e0572b084f8c94c89255c553e2b", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "skew": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_api_usage.sql": "3a453410ffeba21c7c299fe9144f762aab2565cd9f02b32ed3fc6a25c4a7cd4b", "dbt_det/models/staging/stg_api_usage.yml": "67de79479deb6266fc70d5e38651b730a19c2c2ae4641c72ad2a1e2e685d1ccd", "dbt_det/models/staging/stg_audit_events.sql": "062020dbc347feeb7b7b7e9c47e8112691cd434d839d783d147fbbd811c5f040", - "dbt_det/models/staging/stg_audit_events.yml": "cb9f16b3ec30544c0166905dde97589404cd5f653b75d73895971de56762f9c8", + "dbt_det/models/staging/stg_audit_events.yml": "6c6a5d9d56bf277782a723e9e0df2577d3afb46a0370524abb6321bbc4f11320", "dbt_det/models/staging/stg_feature_limits.sql": "8dd6ccb9ee9da75f8102460879f2da89f541e5fc71336b29b9e2372ee917de1d", "dbt_det/models/staging/stg_feature_limits.yml": "381963d063dca0966d5aef605a04b1bb9ca39b6c36fc35e9c53f931ec11dce90", "dbt_det/models/staging/stg_invoices.sql": "6a282cd534a83387c28062ec83924bca249e8466659ed4b7614652a4853e609d", - "dbt_det/models/staging/stg_invoices.yml": "7b72a9cf5ecd110f8a998558f76707cef23fc5a53220cc0b76a1796220dd772e", + "dbt_det/models/staging/stg_invoices.yml": "bde339657aae0e12e3dc820b1f08989ca31719511625ead8189cf7edad5c1089", "dbt_det/models/staging/stg_organizations.sql": "7f29bdc034068d313c826a71bff56d49bfae6039108d376f6034de8c97c9bb19", "dbt_det/models/staging/stg_organizations.yml": "3bcf56cc79a94657ba0c6132ee1eaff87f89b7d3f099c839ade87146e24c4ee5", "dbt_det/models/staging/stg_subscriptions.sql": "2a2d9ee7ba0ae44b610935af21bd4cdd1a38986e1f4fca772fb6775296b0377e", - "dbt_det/models/staging/stg_subscriptions.yml": "30320554d1087f4b2b905d2a8307966d04e1b8cd078b439d52deea368c40d8b1", + "dbt_det/models/staging/stg_subscriptions.yml": "308fc79cb2f277a28ebffd69aab4c9e578ace1512e0c37e7f503b05baf394e63", "dbt_det/models/staging/stg_users.sql": "0327dec706116d8aeb53093adad11614d9b456bc574276a9f44fbaa906be4e65", "dbt_det/models/staging/stg_users.yml": "b162ae9b22d7b026e9f546b58ad06f8bc9313351e2fe3d1fa4fd55eb235e3060", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/saas_platform.model2data.yml": "23a8b7391c75e50ecc975ae77ee8abe0507093492b73d65b990b2c7c82b4e42c", "dbt_det/seeds/raw/__seed_config.yml": "b87c6d38dae29df61e3c7bd66560b0ec505833a04a7e784e3b22142c78f3c4b7", - "dbt_det/seeds/raw/api_usage.csv": "012897da91eba694fb7ca82f7a4cb8a5a8eb5a0bed7a4af0ef3691f5c412036c", - "dbt_det/seeds/raw/audit_events.csv": "a9d6d60a7f0bcba28a1928720da2bd5cfcc9a4bed296eafbc9e1eddd7381475c", - "dbt_det/seeds/raw/feature_limits.csv": "c991751064bf749aa1277c53eca6ba4e5941566f6ab1b083d3e49d14eb4ffab2", - "dbt_det/seeds/raw/invoices.csv": "19489a185cd13149b16f76e62e6b508de437d67648c30f7e6d753bd305ec6268", - "dbt_det/seeds/raw/organizations.csv": "bb9019c41083da011f0512d4a5ea4225fa766a8c9d3feaabdfb6713d19a412e0", - "dbt_det/seeds/raw/subscriptions.csv": "02bb173269e98ca870de3cfc875f23b54630cc11a6162d35d0640ae4193d0296", - "dbt_det/seeds/raw/users.csv": "d55263c0ea25908f11d2a300106749af4022e2f05b0a7e452ef171f2e707ea46" + "dbt_det/seeds/raw/api_usage.csv": "b4bc4b9edc8201abd197aeef3ba03518e33e8c6f0ae5892168e5b417181b6da0", + "dbt_det/seeds/raw/audit_events.csv": "8f67794907cfbe622d8335c52df9c204adf3b96db213b06211e437b981ebea2c", + "dbt_det/seeds/raw/feature_limits.csv": "ddd8ec06508a87a67d6dc3beddd1539b19bdd8617373a6cb88f04a801b69e5d6", + "dbt_det/seeds/raw/invoices.csv": "a266716814be82177203c78fa0ec5f42dc4b05f8f94ff76a32e2598369aa0dc7", + "dbt_det/seeds/raw/organizations.csv": "b12a3a33bd8558d15c4e79f3c1284af6f2101e80bab540ea3457540d5a7abb8e", + "dbt_det/seeds/raw/subscriptions.csv": "13fa0e6f821336ed0afb1721f2d30e83c88938d63e876ebcfaa47e2a234bbe2f", + "dbt_det/seeds/raw/users.csv": "09fdc161f74d9cfa0fa3104973221a5b3ea750009ae20ad3901086e7c850d583" }, "table-seed": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "88ed2f885637e38367e119250b09f764e4570e8ef244ae4bdfa55e0d832d466a", + "dbt_det/models/staging/stg_orders.yml": "cba055fdb3f2d6145cddcf8696242fbfb34463577e681284eca78e6f11b96ede", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", "dbt_det/models/staging/stg_product_reviews.yml": "e7692c4d586af9ea57c9afbb421d5424be907fb28f4464bf82f75033b402ef48", "dbt_det/models/staging/stg_products.sql": "7455128b10e0357213f26c209913a73df3f417be28ddd9d4dd8681390e7dcdab", "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "f10588f38f838c163af5a58349670e4abc2e753619f1ae623b98ef2758d8be08", - "dbt_det/seeds/raw/customers.csv": "0ae988dbb32bcba86047756c3fa4cb22cb74d4b5e6dcc604c18cb56e0507ccf8", - "dbt_det/seeds/raw/order_items.csv": "30fb11a3cc39df8efb4f0e12a712c270e216dce741d6e73b94626810741a950e", - "dbt_det/seeds/raw/orders.csv": "a128378bb766b4a8f451e6993fd38d1c5869ffdbfe343b2fdc1815f8d3a529ed", - "dbt_det/seeds/raw/product_reviews.csv": "e7bd78f152f82b650b2392a38e2bac3f6837a8f3b1cb7818a3f834d88a897768", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "2b34b876f615650d684d7ba23e0cf0f63d8f6addc7f9798c2aa16735c9b6a45d", + "dbt_det/seeds/raw/order_items.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/seeds/raw/orders.csv": "520186d54c48c0b24dab78ba13d8ca2301a2e37da63d7bec8a6436ddb95c1637", + "dbt_det/seeds/raw/product_reviews.csv": "95f9a92f499d61ebc0d9747dd27474e1a5d96eaf3cec91b3d36d1a9b2ceb3b07", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "unit-tests": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", + "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", "dbt_det/models/staging/stg_order_items.yml": "2d90adc3a2e3beeddab1b0b8a0140309a1bb5b667337d1e187b2ab40333cd966", "dbt_det/models/staging/stg_orders.sql": "ce9432edd2e51bd8d6caa6e887268ae57ebad6041881548e140e172fc4420c91", - "dbt_det/models/staging/stg_orders.yml": "88ed2f885637e38367e119250b09f764e4570e8ef244ae4bdfa55e0d832d466a", + "dbt_det/models/staging/stg_orders.yml": "cba055fdb3f2d6145cddcf8696242fbfb34463577e681284eca78e6f11b96ede", "dbt_det/models/staging/stg_product_reviews.sql": "939f77269e38a2b8265ce001e62a8194fd5335d340f97168a67f6915c8881f75", "dbt_det/models/staging/stg_product_reviews.yml": "e7692c4d586af9ea57c9afbb421d5424be907fb28f4464bf82f75033b402ef48", "dbt_det/models/staging/stg_products.sql": "7455128b10e0357213f26c209913a73df3f417be28ddd9d4dd8681390e7dcdab", "dbt_det/models/staging/stg_products.yml": "56af441c9582505bf71fe86b1a6e79fc83db957b945db4f1a5ab1775aecb4e90", - "dbt_det/models/staging/ut_stg_customers.yml": "01243a26d9fee45d1191ba99bdecf627b3db55e5348c19c61f5808e851b2e1e7", - "dbt_det/models/staging/ut_stg_order_items.yml": "bbe8876ac59790159931a024ed93f1e57156e7c758fbab73e24dcab9bc20d81d", - "dbt_det/models/staging/ut_stg_orders.yml": "a3ae649e2a103fafd05e1ed5c31d9b41652915aba8f03a9a7b71ca0b8d16e218", - "dbt_det/models/staging/ut_stg_product_reviews.yml": "b0fcdcaf25839f6777d80bc3bb7c6edd3131b00a9211169f692ee527f5ed38d1", - "dbt_det/models/staging/ut_stg_products.yml": "177fcfbb24678c60d441abf9600f721aa71e7fe7b031e0cd18d6ec4ee238be38", + "dbt_det/models/staging/ut_stg_customers.yml": "6f000efdb9149b85c510481ced1740a311c5f8824d16a369744d52a9fe043287", + "dbt_det/models/staging/ut_stg_order_items.yml": "dd4bc143ad130bca077454784f2fd922b18cf8d6b938082baeefe920aee809f0", + "dbt_det/models/staging/ut_stg_orders.yml": "521350235183639006cb7f0b98c464fd69d87bf912afb814a16c3b27fb7dec4b", + "dbt_det/models/staging/ut_stg_product_reviews.yml": "5297e7929bfdbed3013f60c4be64679e5590f94881c91b5f994295c7723eddc6", + "dbt_det/models/staging/ut_stg_products.yml": "344214cf16d2db60e633acf51e46ba1bdd57a6eb2219bd433f39f23f7eadbb1f", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "f10588f38f838c163af5a58349670e4abc2e753619f1ae623b98ef2758d8be08", - "dbt_det/seeds/raw/customers.csv": "0ae988dbb32bcba86047756c3fa4cb22cb74d4b5e6dcc604c18cb56e0507ccf8", - "dbt_det/seeds/raw/order_items.csv": "77134c2df8747a7fd348846a2cc7a233f6e0b4a8159204ec86aa7484747837c0", - "dbt_det/seeds/raw/orders.csv": "a9f5c94298c8d5d7a90a9a5d8e95f6a5adcaac3fc56badf830c615a0987e0d23", - "dbt_det/seeds/raw/product_reviews.csv": "e7bd78f152f82b650b2392a38e2bac3f6837a8f3b1cb7818a3f834d88a897768", - "dbt_det/seeds/raw/products.csv": "9274598e42761eea40a269c36524bc83eab320de4160c2fe0f0a5584fd8aebb3" + "dbt_det/seeds/raw/customers.csv": "2b34b876f615650d684d7ba23e0cf0f63d8f6addc7f9798c2aa16735c9b6a45d", + "dbt_det/seeds/raw/order_items.csv": "133acea1e749fbafa7d3227b1ce0fe303e4d691dd351723412b8dd686cc430f6", + "dbt_det/seeds/raw/orders.csv": "3124312b95043b7592f149a40fd51ab19a03276313bef80973716e6759376fdb", + "dbt_det/seeds/raw/product_reviews.csv": "95f9a92f499d61ebc0d9747dd27474e1a5d96eaf3cec91b3d36d1a9b2ceb3b07", + "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" } } From 871cf072511c677c4c5f5060e48c0423691f9c50 Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Thu, 8 Oct 2026 09:17:00 +0200 Subject: [PATCH 06/14] Cover the parent-date edges and drop unreachable branches Tests reach the paths the rule's patch left uncovered: an empty child and an all-null creation column, a creation date that `when` fills after the rule ran (the row moves again and its later dates follow), the defects check with no parent frame or no rows, DBML with an `after_parent` note converting to 0.5.0, a model given the hint in code being written as 0.5.0, and an enum named like a date beside the creation column. Branches no input reaches are removed rather than kept: a datetime64 column (generation stores text and dates), a parent frame or foreign key that is missing (a rule only names generated parents), the guards around already-unboxed keys, a window start or end a parent can never pass, a rule without links, a null check `NaT` comparisons already answer, and a parent-less dbt test (`dbt_tests` always names its parent). Co-Authored-By: Claude Opus 5.5 --- model2data/defects/apply.py | 3 +- model2data/defects/checks.py | 3 +- model2data/generate/parents.py | 37 +++----------- tests/test_parent_dates.py | 93 ++++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 33 deletions(-) diff --git a/model2data/defects/apply.py b/model2data/defects/apply.py index f4cfed6..63bae0c 100644 --- a/model2data/defects/apply.py +++ b/model2data/defects/apply.py @@ -657,8 +657,7 @@ def _before_parents_once_late(table: _Table, candidates: list[int], name: str) - cutoffs: dict[int, Any] = {} risky = set() for position in candidates: - if pd.isna(floor[position]) or pd.isna(created[position]): - continue + # A null foreign key or date gives NaT, which compares false below. day = table.inserted_day[position] if day not in cutoffs: cutoffs[day] = table.loaded_max(day - 1, name) diff --git a/model2data/defects/checks.py b/model2data/defects/checks.py index 9fffb79..ab5e076 100644 --- a/model2data/defects/checks.py +++ b/model2data/defects/checks.py @@ -342,8 +342,7 @@ def _not_before(test: DbtTest, frame: pd.DataFrame, seeded: Mapping, cache: Colu def _not_before_parent(test: DbtTest, frame: pd.DataFrame, seeded: Mapping, cache: Columns) -> bool: - if test.parent is None: - return False + assert test.parent is not None # `dbt_tests` names the parent of every such test parent = seeded.get(test.parent[0]) if parent is None: return False diff --git a/model2data/generate/parents.py b/model2data/generate/parents.py index a9e0ef3..5a0bda8 100644 --- a/model2data/generate/parents.py +++ b/model2data/generate/parents.py @@ -175,8 +175,6 @@ def moments(series: pd.Series, kind: str) -> pd.Series: A timestamp column holds `"YYYY-MM-DD HH:MM:SS"` text, a date column `date`s. """ - if pd.api.types.is_datetime64_any_dtype(series): - return series if kind == "timestamp": return pd.to_datetime(series, format=_TS_FORMAT, errors="coerce") return pd.to_datetime(series, errors="coerce") @@ -190,9 +188,7 @@ def parent_floor( NaT where the foreign key is null, points at no row, or the parent's date is null. By day (midnight) when `link.by_day`. """ - parent = parents.get(link.parent_table) - if parent is None or link.fk not in frame.columns: - return pd.Series(pd.NaT, index=frame.index, dtype="datetime64[ns]") + parent = parents[link.parent_table] created = moments(parent[link.parent_column], link.parent_kind) if link.by_day: created = created.dt.normalize() @@ -209,27 +205,15 @@ def _key_index(keys: pd.Series) -> pd.Index: def _key_values(values: pd.Series) -> pd.Series: - """Keys as plain Python values, so an `Int64` parent and an int child match.""" + """Keys as plain Python values (`tolist` unboxes them), so an `Int64` parent and an + int child match, and a missing key (`pd.NA`, NaN) as None.""" return pd.Series( - [None if _is_na(value) else _plain(value) for value in values.tolist()], + [None if pd.isna(value) else value for value in values.tolist()], index=values.index, dtype=object, ) -def _is_na(value: object) -> bool: - try: - return bool(pd.isna(value)) - except (TypeError, ValueError): - return False - - -def _plain(value: object) -> object: - if isinstance(value, np.generic): - return value.item() - return value - - # --------------------------------------------------------- # Day 0: making a table's rows hold # --------------------------------------------------------- @@ -249,8 +233,6 @@ def follow_parents( tables. `repick` false skips step 2 (taking another parent), for a pass that must leave the foreign keys as they are. """ - if rule.column not in df.columns or not len(df): - return [] own = moments(df[rule.column], rule.kind) if not bool(own.notna().any()): return [] @@ -353,9 +335,8 @@ def draw(self, lower: datetime) -> datetime: return self._timestamp(lower) def _date(self, lower: datetime) -> datetime: - first = max(0, (lower.date() - self.start).days) - if first >= len(self.days): - return lower + # A parent is never dated after the anchor, so `first` is inside the window. + first = min(max(0, (lower.date() - self.start).days), len(self.days) - 1) before = self.added_up[first - 1] if first else 0.0 point = before + random.random() * (self.added_up[-1] - before) index = min(bisect_right(self.added_up, point), len(self.days) - 1) @@ -368,9 +349,9 @@ def _timestamp(self, lower: datetime) -> datetime: # A parent created at the very end of the window (a date on the # anchor day): the child is created at the end too, never past it. return lower + # Only a row dated before its parent moves, and it is dated inside the + # window, so the parent is too: `first` is never before the window. first = (lower.date() - self.start).days - if first < 0: - first, lower = 0, datetime.combine(self.start, datetime.min.time()) offset = lower.hour * 3600 + lower.minute * 60 + lower.second first_share = self.weights[first] * self._share_after(offset) rest = self.added_up[-1] - self.added_up[first] @@ -415,6 +396,4 @@ def floor_on_day( ) -> pd.Series: """Per new row: the latest creation moment among the parents it points at (NaT for none).""" floors = [parent_floor(frame, link, parents) for link in rule.links] - if not floors: - return pd.Series(pd.NaT, index=frame.index, dtype="datetime64[ns]") return pd.concat(floors, axis=1).max(axis=1) if len(floors) > 1 else floors[0] diff --git a/tests/test_parent_dates.py b/tests/test_parent_dates.py index aad1e5b..1ee6572 100644 --- a/tests/test_parent_dates.py +++ b/tests/test_parent_dates.py @@ -438,3 +438,96 @@ def test_later_days_continue_the_numbering(): ) day1 = results[1].tables["customers"].inserted assert day1["id"].tolist() == list(range(301, 309)) + + +# --------------------------------------------------------------------------- +# Edges +# --------------------------------------------------------------------------- +def test_an_empty_or_undated_child_is_left_alone(): + document = copy.deepcopy(SHOP) + document["tables"]["reviews"]["columns"]["review_date"] = { + "type": "date", + "generate": {"null_rate": 1.0}, + } + frames = _generate(document, rows={**ROWS, "orders": 0}) + assert frames["orders"].empty + assert frames["reviews"]["review_date"].isna().all() + + +def test_a_date_filled_by_when_still_follows_its_parent(): + """`when` fills a creation date after the rule ran; the row then moves again.""" + document: dict[str, Any] = { + "model2data": "0.5.0", + "enums": {"kind": ["a", "b"]}, + "tables": { + "customers": { + "columns": { + "id": {"type": "int", "pk": True}, + "created_at": {"type": "timestamp", "not_null": True}, + } + }, + "events": { + "columns": { + "id": {"type": "int", "pk": True}, + "customer_id": {"type": "int", "not_null": True, "references": "customers.id"}, + "kind": {"type": "kind", "not_null": True}, + "created_at": {"type": "timestamp", "generate": {"when": {"kind": ["a"]}}}, + "updated_at": {"type": "timestamp"}, + } + }, + }, + } + for seed in range(1, 6): + frames = _generate(document, seed=seed, rows={"customers": 50, "events": 400}) + joined = _joined(frames, "events", "customer_id", "created_at", "customers", "created_at") + assert (joined["_mine"] >= joined["_theirs"]).all() + events = frames["events"].dropna(subset=["created_at", "updated_at"]) + assert (pd.to_datetime(events["updated_at"]) >= pd.to_datetime(events["created_at"])).all() + + +def test_the_parent_check_without_the_parent_or_rows_passes(): + tables, refs = _engine(SHOP) + test = next( + t + for t in dbt_tests(tables, refs, hint_tests="warn") + if t.type == "model2data_not_before_parent" + ) + frames = _generate(SHOP) + seeded = {key: as_seeded(frame) for key, frame in frames.items()} + assert not fails(test, {"orders": seeded["orders"]}) + assert not fails(test, {**seeded, "orders": seeded["orders"].iloc[0:0]}) + + +def test_dbml_with_an_after_parent_note_converts_to_0_5(): + from model2data.model import from_dbml + + model = from_dbml( + "Table customers {\n id int [pk]\n created_at timestamp\n}\n" + "Table orders {\n id int [pk]\n customer_id int [ref: > customers.id]\n" + " order_date timestamp [note: '{\"after_parent\": false}']\n}\n" + ) + assert str(model.version) == "0.5.0" + + +def test_a_model_given_after_parent_is_written_as_0_5(): + from model2data.model import dump + + document = copy.deepcopy(SHOP) + document["model2data"] = "0.4.0" + model = from_dict(document) + model.tables["orders"].columns["order_date"].generate["after_parent"] = False + assert "model2data: 0.5.0" in dump(model) + + +def test_an_enum_named_like_a_date_is_not_a_creation_candidate(): + document = copy.deepcopy(SHOP) + document["enums"] = {"date_bucket": ["early", "late"]} + columns = document["tables"]["orders"]["columns"] + document["tables"]["orders"]["columns"] = { + "id": columns["id"], + "bucket": "date_bucket", + **{k: v for k, v in columns.items() if k != "id"}, + } + columns = document["tables"]["orders"]["columns"] + columns["order_date"]["generate"]["after_parent"] = False + assert _issues(document) == [] From d9ac8226aca08084012e8c733084cf2a002bc640 Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Thu, 8 Oct 2026 21:48:56 +0200 Subject: [PATCH 07/14] Replace the automatic parent rule with a cross-table `after` A child's date no longer follows its parents' on its own, and the `after_parent` opt-out is gone. Instead `after` names what a date follows: a column of the same row as before, a parent's column as `
.`, or a list of either. `after: customers.created_at` on orders.order_date keeps each order on or after the created_at of the customer its foreign key points at. The parent is reached through exactly one one-column foreign key onto its key; validation names what is wrong otherwise (no such table or column, not a date, no foreign key to it, only a composite one, one onto a column that is no key, the column's own table, and two foreign keys to the parent, named), and refuses tables whose afters name each other's columns. The generator refuses a parent drawn after the child. The technique that kept the stories stays: a late row takes an older parent from those the column points at, and only a row with none (or a one-to-one) moves its date within the column's own shape, never past as_of. A list mixing the row's own columns places the value after both. The fix-ups after `when` and composite-key repair, the later days, the by-day comparison of a date with a timestamp, the dbt test `model2data_not_before_parent` (one per parent column) and the late_arriving preference all follow the hint now. Validation warns where a child's first date can fall before a parent's created_at-like date and no `after` names that parent, with the hint to add as a structured `Issue.suggestion` (path, value, and the spec version an older document needs) for a one-click fix. No warning for a parent without such a date, a self-reference, or two foreign keys to one parent. A list or a parent's column in `after` is spec 0.5.0, gated as before. The bundled examples (and the spec's coffee-webshop example) use it where it is plainly right. Co-Authored-By: Claude Opus 5.5 --- examples/advanced_features.dbml | 2 +- examples/advanced_features.model2data.yml | 9 +- examples/ecommerce.dbml | 2 +- examples/ecommerce.model2data.yml | 9 +- examples/ecommerce_daily.model2data.yml | 9 +- examples/ecommerce_training.model2data.yml | 9 +- examples/saas_platform.dbml | 6 +- examples/saas_platform.model2data.yml | 19 +- model2data/cli.py | 5 +- model2data/dbt/hint_tests.py | 34 +- .../hint_macros/model2data_parent_tests.sql | 10 +- model2data/dbt/tests.py | 6 +- model2data/defects/apply.py | 32 +- model2data/generate/core.py | 79 +- model2data/generate/days.py | 30 +- model2data/generate/hints.py | 63 +- model2data/generate/parents.py | 218 +++--- model2data/generate/timeline.py | 91 ++- model2data/guide/tune.md | 36 +- model2data/model/__init__.py | 3 +- model2data/model/dbml.py | 17 +- model2data/model/document.py | 12 +- model2data/model/errors.py | 22 + model2data/model/validate.py | 247 ++++-- model2data/spec/README.md | 61 +- .../examples/coffee_webshop.model2data.yml | 6 +- model2data/spec/model.schema.json | 30 +- tests/test_cli_model.py | 4 +- tests/test_defects.py | 7 +- tests/test_defects_dbt.py | 6 +- tests/test_metrics_cli.py | 2 +- tests/test_model_dbml.py | 4 +- tests/test_parent_dates.py | 711 +++++++++++++----- 33 files changed, 1240 insertions(+), 561 deletions(-) diff --git a/examples/advanced_features.dbml b/examples/advanced_features.dbml index 6229b44..2fbf3b8 100644 --- a/examples/advanced_features.dbml +++ b/examples/advanced_features.dbml @@ -48,7 +48,7 @@ Table time_entries { id bigint [pk, not null] project_id bigint [not null] employee_id bigint [not null] - entry_date date [not null] + entry_date date [not null, note: '{"after": "employees.hire_date"}'] hours_worked numeric [not null, note: '{"min": 0.5, "max": 8}'] billable boolean } diff --git a/examples/advanced_features.model2data.yml b/examples/advanced_features.model2data.yml index b892842..cf5a086 100644 --- a/examples/advanced_features.model2data.yml +++ b/examples/advanced_features.model2data.yml @@ -1,5 +1,5 @@ -# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.2.0/model.schema.json -model2data: 0.2.0 +# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json +model2data: 0.5.0 description: > HR and project management: departments, employees with a self-referencing manager hierarchy, projects, assignments, time entries, expenses and performance reviews. @@ -60,7 +60,10 @@ tables: id: {type: bigint, pk: true, not_null: true} project_id: {type: bigint, not_null: true, references: projects.id} employee_id: {type: bigint, not_null: true, references: employees.id} - entry_date: {type: date, not_null: true} + entry_date: + type: date + not_null: true + generate: {after: employees.hire_date} hours_worked: type: numeric not_null: true diff --git a/examples/ecommerce.dbml b/examples/ecommerce.dbml index 96d2ab7..27e4270 100644 --- a/examples/ecommerce.dbml +++ b/examples/ecommerce.dbml @@ -34,7 +34,7 @@ Table products { Table orders { id bigint [pk, not null] customer_id bigint [not null] - order_date timestamp [not null] + order_date timestamp [not null, note: '{"after": "customers.created_at"}'] total_amount numeric [not null, note: '{"min": 10, "max": 5000}'] status order_status } diff --git a/examples/ecommerce.model2data.yml b/examples/ecommerce.model2data.yml index 01ebdec..64bd2bd 100644 --- a/examples/ecommerce.model2data.yml +++ b/examples/ecommerce.model2data.yml @@ -1,5 +1,5 @@ -# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.2.0/model.schema.json -model2data: 0.2.0 +# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json +model2data: 0.5.0 description: > An online store: customers, a product catalogue, orders and their line items, and product reviews. @@ -37,7 +37,10 @@ tables: columns: id: {type: bigint, pk: true, not_null: true} customer_id: {type: bigint, not_null: true, references: customers.id} - order_date: {type: timestamp, not_null: true} + order_date: + type: timestamp + not_null: true + generate: {after: customers.created_at} total_amount: type: numeric not_null: true diff --git a/examples/ecommerce_daily.model2data.yml b/examples/ecommerce_daily.model2data.yml index 92986b6..5e27007 100644 --- a/examples/ecommerce_daily.model2data.yml +++ b/examples/ecommerce_daily.model2data.yml @@ -1,5 +1,5 @@ -# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.2.0/model.schema.json -model2data: 0.2.0 +# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json +model2data: 0.5.0 description: > The online store of ecommerce, moving on day by day: new customers, orders and line items each day, and orders that move from pending to delivered. Generate the days after the first with `model2data generate --days 7`. @@ -39,7 +39,10 @@ tables: columns: id: {type: bigint, pk: true, not_null: true} customer_id: {type: bigint, not_null: true, references: customers.id} - order_date: {type: timestamp, not_null: true} + order_date: + type: timestamp + not_null: true + generate: {after: customers.created_at} total_amount: type: numeric not_null: true diff --git a/examples/ecommerce_training.model2data.yml b/examples/ecommerce_training.model2data.yml index f977a76..cf533b9 100644 --- a/examples/ecommerce_training.model2data.yml +++ b/examples/ecommerce_training.model2data.yml @@ -1,5 +1,5 @@ -# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.3.0/model.schema.json -model2data: 0.3.0 +# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json +model2data: 0.5.0 description: > The ecommerce_daily store with deliberate defects, for teaching dbt tests: the `training` preset breaks each kind of standard test once, and the tables add defects of their own. Orders keep their history (`orders_history`, one row per version), which the preset breaks with overlapping versions on a run of several days. `model2data generate --days 3` writes defects_report.json and EXPECTED_FAILURES.md into the project, listing every broken row and the tests `dbt build` should fail. @@ -41,7 +41,10 @@ tables: columns: id: {type: bigint, pk: true, not_null: true} customer_id: {type: bigint, not_null: true, references: customers.id} - order_date: {type: timestamp, not_null: true} + order_date: + type: timestamp + not_null: true + generate: {after: customers.created_at} total_amount: type: numeric not_null: true diff --git a/examples/saas_platform.dbml b/examples/saas_platform.dbml index 6409cf0..64476b8 100644 --- a/examples/saas_platform.dbml +++ b/examples/saas_platform.dbml @@ -60,7 +60,7 @@ Table subscriptions { plan_tier plan_tier_type [not null] monthly_price numeric [not null, note: '{"min": 0, "max": 999}'] status subscription_status - current_period_start timestamp [not null] + current_period_start timestamp [not null, note: '{"after": "organizations.created_at"}'] current_period_end timestamp } @@ -70,7 +70,7 @@ Table invoices { subscription_id bigint [not null] amount numeric [not null, note: '{"min": 0, "max": 999}'] status invoice_status - created_at timestamp [not null] + created_at timestamp [not null, note: '{"after": "organizations.created_at"}'] } Table feature_limits { @@ -95,7 +95,7 @@ Table audit_events { event_type audit_event_type [not null] resource_id text resource_type text - created_at timestamp [not null] + created_at timestamp [not null, note: '{"after": "organizations.created_at"}'] } Ref { diff --git a/examples/saas_platform.model2data.yml b/examples/saas_platform.model2data.yml index 00bc1b9..84e524c 100644 --- a/examples/saas_platform.model2data.yml +++ b/examples/saas_platform.model2data.yml @@ -1,5 +1,5 @@ -# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.2.0/model.schema.json -model2data: 0.2.0 +# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json +model2data: 0.5.0 description: > A multi-tenant SaaS platform: organizations, their users, subscriptions, invoices, plan limits, API usage and an audit log. @@ -38,7 +38,10 @@ tables: not_null: true generate: {min: 0, max: 999} status: subscription_status - current_period_start: {type: timestamp, not_null: true} + current_period_start: + type: timestamp + not_null: true + generate: {after: organizations.created_at} current_period_end: timestamp invoices: @@ -51,7 +54,10 @@ tables: not_null: true generate: {min: 0, max: 999} status: invoice_status - created_at: {type: timestamp, not_null: true} + created_at: + type: timestamp + not_null: true + generate: {after: organizations.created_at} feature_limits: columns: @@ -91,4 +97,7 @@ tables: event_type: {type: audit_event_type, not_null: true} resource_id: text resource_type: text - created_at: {type: timestamp, not_null: true} + created_at: + type: timestamp + not_null: true + generate: {after: organizations.created_at} diff --git a/model2data/cli.py b/model2data/cli.py index 9962dd5..65cf1fe 100644 --- a/model2data/cli.py +++ b/model2data/cli.py @@ -838,7 +838,7 @@ def pick(option: Any, from_run: Any, default: Any) -> Any: def _spec_of(file: Path) -> str: """The spec a conforming model is written against: 0.5.0, 0.4.0, 0.3.0, or 0.2.0 (DBML - without `when` or `after_parent` included).""" + without `when` or a 0.5.0 `after` included).""" version = str(load(file).version) for minor in ("0.5", "0.4", "0.3"): if version.startswith(minor): @@ -1030,7 +1030,8 @@ def convert_command( force: bool = typer.Option(False, "--force", help="Overwrite the output file if it exists."), ): """Convert a model -- DBML, typically -- to a .model2data.yml document (spec 0.2.0, or - 0.3.0 when it has defects, 0.4.0 when it has `when`, 0.5.0 when it has `after_parent`).""" + 0.3.0 when it has defects, 0.4.0 when it has `when`, 0.5.0 when an `after` is a list or + names a parent's column).""" try: model = load(file) except ModelError as error: diff --git a/model2data/dbt/hint_tests.py b/model2data/dbt/hint_tests.py index a1b3033..300a68a 100644 --- a/model2data/dbt/hint_tests.py +++ b/model2data/dbt/hint_tests.py @@ -12,17 +12,16 @@ |---------------------------|----------------------------------------|-----------| | `generate.min` / `max` | `model2data_between` | none | | `generate.after: other` | `model2data_not_before` | none | +| `generate.after: table.column` | `model2data_not_before_parent` | none | | `generate.null_rate` | `model2data_max_null_share` | yes | | `generate.distinct: n` | `model2data_max_distinct` | none | | `generate.when` | `model2data_when` | none | | table `grain` | `model2data_unique_combination` | none | -| a creation date after its parents' | `model2data_not_before_parent` | none | -The last one comes from no hint: it is what generation keeps on every table -whose creation date follows a parent's (see `model2data.generate.parents`), one -test per foreign key it follows, on the child's creation column. It needs the -model's references, so a caller passing bare tables passes `refs` too; a column -with `after_parent: false` gets none. +An `after` listing several columns writes one test for each. A parent's column +(`after: customers.created_at`, see `model2data.generate.parents`) is tested +through the foreign key that reaches it, so that test needs the model's +references: a caller passing bare tables passes `refs` too. A column with `when` and `null_rate` has its null share tested among the rows `when` matches (`model2data_when_max_null_share`), since only those rows can @@ -51,6 +50,7 @@ from model2data.generate import kinds from model2data.generate.core import parent_rules_for from model2data.generate.parents import ParentRule +from model2data.generate.timeline import same_row_after from model2data.model.engine import EngineInputs, to_engine from model2data.model.types import Model from model2data.parse.dbml import ColumnDef, TableDef @@ -75,10 +75,9 @@ class HintTest: `column` is None for a table-level test (grain). `test` is the generic test's name, `arguments` its parameters as dbt's `arguments:` block, and `hint` the hint it comes from (`min`, `max`, `after`, `null_rate`, - `distinct`, `when`, `grain`, or `after_parent` for a creation date that - follows a parent's). `min` and `max` of one column share one test. + `distinct`, `when`, `grain`). `min` and `max` of one column share one test. `parent` is the `(table, column)` a test looks rows up in: the parent's key - for `after_parent`, None for every other test. + for `model2data_not_before_parent`, None for every other test. """ table: str @@ -107,8 +106,8 @@ def hint_tests_for( `to_engine(model).tables`, or the renamed tables the CLI hands `generate_dbt_yml`). Table-level tests follow the table's column tests. With `tables`, `refs` are the references as `to_engine` gives them; without - them no `after_parent` test is written (a `Model` or `EngineInputs` brings - its own). + them no test of a parent's column is written (a `Model` or `EngineInputs` + brings its own). `tolerance` is the absolute slack of the statistical test (the null share may be `null_rate + tolerance`); hard constraints take none. Severity is @@ -131,11 +130,11 @@ def hint_tests_for( for key, table in tables.items(): by_name = {column.name: column for column in table.columns} names = set(by_name) - rule = rules.get(key) + parents = {rule.column: rule for rule in rules.get(key, [])} for column in table.columns: found.extend(_column_tests(key, column, by_name, tolerance)) - if rule is not None and rule.column == column.name: - found.extend(_parent_tests(key, rule)) + if column.name in parents: + found.extend(_parent_tests(key, parents[column.name])) grain = (table.note or {}).get("grain") if grain and all(name in names for name in grain): found.append( @@ -173,8 +172,7 @@ def _column_tests( ) ) - after = note.get("after") - if isinstance(after, str) and after in kinds_of and after != column.name: + for after in [a for a in same_row_after(note) if a in kinds_of and a != column.name]: arguments: dict[str, Any] = {"other": after} # The generator compares a date with a timestamp at day granularity. if ( @@ -230,7 +228,7 @@ def _column_tests( def _parent_tests(table: str, rule: ParentRule) -> list[HintTest]: - """One `model2data_not_before_parent` test per foreign key the creation date follows.""" + """One `model2data_not_before_parent` test per parent column the `after` names.""" tests = [] for link in rule.links: arguments: dict[str, Any] = { @@ -247,7 +245,7 @@ def _parent_tests(table: str, rule: ParentRule) -> list[HintTest]: rule.column, "model2data_not_before_parent", arguments, - "after_parent", + "after", parent=(link.parent_table, link.parent_key), ) ) diff --git a/model2data/dbt/templates/hint_macros/model2data_parent_tests.sql b/model2data/dbt/templates/hint_macros/model2data_parent_tests.sql index ada9bbe..633bbdb 100644 --- a/model2data/dbt/templates/hint_macros/model2data_parent_tests.sql +++ b/model2data/dbt/templates/hint_macros/model2data_parent_tests.sql @@ -1,11 +1,11 @@ {#- - The generic test model2data writes where a table's creation date follows its - parents': a row is not dated before the parent row its foreign key points at - (an order not before its customer signed up). Written beside - model2data_hint_tests.sql, and only by a model that has such a table. Plain + The generic test model2data writes for a cross-table `after` (`after: + customers.created_at` on orders.order_date): a row is not dated before the + parent row its foreign key points at. Written beside + model2data_hint_tests.sql, and only by a model that has such a hint. Plain ANSI SQL, runs on DuckDB and Postgres. Returns the rows that break it. `column_name` arrives already quoted by the schema YAML; `foreign_key`, - `field` (the parent's key) and `parent_column` (the parent's creation date) + `field` (the parent's key) and `parent_column` (the column `after` names) are raw names and are quoted here. A row whose foreign key is null, or points at no parent row, is left to the not_null and relationships tests, and a key several parent rows share (left to the unique test) counts its earliest one. diff --git a/model2data/dbt/tests.py b/model2data/dbt/tests.py index 53cff78..85826db 100644 --- a/model2data/dbt/tests.py +++ b/model2data/dbt/tests.py @@ -68,8 +68,10 @@ def generate_dbt_yml( hinted = _hinted(tables, hint_tests, test_tolerance, refs) if hinted: - hints = {test.hint for tests in hinted.values() for test in tests} - write_hint_macros(dest, when="when" in hints, parents="after_parent" in hints) + names = {test.test for tests in hinted.values() for test in tests} + write_hint_macros( + dest, when="model2data_when" in names, parents="model2data_not_before_parent" in names + ) if any(_history_entries(table) for table in tables.values()): write_history_macros(dest) fk_map = _fk_map(tables, refs) diff --git a/model2data/defects/apply.py b/model2data/defects/apply.py index 63bae0c..66625f4 100644 --- a/model2data/defects/apply.py +++ b/model2data/defects/apply.py @@ -113,11 +113,11 @@ def __init__( refs: list[dict], results: Optional[list[DayResult]], incremental: Any, - rule: Optional[ParentRule] = None, + rules: Sequence[ParentRule] = (), ): self.key = key - # The parents the table's creation date follows (see generate.parents), if any. - self.rule = rule + # The columns whose `after` names a parent's column (see generate.parents). + self.rules = list(rules) self.table = table self.frames = frames self.frame = frames[key] @@ -643,28 +643,32 @@ def _before_parents_once_late(table: _Table, candidates: list[int], name: str) - """The candidates moving back would date before a parent row (see generate.parents). A late-arriving row is moved back past the previous load's cutoff, every date of - it; when the table's creation date follows its parents', that can put it before - one, which the dbt test `model2data_not_before_parent` then reports. Rows that - stay clear of their parents are taken first, so the defect breaks the - incremental model it is meant to and no test besides. + it; a date with a cross-table `after` can then fall before its parent, which the + dbt test `model2data_not_before_parent` reports. Rows that stay clear of their + parents are taken first, so the defect breaks the incremental model it is meant + to and no test besides. """ - rule = table.rule - if rule is None or not candidates: + if not table.rules or not candidates: return set() - floor = floor_on_day(table.frame, rule, table.frames).tolist() - created = moments(table.frame[rule.column], rule.kind).tolist() + checks = [ + ( + floor_on_day(table.frame, rule, table.frames).tolist(), + moments(table.frame[rule.column], rule.kind).tolist(), + ) + for rule in table.rules + ] values = table.frame[name].tolist() cutoffs: dict[int, Any] = {} risky = set() for position in candidates: - # A null foreign key or date gives NaT, which compares false below. day = table.inserted_day[position] if day not in cutoffs: cutoffs[day] = table.loaded_max(day - 1, name) cutoff, moment, back = cutoffs[day], _moment(values[position]), 1 while cutoff is not None and moment - timedelta(days=back) >= cutoff: back += 1 - if created[position] - pd.Timedelta(days=back) < floor[position]: + # A null foreign key or date gives NaT, which compares false. + if any(own[position] - pd.Timedelta(days=back) < floor[position] for floor, own in checks): risky.add(position) return risky @@ -876,7 +880,7 @@ def break_tables( inputs.refs, results, inputs.incremental.get(target), - rules.get(target), + rules.get(target, []), ) table = tables[target] rng = random.Random(defect_stream_seed(base_seed, key, defect)) diff --git a/model2data/generate/core.py b/model2data/generate/core.py index 8de2e4a..9b8e3a0 100644 --- a/model2data/generate/core.py +++ b/model2data/generate/core.py @@ -158,7 +158,7 @@ def generate_data_from_dbml( # Generate tables in dependency order # --------------------------------------------------------- ordered_tables, deferred_edges = _table_order(tables, fk_refs) - # Which tables' creation dates follow their parents' (see generate.parents). + # The columns whose `after` names a parent's column (see generate.parents). rules = parent_rules(tables, fk_refs, ordered_tables) generated: dict[str, pd.DataFrame] = {} @@ -247,18 +247,26 @@ def generate_data_from_dbml( ) df = pd.DataFrame(data) - # A row comes into being on or after the parent rows it points at: - # before the row's own later dates are ordered after its creation date, - # so they follow it wherever it ends up (see generate.parents). - rule = rules.get(table_name) - if rule is not None: - creation = next(c for c in table_def.columns if c.name == rule.column) - follow_parents(df, rule, creation, generated, as_of=as_of, time_profile=profile) + # A cross-table `after` keeps a row's date on or after the parent row + # it points at: before the row's own later dates are ordered, so they + # follow it wherever it ends up (see generate.parents). + table_rules = rules.get(table_name, []) + floors = { + rule.column: follow_parents( + df, + rule, + _column(table_def, rule.column), + generated, + as_of=as_of, + time_profile=profile, + )[0] + for rule in table_rules + } # Ordering runs before FK resolution/dedup so a self-ref repair or a # composite-key retry regenerates a temporal column's value into a # frame that already respects created/updated/closed ordering, rather # than one where only the untouched columns do. - df = order_row_times(df, table_def, as_of=as_of, time_profile=profile) + df = order_row_times(df, table_def, as_of=as_of, time_profile=profile, floors=floors) df = _resolve_self_referencing_fks( df, table_def, @@ -295,8 +303,8 @@ def generate_data_from_dbml( as_of=as_of, time_profile=profile, ) - if rule is not None: - df = _hold_parent_rule(df, table_def, rule, generated, as_of, profile) + if table_rules: + df = _hold_parent_rules(df, table_def, table_rules, generated, as_of, profile) df = _coerce_integer_dtypes(df, table_def) generated[table_name] = df @@ -382,12 +390,11 @@ def _mirror_attributes( return df -def parent_rules_for(tables: dict[str, TableDef], refs: list[dict]) -> dict[str, ParentRule]: - """The tables whose creation date `generate_data_from_dbml` keeps after their parents'. +def parent_rules_for(tables: dict[str, TableDef], refs: list[dict]) -> dict[str, list[ParentRule]]: + """The columns whose cross-table `after` `generate_data_from_dbml` keeps, by table key. - By table key, each with the foreign keys it follows (see - `generate.parents`). What the dbt export tests, so it is worked out exactly - as generation works it out. + Each with the foreign keys it follows (see `generate.parents`). What the + dbt export tests, so it is worked out exactly as generation works it out. """ fk_refs, _ = classify_refs(tables, refs) order, _broken, _leftover = _plan_table_order(tables, fk_refs) @@ -397,28 +404,44 @@ def parent_rules_for(tables: dict[str, TableDef], refs: list[dict]) -> dict[str, # --------------------------------------------------------- # Internal helpers # --------------------------------------------------------- -def _hold_parent_rule( +def _column(table_def: TableDef, name: str) -> ColumnDef: + return next(c for c in table_def.columns if c.name == name) + + +def _hold_parent_rules( df: pd.DataFrame, table_def: TableDef, - rule: ParentRule, + rules: list[ParentRule], generated: dict[str, pd.DataFrame], as_of: AsOf, profile: TimeProfile, ) -> pd.DataFrame: - """Keep the parent rule after the passes that run once it was applied. + """Keep the cross-table `after` after the passes that run once it was applied. A composite key's repair can give a row another parent, and `when` can fill - a creation date the null pass had left empty; either can leave a row dated - before its parent. Such a row's date moves (its parents stay: the - attributes mirrored through them are final by now), and the row's later - dates are placed after it again. A table neither touched comes back as it was. + a date the null pass had left empty; either can leave a row dated before + its parent. Such a row's date moves (its parents stay: the attributes + mirrored through them are final by now), and the row's dates that follow + something are placed again. A table neither touched comes back as it was. """ - creation = next(c for c in table_def.columns if c.name == rule.column) - moved = follow_parents( - df, rule, creation, generated, as_of=as_of, time_profile=profile, repick=False - ) + floors: dict[str, pd.Series] = {} + moved: set = set() + for rule in rules: + floor, rows = follow_parents( + df, + rule, + _column(table_def, rule.column), + generated, + as_of=as_of, + time_profile=profile, + repick=False, + ) + floors[rule.column] = floor + moved.update(rows) if moved: - df = order_row_times(df, table_def, as_of=as_of, time_profile=profile, rows=moved) + df = order_row_times( + df, table_def, as_of=as_of, time_profile=profile, rows=sorted(moved), floors=floors + ) return df diff --git a/model2data/generate/days.py b/model2data/generate/days.py index 52f49ab..d481210 100644 --- a/model2data/generate/days.py +++ b/model2data/generate/days.py @@ -334,7 +334,7 @@ class _Engine: day_date: date = field(default_factory=date.today) warnings: list[str] = field(default_factory=list) states: dict[str, pd.DataFrame] = field(default_factory=dict) - rules: dict[str, ParentRule] = field(default_factory=dict) + rules: dict[str, list[ParentRule]] = field(default_factory=dict) # -- one table, one day --------------------------------------------- def advance( @@ -733,28 +733,26 @@ def _place_in_day( def _after_parents( self, table: TableDef, frame: pd.DataFrame, stamps: dict[str, pd.Series] ) -> None: - """A new row's creation timestamp is not before its parents' (see generate.parents). + """A new row's timestamp is not before the parent its `after` names (generate.parents). Only a parent inserted the same day can be later: every other one is from an earlier day. Such a row is moved to a time between its latest parent and the end of the day, evenly. A date column falls on the day, which is never before a parent's. """ - rule = self.rules.get(table.name) - if rule is None or rule.column not in stamps: - return - floor = floor_on_day(frame, rule, self.states) - current = stamps[rule.column] - late = current.notna() & floor.notna() & (current < floor) - if not bool(late.any()): - return end = pd.Timestamp(self._day_start()) + pd.Timedelta(seconds=86399) - moved = current.copy() - for label in frame.index[late.to_numpy()]: - lower = floor.at[label] - room = max(0, int((end - lower).total_seconds())) - moved.at[label] = lower + pd.Timedelta(seconds=random.randint(0, room)) - stamps[rule.column] = moved + for rule in self.rules.get(table.name, []): + if rule.column not in stamps: + continue + floor = floor_on_day(frame, rule, self.states) + current = stamps[rule.column] + late = current.notna() & floor.notna() & (current < floor) + moved = current.copy() + for label in frame.index[late.to_numpy()]: + lower = floor.at[label] + room = max(0, int((end - lower).total_seconds())) + moved.at[label] = lower + pd.Timedelta(seconds=random.randint(0, room)) + stamps[rule.column] = moved def _timestamps(self, count: int, profile: TimeProfile) -> pd.DatetimeIndex: if profile.business_hours: diff --git a/model2data/generate/hints.py b/model2data/generate/hints.py index 13fd0ab..dc758ba 100644 --- a/model2data/generate/hints.py +++ b/model2data/generate/hints.py @@ -3,7 +3,7 @@ A note has always been either plain text (a comment, ignored by generation) or a JSON object read for `min`/`max`. This module documents the rest of that object's vocabulary -- `null_rate`, `weights`, `true_rate`, `distinct`, `skew`, -`after`, `business_hours`, `growth`, `seasonality`, `after_parent`, `distribution` and its +`after`, `business_hours`, `growth`, `seasonality`, `distribution` and its parameters -- and checks it once, before a single row is generated, so a typo'd enum value or a hint on the wrong kind of column fails with a message naming the table and column rather than surfacing as a wrong-looking dataset @@ -17,11 +17,10 @@ | `true_rate` | boolean columns | fraction of non-null rows that are true | | `distinct` | columns that aren't an FK, `pk`, `unique`, enum | positive integer: draw from a pool that size | | `skew` | foreign-key columns | overrides the run-level `skew` for this column | -| `after` | date/timestamp columns | name of another date/timestamp column, read by the time-aware generator | +| `after` | date/timestamp columns | another date/timestamp column of the row, a parent's `
.` (see generate.parents), or a list of them | | `business_hours` | date/timestamp columns | overrides the run-level `TimeProfile.business_hours` for this column | | `growth` | date/timestamp columns | overrides the run-level `TimeProfile.growth` for this column | | `seasonality` | date/timestamp columns | overrides the run-level `TimeProfile.seasonality` for this column | -| `after_parent` | a table's creation column (see generate.timeline.creation_column) | `false`: not kept on or after its parents' creation dates (see generate.parents) | | `distribution` | numeric columns | draw shape: `uniform` (default), `normal`, `lognormal`, `exponential` | | `mean` | numeric columns, with `distribution: normal` or `exponential` | normal: the centre; exponential: the average (scale) | | `stddev` | numeric columns, with `distribution: normal` | normal: the spread | @@ -35,8 +34,9 @@ from collections.abc import Mapping from model2data.generate import kinds +from model2data.generate.parents import parent_link from model2data.generate.relationships import build_fk_lookup, classify_refs -from model2data.generate.timeline import creation_column +from model2data.generate.timeline import parent_after from model2data.parse.dbml import ColumnDef, TableDef @@ -84,8 +84,12 @@ def validate_hints(tables: Mapping[str, TableDef], refs: list[dict]) -> None: is_unique="unique" in column.settings or column.name in composite_unique_columns, columns_by_name=columns_by_name, ) - if "after_parent" in note: - _check_after_parent(table_def, column, note["after_parent"]) + for entry in parent_after(note): + link, reason = parent_link(tables, fk_refs, table_name, entry) + if link is None: + raise ValueError( + f'{table_name}.{column.name}: "after" names {".".join(entry)}, {reason}.' + ) # --------------------------------------------------------- @@ -295,35 +299,30 @@ def _check_after( after: object, columns_by_name: dict[str, ColumnDef], ) -> None: - if not isinstance(after, str): - raise ValueError(f'{label}: "after" must be a column name (got {after!r}).') - other = columns_by_name.get(after) - if other is None: - raise ValueError( - f'{label}: "after" names {after!r}, which is not a column of {table_name}.' - ) - if not _is_temporal_type(_base_type(other.data_type)): - raise ValueError( - f'{label}: "after" names {after!r}, which is not a date or timestamp column.' - ) - + """A column of the same row, a parent's `
.`, or a non-empty list of them. -def _check_after_parent(table_def: TableDef, column: ColumnDef, value: object) -> None: - label = f"{table_def.name}.{column.name}" - if column.enum_values or not _is_temporal_type(_base_type(column.data_type)): - raise ValueError(f'{label}: "after_parent" only applies to date/timestamp columns.') - _check_bool(label, "after_parent", value) - creation = creation_column(table_def) - if creation is None or creation.name != column.name: - where = ( - f"which in {table_def.name} is {creation.name}" - if creation is not None - else f"and {table_def.name} has none (every date follows another column)" - ) + A parent's column is checked by `validate_hints` itself, which knows the + other tables and the foreign keys. + """ + raw: list = after if isinstance(after, list) else [after] + entries = [entry for entry in raw if isinstance(entry, str)] + if not entries or len(entries) != len(raw): raise ValueError( - f'{label}: "after_parent" sits on the date a row comes into being with, {where}: ' - "only that column is kept on or after the parent rows' own." + f'{label}: "after" must be a column name, `
.`, or a list of them ' + f"(got {after!r})." ) + for entry in entries: + if "." in entry: + continue + other = columns_by_name.get(entry) + if other is None: + raise ValueError( + f'{label}: "after" names {entry!r}, which is not a column of {table_name}.' + ) + if not _is_temporal_type(_base_type(other.data_type)): + raise ValueError( + f'{label}: "after" names {entry!r}, which is not a date or timestamp column.' + ) def _check_when( diff --git a/model2data/generate/parents.py b/model2data/generate/parents.py index 5a0bda8..a4bd804 100644 --- a/model2data/generate/parents.py +++ b/model2data/generate/parents.py @@ -1,28 +1,27 @@ -"""A child row comes into being on or after the parent rows it points at. +"""A cross-table `after`: a child's date on or after the parent row it points at. An order placed before its customer signed up, a review written before the -product existed: each column is plausible on its own and the pair is not. So a -table's **creation column** (see `generate.timeline.creation_column`: its -`created_at`, `signup_date`, or else its first date that follows nothing, such -as `order_date`) is kept on or after the creation column of every parent row -its foreign keys point at, on every row, by the rule `parent_rules` works out: - -- The rule follows a foreign key onto a parent's primary or unique key (one - parent row per value), to a parent that has a creation column and is - generated before the child. A self-reference, a foreign key that breaks a - cycle (its parent is generated after the child), a parent without a date - and a reference onto a column that is no key are not followed; a null - foreign key constrains nothing. -- `after_parent: false` on the child's creation column turns the rule off for - the table. +product existed: each column is plausible on its own and the pair is not. +`after: customers.created_at` on `orders.order_date` keeps every order on or +after the `created_at` of the customer its foreign key points at; a list +(`after: [orders.ordered_at, products.launched_at]`) keeps it after each, and +may mix in columns of the row itself (`after: [placed_at, customers.created_at]`). +`parent_rules` works out, per column, which foreign key reaches each parent: + +- The parent is reached through exactly one one-column foreign key onto its + primary key or a unique column (one parent row per value), and is generated + before the child. Anything else is refused, naming the foreign keys when + there are several. A self-reference is not a parent: `after` names a column + of the row itself without a table. A null foreign key constrains nothing on + its row. - A date compared with a timestamp compares by day: a date is not before the timestamp's day, and a timestamp is not before the date's midnight. How a row is made to hold (`follow_parents`), keeping the child's own shape: 1. A row that already holds keeps every value: the column was drawn exactly - as it is without the rule. -2. A row whose date is before a parent's takes another parent through that + as it is without the hint. +2. A row whose date is before its parent's takes another parent through that foreign key, one created on or before its date, drawn from the parents the column already points at (so a popular customer stays popular, and one created earlier has had longer to order). The child's date keeps its draw, @@ -35,13 +34,12 @@ weekdays) and, for a timestamp, the same hours. A parent created at the very end of the window gives a child at the very end of it, never past it. -Because a kept row is a draw from the column's shape and a moved one is a draw -from that shape cut at the parent, step 3 alone gives each row exactly the -shape cut at its parent; step 2 keeps the column's shape whole wherever a -parent allows it. +A column that also follows a column of its own row (by `after` or by its +name's stage) is then placed after both by `timeline.order_row_times`, which +reads the parents' moments as one more thing to follow. All draws come from the table's own stream (seeded by `generate.core`), so the -rule is as reproducible as the rest of the table. +hint is as reproducible as the rest of the table. """ from __future__ import annotations @@ -49,7 +47,7 @@ import random from bisect import bisect_right from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import datetime, timedelta from itertools import accumulate from typing import Optional @@ -69,7 +67,7 @@ _resolve_anchor, _window_days, column_business_hours, - creation_column, + parent_after, ) from model2data.parse.dbml import ColumnDef, TableDef @@ -78,13 +76,13 @@ @dataclass(frozen=True) class ParentLink: - """One foreign key of a child along which its creation column follows a parent's. + """One parent column a child's `after` names, and the foreign key that reaches it. `fk` is the child's foreign-key column, `parent_key` the parent column it - references, `parent_column` the parent's creation column, of `parent_kind` - (`"date"` or `"timestamp"`). `unique` is true - when the foreign key takes each parent at most once (a one-to-one), so a - row cannot simply take another parent. `by_day` is true when the two + references, `parent_column` the column `after` names, of `parent_kind` + (`"date"` or `"timestamp"`). `unique` is true when the foreign key takes + each parent at most once (a one-to-one), so a row cannot simply take + another parent. `by_day` is true when the two columns compare by day (one is a date, the other a timestamp). """ @@ -99,7 +97,7 @@ class ParentLink: @dataclass(frozen=True) class ParentRule: - """A table's creation column (`column`, of `kind`) and the parents it follows.""" + """A column (`column`, of `kind`) of `table` and the parent columns its `after` names.""" table: str column: str @@ -107,63 +105,104 @@ class ParentRule: links: tuple[ParentLink, ...] -def after_parent_enabled(column: ColumnDef) -> bool: - """False when the column opts out with `after_parent: false`.""" - return (column.note or {}).get("after_parent") is not False +def parent_link( + tables: Mapping[str, TableDef], + fk_refs: list[dict], + table_name: str, + entry: tuple[str, str], +) -> tuple[Optional[ParentLink], str]: + """The foreign key `table_name` reaches `entry`'s parent column through, or why none. + + `entry` is `(parent table, parent column)` from a cross-table `after`. + Returns the link and an empty reason, or None and the reason, worded to + follow `names
., `. + """ + parent, column = entry + parent_def = tables.get(parent) + if parent_def is None: + return None, f"and there is no table {parent}" + if parent == table_name: + return None, ( + f"a column of {table_name} itself: name a column of the same row without its " + f"table (`after: {column}`); a parent row in the same table is not followed" + ) + target = next((c for c in parent_def.columns if c.name == column), None) + if target is None: + return None, f"and {parent} has no column {column}" + parent_kind = kinds.temporal_kind(target.data_type) + if target.enum_values or parent_kind is None: + return None, "which is not a date or timestamp column" + refs = [r for r in fk_refs if r["source_table"] == table_name and r["target_table"] == parent] + if not refs: + return None, ( + f"and {table_name} has no foreign key to {parent}: `after` reaches another table " + "only through a foreign key of the row" + ) + if len(refs) > 1: + names = ", ".join(r["source_column"] for r in refs) + return None, ( + f"and {table_name} reaches {parent} through more than one foreign key ({names}): " + f"`after` cannot tell which {parent} row it means" + ) + (ref,) = refs + if _key_kind(dict(tables), ref) not in ("pk", "unique"): + return None, ( + f"and {ref['source_column']} references {parent}.{ref['target_column']}, which is " + f"not a key: `after` needs the one {parent} row the foreign key points at" + ) + fk_column = next(c for c in tables[table_name].columns if c.name == ref["source_column"]) + link = ParentLink( + fk=fk_column.name, + parent_table=parent, + parent_key=ref["target_column"], + parent_column=column, + parent_kind=parent_kind, + unique="pk" in fk_column.settings or "unique" in fk_column.settings, + by_day=False, + ) + return link, "" def parent_rules( tables: Mapping[str, TableDef], fk_refs: list[dict], order: list[str] -) -> dict[str, ParentRule]: - """The rule of every table it applies to, by table key. +) -> dict[str, list[ParentRule]]: + """The cross-table `after` of every column that has one, by table key. `fk_refs` are the foreign keys as `generate.relationships.classify_refs` - finds them, `order` the order tables are generated in: a parent counts - only when it comes before the child, which leaves out a self-reference and - a foreign key that breaks a cycle (its parent is drawn after the child). + finds them, `order` the order tables are generated in; the hints are the + ones `generate.hints.validate_hints` accepted. Raises ValueError for a + parent generated after the table (their foreign keys form a cycle, broken + at the very key followed). """ position = {name: index for index, name in enumerate(order)} - rules: dict[str, ParentRule] = {} + rules: dict[str, list[ParentRule]] = {} for table_name, table_def in tables.items(): - column = creation_column(table_def) - if column is None or not after_parent_enabled(column): - continue - kind = kinds.temporal_kind(column.data_type) - assert kind is not None - by_name = {c.name: c for c in table_def.columns} - links: list[ParentLink] = [] - for ref in fk_refs: - if ref["source_table"] != table_name: + for column in table_def.columns: + entries = parent_after(column.note) + if not entries: continue - parent = ref["target_table"] - parent_def = tables.get(parent) - fk_column = by_name.get(ref["source_column"]) - if ( - parent_def is None - or fk_column is None - or parent == table_name - or position.get(parent, len(order)) >= position.get(table_name, -1) - or _key_kind(dict(tables), ref) not in ("pk", "unique") - ): - continue - parent_column = creation_column(parent_def) - if parent_column is None: - continue - parent_kind = kinds.temporal_kind(parent_column.data_type) - assert parent_kind is not None - link = ParentLink( - fk=fk_column.name, - parent_table=parent, - parent_key=ref["target_column"], - parent_column=parent_column.name, - parent_kind=parent_kind, - unique="pk" in fk_column.settings or "unique" in fk_column.settings, - by_day=parent_kind != kind, + # `validate_hints` has refused an `after` on any other column, and + # any entry `parent_link` finds no link for. + kind = kinds.temporal_kind(column.data_type) + assert kind is not None + label = f"{table_name}.{column.name}" + links: list[ParentLink] = [] + for entry in entries: + link, _ = parent_link(tables, fk_refs, table_name, entry) + assert link is not None + named = f"{entry[0]}.{entry[1]}" + if position[link.parent_table] > position[table_name]: + raise ValueError( + f'{label}: "after" names {named}, and {link.parent_table} is generated ' + f"after {table_name}: their foreign keys form a cycle, broken at " + f"{link.fk}, whose parents are drawn once {table_name} exists." + ) + link = replace(link, by_day=link.parent_kind != kind) + if link not in links: + links.append(link) + rules.setdefault(table_name, []).append( + ParentRule(table_name, column.name, kind, tuple(links)) ) - if link not in links: - links.append(link) - if links: - rules[table_name] = ParentRule(table_name, column.name, kind, tuple(links)) return rules @@ -183,7 +222,7 @@ def moments(series: pd.Series, kind: str) -> pd.Series: def parent_floor( frame: pd.DataFrame, link: ParentLink, parents: Mapping[str, pd.DataFrame] ) -> pd.Series: - """Per row of `frame`: the creation moment of the parent its `link` points at. + """Per row of `frame`: the moment of `link`'s parent column on the row it points at. NaT where the foreign key is null, points at no row, or the parent's date is null. By day (midnight) when `link.by_day`. @@ -226,17 +265,16 @@ def follow_parents( as_of: AsOf = None, time_profile: Optional[TimeProfile] = None, repick: bool = True, -) -> list: - """Make every row of `df` hold `rule`, in place; return the index labels whose date moved. - - See the module docstring for how. `parents` holds the generated parent - tables. `repick` false skips step 2 (taking another parent), for a pass - that must leave the foreign keys as they are. +) -> tuple[pd.Series, list]: + """Make every row of `df` hold `rule`, in place. + + Returns each row's latest parent moment (NaT where none applies), for + `timeline.order_row_times` to place the column after too, and the index + labels whose date moved. See the module docstring for how. `parents` + holds the generated parent tables. `repick` false skips step 2 (taking + another parent), for a pass that must leave the foreign keys as they are. """ own = moments(df[rule.column], rule.kind) - if not bool(own.notna().any()): - return [] - floors: list[pd.Series] = [] for link in rule.links: floor = parent_floor(df, link, parents) @@ -248,7 +286,7 @@ def follow_parents( lowest = pd.concat(floors, axis=1).max(axis=1) if len(floors) > 1 else floors[0] late = own.notna() & lowest.notna() & (own < lowest) if not bool(late.any()): - return [] + return lowest, [] window = _Window(rule.kind, column, as_of, time_profile) positions = np.flatnonzero(late.to_numpy()) @@ -258,7 +296,7 @@ def follow_parents( lower = pd.Timestamp(floor_values[position]).to_pydatetime() values[position] = _stored(window.draw(lower), rule.kind) df[rule.column] = pd.Series(values, index=df.index, dtype=object) - return df.index[positions].tolist() + return lowest, df.index[positions].tolist() def _repick( @@ -389,11 +427,11 @@ def _time_after(self, offset: int) -> int: # --------------------------------------------------------- -# Later days: a new row's timestamp on its day +# Later days, and the defects: the parents' moments per row # --------------------------------------------------------- def floor_on_day( frame: pd.DataFrame, rule: ParentRule, parents: Mapping[str, pd.DataFrame] ) -> pd.Series: - """Per new row: the latest creation moment among the parents it points at (NaT for none).""" + """Per row: the latest moment among the parent columns `rule` names (NaT for none).""" floors = [parent_floor(frame, link, parents) for link in rule.links] return pd.concat(floors, axis=1).max(axis=1) if len(floors) > 1 else floors[0] diff --git a/model2data/generate/timeline.py b/model2data/generate/timeline.py index 798dd7c..43409f2 100644 --- a/model2data/generate/timeline.py +++ b/model2data/generate/timeline.py @@ -32,6 +32,8 @@ from itertools import accumulate from typing import Optional, Union +from pandas import NaT + from model2data.generate.kinds import temporal_kind from model2data.generate.options import TimeProfile from model2data.parse.dbml import ColumnDef, TableDef @@ -340,13 +342,42 @@ def _infer_stage(column_name: str) -> Optional[int]: return None +def after_entries(note: Optional[dict]) -> list[str]: + """A column's `after` as a list: one name (`created_at`, `customers.created_at`) or several.""" + after = (note or {}).get("after") + if isinstance(after, str): + return [after] + if isinstance(after, list): + return [entry for entry in after if isinstance(entry, str)] + return [] + + +def same_row_after(note: Optional[dict]) -> list[str]: + """The `after` entries naming a column of the same row (no table before a `.`).""" + return [entry for entry in after_entries(note) if "." not in entry] + + +def parent_after(note: Optional[dict]) -> list[tuple[str, str]]: + """The `after` entries naming a parent's column, as `(table key, column)`. + + The last `.` separates the column, so a table outside the default schema + reads as `raw.customers.created_at`. + """ + found = [] + for entry in after_entries(note): + if "." in entry: + table, _, column = entry.rpartition(".") + found.append((table, column)) + return found + + def creation_column_name(columns: list[tuple[str, bool]]) -> Optional[str]: """Which of a table's date/timestamp columns the row comes into being with. `columns` are the table's date and timestamp columns in table order, each - as `(name, follows)`, `follows` being whether it has an `after` hint. A - column with `after` follows another column of its row, so it is never the - one the row starts with. Of the rest, the first whose name puts it at the + as `(name, follows)`, `follows` being whether its `after` names a column of + its own row. Such a column follows another one, so it is never the one the + row starts with. Of the rest, the first whose name puts it at the created stage (`created_at`, `signup_date`, `start_date`: see `_STAGE_WORDS`) is the creation column; failing that, the first whose name puts it at no stage at all (`order_date`, `hire_date`), a birth date @@ -366,8 +397,9 @@ def creation_column_name(columns: list[tuple[str, bool]]) -> Optional[str]: def creation_column(table_def: TableDef) -> Optional[ColumnDef]: """The date/timestamp column a row of `table_def` comes into being with, or None. - See `creation_column_name`. It is the column a child row's own date is - kept on or after its parent's by (see `generate.parents`). + See `creation_column_name`. Generation does not act on it: the model's + validation reads it to suggest a cross-table `after` where a child's date + can fall before its parent's (see `model.validate`). """ temporal = [ column @@ -375,7 +407,7 @@ def creation_column(table_def: TableDef) -> Optional[ColumnDef]: if not column.enum_values and _column_kind(column.data_type) is not None ] name = creation_column_name( - [(column.name, bool((column.note or {}).get("after"))) for column in temporal] + [(column.name, bool(same_row_after(column.note))) for column in temporal] ) return next((column for column in temporal if column.name == name), None) @@ -424,6 +456,10 @@ def _format_value(moment: datetime, kind: str) -> Union[str, date]: return moment.date() +def _is_nat(value: object) -> bool: + return value is None or value is NaT + + def column_business_hours(column: ColumnDef, time_profile: Optional[TimeProfile]) -> bool: """Whether `column` is drawn in business hours: its own hint, else the run's.""" note = column.note or {} @@ -446,22 +482,24 @@ def _build_dependencies( explicit: set[str] = set() for column in temporal_columns: - after = (column.note or {}).get("after") if column.note else None - if not after: + # A parent's column (`customers.created_at`) is no column of this row: + # generate.parents keeps it, and the column stays in the stage system. + names = same_row_after(column.note) + if not names: continue - after_name = str(after) - target = columns_by_name.get(after_name) - if target is None: - raise ValueError( - f"Table '{table_def.name}' column '{column.name}': after references " - f"unknown column '{after_name}'." - ) - if _column_kind(target.data_type) is None: - raise ValueError( - f"Table '{table_def.name}' column '{column.name}': after column " - f"'{after_name}' is not a date or timestamp column." - ) - deps[column.name] = {after_name} + for after_name in names: + target = columns_by_name.get(after_name) + if target is None: + raise ValueError( + f"Table '{table_def.name}' column '{column.name}': after references " + f"unknown column '{after_name}'." + ) + if _column_kind(target.data_type) is None: + raise ValueError( + f"Table '{table_def.name}' column '{column.name}': after column " + f"'{after_name}' is not a date or timestamp column." + ) + deps[column.name] = set(names) explicit.add(column.name) stages = { @@ -521,6 +559,7 @@ def order_row_times( as_of: AsOf = None, time_profile: Optional[TimeProfile] = None, rows: Optional[list] = None, + floors: Optional[dict] = None, ): """Fix up temporal columns so an earlier-stage column never lands later. @@ -537,8 +576,11 @@ def order_row_times( Leaves `df` untouched when the table holds no `after` hint and no recognisable created/updated/closed-style name pair, and never modifies a column that isn't a date/timestamp/datetime type. `rows`, when given, are - the only index labels it places again (a row whose creation date moved - after the table was ordered); every row otherwise. + the only index labels it places again (a row whose date moved after the + table was ordered); every row otherwise. `floors` maps a column to a + `datetime64` Series of the latest parent moment each row must not precede + (a cross-table `after`, see generate.parents): a column placed here lands + after that too. """ columns_by_name = {column.name: column for column in table_def.columns} temporal_columns = [ @@ -570,6 +612,7 @@ def order_row_times( assert other_kind is not None required_kinds[name] = other_kind business_hours = column_business_hours(columns_by_name[column_name], time_profile) + floor = (floors or {}).get(column_name) for idx in df.index if rows is None else rows: if _parse_value(df.at[idx, column_name], kind) is None: @@ -584,6 +627,8 @@ def order_row_times( for parsed in [_parse_value(df.at[idx, name], other_kind)] if parsed is not None ] + if floor is not None and not _is_nat(floor.at[idx]): + earlier.append(floor.at[idx].to_pydatetime()) if not earlier: continue latest_required = max(earlier) diff --git a/model2data/guide/tune.md b/model2data/guide/tune.md index e0cdab0..09205f8 100644 --- a/model2data/guide/tune.md +++ b/model2data/guide/tune.md @@ -62,7 +62,7 @@ columns. Add `not_null` to every column that must always have a value. `references`, `measure`, `generate`. - `generate`: `min`, `max`, `distribution`, `null_rate`, `weights`, `true_rate`, `distinct`, `skew`, `after`, `business_hours`, `growth`, `seasonality`, `transitions`, - `when`, `after_parent`. + `when`. - Table: `columns`, `description`, `color`, `role`, `grain`, `keys`, `foreign_keys`, `incremental`, `defects`. - Top level: `model2data` (spec version), `name`, `description`, `enums`, `tables`, @@ -95,16 +95,14 @@ tables: type: sub_status generate: {transitions: {trial: [active, cancelled], active: [cancelled]}} seats: {type: integer, generate: {min: 1, max: 500, distribution: {kind: lognormal, median: 10}}} - started_at: {type: timestamp, not_null: true, generate: {business_hours: true, growth: 0.5}} + started_at: + type: timestamp + not_null: true + generate: {business_hours: true, growth: 0.5, after: accounts.created_at} cancelled_at: {type: timestamp, generate: {after: started_at, when: {status: [cancelled]}}} updated_at: {type: timestamp, generate: {after: started_at}} defects: - {type: nulls, column: account_id, count: 2} - audit_events: - columns: - id: {type: bigint, pk: true} - account_id: {type: bigint, references: accounts.id} - logged_at: {type: timestamp, not_null: true, generate: {after_parent: false}} run: rows: 100 rows_per_table: {accounts: 20} @@ -127,19 +125,17 @@ only the matching rows. On later days a row whose status moves into a listed val value that day (its `updated_at` when the table has one), and one moving out loses it. It writes the dbt test `model2data_when`. -A row is never created before the parent rows it points at: `started_at` above (the -subscription's creation column) is on or after its account's `created_at`, on every row, -with no hint. The creation column is a table's first date or timestamp without `after` -whose name says created (`created_at`, `signup_date`, `start_date`), else its first that -names no later stage (`order_date`); a child follows each parent that has one, through a -foreign key onto the parent's key. A row dated too early takes another parent created by -then, so the child's growth and seasonality stay as asked; with no such parent its date -moves. Self-references, nullable foreign keys left null, parents without a date and -foreign keys that break a cycle are not followed. `after_parent: false` on the creation -column (spec 0.5.0) turns it off for that table: `audit_events` above copies old events, so -its `logged_at` may predate the account. With `--hint-tests` each followed foreign key -writes the dbt test `model2data_not_before_parent`. Integer primary keys number the rows -1..N, and every output lists the rows in key order. +`after` also names a parent's column as `
.`: `started_at` above is on or after +the `created_at` of the account its `account_id` points at (spec 0.5.0, so the model says +`model2data: 0.5.0`). The table must reach that parent through exactly one one-column foreign +key onto its key; a list (`after: [started_at, accounts.created_at]`) follows each entry. A +row dated before its parent takes another parent created by then, so the column keeps its +growth and seasonality; with none (or a one-to-one) its date moves. A null foreign key +constrains nothing. `validate` warns where a child's first date can fall before its parent's +`created_at`-like date and names the `after` to add. It writes the dbt test +`model2data_not_before_parent`. Integer primary keys number the rows 1..N, and every output +lists the rows in key order. + The full spec and JSON Schema ship in the package: `model2data/spec/README.md`, `model2data/spec/model.schema.json`. diff --git a/model2data/model/__init__.py b/model2data/model/__init__.py index 940d8d5..db7b1a8 100644 --- a/model2data/model/__init__.py +++ b/model2data/model/__init__.py @@ -17,7 +17,7 @@ from model2data.model.document import from_dict, to_dict from model2data.model.dump import dump from model2data.model.engine import EngineInputs, run_as_of, to_engine -from model2data.model.errors import Issue, ModelError +from model2data.model.errors import Issue, ModelError, Suggestion from model2data.model.reader import load, validate from model2data.model.types import ( DEFECT_PRESETS, @@ -58,6 +58,7 @@ "Relationship", "Run", "Shape", + "Suggestion", "Table", "dump", "from_dbml", diff --git a/model2data/model/dbml.py b/model2data/model/dbml.py index 1b2b9fd..a0be792 100644 --- a/model2data/model/dbml.py +++ b/model2data/model/dbml.py @@ -39,7 +39,7 @@ import warnings from typing import Any, Optional -from model2data.model.document import from_dict +from model2data.model.document import from_dict, uses_0_5_after from model2data.model.errors import Issue, ModelError from model2data.model.types import Model @@ -197,18 +197,17 @@ def document(self) -> dict[str, Any]: if self._named("the table", table.name, f"tables.{key}"): self.tables[key] = self._table(key, table) document["tables"] = self.tables - # A note hint `when` is spec 0.4.0 and `after_parent` 0.5.0: the document says so, - # and any other stays 0.2.0. - hints = { - hint + # A note hint `when` is spec 0.4.0, and an `after` that is a list or names a + # parent's column 0.5.0: the document says so, and any other stays 0.2.0. + generates = [ + column.get("generate") or {} for table in self.tables.values() for column in (table.get("columns") or {}).values() if isinstance(column, dict) - for hint in (column.get("generate") or {}) - } - if "after_parent" in hints: + ] + if any(uses_0_5_after(generate.get("after")) for generate in generates): document["model2data"] = "0.5.0" - elif "when" in hints: + elif any("when" in generate for generate in generates): document["model2data"] = "0.4.0" relationships = self._refs() diff --git a/model2data/model/document.py b/model2data/model/document.py index 3822c08..f4fc308 100644 --- a/model2data/model/document.py +++ b/model2data/model/document.py @@ -247,19 +247,25 @@ def uses_0_4(model: Model) -> bool: def uses_0_5(model: Model) -> bool: - """Whether the model uses what spec 0.5.0 added: a column's `after_parent`.""" + """Whether the model uses what spec 0.5.0 added: a list, or a parent's column, in `after`.""" return any( - "after_parent" in column.generate + uses_0_5_after(column.generate.get("after")) for table in model.tables.values() for column in table.columns.values() ) +def uses_0_5_after(after: Any) -> bool: + """Whether an `after` is in a form spec 0.5.0 added: a list, or `
.`.""" + return isinstance(after, list) or (isinstance(after, str) and "." in after) + + def document_version(model: Model) -> Any: """The version the model's document is written against. The version the model was read with, unless the model uses what that - version does not have: 0.5.0 when it uses `after_parent` and was read as + version does not have: 0.5.0 when it uses a list or a parent's column in + `after` and was read as 0.2-0.4, 0.4.0 when it uses `when` and was read as 0.2 or 0.3, 0.3.0 when it uses `defects` or `incremental.history` and was read as 0.2. A document without them is written as the version it was, as always. """ diff --git a/model2data/model/errors.py b/model2data/model/errors.py index 4309163..edb36d4 100644 --- a/model2data/model/errors.py +++ b/model2data/model/errors.py @@ -14,6 +14,25 @@ def format_path(parts: Iterable[PathPart]) -> str: return ".".join(str(part) for part in parts) +@dataclass(frozen=True) +class Suggestion: + """A change that resolves a warning, for a tool to offer as a one-click fix. + + Set the value at `path` (a document path, as `Issue.path`) to `value`: a + string, or a tuple a tool writes as a YAML list. `spec`, when set, is the + spec version the document must then say (`model2data:`), because `value` + uses what that version added. + """ + + path: str + value: Union[str, tuple[str, ...]] + spec: Union[str, None] = None + + def to_dict(self) -> dict: + value = list(self.value) if isinstance(self.value, tuple) else self.value + return {"path": self.path, "value": value, "spec": self.spec} + + @dataclass(frozen=True) class Issue: """One thing wrong with a document: where it is, and what is wrong with it. @@ -26,12 +45,15 @@ class Issue: `severity` is `"error"` for a way the document does not conform, and `"warning"` for one the spec asks a reader to point out in a document that still conforms (a reference onto a column that is not a key). + `suggestion`, on some warnings, is the change that resolves it. """ path: str message: str line: Union[int, None] = None severity: Literal["error", "warning"] = "error" + # A fix a tool can apply for the user (a warning's suggested hint). + suggestion: Union[Suggestion, None] = None @property def is_error(self) -> bool: diff --git a/model2data/model/validate.py b/model2data/model/validate.py index ed4fef8..35aaafa 100644 --- a/model2data/model/validate.py +++ b/model2data/model/validate.py @@ -18,7 +18,7 @@ from collections.abc import Mapping from functools import lru_cache from importlib import resources -from typing import Any, Optional +from typing import Any, Optional, Union import jsonschema from jsonschema.exceptions import ValidationError @@ -26,13 +26,14 @@ from model2data.dbt.naming import dbt_identifier from model2data.generate import kinds from model2data.generate.timeline import creation_column_name -from model2data.model.errors import Issue, PathPart, format_path +from model2data.model.errors import Issue, PathPart, Suggestion, format_path SPEC_VERSION = "0.5.0" # The columns `incremental.history` adds to `
_history`. HISTORY_COLUMNS = ("valid_from", "valid_to", "is_current") # The minor versions this reader implements. 0.3.0 only adds `defects` and -# `incremental.history`, 0.4.0 only `when`, 0.5.0 only `after_parent`, so an older +# `incremental.history`, 0.4.0 only `when`, 0.5.0 only a list or a parent's +# `
.` in `after`, so an older # document reads exactly as it did; it just cannot use them. READS = ("0.2", "0.3", "0.4", "0.5") _URL = "https://www.jbanalytica.com/model2data/spec/{}/model.schema.json" @@ -422,6 +423,9 @@ def run(self) -> list[Issue]: self._relationships() self._groups() self._run() + if self.minor not in ("0.2", "0.3", "0.4"): + self._parent_after_cycles() + self._suggest_parent_after() return self.issues def add(self, path: list[PathPart], message: str, *, warning: bool = False) -> None: @@ -430,14 +434,18 @@ def add(self, path: list[PathPart], message: str, *, warning: bool = False) -> N # -- enums --------------------------------------------------------- def _creation_column(self, key: str) -> Optional[str]: - """The date a row of `key` comes into being with (`generate.timeline.creation_column`).""" + """The date a row of `key` comes into being with (`generate.timeline.creation_column`). + + Read only to suggest a cross-table `after` (`_suggest_parent_after`).""" temporal = [] for name, column in self.columns[key].items(): type_text = column.get("type") if not isinstance(type_text, str) or self.enum_named(type_text): continue if kinds.is_temporal_type(type_text): - follows = isinstance(_mapping(column.get("generate")).get("after"), str) + after = _mapping(column.get("generate")).get("after") + entries = after if isinstance(after, list) else [after] + follows = any(isinstance(e, str) and "." not in e for e in entries) temporal.append((name, follows)) return creation_column_name(temporal) @@ -665,7 +673,7 @@ def _needs_0_4(self, path: list[PathPart], what: str = "when") -> bool: ) return True - def _needs_0_5(self, path: list[PathPart], what: str = "after_parent") -> bool: + def _needs_0_5(self, path: list[PathPart], what: str) -> bool: """Report what 0.5.0 added (`what`) in a 0.2-0.4 document; True when it was reported.""" if self.minor not in ("0.2", "0.3", "0.4"): return False @@ -1080,41 +1088,8 @@ def _hints( self._when(key, name, column, when, [*gen_path, "when"], is_fk, in_pk or in_key) after = generate.get("after") - if temporal and isinstance(after, str): - other = self.columns[key].get(after) - if after == name: - self.add( - [*gen_path, "after"], "names the column itself: `after` names another column" - ) - elif other is None: - self.add( - [*gen_path, "after"], f"names {_show(after)}, which is not a column of {key}" - ) - elif not kinds.is_temporal_type(str(other.get("type", ""))) or self.enum_named( - other.get("type") - ): - self.add( - [*gen_path, "after"], - f"names {after}, which is not a date or timestamp column", - ) - - if ( - "after_parent" in generate - and temporal - and not self._needs_0_5([*gen_path, "after_parent"]) - ): - creation = self._creation_column(key) - if creation != name: - where = ( - f"which in {key} is {creation}" - if creation is not None - else f"and every date of {key} follows another column" - ) - self.add( - [*gen_path, "after_parent"], - f"sits on the date a row comes into being with, {where}: only that " - "column is kept on or after its parent rows' own", - ) + if temporal and after is not None: + self._after(key, name, after, [*gen_path, "after"]) if integer: bounds = {} @@ -1294,18 +1269,185 @@ def _misplaced( } return "only sits on " + " or ".join(words.get(kind, kind) for kind in allowed) + # -- after --------------------------------------------------------- + def _after(self, key: str, name: str, after: Any, path: list[PathPart]) -> None: + """Each entry of an `after`: a column of the row, or a parent's `
.`.""" + entries = after if isinstance(after, list) else [after] + if not all(isinstance(entry, str) for entry in entries): + return # the schema has reported it + if (isinstance(after, list) or any("." in entry for entry in entries)) and self._needs_0_5( + path, "a list or a `
.` in `after`" + ): + return + for position, entry in enumerate(entries): + at = [*path, position] if isinstance(after, list) else path + if "." in entry: + problem = self._parent_after_issue(key, entry) + if problem: + self.add(at, f"names {entry}, {problem}") + continue + other = self.columns[key].get(entry) + if entry == name: + self.add(at, "names the column itself: `after` names another column") + elif other is None: + self.add(at, f"names {_show(entry)}, which is not a column of {key}") + elif not self._is_temporal(other): + self.add(at, f"names {entry}, which is not a date or timestamp column") + + def _is_temporal(self, column: Mapping) -> bool: + type_text = column.get("type") + return ( + isinstance(type_text, str) + and kinds.is_temporal_type(type_text) + and not self.enum_named(type_text) + ) + + def parent_keys(self, key: str) -> dict[str, list[tuple[str, str]]]: + """The `references` of `key`'s columns by parent table: `(child column, parent column)`.""" + found: dict[str, list[tuple[str, str]]] = {} + for name, column in self.columns.get(key, {}).items(): + reference = column.get("references") + to = reference.get("to") if isinstance(reference, Mapping) else reference + if isinstance(to, str) and "." in to: + parent, parent_column = split_column_path(to) + found.setdefault(parent, []).append((name, parent_column)) + # A `foreign_keys` entry has two columns or more: it identifies no row by one. + return found + + def _parent_after_issue(self, key: str, entry: str) -> Optional[str]: + """Why `entry`, a `
.` in an `after` of `key`, names no parent's date.""" + parent, column = split_column_path(entry) + if parent not in self.tables: + return f"and there is no table {_show(parent)}" + if parent == key: + return ( + f"a column of {key} itself: name a column of the same row without its table " + f"(`after: {column}`); a parent row in the same table is not followed" + ) + target = self.columns.get(parent, {}).get(column) + if target is None: + return f"and {parent} has no column {_show(column)}" + if not self._is_temporal(target): + return "which is not a date or timestamp column" + keys = self.parent_keys(key).get(parent, []) + if not keys: + composite = any( + _mapping(fk).get("references") == parent + for fk in _list(_mapping(self.tables.get(key)).get("foreign_keys")) + ) + if composite: + return ( + f"and {key} reaches {parent} only through a foreign key of several " + "columns: `after` follows a one-column foreign key" + ) + return ( + f"and {key} has no foreign key to {parent}: `after` reaches another table " + "only through a foreign key of the row" + ) + if len(keys) > 1: + names = ", ".join(child for child, _ in keys) + return ( + f"and {key} reaches {parent} through more than one foreign key ({names}): " + f"`after` cannot tell which {parent} row it means" + ) + ((child, parent_column),) = keys + if {parent_column} not in self.key_sets(parent): + return ( + f"and {child} references {parent}.{parent_column}, which is not a key: " + f"`after` needs the one {parent} row the foreign key points at" + ) + return None + + def _parent_after_cycles(self) -> None: + """Tables whose `after`s name each other's columns: neither can be drawn first.""" + edges: dict[str, list[tuple[str, list[PathPart], str]]] = {} + for key, columns in self.columns.items(): + for name, column in columns.items(): + after = _mapping(column.get("generate")).get("after") + entries = after if isinstance(after, list) else [after] + for position, entry in enumerate(entries): + if not isinstance(entry, str) or "." not in entry: + continue + parent = split_column_path(entry)[0] + if parent in self.tables and parent != key: + path: list[PathPart] = ["tables", key, "columns", name, "generate", "after"] + if isinstance(after, list): + path.append(position) + edges.setdefault(key, []).append((parent, path, entry)) + + def reaches(start: str, goal: str) -> bool: + seen, stack = set(), [start] + while stack: + node = stack.pop() + if node == goal: + return True + if node not in seen: + seen.add(node) + stack.extend(parent for parent, _, _ in edges.get(node, [])) + return False + + for key, outgoing in edges.items(): + for parent, path, entry in outgoing: + if reaches(parent, key): + self.add( + path, + f"names {entry}, and an `after` of {parent} (or of a table it names) " + f"names a column of {key}: the tables' dates cannot each follow the other", + ) + + def _suggest_parent_after(self) -> None: + """Warn where a child's first date can fall before the parent row it points at.""" + for key, columns in self.columns.items(): + named = set() + for column in columns.values(): + after = _mapping(column.get("generate")).get("after") + for entry in after if isinstance(after, list) else [after]: + if isinstance(entry, str) and "." in entry: + named.add(split_column_path(entry)[0]) + child = self._creation_column(key) + if child is None: + continue + for parent, keys in self.parent_keys(key).items(): + if parent == key or parent in named or parent not in self.tables or len(keys) != 1: + continue + ((fk, parent_column),) = keys + if {parent_column} not in self.key_sets(parent): + continue + created = self._creation_column(parent) + if created is None: + continue + target = f"{parent}.{created}" + current = _mapping(columns[child].get("generate")).get("after") + if current is None: + value: Union[str, tuple[str, ...]] = target + else: + value = (*(current if isinstance(current, list) else [current]), target) + path = ["tables", key, "columns", child, "generate", "after"] + self.issues.append( + Issue( + format_path(["tables", key, "columns", child]), + f"can fall before {target} (the {parent} row it points at through " + f"{fk}); add `after: {target}` to keep it after", + severity="warning", + suggestion=Suggestion( + format_path(path), + value, + None if self.minor in (None, "0.5") else SPEC_VERSION, + ), + ) + ) + def _after_cycles(self, key: str, columns: dict[str, dict]) -> None: - edges: dict[str, str] = {} + edges: dict[str, list[str]] = {} for name, column in columns.items(): after = _mapping(column.get("generate")).get("after") - if isinstance(after, str) and after in columns and after != name: - edges[name] = after + for entry in after if isinstance(after, list) else [after]: + if isinstance(entry, str) and entry in columns and entry != name: + edges.setdefault(name, []).append(entry) reported: set = set() - for start in edges: - chain = [start] - node = start - while node in edges: - node = edges[node] + + def walk(chain: list[str]) -> None: + for node in edges.get(chain[-1], []): if node in chain: cycle = chain[chain.index(node) :] if not reported.intersection(cycle): @@ -1316,8 +1458,11 @@ def _after_cycles(self, key: str, columns: dict[str, dict]) -> None: ["tables", key, "columns", first, "generate", "after"], f"the after hints of {key} form a cycle: {' -> '.join([*loop, first])}", ) - break - chain.append(node) + elif not reported.intersection(chain): + walk([*chain, node]) + + for start in edges: + walk([start]) # -- elsewhere ----------------------------------------------------- def _column_path(self, path: list[PathPart], value: Any) -> None: diff --git a/model2data/spec/README.md b/model2data/spec/README.md index 8cbbca3..175eb4d 100644 --- a/model2data/spec/README.md +++ b/model2data/spec/README.md @@ -97,14 +97,13 @@ document keeps saying the version it says and pointing at that version's schema; uses `when` says `model2data: 0.4.0`, and `when` in a document that says 0.2 or 0.3 is an error. A writer writes the version the document was read with, and 0.4.0 once it uses `when`. -0.5.0 adds a column's [`after_parent`](#references), and nothing else in the document: a -0.2, 0.3 or 0.4 document is a 0.5.0 document that does not use it. A document that uses it says -`model2data: 0.5.0`, and `after_parent` in a document that says an earlier version is an -error. A writer writes the version the document was read with, and 0.5.0 once it uses -`after_parent`. 0.5.0 also says two things about the generated data of every document, as -until 1.0.0 a minor release may: a child row's creation date is not before its parents' (see -[References](#references)), and an integer primary key numbers the rows 1 to n, in order (see -[Keys](#keys)). +0.5.0 adds two forms of a column's [`after`](#references): a parent's column, +`
.`, and a list. A 0.2, 0.3 or 0.4 document is a 0.5.0 document that uses +neither, and reads as it always read. A document that uses either says `model2data: 0.5.0`, +and either in a document that says an earlier version is an error. A writer writes the +version the document was read with, and 0.5.0 once it uses either. 0.5.0 also says, of the +generated data of every document, as until 1.0.0 a minor release may, that an integer primary +key numbers the rows 1 to n, in order (see [Keys](#keys)). 0.2.0 replaced 0.1.0, which carried hints as JSON inside DBML notes. See [From 0.1](#from-01). @@ -185,19 +184,25 @@ reference is generated like any foreign key -- every non-null child value is dra values the parent column actually holds, so each one exists in the parent -- but it does not imply one parent row per value, as a reference onto a key does. -A child row comes into being on or after the parent rows it points at (spec 0.5.0): an order -is not placed before its customer signed up. A table's **creation column** is its first date -or timestamp column without `after` whose name puts it at the created stage (`created_at`, -`signup_date`, `start_date`: the engine's stage words), or, when none does, its first without -`after` whose name puts it at no stage (`order_date`), a birth date excepted; a table whose -dates all follow something, or are all updated- or closed-stage, has none. On every row, the -child's creation column is not before the creation column of the parent row each foreign key -points at, when the key references the parent's primary key or a unique column and the parent -has a creation column. A date compared with a timestamp compares by day. A null foreign key, -a self-reference, a foreign key that breaks a cycle of references and a parent without a -creation column constrain nothing. The child's later dates follow its creation date as they -always do. `after_parent: false` on the creation column leaves it free of its parents. How -the engine keeps the child's own shape while doing so is the engine's (see +A date can be kept on or after a date of the parent row it points at: `after: +customers.created_at` on `orders.order_date` places no order before its customer signed up +(spec 0.5.0). `after` names a column of the same row, a parent's column as `
.` +(a column path: the last `.` separates the column), or a list of them, all of which the value +is not before. A parent's column is read on the row the child's foreign key points at, so: + +- the named table is reached through exactly one foreign key of one column, onto the parent's + primary key, a one-column key, or a unique column. A table with two foreign keys to the parent + (`sender_id` and `receiver_id` onto `users`) cannot say which row it means, and it is an + error naming them; so is a parent reached only through a foreign key of several columns, or + through a reference onto a column that is no key; +- the named column is a date or timestamp; a date compared with a timestamp compares by day; +- a row whose foreign key is null is not constrained; +- the named table is not the column's own: a column of the same row is named without a table, + and a parent row in the same table (a self-reference) is not followed; +- two tables' `after`s must not name each other's columns (a cycle), and the parent must not be + generated after the child (a cycle of foreign keys broken at the very foreign key followed). + +How the engine keeps the child's own shape while doing so is the engine's (see `model2data.generate.parents`). A foreign key over several columns is written on its table under `foreign_keys`, pairing @@ -298,7 +303,7 @@ orders: - A day's new rows continue the existing ones: an integer key continues after the largest value held, any other unique value differs from every value held, and a foreign key points at a row that exists by then, one inserted the same day included (parents are inserted before - children), and a new row's creation column is not before that parent's. A foreign key that + children), and a new row's `after` of a parent's column holds on that parent. A foreign key that must be unique takes each parent once, so a table whose parents run out inserts fewer rows that day. Every date and timestamp column of a new row falls on the day (a timestamp at any time of it, weighted toward working hours when the run shape has @@ -419,8 +424,9 @@ reports each failure with the path of the value (`tables.orders.columns.status.g 3. A `keys` entry names columns of its own table. 4. Every hint sits on a column of a kind it applies to. 5. Every key of `weights` is a member of the column's enum. -6. `after` names another temporal column of the same table, and the `after` hints of a table do - not form a cycle. +6. Each entry of an `after` names another temporal column of the same table, or a temporal + column of a parent table reached as [References](#references) says; the `after` hints of a + table do not form a cycle, and two tables' `after`s do not name each other's columns. 7. On an integer column, `min` and `max` are whole numbers, and `min` does not exceed `max` once a bound left out takes its default (0 and 100). 8. `null_rate` is only on a nullable column (it would otherwise have no rows to null). @@ -460,6 +466,13 @@ conforms. 2. A `grain` contains no key of its table (its primary key, a `keys` entry, or a `unique` column). The generator does not read `grain`, so generated rows may repeat it; declaring it as a key too makes them unique. +3. A table reaches a parent through one one-column foreign key onto a key, the parent has a + date or timestamp that reads as its creation (by the engine's stage words, as `created_at`, + or else its first that names no later stage), the table's own first such date has no `after` + naming that parent, and no other column's `after` names it: the child's date can fall + before its parent's. The warning names the `after` to add. A tool may offer it as a fix: the + engine's warning carries the path and value to set (and the spec version the document then + needs). ## From 0.1 diff --git a/model2data/spec/examples/coffee_webshop.model2data.yml b/model2data/spec/examples/coffee_webshop.model2data.yml index e6e2669..8c5acb4 100644 --- a/model2data/spec/examples/coffee_webshop.model2data.yml +++ b/model2data/spec/examples/coffee_webshop.model2data.yml @@ -1,5 +1,5 @@ -# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.2.0/model.schema.json -model2data: 0.2.0 +# yaml-language-server: $schema=https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json +model2data: 0.5.0 name: coffee_webshop description: > A Belgian specialty-coffee roaster's customers, catalogue, orders and reviews. @@ -80,7 +80,7 @@ tables: order_date: type: timestamp not_null: true - generate: {business_hours: true, seasonality: 0.5} + generate: {business_hours: true, seasonality: 0.5, after: customers.created_at} shipped_at: type: timestamp generate: {after: order_date} diff --git a/model2data/spec/model.schema.json b/model2data/spec/model.schema.json index 05e208a..ebd3bbe 100644 --- a/model2data/spec/model.schema.json +++ b/model2data/spec/model.schema.json @@ -7,7 +7,7 @@ "required": ["model2data", "tables"], "properties": { "model2data": { - "description": "The spec version the document is written against, `MAJOR.MINOR` or `MAJOR.MINOR.PATCH`. Write it as `0.5.0`. A 0.2, 0.3 or 0.4 document is still read, and reads as it did: 0.3.0 only adds `defects` and `incremental.history`, which a 0.2 document cannot use, 0.4.0 only a column's `when`, which a 0.2 or 0.3 document cannot use, and 0.5.0 only a column's `after_parent`, which a 0.2, 0.3 or 0.4 document cannot use. A reader that met the YAML number 0.5 reads it as `0.5`.", + "description": "The spec version the document is written against, `MAJOR.MINOR` or `MAJOR.MINOR.PATCH`. Write it as `0.5.0`. A 0.2, 0.3 or 0.4 document is still read, and reads as it did: 0.3.0 only adds `defects` and `incremental.history`, which a 0.2 document cannot use, 0.4.0 only a column's `when`, which a 0.2 or 0.3 document cannot use, and 0.5.0 only an `after` that is a list or names a parent's column, which a 0.2, 0.3 or 0.4 document cannot use. A reader that met the YAML number 0.5 reads it as `0.5`.", "oneOf": [ { "type": "string", "pattern": "^0\\.[2345](\\.(0|[1-9][0-9]*))?$" }, { "enum": [0.2, 0.3, 0.4, 0.5] } @@ -76,6 +76,12 @@ "pattern": "^[^./\\\\\\u0000-\\u001f\\u007f]+(\\.[^./\\\\\\u0000-\\u001f\\u007f]+){1,2}$", "examples": ["customers.id", "raw.users.id"] }, + "afterTarget": { + "description": "What an `after` names: a column of the same table (a name), or a column of a parent table (a column path: `table.column` or `schema.table.column`).", + "type": "string", + "pattern": "^[^./\\\\\\u0000-\\u001f\\u007f]+(\\.[^./\\\\\\u0000-\\u001f\\u007f]+){0,2}$", + "examples": ["placed_at", "customers.created_at"] + }, "color": { "description": "A hex colour literal: `#rgb`, `#rgba`, `#rrggbb` or `#rrggbbaa`. Quote it in YAML, where `#` starts a comment.", "type": "string", @@ -486,14 +492,23 @@ "x-model2data-applies-to": ["non-key"] }, "skew": { - "description": "How unevenly the foreign key's values spread over the parent rows, replacing the run's skew for this column. 0: every parent equally likely. Above 0: the parents, in a seeded random order, are weighted exp(−8 · skew² · i / n) by rank i of n; at 0.8 the top fifth of parents hold roughly 65–75% of the rows, at 1.0 roughly 80%. Where the child's creation date follows its parent's (see `after_parent`), the parents created earliest also take the rows dated before later ones existed, so the spread is steeper.", + "description": "How unevenly the foreign key's values spread over the parent rows, replacing the run's skew for this column. 0: every parent equally likely. Above 0: the parents, in a seeded random order, are weighted exp(−8 · skew² · i / n) by rank i of n; at 0.8 the top fifth of parents hold roughly 65–75% of the rows, at 1.0 roughly 80%. Where the child's date has an `after` naming the parent's date, the parents created earliest also take the rows dated before later ones existed, so the spread is steeper.", "$ref": "#/$defs/fraction", "x-model2data-applies-to": ["foreign-key"] }, "after": { - "description": "Another temporal column of the same table. In every row, this column's value is not earlier than that column's. A date column that follows a timestamp compares by day: it is not before the timestamp's date. The `after` hints of one table must not form a cycle.", - "$ref": "#/$defs/name", - "x-model2data-applies-to": ["temporal"] + "description": "What this column is not earlier than, in every row: another temporal column of the same table (`placed_at`), a temporal column of a parent table as `
.` (`customers.created_at`), or a list of them, every one of which it is not earlier than. A parent's column is read on the parent row the row's foreign key points at, so the table reaches that parent through exactly one one-column foreign key onto its primary key or a unique column; a null foreign key constrains nothing. A date compared with a timestamp compares by day: a date is not before the timestamp's date, a timestamp not before the date's midnight. The `after` hints of one table must not form a cycle, nor may two tables' `after`s name each other's columns. A list or a parent's column is spec 0.5.0.", + "oneOf": [ + { "$ref": "#/$defs/afterTarget" }, + { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/afterTarget" } + } + ], + "x-model2data-applies-to": ["temporal"], + "examples": ["placed_at", "customers.created_at", ["placed_at", "customers.created_at"]] }, "business_hours": { "description": "Replaces the run's business-hours setting for this column. When true, weekend days weigh 0.3 against a weekday's 1, and a timestamp's hour is drawn from a working-day profile (busiest 08:00–17:59, a lunch dip, rare at night) rather than evenly.", @@ -511,11 +526,6 @@ "$ref": "#/$defs/fraction", "x-model2data-applies-to": ["temporal"] }, - "after_parent": { - "description": "Whether this column, the date a row comes into being with, is kept on or after the creation date of every parent row the row's foreign keys point at: an order is not placed before its customer signed up (see References in the spec README). `true` is the default; `false` draws the column on its own, its parents' dates aside. It sits only on its table's creation column: the first date or timestamp column without `after` whose name puts it at the created stage (`created_at`, `signup_date`, `start_date`), or, when none does, the first without `after` whose name puts it at no stage (`order_date`), a birth date excepted. Spec 0.5.0.", - "type": "boolean", - "x-model2data-applies-to": ["temporal"] - }, "transitions": { "description": "The states each enum member may move to when the row is updated on a later generated day (see a table's `incremental`): a mapping of member to the members that may follow it. A member with no entry, or an empty list, does not change. Every key and every target is a member of the column's enum.", "type": "object", diff --git a/tests/test_cli_model.py b/tests/test_cli_model.py index 004f3ea..39cff74 100644 --- a/tests/test_cli_model.py +++ b/tests/test_cli_model.py @@ -160,7 +160,7 @@ def test_two_table_keys_that_would_be_one_dbt_name_are_refused(in_tmp): def test_generate_prints_warnings_and_goes_on(in_tmp): result = runner.invoke(app, ["--file", str(EXAMPLES / "hackernews.dbml"), "--rows", "10"]) assert result.exit_code == 0, result.output - assert "⚠️ hackernews.dbml: 1 warning" in result.output + assert "⚠️ hackernews.dbml: 2 warnings" in result.output assert "tables._dlt_loads.columns.schema_version_hash.references" in result.output @@ -194,7 +194,7 @@ def test_validate_prints_every_issue_with_its_path_and_exits_1(tmp_path): def test_validate_exits_0_on_warnings_alone(tmp_path): result = runner.invoke(app, ["validate", str(EXAMPLES / "hackernews.dbml")]) assert result.exit_code == 0 - assert "⚠️ hackernews.dbml: 1 warning" in result.output + assert "⚠️ hackernews.dbml: 2 warnings" in result.output assert "conforms to spec 0.2.0" in result.output diff --git a/tests/test_defects.py b/tests/test_defects.py index c221cde..955e830 100644 --- a/tests/test_defects.py +++ b/tests/test_defects.py @@ -146,7 +146,12 @@ def test_an_empty_defects_list_is_kept(): def test_a_0_2_model_given_defects_is_written_as_0_3(): - model = load((EXAMPLES / "ecommerce.model2data.yml").read_text()) + model = load( + (EXAMPLES / "ecommerce.model2data.yml") + .read_text() + .replace("0.5.0", "0.2.0") + .replace("generate: {after: customers.created_at}", "description: placed") + ) assert model.version == "0.2.0" assert to_dict(model)["model2data"] == "0.2.0" assert dump(model).splitlines()[0].endswith("/spec/0.2.0/model.schema.json") diff --git a/tests/test_defects_dbt.py b/tests/test_defects_dbt.py index 250cb7f..654c805 100644 --- a/tests/test_defects_dbt.py +++ b/tests/test_defects_dbt.py @@ -191,8 +191,10 @@ def test_a_preset_fails_exactly_the_tests_the_report_names( def test_the_parent_dates_test_passes_clean_and_fails_a_customer_created_late( tmp_path, monkeypatch ): - """Orders follow their customers' `created_at`: dbt agrees, and catches one that does not.""" - project = _generate(tmp_path, monkeypatch, SHOP) + """`after: customers.created_at` holds: dbt agrees, and catches an order that does not.""" + model = load(SHOP) + model.tables["orders"].columns["ordered_at"].generate["after"] = "customers.created_at" + project = _generate(tmp_path, monkeypatch, dump(model)) assert (project / "macros" / "model2data_parent_tests.sql").exists() assert _build(project) == set() diff --git a/tests/test_metrics_cli.py b/tests/test_metrics_cli.py index 08ca607..62e336e 100644 --- a/tests/test_metrics_cli.py +++ b/tests/test_metrics_cli.py @@ -99,7 +99,7 @@ def test_validate_refuses_a_metrics_file_whose_model_does_not_conform(tmp_path): def test_validate_mixes_models_and_metrics_files(tmp_path): result = _invoke(["validate", COFFEE, COFFEE_METRICS], tmp_path) assert result.exit_code == 0, result.output - assert "conforms to spec 0.2.0" in result.output + assert "conforms to spec 0.5.0" in result.output assert "conforms to metrics spec 0.1.0" in result.output assert "✅ 2 model files conform." in result.output diff --git a/tests/test_model_dbml.py b/tests/test_model_dbml.py index 43b10a1..a5feff0 100644 --- a/tests/test_model_dbml.py +++ b/tests/test_model_dbml.py @@ -350,7 +350,9 @@ def test_a_ref_onto_a_column_that_is_no_key_draws_from_the_parents_values(): a warning, and every hash a load names is a version's hash.""" model = load(EXAMPLES / "hackernews.dbml") assert [issue.path for issue in model.warnings] == [ - "tables._dlt_loads.columns.schema_version_hash.references" + "tables._dlt_loads.columns.schema_version_hash.references", + # A load's pipeline state can predate the load: the suggested `after`. + "tables._dlt_pipeline_state.columns.created_at", ] inputs = to_engine(model) for seed in (1, 7, 42): diff --git a/tests/test_parent_dates.py b/tests/test_parent_dates.py index 1ee6572..5db9924 100644 --- a/tests/test_parent_dates.py +++ b/tests/test_parent_dates.py @@ -1,4 +1,4 @@ -"""A child row is created on or after its parents, and integer keys number the rows 1..N.""" +"""Cross-table `after` (a child's date after its parent row's), its warning, and keys 1..N.""" from __future__ import annotations @@ -18,7 +18,7 @@ from model2data.generate.options import TimeProfile from model2data.generate.parents import _Window from model2data.generate.timeline import creation_column -from model2data.model import from_dict, to_engine +from model2data.model import Suggestion, dump, from_dbml, from_dict, to_engine from model2data.model.errors import ModelError from model2data.parse.dbml import ColumnDef, TableDef @@ -59,7 +59,12 @@ "order_date": { "type": "timestamp", "not_null": True, - "generate": {"business_hours": True, "growth": 0.5, "seasonality": 0.6}, + "generate": { + "business_hours": True, + "growth": 0.5, + "seasonality": 0.6, + "after": "customers.created_at", + }, }, "shipped_at": {"type": "timestamp", "generate": {"after": "order_date"}}, }, @@ -69,7 +74,11 @@ "id": {"type": "bigint", "pk": True}, "product_id": {"type": "bigint", "not_null": True, "references": "products.id"}, "customer_id": {"type": "bigint", "references": "customers.id"}, - "review_date": {"type": "date", "not_null": True}, + "review_date": { + "type": "date", + "not_null": True, + "generate": {"after": ["products.launched_on", "customers.created_at"]}, + }, } }, }, @@ -91,6 +100,11 @@ def _generate(document: dict, seed: int = 3, **kwargs) -> dict[str, pd.DataFrame ) +def _engine(document: dict) -> tuple[dict, list]: + inputs = to_engine(from_dict(document)) + return inputs.tables, inputs.refs + + def _joined( frames: dict[str, pd.DataFrame], child: str, fk: str, column: str, parent: str, created: str ) -> pd.DataFrame: @@ -106,35 +120,18 @@ def _joined( return joined -# --------------------------------------------------------------------------- -# Which column a row comes into being with -# --------------------------------------------------------------------------- -@pytest.mark.parametrize( - ("columns", "expected"), - [ - ([("updated_at", "timestamp"), ("created_at", "timestamp")], "created_at"), - ([("order_date", "timestamp"), ("shipped_at", "timestamp")], "order_date"), - ([("birth_date", "date"), ("hire_date", "date")], "hire_date"), - ([("updated_at", "timestamp"), ("deleted_at", "timestamp")], None), - ([("name", "varchar")], None), - ], -) -def test_the_creation_column_is_the_created_stage_else_the_first_unstaged_date(columns, expected): - table = TableDef("t", [ColumnDef(name, kind) for name, kind in columns]) - found = creation_column(table) - assert (found.name if found else None) == expected +def _without(document: dict, table: str, column: str) -> dict: + plain = copy.deepcopy(document) + plain["tables"][table]["columns"][column]["generate"].pop("after") + return plain -def test_a_column_with_after_is_never_the_creation_column(): - table = TableDef( - "t", - [ - ColumnDef("event_date", "date", note={"after": "logged_at"}), - ColumnDef("logged_at", "timestamp"), - ], - ) - found = creation_column(table) - assert found is not None and found.name == "logged_at" +def _issues(document: dict) -> list[str]: + try: + model = from_dict(document) + except ModelError as error: + return [str(issue) for issue in error.issues] + return [str(issue) for issue in model.warnings] # --------------------------------------------------------------------------- @@ -150,7 +147,13 @@ def test_no_order_is_placed_before_its_customer_signed_up(seed): assert (pd.to_datetime(shipped["shipped_at"]) >= pd.to_datetime(shipped["order_date"])).all() -def test_a_date_follows_every_parent_and_compares_with_a_timestamp_by_day(): +def test_without_the_hint_orders_predate_their_customers(): + frames = _generate(_without(SHOP, "orders", "order_date")) + joined = _joined(frames, "orders", "customer_id", "order_date", "customers", "created_at") + assert (joined["_mine"] < joined["_theirs"]).mean() > 0.3 + + +def test_a_list_follows_every_parent_and_compares_a_date_with_a_timestamp_by_day(): frames = _generate(SHOP) by_product = _joined(frames, "reviews", "product_id", "review_date", "products", "launched_on") assert (by_product["_mine"] >= by_product["_theirs"]).all() @@ -162,9 +165,53 @@ def test_a_date_follows_every_parent_and_compares_with_a_timestamp_by_day(): assert frames["reviews"]["customer_id"].isna().any() +def test_a_list_may_mix_a_column_of_the_row_with_a_parents(): + document = copy.deepcopy(SHOP) + columns = document["tables"]["orders"]["columns"] + columns["shipped_at"]["generate"]["after"] = ["order_date", "customers.created_at"] + columns["order_date"]["generate"].pop("after") + frames = _generate(document) + orders = frames["orders"].dropna(subset=["shipped_at"]) + assert (pd.to_datetime(orders["shipped_at"]) >= pd.to_datetime(orders["order_date"])).all() + joined = _joined(frames, "orders", "customer_id", "shipped_at", "customers", "created_at") + assert (joined["_mine"] >= joined["_theirs"]).all() + + +def test_a_parent_after_keeps_the_column_in_its_stage_chain(): + """`created_at` with a parent's `after` still comes before the row's `updated_at`.""" + document: dict[str, Any] = { + "model2data": "0.5.0", + "tables": { + "accounts": { + "columns": { + "id": {"type": "int", "pk": True}, + "created_at": {"type": "timestamp", "not_null": True}, + } + }, + "users": { + "columns": { + "id": {"type": "int", "pk": True}, + "account_id": {"type": "int", "not_null": True, "references": "accounts.id"}, + "created_at": { + "type": "timestamp", + "not_null": True, + "generate": {"after": "accounts.created_at"}, + }, + "updated_at": {"type": "timestamp", "not_null": True}, + } + }, + }, + } + frames = _generate(document, rows={"accounts": 30, "users": 500}) + users = frames["users"] + assert (pd.to_datetime(users["updated_at"]) >= pd.to_datetime(users["created_at"])).all() + joined = _joined(frames, "users", "account_id", "created_at", "accounts", "created_at") + assert (joined["_mine"] >= joined["_theirs"]).all() + + def test_the_orders_keep_their_own_shape(): """Late rows take an older parent rather than moving their date: the order - dates' seasonal peak and growth are what the column asks for.""" + dates' seasonal peak and working hours are what the column asks for.""" frames = _generate(SHOP, rows={**ROWS, "orders": 20000}) moments = pd.to_datetime(frames["orders"]["order_date"]) gift = ((moments >= "2025-11-28") & (moments <= "2025-12-24")).sum() / 27 @@ -174,20 +221,6 @@ def test_the_orders_keep_their_own_shape(): assert ((hours >= 8) & (hours < 18)).mean() > 0.7 -def test_after_parent_false_draws_the_column_on_its_own(): - document = copy.deepcopy(SHOP) - document["tables"]["orders"]["columns"]["order_date"]["generate"]["after_parent"] = False - frames = _generate(document) - joined = _joined(frames, "orders", "customer_id", "order_date", "customers", "created_at") - assert (joined["_mine"] < joined["_theirs"]).any() - assert "orders" not in parent_rules_for(*_engine(document)) - - -def _engine(document: dict) -> tuple[dict, list]: - inputs = to_engine(from_dict(document)) - return inputs.tables, inputs.refs - - def test_a_one_to_one_child_keeps_its_parent_and_moves_its_date(): document: dict[str, Any] = { "model2data": "0.5.0", @@ -206,7 +239,11 @@ def test_a_one_to_one_child_keeps_its_parent_and_moves_its_date(): "not_null": True, "references": {"to": "users.id", "one_to_one": True}, }, - "created_at": {"type": "timestamp", "not_null": True}, + "created_at": { + "type": "timestamp", + "not_null": True, + "generate": {"after": "users.created_at"}, + }, } }, }, @@ -216,27 +253,7 @@ def test_a_one_to_one_child_keeps_its_parent_and_moves_its_date(): assert profiles["user_id"].is_unique joined = _joined(frames, "profiles", "user_id", "created_at", "users", "created_at") assert (joined["_mine"] >= joined["_theirs"]).all() - assert (joined["_mine"] < MIDNIGHT + pd.Timedelta(seconds=1)).all() - - -def test_self_references_and_parents_without_a_date_are_not_followed(): - document: dict[str, Any] = { - "model2data": "0.5.0", - "tables": { - "teams": {"columns": {"id": {"type": "int", "pk": True}, "name": "word"}}, - "employees": { - "columns": { - "id": {"type": "int", "pk": True}, - "team_id": {"type": "int", "not_null": True, "references": "teams.id"}, - "manager_id": {"type": "int", "references": "employees.id"}, - "hired_on": {"type": "date", "not_null": True}, - } - }, - }, - } - assert parent_rules_for(*_engine(document)) == {} - frames = _generate(document, rows={"teams": 5, "employees": 50}) - assert set(frames["employees"]["team_id"]) <= set(frames["teams"]["id"]) + assert (joined["_mine"] <= MIDNIGHT).all() @pytest.mark.parametrize("kind", ["date", "timestamp"]) @@ -260,13 +277,58 @@ def test_a_moved_timestamp_keeps_business_hours(): assert sum(8 <= m.hour < 18 for m in moments) / len(moments) > 0.7 -def test_the_rule_is_deterministic(): +def test_the_hint_is_deterministic(): first = _generate(SHOP, seed=11) second = _generate(SHOP, seed=11) for key, frame in first.items(): pd.testing.assert_frame_equal(frame, second[key]) +def test_an_empty_or_undated_child_is_left_alone(): + document = copy.deepcopy(SHOP) + document["tables"]["reviews"]["columns"]["review_date"] = { + "type": "date", + "generate": {"null_rate": 1.0, "after": "products.launched_on"}, + } + frames = _generate(document, rows={**ROWS, "orders": 0}) + assert frames["orders"].empty + assert frames["reviews"]["review_date"].isna().all() + + +def test_a_date_filled_by_when_still_follows_its_parent(): + """`when` fills a date after the hint ran; the row then moves again.""" + document: dict[str, Any] = { + "model2data": "0.5.0", + "enums": {"kind": ["a", "b"]}, + "tables": { + "customers": { + "columns": { + "id": {"type": "int", "pk": True}, + "created_at": {"type": "timestamp", "not_null": True}, + } + }, + "events": { + "columns": { + "id": {"type": "int", "pk": True}, + "customer_id": {"type": "int", "not_null": True, "references": "customers.id"}, + "kind": {"type": "kind", "not_null": True}, + "created_at": { + "type": "timestamp", + "generate": {"when": {"kind": ["a"]}, "after": "customers.created_at"}, + }, + "updated_at": {"type": "timestamp"}, + } + }, + }, + } + for seed in range(1, 6): + frames = _generate(document, seed=seed, rows={"customers": 50, "events": 400}) + joined = _joined(frames, "events", "customer_id", "created_at", "customers", "created_at") + assert (joined["_mine"] >= joined["_theirs"]).all() + events = frames["events"].dropna(subset=["created_at", "updated_at"]) + assert (pd.to_datetime(events["updated_at"]) >= pd.to_datetime(events["created_at"])).all() + + # --------------------------------------------------------------------------- # The days after the first # --------------------------------------------------------------------------- @@ -284,67 +346,347 @@ def test_every_day_keeps_new_orders_after_their_customers(): # --------------------------------------------------------------------------- -# Validation of the hint +# Validation # --------------------------------------------------------------------------- -def _issues(document: dict) -> list[str]: - try: - from_dict(document) - except ModelError as error: - return [str(issue) for issue in error.issues] - return [] +def _with_after(after: Any, *, table: str = "orders", column: str = "order_date") -> dict: + document = copy.deepcopy(SHOP) + document["tables"][table]["columns"][column]["generate"]["after"] = after + return document -def test_after_parent_on_another_date_names_the_creation_column(): - document = copy.deepcopy(SHOP) - document["tables"]["orders"]["columns"]["shipped_at"]["generate"]["after_parent"] = False +@pytest.mark.parametrize( + ("after", "message"), + [ + ("shops.opened_at", 'names shops.opened_at, and there is no table "shops"'), + ( + "customers.signed_up", + 'names customers.signed_up, and customers has no column "signed_up"', + ), + ("customers.email", "names customers.email, which is not a date or timestamp column"), + ( + "products.launched_on", + "names products.launched_on, and orders has no foreign key to products: `after` " + "reaches another table only through a foreign key of the row", + ), + ( + "orders.shipped_at", + "names orders.shipped_at, a column of orders itself: name a column of the same row " + "without its table (`after: shipped_at`)", + ), + (["customers.created_at", "placed"], 'names "placed", which is not a column of orders'), + ], +) +def test_a_wrong_after_says_what_is_wrong(after, message): + issues = _issues(_with_after(after)) + assert any(message in issue for issue in issues), issues + + +def test_two_foreign_keys_to_the_parent_are_named(): + document: dict[str, Any] = { + "model2data": "0.5.0", + "tables": { + "users": { + "columns": { + "id": {"type": "int", "pk": True}, + "created_at": {"type": "timestamp", "not_null": True}, + } + }, + "messages": { + "columns": { + "id": {"type": "int", "pk": True}, + "sender_id": {"type": "int", "references": "users.id"}, + "receiver_id": {"type": "int", "references": "users.id"}, + "sent_at": {"type": "timestamp", "generate": {"after": "users.created_at"}}, + } + }, + }, + } (issue,) = _issues(document) - assert "tables.orders.columns.shipped_at.generate.after_parent" in issue - assert "which in orders is order_date" in issue + assert issue == ( + "tables.messages.columns.sent_at.generate.after: names users.created_at, and messages " + "reaches users through more than one foreign key (sender_id, receiver_id): `after` " + "cannot tell which users row it means" + ) -def test_after_parent_on_a_non_date_and_a_non_boolean_are_errors(): - document = copy.deepcopy(SHOP) - document["tables"]["orders"]["columns"]["customer_id"]["generate"]["after_parent"] = False - document["tables"]["orders"]["columns"]["order_date"]["generate"]["after_parent"] = "no" +def test_a_reference_onto_no_key_or_a_composite_one_is_refused(): + document: dict[str, Any] = { + "model2data": "0.5.0", + "tables": { + "batches": { + "keys": [{"pk": ["site", "number"]}], + "columns": { + "site": "int", + "number": "int", + "code": "varchar", + "made_at": {"type": "timestamp"}, + }, + }, + "items": { + "foreign_keys": [ + { + "columns": ["site", "number"], + "references": "batches", + "to_columns": ["site", "number"], + } + ], + "columns": { + "id": {"type": "int", "pk": True}, + "site": "int", + "number": "int", + "made_at": {"type": "timestamp", "generate": {"after": "batches.made_at"}}, + }, + }, + "labels": { + "columns": { + "id": {"type": "int", "pk": True}, + "code": {"type": "varchar", "references": "batches.code"}, + "printed_at": {"type": "timestamp", "generate": {"after": "batches.made_at"}}, + }, + }, + }, + } + issues = _issues(document) + assert any("only through a foreign key of several columns" in i for i in issues), issues + assert any( + "code references batches.code, which is not a key: `after` needs the one batches row" in i + for i in issues + ), issues + + +def test_tables_whose_afters_name_each_other_are_a_cycle(): + document: dict[str, Any] = { + "model2data": "0.5.0", + "tables": { + "a": { + "columns": { + "id": {"type": "int", "pk": True}, + "b_id": {"type": "int", "references": "b.id"}, + "at": {"type": "timestamp", "generate": {"after": "b.at"}}, + } + }, + "b": { + "columns": { + "id": {"type": "int", "pk": True}, + "a_id": {"type": "int", "references": "a.id"}, + "at": {"type": "timestamp", "generate": {"after": "a.at"}}, + } + }, + }, + } issues = _issues(document) + assert len([i for i in issues if "cannot each follow the other" in i]) == 2, issues + + +def test_a_same_table_cycle_through_a_list_is_found(): + document = _with_after(["customers.created_at", "shipped_at"]) assert any( - "customer_id.generate.after_parent" in i and "date or timestamp" in i for i in issues + "form a cycle: order_date -> shipped_at -> order_date" in i for i in _issues(document) ) - assert any("order_date.generate.after_parent" in i for i in issues) -def test_after_parent_needs_spec_0_5(): - document = copy.deepcopy(SHOP) +@pytest.mark.parametrize("after", ["customers.created_at", ["shipped_at"]]) +def test_the_new_forms_need_spec_0_5(after): + document = _with_after(after) document["model2data"] = "0.4.0" - document["tables"]["orders"]["columns"]["order_date"]["generate"]["after_parent"] = False - (issue,) = _issues(document) - assert "write `model2data: 0.5.0`" in issue + document["tables"]["reviews"]["columns"]["review_date"]["generate"].pop("after") + issues = _issues(document) + assert any("is spec 0.5.0" in i and "write `model2data: 0.5.0`" in i for i in issues), issues -def test_the_generator_checks_after_parent_on_bare_tables(): +def test_the_generator_checks_after_on_bare_tables(): tables = { + "customers": TableDef( + "customers", [ColumnDef("id", "int", {"pk"}), ColumnDef("created_at", "timestamp")] + ), "orders": TableDef( "orders", [ ColumnDef("id", "int", {"pk"}), - ColumnDef("order_date", "date"), - ColumnDef("total", "int", note={"after_parent": False}), + ColumnDef("order_date", "date", note={"after": "customers.created_at"}), ], - ) + ), } - with pytest.raises(ValueError, match="only applies to date/timestamp columns"): + with pytest.raises(ValueError, match="orders has no foreign key to customers"): validate_hints(tables, []) - tables["orders"].columns[2] = ColumnDef("shipped_on", "date", note={"after_parent": False}) - with pytest.raises(ValueError, match="which in orders is order_date"): + tables["orders"].columns[1] = ColumnDef("order_date", "date", note={"after": [3]}) + with pytest.raises(ValueError, match="must be a column name"): validate_hints(tables, []) +def test_a_parent_drawn_after_the_child_is_refused_at_generation(): + """A foreign-key cycle broken at the very key the `after` follows.""" + tables = { + "users": TableDef( + "users", + [ + ColumnDef("id", "int", {"pk"}), + ColumnDef("post_id", "int"), + ColumnDef("created_at", "timestamp", note={"after": "posts.posted_at"}), + ], + ), + "posts": TableDef( + "posts", + [ + ColumnDef("id", "int", {"pk"}), + ColumnDef("user_id", "int", {"not null"}), + ColumnDef("posted_at", "timestamp"), + ], + ), + } + refs = [ + { + "source_table": "users", + "source_column": "post_id", + "target_table": "posts", + "target_column": "id", + }, + { + "source_table": "posts", + "source_column": "user_id", + "target_table": "users", + "target_column": "id", + }, + ] + with pytest.raises(ValueError, match="posts is generated after users"): + generate_data_from_dbml(tables, refs, base_rows=5, seed=1, as_of=AS_OF) + + +# --------------------------------------------------------------------------- +# The warning +# --------------------------------------------------------------------------- +def test_a_child_without_the_hint_is_warned_with_the_fix(): + document = _without(SHOP, "orders", "order_date") + model = from_dict(document) + (warning,) = [w for w in model.warnings if "orders" in w.path] + assert warning.severity == "warning" + assert warning.path == "tables.orders.columns.order_date" + assert warning.message == ( + "can fall before customers.created_at (the customers row it points at through " + "customer_id); add `after: customers.created_at` to keep it after" + ) + assert warning.suggestion == Suggestion( + "tables.orders.columns.order_date.generate.after", "customers.created_at" + ) + assert warning.suggestion.to_dict() == { + "path": "tables.orders.columns.order_date.generate.after", + "value": "customers.created_at", + "spec": None, + } + + +def test_the_fix_extends_an_after_and_names_the_spec_an_older_document_needs(): + document = _without(SHOP, "orders", "order_date") + document["model2data"] = "0.4.0" + document["tables"]["reviews"]["columns"]["review_date"]["generate"].pop("after") + model = from_dict(document) + suggestions = {w.path: w.suggestion for w in model.warnings} + assert suggestions["tables.orders.columns.order_date"] == Suggestion( + "tables.orders.columns.order_date.generate.after", "customers.created_at", "0.5.0" + ) + # reviews has two parents with a date: a warning for each. + assert len([w for w in model.warnings if w.path.startswith("tables.reviews")]) == 2 + + listed = copy.deepcopy(SHOP) + listed["tables"]["reviews"]["columns"]["review_date"]["generate"]["after"] = [ + "products.launched_on" + ] + (warning,) = from_dict(listed).warnings + assert warning.suggestion is not None + assert warning.suggestion.value == ("products.launched_on", "customers.created_at") + assert warning.suggestion.to_dict()["value"] == ["products.launched_on", "customers.created_at"] + + +def test_no_warning_where_it_would_be_noise(): + document: dict[str, Any] = { + "model2data": "0.5.0", + "tables": { + "teams": {"columns": {"id": {"type": "int", "pk": True}, "name": "word"}}, + "users": { + "columns": { + "id": {"type": "int", "pk": True}, + "created_at": {"type": "timestamp"}, + } + }, + "employees": { + "columns": { + "id": {"type": "int", "pk": True}, + "team_id": {"type": "int", "references": "teams.id"}, + "manager_id": {"type": "int", "references": "employees.id"}, + "hired_on": {"type": "date"}, + } + }, + "messages": { + "columns": { + "id": {"type": "int", "pk": True}, + "sender_id": {"type": "int", "references": "users.id"}, + "receiver_id": {"type": "int", "references": "users.id"}, + "sent_at": {"type": "timestamp"}, + } + }, + }, + } + # A parent without a date, a self-reference, two foreign keys to one parent. + assert from_dict(document).warnings == [] + assert from_dict(SHOP).warnings == [] + + +def test_an_enum_named_like_a_date_is_not_what_the_warning_names(): + document = _without(SHOP, "orders", "order_date") + document["enums"] = {"date_bucket": ["early", "late"]} + columns = document["tables"]["orders"]["columns"] + document["tables"]["orders"]["columns"] = { + "id": columns["id"], + "bucket": "date_bucket", + **{k: v for k, v in columns.items() if k != "id"}, + } + (warning,) = [w for w in from_dict(document).warnings if "orders" in w.path] + assert warning.path == "tables.orders.columns.order_date" + + +@pytest.mark.parametrize( + ("columns", "expected"), + [ + ([("updated_at", "timestamp"), ("created_at", "timestamp")], "created_at"), + ([("order_date", "timestamp"), ("shipped_at", "timestamp")], "order_date"), + ([("birth_date", "date"), ("hire_date", "date")], "hire_date"), + ([("updated_at", "timestamp"), ("deleted_at", "timestamp")], None), + ([("name", "varchar")], None), + ], +) +def test_the_warning_reads_the_created_stage_else_the_first_unstaged_date(columns, expected): + table = TableDef("t", [ColumnDef(name, kind) for name, kind in columns]) + found = creation_column(table) + assert (found.name if found else None) == expected + + +# --------------------------------------------------------------------------- +# Conversion and writing +# --------------------------------------------------------------------------- +def test_dbml_with_a_parent_after_converts_to_0_5(): + model = from_dbml( + "Table customers {\n id int [pk]\n created_at timestamp\n}\n" + "Table orders {\n id int [pk]\n customer_id int [ref: > customers.id]\n" + ' order_date timestamp [note: \'{"after": "customers.created_at"}\']\n}\n' + ) + assert str(model.version) == "0.5.0" + + +def test_a_model_given_a_parent_after_is_written_as_0_5(): + document = _without(SHOP, "orders", "order_date") + document["model2data"] = "0.4.0" + document["tables"]["reviews"]["columns"]["review_date"]["generate"].pop("after") + model = from_dict(document) + model.tables["orders"].columns["order_date"].generate["after"] = "customers.created_at" + assert "model2data: 0.5.0" in dump(model) + + # --------------------------------------------------------------------------- # dbt # --------------------------------------------------------------------------- -def test_dbt_gets_one_parent_test_per_followed_foreign_key(tmp_path): +def test_dbt_gets_one_test_per_parent_column(tmp_path): tables, refs = _engine(SHOP) - tests = [t for t in hint_tests_for(from_dict(SHOP)) if t.hint == "after_parent"] + tests = [t for t in hint_tests_for(from_dict(SHOP)) if t.test == "model2data_not_before_parent"] assert [(t.table, t.column, t.arguments["foreign_key"]) for t in tests] == [ ("orders", "order_date", "customer_id"), ("reviews", "review_date", "product_id"), @@ -360,7 +702,7 @@ def test_dbt_gets_one_parent_test_per_followed_foreign_key(tmp_path): "parent_column": "created_at", } # Bare tables need their refs for it. - assert not [t for t in hint_tests_for(tables) if t.hint == "after_parent"] + assert not [t for t in hint_tests_for(tables) if t.test == "model2data_not_before_parent"] generate_dbt_yml(tmp_path, tables, refs, hint_tests="warn") assert (tmp_path / "macros" / PARENT_MACROS_FILE).exists() @@ -372,6 +714,15 @@ def test_dbt_gets_one_parent_test_per_followed_foreign_key(tmp_path): assert not (plain / "macros" / PARENT_MACROS_FILE).exists() +def test_a_list_writes_a_test_for_each_column_of_the_row_too(): + document = copy.deepcopy(SHOP) + columns = document["tables"]["orders"]["columns"] + columns["shipped_at"]["generate"]["after"] = ["order_date", "customers.created_at"] + names = [(t.column, t.test) for t in hint_tests_for(from_dict(document)) if t.table == "orders"] + assert ("shipped_at", "model2data_not_before") in names + assert ("shipped_at", "model2data_not_before_parent") in names + + def test_the_defects_check_reads_the_parent_test_as_its_sql_does(): tables, refs = _engine(SHOP) frames = _generate(SHOP) @@ -384,13 +735,24 @@ def test_the_defects_check_reads_the_parent_test_as_its_sql_does(): seeded = {key: as_seeded(frame) for key, frame in frames.items()} assert not any(fails(t, seeded) for t in tests) + orders_test = next(t for t in tests if t.table == "orders") + assert not fails(orders_test, {"orders": seeded["orders"]}) + assert not fails(orders_test, {**seeded, "orders": seeded["orders"].iloc[0:0]}) + broken = frames["orders"].copy() broken.loc[0, "order_date"] = "2000-01-01 00:00:00" seeded["orders"] = as_seeded(broken) - orders_test = next(t for t in tests if t.table == "orders") assert fails(orders_test, seeded) +def test_parent_rules_name_the_followed_columns(): + rules = parent_rules_for(*_engine(SHOP)) + assert [(r.table, r.column) for found in rules.values() for r in found] == [ + ("orders", "order_date"), + ("reviews", "review_date"), + ] + + # --------------------------------------------------------------------------- # Integer keys # --------------------------------------------------------------------------- @@ -440,52 +802,73 @@ def test_later_days_continue_the_numbering(): assert day1["id"].tolist() == list(range(301, 309)) -# --------------------------------------------------------------------------- -# Edges -# --------------------------------------------------------------------------- -def test_an_empty_or_undated_child_is_left_alone(): - document = copy.deepcopy(SHOP) - document["tables"]["reviews"]["columns"]["review_date"] = { - "type": "date", - "generate": {"null_rate": 1.0}, +def _bare(parent_columns: list[ColumnDef], refs: list[dict]) -> tuple[dict, list]: + tables = { + "users": TableDef("users", [ColumnDef("id", "int", {"pk"}), *parent_columns]), + "messages": TableDef( + "messages", + [ + ColumnDef("id", "int", {"pk"}), + ColumnDef("sender_id", "int"), + ColumnDef("receiver_id", "int"), + ColumnDef("code", "varchar"), + ColumnDef("sent_at", "timestamp"), + ], + ), } - frames = _generate(document, rows={**ROWS, "orders": 0}) - assert frames["orders"].empty - assert frames["reviews"]["review_date"].isna().all() + return tables, refs -def test_a_date_filled_by_when_still_follows_its_parent(): - """`when` fills a creation date after the rule ran; the row then moves again.""" - document: dict[str, Any] = { - "model2data": "0.5.0", - "enums": {"kind": ["a", "b"]}, - "tables": { - "customers": { - "columns": { - "id": {"type": "int", "pk": True}, - "created_at": {"type": "timestamp", "not_null": True}, - } - }, - "events": { - "columns": { - "id": {"type": "int", "pk": True}, - "customer_id": {"type": "int", "not_null": True, "references": "customers.id"}, - "kind": {"type": "kind", "not_null": True}, - "created_at": {"type": "timestamp", "generate": {"when": {"kind": ["a"]}}}, - "updated_at": {"type": "timestamp"}, - } - }, - }, +def _ref(column: str, target: str = "id") -> dict: + return { + "source_table": "messages", + "source_column": column, + "target_table": "users", + "target_column": target, } - for seed in range(1, 6): - frames = _generate(document, seed=seed, rows={"customers": 50, "events": 400}) - joined = _joined(frames, "events", "customer_id", "created_at", "customers", "created_at") - assert (joined["_mine"] >= joined["_theirs"]).all() - events = frames["events"].dropna(subset=["created_at", "updated_at"]) - assert (pd.to_datetime(events["updated_at"]) >= pd.to_datetime(events["created_at"])).all() -def test_the_parent_check_without_the_parent_or_rows_passes(): +@pytest.mark.parametrize( + ("after", "parent_columns", "refs", "message"), + [ + ("staff.created_at", [], [], "there is no table staff"), + ("messages.code", [], [], "a column of messages itself"), + ("users.joined", [], [_ref("sender_id")], "users has no column joined"), + ( + "users.handle", + [ColumnDef("handle", "varchar")], + [_ref("sender_id")], + "which is not a date or timestamp column", + ), + ( + "users.created_at", + [ColumnDef("created_at", "timestamp")], + [_ref("sender_id"), _ref("receiver_id")], + r"more than one foreign key \(sender_id, receiver_id\)", + ), + ( + "users.created_at", + [ColumnDef("created_at", "timestamp"), ColumnDef("handle", "varchar")], + [_ref("code", "handle")], + "code references users.handle, which is not a key", + ), + ], +) +def test_the_generator_names_what_is_wrong_with_a_parent_after( + after, parent_columns, refs, message +): + tables, refs = _bare(parent_columns, refs) + tables["messages"].columns[4] = ColumnDef("sent_at", "timestamp", note={"after": after}) + with pytest.raises(ValueError, match=message): + validate_hints(tables, refs) + + +def test_an_after_that_is_no_name_is_left_to_the_schema(): + issues = _issues(_with_after(["customers.created_at", 3])) + assert any("generate.after" in issue for issue in issues), issues + + +def test_the_parent_check_passes_rows_pointing_at_no_parent(): tables, refs = _engine(SHOP) test = next( t @@ -494,40 +877,6 @@ def test_the_parent_check_without_the_parent_or_rows_passes(): ) frames = _generate(SHOP) seeded = {key: as_seeded(frame) for key, frame in frames.items()} - assert not fails(test, {"orders": seeded["orders"]}) - assert not fails(test, {**seeded, "orders": seeded["orders"].iloc[0:0]}) - - -def test_dbml_with_an_after_parent_note_converts_to_0_5(): - from model2data.model import from_dbml - - model = from_dbml( - "Table customers {\n id int [pk]\n created_at timestamp\n}\n" - "Table orders {\n id int [pk]\n customer_id int [ref: > customers.id]\n" - " order_date timestamp [note: '{\"after_parent\": false}']\n}\n" - ) - assert str(model.version) == "0.5.0" - - -def test_a_model_given_after_parent_is_written_as_0_5(): - from model2data.model import dump - - document = copy.deepcopy(SHOP) - document["model2data"] = "0.4.0" - model = from_dict(document) - model.tables["orders"].columns["order_date"].generate["after_parent"] = False - assert "model2data: 0.5.0" in dump(model) - - -def test_an_enum_named_like_a_date_is_not_a_creation_candidate(): - document = copy.deepcopy(SHOP) - document["enums"] = {"date_bucket": ["early", "late"]} - columns = document["tables"]["orders"]["columns"] - document["tables"]["orders"]["columns"] = { - "id": columns["id"], - "bucket": "date_bucket", - **{k: v for k, v in columns.items() if k != "id"}, - } - columns = document["tables"]["orders"]["columns"] - columns["order_date"]["generate"]["after_parent"] = False - assert _issues(document) == [] + customers = seeded["customers"].copy() + customers["id"] = [str(10_000 + n) for n in range(len(customers))] + assert not fails(test, {**seeded, "customers": customers}) From 5fc5f59b9266c549ac92fdce95d5c4595b7e1e1d Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Thu, 8 Oct 2026 21:48:57 +0200 Subject: [PATCH 08/14] Document the cross-table `after` and re-record determinism The changelog's Unreleased section, the README, LLMS.md and the time-shapes, distributions and dbt-project pages describe `after` naming a parent's column, its errors and the warning that suggests it, in place of the automatic rule. Dates change only for models that use it (the examples now do); ids change for every model with an integer primary key. The determinism digests are recorded again for these deliberate changes. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 87 +++++++++++++---------- LLMS.md | 25 ++++--- README.md | 10 +-- README_PYPI.md | 10 +-- docs/dbt-project.md | 2 +- docs/distributions.md | 6 +- docs/time-shapes.md | 43 +++++++----- tests/fixtures/determinism.json | 120 +++++++++++++++----------------- 8 files changed, 165 insertions(+), 138 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d28be1..02612df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,18 +8,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added -- **A child row is created on or after its parents.** An order is no longer placed before its - customer signed up (in `examples/ecommerce.model2data.yml` half the orders were). A table's - **creation column** is its first date or timestamp column without `after` whose name says the - row is created (`created_at`, `signup_date`, `start_date`, ...), or, when none does, its first - that names no later stage (`order_date`, `hire_date`); a birth date never is. On every row, - a child's creation column is on or after the creation column of each parent row its foreign - keys point at (a foreign key onto the parent's primary key or a unique column, to a parent - that has a creation column); with several parents, after the latest. A date compared with a - timestamp compares by day, and the child's later dates (`after`, updated/closed names) follow - its creation date as before. Not followed, and documented as such: self-references, nullable - foreign keys left null, parents without a creation column, and foreign keys that break a - cycle of references. +- **`after` can name a parent's column.** `after: customers.created_at` on `orders.order_date` + keeps every order on or after the `created_at` of the customer its foreign key points at (in + `examples/ecommerce.model2data.yml`, half the orders used to predate their customer). `after` + takes a column of the same row as before, a parent's column as `
.` (the last + `.` separates the column, so `raw.customers.created_at` works), or a list of either, all of + which the value is not before: `after: [orders.ordered_at, products.launched_at]`, or + `after: [placed_at, customers.created_at]`. A date compared with a timestamp compares by day. + - **Reaching the parent:** through exactly one one-column foreign key onto the parent's + primary key or a unique column. A null foreign key constrains nothing on its row. Errors, + in the style of the others, for an unknown table or column, a column that is no date, a + table the row has no foreign key to (or only a composite one, or one onto a column that is + no key), the column's own table (a column of the row is named without its table; a + self-reference is not followed), more than one foreign key to the parent (`messages` + reaching `users` through `sender_id` and `receiver_id` is an error naming both), two + tables whose `after`s name each other's columns, and a parent generated after the child + (a cycle of foreign keys broken at the very key followed). - **The child keeps its own shape.** A row whose date falls before its parent's takes another parent, one created by then, drawn from the parents the column already points at; its date keeps its draw, so growth, seasonality, business hours and the `as_of` window hold exactly. @@ -27,27 +31,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/). parent) moves its date instead, drawn again from the column's own shape cut to the part of the window on or after the parent's; a parent created on the window's last day gives a child on that day, never past `as_of`. Parents created early collect more children, so a skewed - foreign key is steeper than before: on a coffee-webshop model with `skew: 0.8` and - customers growing over the year, the top fifth of customers hold about 80% of the orders - rather than 65%, while the orders' gift-season peak and growth are unchanged. - - **Days after the first** keep it too: a new row whose parent was inserted the same day lands - later that day than its parent. - - **`after_parent: false`** on the creation column draws it on its own, its parents aside, for - the table whose rows really can predate them (imported history, say). It is validated like - any hint: on a column that is not a date or timestamp, or a date that is not the table's - creation column, it is an error naming the column that is. It is new vocabulary, so it - needs **spec 0.5.0** (`model2data: 0.5.0`); a 0.2, 0.3 or 0.4 document using it is an error - naming the version to write, and reads exactly as before otherwise. DBML with an - `after_parent` note converts to 0.5.0. The schema's `$id` moves to `/spec/0.5.0/`. - - **dbt: `model2data_not_before_parent`.** With hint tests on (`--hint-tests`), each foreign key - a creation date follows writes a relationship-aware test on the child's creation column: no - row dated before the parent row it joins to (by day for a date against a timestamp; a key - several parent rows share counts its earliest, leaving duplicates to the `unique` test). - The macro ships in `macros/model2data_parent_tests.sql`, written only when a project has - such a test. `hint_tests_for(tables, refs=...)` takes the references for it when given bare - tables. The defects report checks it like every other test, so a defect that dates a row - before its parent lists it in `EXPECTED_FAILURES.md`; `late_arriving` takes rows that stay - clear of their parents first, so it still breaks no test where it can. + foreign key is steeper: on a coffee-webshop model with `skew: 0.8`, customers growing over + the year and `after: customers.created_at` on the orders, the top fifth of customers hold + about 80% of the orders rather than 65%, while the orders' gift-season peak and growth are + unchanged. A composite-key repair or a `when` fill that dates a row before its parent + afterwards moves only that row's date, and its later dates follow it again. + - **Days after the first:** a new row whose parent was inserted the same day lands later that + day than its parent. + - **Spec 0.5.0.** A list or a parent's column in `after` is new vocabulary: a document using + either says `model2data: 0.5.0`, and a 0.2, 0.3 or 0.4 document using it gets an error + naming the version to write (and otherwise reads exactly as before). DBML with such a note + converts to 0.5.0. The schema's `$id` moves to `/spec/0.5.0/`. + - **dbt: `model2data_not_before_parent`.** With hint tests on (`--hint-tests`), each parent + column an `after` names writes a test that joins the child to the parent model through the + foreign key and returns the rows dated before their parent row (by day for a date against a + timestamp; a key several parent rows share counts its earliest, leaving duplicates to the + `unique` test). The macro ships in `macros/model2data_parent_tests.sql`, written only when a + project has such a test. `hint_tests_for(tables, refs=...)` takes the references for it when + given bare tables. The defects report checks it like every other test, so a defect that + dates a row before its parent lists it in `EXPECTED_FAILURES.md`; `late_arriving` takes + rows that stay clear of their parents first, so it still breaks no test where it can. +- **A warning where a child's date can fall before its parent's.** When a table reaches a parent + through one foreign key, the parent has a date that reads as its creation (`created_at`, + `signup_date`, or its first date naming no later stage), and the table's own first date has no + `after` naming that parent, validation (and so every run) warns, on + `tables.orders.columns.order_date`: "can fall before customers.created_at (the customers row + it points at through customer_id); add `after: customers.created_at` to keep it after". The + warning's `Issue` carries a `suggestion` (`model2data.model.Suggestion`: the `path` to set, + the `value`, and the `spec` version an older document must then say) for a tool to offer as a + fix. No warning for a parent without such a date, a self-reference, two foreign keys to one + parent, or a table whose `after` already names that parent. Generation itself does not change. +- The bundled examples use it where it is plainly right: `ecommerce`, `ecommerce_daily`, + `ecommerce_training` (orders after their customer), `advanced_features` (time entries after + the employee's hire date), `saas_platform` (subscriptions, invoices and audit events after + their organisation), and the spec's coffee-webshop example; those documents now say 0.5.0. ### Changed - **Integer primary keys number the rows 1..N, and the rows come out in key order.** A table's @@ -63,8 +80,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - **Output changes for the same seed (the determinism promise is per version).** Every model with an integer primary key generates different bytes than 1.14.0: the key no longer draws from the table's random stream, so every column after it draws from a different point of it. - A model whose child tables have a creation date and a dated parent also changes where the - rule applies. The same model, seed, `--as-of` and options still give the same bytes on any + Dates change only where a model uses a parent's column in `after` (the examples listed + above now do). The same model, seed, `--as-of` and options still give the same bytes on any machine and any day with 1.15.0. Pin `model2data==1.14.0` to keep fixtures of the old shape. ## [1.14.0] - 2026-10-07 diff --git a/LLMS.md b/LLMS.md index f9868ab..a53da66 100644 --- a/LLMS.md +++ b/LLMS.md @@ -161,19 +161,22 @@ shipped_at timestamp [note: '{"after": "ordered_at"}'] A birth-date-style column (`birth_date`, `date_of_birth`, `dob`) is never folded into this chain — it describes the person, not the record. -**A child row is never created before its parents.** A table's creation column (its first -date/timestamp without `after` named like `created_at`/`signup_date`/`start_date`, else its first -named for no stage, like `order_date`) is on or after the creation column of every parent row its -foreign keys point at (onto a pk or unique key): no order before its customer's `created_at`. -Rows drawn too early take another, older parent, so the child's own growth/seasonality/business -hours hold; a row with none (or a one-to-one) moves its date. Not followed: self-references, null -foreign keys, parents without a date, cycle-breaking foreign keys. Opt out per table on the -creation column, spec 0.5.0 (`model2data: 0.5.0` in YAML): +**`after` can name a parent's column.** `
.` reaches the parent row through the +row's foreign key: `after: customers.created_at` keeps every order on or after its customer's +sign-up. A list follows every entry, and may mix the row's own columns: ```dbml -logged_at timestamp [note: '{"after_parent": false}'] +order_date timestamp [note: '{"after": "customers.created_at"}'] +shipped_at timestamp [note: '{"after": ["order_date", "customers.created_at"]}'] ``` -With `--hint-tests`, each followed foreign key writes the dbt test `model2data_not_before_parent`. -Integer primary keys are 1..N and every output lists rows in key order. +The table must reach the parent through exactly one one-column foreign key onto its pk or a unique +column (two, like `sender_id`/`receiver_id` to `users`, is an error naming them). Rows dated too +early take an older parent, so the column's growth/seasonality/business hours hold; a row with +none (or a one-to-one) moves its date. A null foreign key constrains nothing; a self-reference is +not a parent (name the column without a table). In YAML it is spec 0.5.0 (`model2data: 0.5.0`). +Validation warns where a child's first date can fall before a parent's `created_at`-like date, +with the `after` to add. With `--hint-tests` each parent column writes the dbt test +`model2data_not_before_parent`. Integer primary keys are 1..N and every output lists rows in key +order. **Tie lifecycle columns to the status they belong to with `when`.** Without it, every nullable column is filled or nulled independently of the others, so a `todo` task gets a `completed_at` diff --git a/README.md b/README.md index 1f4dab2..7be20e1 100644 --- a/README.md +++ b/README.md @@ -42,9 +42,10 @@ emails, not `Lorem ipsum`. Type a column with any Faker provider (`billing_count come from one country. → [The model file](docs/model-file.md) **Foreign keys that resolve.** Tables are generated in dependency order and every foreign key -points at a parent row that exists — onto a primary key or a unique one — created no later than -the child: no order is placed before its customer signed up. Integer primary keys number the -rows 1..N, in order. Composite primary and unique keys hold, and a column that repeats a +points at a parent row that exists — onto a primary key or a unique one — and +`after: customers.created_at` keeps an order on or after the sign-up of the customer it points +at (`validate` suggests it where a date could fall before its parent's). Integer primary keys +number the rows 1..N, in order. Composite primary and unique keys hold, and a column that repeats a parent's value through a foreign key (`orders.customer_email`) holds the email of the order's own customer. → [The model file](docs/model-file.md) @@ -70,8 +71,7 @@ byte-identical. → [The determinism promise](docs/generating.md#the-determinism **When things happen.** `--business-hours`, `--growth` and `--seasonality` shape every timestamp (weekdays and working hours, a trend, a Q4 peak), or one column at a time. `updated_at` never -lands before `created_at`, a child's `created_at` never before its parent's, `after` orders any -two columns, and `when` gives `shipped_at` a value on shipped and delivered orders only. → [When things happen](docs/time-shapes.md) +lands before `created_at`, `after` orders any two columns, or a column after a parent's, and `when` gives `shipped_at` a value on shipped and delivered orders only. → [When things happen](docs/time-shapes.md) **How the data is spread.** `--skew` lets a few customers place most of the orders; `weights`, `true_rate`, `null_rate` and `distinct` shape a column; `distribution` draws a number from a diff --git a/README_PYPI.md b/README_PYPI.md index 421fed2..c794737 100644 --- a/README_PYPI.md +++ b/README_PYPI.md @@ -42,9 +42,10 @@ emails, not `Lorem ipsum`. Type a column with any Faker provider (`billing_count come from one country. → [The model file](https://github.com/JB-Analytica/model2data/blob/main/docs/model-file.md) **Foreign keys that resolve.** Tables are generated in dependency order and every foreign key -points at a parent row that exists — onto a primary key or a unique one — created no later than -the child: no order is placed before its customer signed up. Integer primary keys number the -rows 1..N, in order. Composite primary and unique keys hold, and a column that repeats a +points at a parent row that exists — onto a primary key or a unique one — and +`after: customers.created_at` keeps an order on or after the sign-up of the customer it points +at (`validate` suggests it where a date could fall before its parent's). Integer primary keys +number the rows 1..N, in order. Composite primary and unique keys hold, and a column that repeats a parent's value through a foreign key (`orders.customer_email`) holds the email of the order's own customer. → [The model file](https://github.com/JB-Analytica/model2data/blob/main/docs/model-file.md) @@ -70,8 +71,7 @@ byte-identical. → [The determinism promise](https://github.com/JB-Analytica/mo **When things happen.** `--business-hours`, `--growth` and `--seasonality` shape every timestamp (weekdays and working hours, a trend, a Q4 peak), or one column at a time. `updated_at` never -lands before `created_at`, a child's `created_at` never before its parent's, `after` orders any -two columns, and `when` gives `shipped_at` a value on shipped and delivered orders only. → [When things happen](https://github.com/JB-Analytica/model2data/blob/main/docs/time-shapes.md) +lands before `created_at`, `after` orders any two columns, or a column after a parent's, and `when` gives `shipped_at` a value on shipped and delivered orders only. → [When things happen](https://github.com/JB-Analytica/model2data/blob/main/docs/time-shapes.md) **How the data is spread.** `--skew` lets a few customers place most of the orders; `weights`, `true_rate`, `null_rate` and `distinct` shape a column; `distribution` draws a number from a diff --git a/docs/dbt-project.md b/docs/dbt-project.md index 52e42c5..548d60e 100644 --- a/docs/dbt-project.md +++ b/docs/dbt-project.md @@ -52,7 +52,7 @@ dbt package, so `dbt build` still works offline): | `generate: {distinct: n}` | `model2data_max_distinct`: at most `n` distinct values | none | | `generate: {when}` | `model2data_when`: null exactly where the condition does not hold (with `null_rate`, only there; its null share is then tested among the matching rows) | none | | `grain` on a table | `model2data_unique_combination` | none | -| a creation date that follows its parents' (no hint: see [When things happen](time-shapes.md)) | `model2data_not_before_parent`, one per foreign key it follows, in `macros/model2data_parent_tests.sql`: no row dated before the parent row it joins to | none | +| `generate: {after: table.column}` (a parent's column, see [When things happen](time-shapes.md)) | `model2data_not_before_parent`, one per parent column, in `macros/model2data_parent_tests.sql`: no row dated before the parent row it joins to | none | | an enum-typed column | `accepted_values` (always written) | none | `true_rate`, `weights`, `skew`, `distribution` and the temporal shape hints (`business_hours`, diff --git a/docs/distributions.md b/docs/distributions.md index 1e5d99f..3d6f91b 100644 --- a/docs/distributions.md +++ b/docs/distributions.md @@ -34,9 +34,9 @@ orders: generate: {distinct: 12} ``` -`skew` on a foreign key overrides `--skew` for just that column. When the child's creation date -follows its parent's (see [When things happen](time-shapes.md)), the parents created early also -pick up the children dated before the later ones existed, so the spread is steeper than `skew` +`skew` on a foreign key overrides `--skew` for just that column. When the child's date has an +`after` naming its parent's (see [When things happen](time-shapes.md)), the parents created +early also pick up the children dated before the later ones existed, so the spread is steeper than `skew` alone: on the coffee-webshop example at `skew: 0.8`, the top fifth of customers hold about 80% of the orders rather than 65%. `weights` biases an enum column toward the values named (unnamed values still appear, at weight 1). `true_rate` is the fraction of diff --git a/docs/time-shapes.md b/docs/time-shapes.md index 0579c0d..eae62c6 100644 --- a/docs/time-shapes.md +++ b/docs/time-shapes.md @@ -19,31 +19,42 @@ shipped_at: generate: {after: ordered_at} ``` -A row is never created before the parent rows it points at: an order's `order_date` is on or -after its customer's `created_at`, a review's `review_date` on or after the product's launch. -It needs no hint. A table's **creation column** is its first date or timestamp without `after` -whose name says created (`created_at`, `signup_date`, `start_date`, ...), else its first that -names no later stage (`order_date`, `hire_date`); a child's creation column follows the creation -column of each parent its foreign keys point at, through a foreign key onto the parent's -primary or unique key, and its own later dates (`shipped_at`, `updated_at`) follow it as -always. A date compared with a timestamp compares by day. +`after` can also name a column of a parent table, as `
.`: the value is then on +or after that column on the parent row the row's foreign key points at, so no order is placed +before its customer signed up (spec 0.5.0: the model says `model2data: 0.5.0`): + +```yaml +order_date: + type: timestamp + generate: {after: customers.created_at} +review_date: + type: date + generate: {after: [orders.ordered_at, products.launched_at]} +``` + +A list follows every entry, and may mix columns of the row itself (`after: [placed_at, +customers.created_at]`). The table must reach each parent through exactly one one-column +foreign key onto the parent's primary key or a unique column: a table with `sender_id` and +`receiver_id` both onto `users` cannot say which user it means, and validation says so, naming +both. A date compared with a timestamp compares by day. A null foreign key constrains nothing, +and a self-reference is not a parent: a column of the same row is named without its table. The child keeps its own shape. A row drawn before its parent takes another parent, one created -by then, from the parents the column already points at, so the order dates keep their growth, +by then, from the parents the column already points at, so the dates keep their growth, seasonality and working hours exactly, and the customers who signed up early end up with more orders, as they would. Only a row no parent was created early enough for (or a one-to-one, which cannot change parent) moves its date, drawn again from the column's own shape on or after the parent's, and never past `as_of`. On days after the first, a new row whose parent was inserted the same day lands later that day than the parent. -Self-references, nullable foreign keys left null, parents without a creation column and -foreign keys that break a cycle are not followed. To draw a creation column on its own, as -before 1.15, say so on it (spec 0.5.0: the model says `model2data: 0.5.0`): +Without the hint, a child's dates are drawn on their own. `model2data validate` (and every run) +warns where that can put a child's first date before its parent's `created_at`-like date, and +names the `after` to add: -```yaml -logged_at: - type: timestamp - generate: {after_parent: false} +``` +⚠️ shop.model2data.yml: 1 warning + - tables.orders.columns.order_date: can fall before customers.created_at (the customers row + it points at through customer_id); add `after: customers.created_at` to keep it after ``` A column that only has a value in some states says which with `when` (spec 0.4.0: the model says diff --git a/tests/fixtures/determinism.json b/tests/fixtures/determinism.json index 0e79fe2..c1664ea 100644 --- a/tests/fixtures/determinism.json +++ b/tests/fixtures/determinism.json @@ -23,10 +23,10 @@ "dbt_det/days/product_reviews/day_000.csv": "95f9a92f499d61ebc0d9747dd27474e1a5d96eaf3cec91b3d36d1a9b2ceb3b07", "dbt_det/days/products/day_000.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/ecommerce_daily.model2data.yml": "45723a5c83cf7058915b01c881ad18720de7d9c795a2e21abe1debde60141053", + "dbt_det/ecommerce_daily.model2data.yml": "fd289ec07740f2323fa65e9b67843004954b8758c7b0568cb566b57fabe18c80", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -54,11 +54,11 @@ "dbt_det/changelog/products.csv": "5f9013e0977d99598a2228ef69f0cecd153ae2ad8461739f2ae88ca6ec7a63f0", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/defects_report.json": "b4213c8e2027d566e6d266551c9bf28e27e07f01584a9f83c069c9df5137b3cb", - "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", + "dbt_det/ecommerce_training.model2data.yml": "6d2e078504e72cf8192a829c7cda2ac0d7be9855336c116cb5b0643db269cee1", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -81,12 +81,12 @@ "dbt_det/seeds/raw/products.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747" }, "example-advanced_features.dbml": { - "dbt_det/advanced_features.dbml": "3e4a90555639ce29adf33441aab4c4a6aefcfe2ecebd79b1aa65f575312c7379", + "dbt_det/advanced_features.dbml": "0ac3bc1dbb95d8283d4b7589928f8b107ef736493d849ccf9743e6bf17f10a0d", "dbt_det/data-tests/unique_combination_stg_project_assignments_project_id_employee_id.sql": "6df209ba797e1d4c1d2fa72053cec3d1641673d0891fe080ce5edb9e264b8852", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_departments.sql": "78a2fdafd3f86532cf31320c9a0159b01598f07b3eeeadfc9ec4f59548617946", "dbt_det/models/staging/stg_departments.yml": "75b04766a3b24c5af493b7d4c604657263efdd2de62474b72e8b4d0fd5a7bc6e", "dbt_det/models/staging/stg_employees.sql": "c63a2b2b4ed86d0b3c2cce77aa5d4dfb032c229a23013c6e65dd396588b53109", @@ -112,12 +112,12 @@ "dbt_det/seeds/raw/time_entries.csv": "20c8eda681fb6df63b2155d3143567ef14353163a389e52698cb555005ff82f2" }, "example-advanced_features.model2data.yml": { - "dbt_det/advanced_features.model2data.yml": "fc0727267e3b7d123a5666b759ed245a40e120562fa63aad5f79dcca28191d71", + "dbt_det/advanced_features.model2data.yml": "f037982ee884de807651b3b6d85f1ef21cc0e0f4652171bd18688cd3ef1f6007", "dbt_det/data-tests/unique_combination_stg_project_assignments_project_id_employee_id.sql": "6df209ba797e1d4c1d2fa72053cec3d1641673d0891fe080ce5edb9e264b8852", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_departments.sql": "78a2fdafd3f86532cf31320c9a0159b01598f07b3eeeadfc9ec4f59548617946", "dbt_det/models/staging/stg_departments.yml": "75b04766a3b24c5af493b7d4c604657263efdd2de62474b72e8b4d0fd5a7bc6e", "dbt_det/models/staging/stg_employees.sql": "c63a2b2b4ed86d0b3c2cce77aa5d4dfb032c229a23013c6e65dd396588b53109", @@ -144,10 +144,10 @@ }, "example-ecommerce.dbml": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/ecommerce.dbml": "b72b2b7be5459c597f8cc9c6787ee24066e1ff12ef5265e2e13c7320cb5deab9", + "dbt_det/ecommerce.dbml": "ab6b58aee406cb99da6d07121700e81c34d831f1d9a857996e6be7657cc5d1d7", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -168,10 +168,10 @@ }, "example-ecommerce.model2data.yml": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", + "dbt_det/ecommerce.model2data.yml": "7d132e7e2d8c4c80b436f4cbc4be13681e73af3fae1a715e38fe4d7d62638a8f", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -192,10 +192,10 @@ }, "example-ecommerce_daily.model2data.yml": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/ecommerce_daily.model2data.yml": "45723a5c83cf7058915b01c881ad18720de7d9c795a2e21abe1debde60141053", + "dbt_det/ecommerce_daily.model2data.yml": "fd289ec07740f2323fa65e9b67843004954b8758c7b0568cb566b57fabe18c80", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -218,11 +218,11 @@ "dbt_det/EXPECTED_FAILURES.md": "d786b65ca362828c22ce6b6cad9f8c189dc98286a6b2c898dd6a9a90c0648187", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/defects_report.json": "4f4f15d9a12bf2b8c2926b74db6df1824e8a277a231535b296390ed24f9c6d1e", - "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", + "dbt_det/ecommerce_training.model2data.yml": "6d2e078504e72cf8192a829c7cda2ac0d7be9855336c116cb5b0643db269cee1", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -249,12 +249,10 @@ "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/hackernews.dbml": "06ec4f08ac1fdb4617e82ebfbd6ca53f546530cac938438edcff152215f357a8", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", - "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg__dlt_loads.sql": "bf6c0705877e236e8feb73268a9fd8cba50b5ba50330eda30e7a1b77f1a780cb", "dbt_det/models/staging/stg__dlt_loads.yml": "69bf37a12589b0b10bbfea9a6d5aab046b16234313306a698235ab83b816e9c5", "dbt_det/models/staging/stg__dlt_pipeline_state.sql": "921824090c1985a61a063350b9c87a296c28d5dafe4d26a0fd9c3a1a13e98624", - "dbt_det/models/staging/stg__dlt_pipeline_state.yml": "01d85053a17d1aa27f2a17537fdcc5affb0f394212d579d7a05217576a7237a1", + "dbt_det/models/staging/stg__dlt_pipeline_state.yml": "aeaa684b4339d83bebcd708fff826ec413ed608ed53d8200320a853cb8f7dd72", "dbt_det/models/staging/stg__dlt_version.sql": "5d279d508581eae585acdbbeaa36622b1faab61996b3956c6e2373c0d1294a8d", "dbt_det/models/staging/stg__dlt_version.yml": "dec6894006cce86c5950e66e5939ea8d16ebacf5722509ff8235e7fe9710b1a9", "dbt_det/models/staging/stg_stories.sql": "1bef38284308be0963e9933f19f53339e110020d4d2993766d48a7c452930397", @@ -264,7 +262,7 @@ "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "2c712c4932785690fc4d9d9117d964a0e784989b1bc65cb8e2afaff4a18bcc16", "dbt_det/seeds/raw/_dlt_loads.csv": "4bc16cd04e968225f6fb788f9f48ea51282e3d089e40dc369b71db1f9b433107", - "dbt_det/seeds/raw/_dlt_pipeline_state.csv": "9387c8e111defbf0cf0cd7f36093c205ebf53997e398b3151b08f077e9837ca4", + "dbt_det/seeds/raw/_dlt_pipeline_state.csv": "67f4f6828c4340bb33744fa1784bca9a1c2ff9093f415ad1aa16cae27edb67e6", "dbt_det/seeds/raw/_dlt_version.csv": "bddf097235105ad5ced6c56380c70d2b9bc915e1a0e4a1368be3d48e534b7d3f", "dbt_det/seeds/raw/stories.csv": "2869db5e096a04b0092bf2a97266cf7969ec40bef9fd45efc0540a1db178a362", "dbt_det/seeds/raw/stories__kids.csv": "c65b11d84a910e24ee31f22eb44df742663e65a175b1b8c2083fa1e8e95f9005" @@ -274,12 +272,10 @@ "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/hackernews.model2data.yml": "f74d4df703b909d58d082b42e29623b8ca0c326c46c5d37fb144e8eb11992c3b", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", - "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", "dbt_det/models/staging/stg__dlt_loads.sql": "bf6c0705877e236e8feb73268a9fd8cba50b5ba50330eda30e7a1b77f1a780cb", "dbt_det/models/staging/stg__dlt_loads.yml": "69bf37a12589b0b10bbfea9a6d5aab046b16234313306a698235ab83b816e9c5", "dbt_det/models/staging/stg__dlt_pipeline_state.sql": "921824090c1985a61a063350b9c87a296c28d5dafe4d26a0fd9c3a1a13e98624", - "dbt_det/models/staging/stg__dlt_pipeline_state.yml": "01d85053a17d1aa27f2a17537fdcc5affb0f394212d579d7a05217576a7237a1", + "dbt_det/models/staging/stg__dlt_pipeline_state.yml": "aeaa684b4339d83bebcd708fff826ec413ed608ed53d8200320a853cb8f7dd72", "dbt_det/models/staging/stg__dlt_version.sql": "5d279d508581eae585acdbbeaa36622b1faab61996b3956c6e2373c0d1294a8d", "dbt_det/models/staging/stg__dlt_version.yml": "dec6894006cce86c5950e66e5939ea8d16ebacf5722509ff8235e7fe9710b1a9", "dbt_det/models/staging/stg_stories.sql": "1bef38284308be0963e9933f19f53339e110020d4d2993766d48a7c452930397", @@ -289,7 +285,7 @@ "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", "dbt_det/seeds/raw/__seed_config.yml": "2c712c4932785690fc4d9d9117d964a0e784989b1bc65cb8e2afaff4a18bcc16", "dbt_det/seeds/raw/_dlt_loads.csv": "4bc16cd04e968225f6fb788f9f48ea51282e3d089e40dc369b71db1f9b433107", - "dbt_det/seeds/raw/_dlt_pipeline_state.csv": "9387c8e111defbf0cf0cd7f36093c205ebf53997e398b3151b08f077e9837ca4", + "dbt_det/seeds/raw/_dlt_pipeline_state.csv": "67f4f6828c4340bb33744fa1784bca9a1c2ff9093f415ad1aa16cae27edb67e6", "dbt_det/seeds/raw/_dlt_version.csv": "bddf097235105ad5ced6c56380c70d2b9bc915e1a0e4a1368be3d48e534b7d3f", "dbt_det/seeds/raw/stories.csv": "2869db5e096a04b0092bf2a97266cf7969ec40bef9fd45efc0540a1db178a362", "dbt_det/seeds/raw/stories__kids.csv": "c65b11d84a910e24ee31f22eb44df742663e65a175b1b8c2083fa1e8e95f9005" @@ -324,7 +320,7 @@ "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_api_usage.sql": "3a453410ffeba21c7c299fe9144f762aab2565cd9f02b32ed3fc6a25c4a7cd4b", "dbt_det/models/staging/stg_api_usage.yml": "67de79479deb6266fc70d5e38651b730a19c2c2ae4641c72ad2a1e2e685d1ccd", "dbt_det/models/staging/stg_audit_events.sql": "062020dbc347feeb7b7b7e9c47e8112691cd434d839d783d147fbbd811c5f040", @@ -332,7 +328,7 @@ "dbt_det/models/staging/stg_feature_limits.sql": "8dd6ccb9ee9da75f8102460879f2da89f541e5fc71336b29b9e2372ee917de1d", "dbt_det/models/staging/stg_feature_limits.yml": "381963d063dca0966d5aef605a04b1bb9ca39b6c36fc35e9c53f931ec11dce90", "dbt_det/models/staging/stg_invoices.sql": "6a282cd534a83387c28062ec83924bca249e8466659ed4b7614652a4853e609d", - "dbt_det/models/staging/stg_invoices.yml": "bde339657aae0e12e3dc820b1f08989ca31719511625ead8189cf7edad5c1089", + "dbt_det/models/staging/stg_invoices.yml": "e1070f6c8d779c2f0927b5e914d277d81dae985aaaedf1cdf8dd3822adf2fa6e", "dbt_det/models/staging/stg_organizations.sql": "7f29bdc034068d313c826a71bff56d49bfae6039108d376f6034de8c97c9bb19", "dbt_det/models/staging/stg_organizations.yml": "3bcf56cc79a94657ba0c6132ee1eaff87f89b7d3f099c839ade87146e24c4ee5", "dbt_det/models/staging/stg_subscriptions.sql": "2a2d9ee7ba0ae44b610935af21bd4cdd1a38986e1f4fca772fb6775296b0377e", @@ -340,12 +336,12 @@ "dbt_det/models/staging/stg_users.sql": "0327dec706116d8aeb53093adad11614d9b456bc574276a9f44fbaa906be4e65", "dbt_det/models/staging/stg_users.yml": "b162ae9b22d7b026e9f546b58ad06f8bc9313351e2fe3d1fa4fd55eb235e3060", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", - "dbt_det/saas_platform.dbml": "a444d3d626b2a104920a5a2af9512ec311ac02e7260e18cb11750fc09adf8735", + "dbt_det/saas_platform.dbml": "885d9d3d6899b9246a6165c63d475db34c58c8bea431976280bc5a54426d60ea", "dbt_det/seeds/raw/__seed_config.yml": "b87c6d38dae29df61e3c7bd66560b0ec505833a04a7e784e3b22142c78f3c4b7", "dbt_det/seeds/raw/api_usage.csv": "5494440e04f34c2e9c7b672c1d9993ff4042b0bd88963e5307e22e9acb27811b", "dbt_det/seeds/raw/audit_events.csv": "ae4f86727f8fc3cbff817811812f2e529cd072ceef427118ee5f8bc30fac06c2", "dbt_det/seeds/raw/feature_limits.csv": "e57be8667393acd8745eba627d0fb3f2b218df515d1c472c8a011eeee3a71926", - "dbt_det/seeds/raw/invoices.csv": "0be42f028ba7337159c335b8f53cbc49ea5c4458dabcf206dac2ad2bc827e31d", + "dbt_det/seeds/raw/invoices.csv": "96fe4a6c4857832c21f8065fb65d21aee1bdff7031d2f3a9b96848767cc2dd51", "dbt_det/seeds/raw/organizations.csv": "b12a3a33bd8558d15c4e79f3c1284af6f2101e80bab540ea3457540d5a7abb8e", "dbt_det/seeds/raw/subscriptions.csv": "e5af6aefaf597fefd16cb1276e388146c54741f2483e6de7deb6e5527b73ddae", "dbt_det/seeds/raw/users.csv": "f6f7dac78e0631991ef382e0fc9331df6d2f137b497f8ccb9a22a22c8d5376df" @@ -354,7 +350,7 @@ "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_api_usage.sql": "3a453410ffeba21c7c299fe9144f762aab2565cd9f02b32ed3fc6a25c4a7cd4b", "dbt_det/models/staging/stg_api_usage.yml": "67de79479deb6266fc70d5e38651b730a19c2c2ae4641c72ad2a1e2e685d1ccd", "dbt_det/models/staging/stg_audit_events.sql": "062020dbc347feeb7b7b7e9c47e8112691cd434d839d783d147fbbd811c5f040", @@ -362,7 +358,7 @@ "dbt_det/models/staging/stg_feature_limits.sql": "8dd6ccb9ee9da75f8102460879f2da89f541e5fc71336b29b9e2372ee917de1d", "dbt_det/models/staging/stg_feature_limits.yml": "381963d063dca0966d5aef605a04b1bb9ca39b6c36fc35e9c53f931ec11dce90", "dbt_det/models/staging/stg_invoices.sql": "6a282cd534a83387c28062ec83924bca249e8466659ed4b7614652a4853e609d", - "dbt_det/models/staging/stg_invoices.yml": "bde339657aae0e12e3dc820b1f08989ca31719511625ead8189cf7edad5c1089", + "dbt_det/models/staging/stg_invoices.yml": "e1070f6c8d779c2f0927b5e914d277d81dae985aaaedf1cdf8dd3822adf2fa6e", "dbt_det/models/staging/stg_organizations.sql": "7f29bdc034068d313c826a71bff56d49bfae6039108d376f6034de8c97c9bb19", "dbt_det/models/staging/stg_organizations.yml": "3bcf56cc79a94657ba0c6132ee1eaff87f89b7d3f099c839ade87146e24c4ee5", "dbt_det/models/staging/stg_subscriptions.sql": "2a2d9ee7ba0ae44b610935af21bd4cdd1a38986e1f4fca772fb6775296b0377e", @@ -370,12 +366,12 @@ "dbt_det/models/staging/stg_users.sql": "0327dec706116d8aeb53093adad11614d9b456bc574276a9f44fbaa906be4e65", "dbt_det/models/staging/stg_users.yml": "b162ae9b22d7b026e9f546b58ad06f8bc9313351e2fe3d1fa4fd55eb235e3060", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", - "dbt_det/saas_platform.model2data.yml": "23a8b7391c75e50ecc975ae77ee8abe0507093492b73d65b990b2c7c82b4e42c", + "dbt_det/saas_platform.model2data.yml": "c95539a326d3163fd7e7d67c245f9e8021ce55a554fff39be46d40dd06ee97de", "dbt_det/seeds/raw/__seed_config.yml": "b87c6d38dae29df61e3c7bd66560b0ec505833a04a7e784e3b22142c78f3c4b7", "dbt_det/seeds/raw/api_usage.csv": "5494440e04f34c2e9c7b672c1d9993ff4042b0bd88963e5307e22e9acb27811b", "dbt_det/seeds/raw/audit_events.csv": "ae4f86727f8fc3cbff817811812f2e529cd072ceef427118ee5f8bc30fac06c2", "dbt_det/seeds/raw/feature_limits.csv": "e57be8667393acd8745eba627d0fb3f2b218df515d1c472c8a011eeee3a71926", - "dbt_det/seeds/raw/invoices.csv": "0be42f028ba7337159c335b8f53cbc49ea5c4458dabcf206dac2ad2bc827e31d", + "dbt_det/seeds/raw/invoices.csv": "96fe4a6c4857832c21f8065fb65d21aee1bdff7031d2f3a9b96848767cc2dd51", "dbt_det/seeds/raw/organizations.csv": "b12a3a33bd8558d15c4e79f3c1284af6f2101e80bab540ea3457540d5a7abb8e", "dbt_det/seeds/raw/subscriptions.csv": "e5af6aefaf597fefd16cb1276e388146c54741f2483e6de7deb6e5527b73ddae", "dbt_det/seeds/raw/users.csv": "f6f7dac78e0631991ef382e0fc9331df6d2f137b497f8ccb9a22a22c8d5376df" @@ -434,10 +430,10 @@ }, "locale-de_DE": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", + "dbt_det/ecommerce.model2data.yml": "7d132e7e2d8c4c80b436f4cbc4be13681e73af3fae1a715e38fe4d7d62638a8f", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -458,10 +454,10 @@ }, "locale-en_US": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", + "dbt_det/ecommerce.model2data.yml": "7d132e7e2d8c4c80b436f4cbc4be13681e73af3fae1a715e38fe4d7d62638a8f", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -482,10 +478,10 @@ }, "locale-fr_FR": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", + "dbt_det/ecommerce.model2data.yml": "7d132e7e2d8c4c80b436f4cbc4be13681e73af3fae1a715e38fe4d7d62638a8f", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -506,10 +502,10 @@ }, "locale-ja_JP": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", + "dbt_det/ecommerce.model2data.yml": "7d132e7e2d8c4c80b436f4cbc4be13681e73af3fae1a715e38fe4d7d62638a8f", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -530,10 +526,10 @@ }, "locale-nl_BE": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", + "dbt_det/ecommerce.model2data.yml": "7d132e7e2d8c4c80b436f4cbc4be13681e73af3fae1a715e38fe4d7d62638a8f", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -554,7 +550,7 @@ }, "metrics": { "dbt_det/coffee_webshop.metrics.yml": "16981620dc46a34c01f2c6cfc36644522bf9c5a80defb49206cc2a82eca67c32", - "dbt_det/coffee_webshop.model2data.yml": "ae20f85480b4e5503e2f54ea5a71d24673b8bb5990644c850e51f91c5eddbc4a", + "dbt_det/coffee_webshop.model2data.yml": "596c24c87f0fed5b4c18a32c3f14af162ccc8c0dc3a278df1f73cd3f4b434079", "dbt_det/data-tests/metrics/metric_average_order_value.sql": "9292cf214a82e05011817057157da7eaa467baf5fda2443a80a826b2302c0bd6", "dbt_det/data-tests/metrics/metric_coffee_units_sold.sql": "6e9d8b6182348163d5311790f71c5b41875c26b8cd7e96aa064075ba1849d19f", "dbt_det/data-tests/metrics/metric_order_items_count.sql": "3d0e8d87ceb9c396106908800b18872c499f9927b634780868271d003df3a415", @@ -569,7 +565,7 @@ "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/metric_values.json": "36848fdba7e43423ea8ab71d2d03eae08c6e20742c1378038caf3871625feadc", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "02f18d31e87a4bfb8314588180ef3382b8db5e5b5460e4222f6e094e4006582c", @@ -621,11 +617,11 @@ "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/defects_report.json": "b4213c8e2027d566e6d266551c9bf28e27e07f01584a9f83c069c9df5137b3cb", "dbt_det/ecommerce_training.metrics.yml": "01324c4ec6103dee37f66d254c82d85515e894ccabd13262368814ab2fc9c6c1", - "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", + "dbt_det/ecommerce_training.model2data.yml": "6d2e078504e72cf8192a829c7cda2ac0d7be9855336c116cb5b0643db269cee1", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/metric_values.json": "f47fa930aa4e3bb4ff42a8cb8f2505355dd7047c96a6833f7102fbb9c75b85a2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", @@ -667,11 +663,11 @@ "dbt_det/days/products/day_000.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/defects_report.json": "1d1826d7a4de48153a39ca9723ddf7328f7497c023225bee1a77c20581182858", - "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", + "dbt_det/ecommerce_training.model2data.yml": "6d2e078504e72cf8192a829c7cda2ac0d7be9855336c116cb5b0643db269cee1", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -711,11 +707,11 @@ "dbt_det/days/products/day_000.csv": "5a494e7fbd0e86dca2ed3d22d57e000f4b774206cfa2bcfeab0e3aae6713b589", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/defects_report.json": "34b8c5d248468829a7e69654b43c45d42b16c5c4c1c7f185a175533790ba096e", - "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", + "dbt_det/ecommerce_training.model2data.yml": "6d2e078504e72cf8192a829c7cda2ac0d7be9855336c116cb5b0643db269cee1", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -753,11 +749,11 @@ "dbt_det/days/product_reviews/day_000.csv": "95f9a92f499d61ebc0d9747dd27474e1a5d96eaf3cec91b3d36d1a9b2ceb3b07", "dbt_det/days/products/day_000.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", + "dbt_det/ecommerce_training.model2data.yml": "6d2e078504e72cf8192a829c7cda2ac0d7be9855336c116cb5b0643db269cee1", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -797,11 +793,11 @@ "dbt_det/days/products/day_000.csv": "8de823f036c4b3e3f848547c24e64245d90e53fd8712ea05100ca10a81986747", "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/defects_report.json": "a044d1215bd9a47ee1c430f6cbf0344ee8aa3ac202bab5e7e2afd79094c54cc3", - "dbt_det/ecommerce_training.model2data.yml": "ab1cfa8d1b9b9638a4e8767cb02209ab2033beb1bdeff470f4bf0b2e596612ed", + "dbt_det/ecommerce_training.model2data.yml": "6d2e078504e72cf8192a829c7cda2ac0d7be9855336c116cb5b0643db269cee1", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", "dbt_det/macros/model2data_history_tests.sql": "f202816bc0a415be7f3a2059bda83233cce37a88004d8c7d6a5fcf24964355ad", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -827,7 +823,7 @@ "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_api_usage.sql": "3a453410ffeba21c7c299fe9144f762aab2565cd9f02b32ed3fc6a25c4a7cd4b", "dbt_det/models/staging/stg_api_usage.yml": "67de79479deb6266fc70d5e38651b730a19c2c2ae4641c72ad2a1e2e685d1ccd", "dbt_det/models/staging/stg_audit_events.sql": "062020dbc347feeb7b7b7e9c47e8112691cd434d839d783d147fbbd811c5f040", @@ -835,7 +831,7 @@ "dbt_det/models/staging/stg_feature_limits.sql": "8dd6ccb9ee9da75f8102460879f2da89f541e5fc71336b29b9e2372ee917de1d", "dbt_det/models/staging/stg_feature_limits.yml": "381963d063dca0966d5aef605a04b1bb9ca39b6c36fc35e9c53f931ec11dce90", "dbt_det/models/staging/stg_invoices.sql": "6a282cd534a83387c28062ec83924bca249e8466659ed4b7614652a4853e609d", - "dbt_det/models/staging/stg_invoices.yml": "bde339657aae0e12e3dc820b1f08989ca31719511625ead8189cf7edad5c1089", + "dbt_det/models/staging/stg_invoices.yml": "e1070f6c8d779c2f0927b5e914d277d81dae985aaaedf1cdf8dd3822adf2fa6e", "dbt_det/models/staging/stg_organizations.sql": "7f29bdc034068d313c826a71bff56d49bfae6039108d376f6034de8c97c9bb19", "dbt_det/models/staging/stg_organizations.yml": "3bcf56cc79a94657ba0c6132ee1eaff87f89b7d3f099c839ade87146e24c4ee5", "dbt_det/models/staging/stg_subscriptions.sql": "2a2d9ee7ba0ae44b610935af21bd4cdd1a38986e1f4fca772fb6775296b0377e", @@ -843,22 +839,22 @@ "dbt_det/models/staging/stg_users.sql": "0327dec706116d8aeb53093adad11614d9b456bc574276a9f44fbaa906be4e65", "dbt_det/models/staging/stg_users.yml": "b162ae9b22d7b026e9f546b58ad06f8bc9313351e2fe3d1fa4fd55eb235e3060", "dbt_det/profiles.yml": "c40b04badc35b941a8de1079c8d0a298f6446f0d5caf5bd50dbe3797ab2a2255", - "dbt_det/saas_platform.model2data.yml": "23a8b7391c75e50ecc975ae77ee8abe0507093492b73d65b990b2c7c82b4e42c", + "dbt_det/saas_platform.model2data.yml": "c95539a326d3163fd7e7d67c245f9e8021ce55a554fff39be46d40dd06ee97de", "dbt_det/seeds/raw/__seed_config.yml": "b87c6d38dae29df61e3c7bd66560b0ec505833a04a7e784e3b22142c78f3c4b7", "dbt_det/seeds/raw/api_usage.csv": "b4bc4b9edc8201abd197aeef3ba03518e33e8c6f0ae5892168e5b417181b6da0", "dbt_det/seeds/raw/audit_events.csv": "8f67794907cfbe622d8335c52df9c204adf3b96db213b06211e437b981ebea2c", "dbt_det/seeds/raw/feature_limits.csv": "ddd8ec06508a87a67d6dc3beddd1539b19bdd8617373a6cb88f04a801b69e5d6", - "dbt_det/seeds/raw/invoices.csv": "a266716814be82177203c78fa0ec5f42dc4b05f8f94ff76a32e2598369aa0dc7", + "dbt_det/seeds/raw/invoices.csv": "cf1e8b2f077e4217fdfba2b9576205d4d2db3c1615a0b380949989c843eaf3db", "dbt_det/seeds/raw/organizations.csv": "b12a3a33bd8558d15c4e79f3c1284af6f2101e80bab540ea3457540d5a7abb8e", "dbt_det/seeds/raw/subscriptions.csv": "13fa0e6f821336ed0afb1721f2d30e83c88938d63e876ebcfaa47e2a234bbe2f", "dbt_det/seeds/raw/users.csv": "09fdc161f74d9cfa0fa3104973221a5b3ea750009ae20ad3901086e7c850d583" }, "table-seed": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", + "dbt_det/ecommerce.model2data.yml": "7d132e7e2d8c4c80b436f4cbc4be13681e73af3fae1a715e38fe4d7d62638a8f", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", @@ -879,10 +875,10 @@ }, "unit-tests": { "dbt_det/dbt_project.yml": "631ba06ded8db3ebbfa85351d29ef0953dc5b92992235b786a21424e1db3ddda", - "dbt_det/ecommerce.model2data.yml": "d2e723fd63e350c1eaaa4b8d246b9284e8ced43e657b7bef2831083a43dab59b", + "dbt_det/ecommerce.model2data.yml": "7d132e7e2d8c4c80b436f4cbc4be13681e73af3fae1a715e38fe4d7d62638a8f", "dbt_det/macros/generate_schema_name.sql": "60f37d9f6fc022c0f482c2699a2e1a4dc61207cbb0a7f355788b4c1d43066438", "dbt_det/macros/model2data_hint_tests.sql": "d0b45f48d4740c22b3881654fc704f48ca08404b8d574f37680c7237ba4b08f8", - "dbt_det/macros/model2data_parent_tests.sql": "0c280806251370760554e3211faf9344d9c091bd8578a6ad4b481c0d69c169bd", + "dbt_det/macros/model2data_parent_tests.sql": "0c1b03fb9492873685dc2a7beb5d91574ff05b604dca24220943c690282424d2", "dbt_det/models/staging/stg_customers.sql": "44df48978d6ac653bdf1c8ed58d8884068c066f5131aeb8f4822f8dc4c424b0a", "dbt_det/models/staging/stg_customers.yml": "55ce32de721f6a366ad5c8c2d3036f7eab2719b6ebe9d7b0bf381afadf58fc6b", "dbt_det/models/staging/stg_order_items.sql": "d9ffb6428433e8ad821637b07a40f88037201b1ea3ad4d05c06ace185f496b39", From 1c656337b5384f930b9b401409012219e7bbdf44 Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Thu, 8 Oct 2026 22:12:22 +0200 Subject: [PATCH 09/14] Teach agents when and how to use a parent's column in after Co-Authored-By: Claude Opus 5.5 Spec 0.5.0 in the summary, the hint line, when to use it, how to act on the warning, and the two-foreign-keys case that is not built yet. --- LLMS.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/LLMS.md b/LLMS.md index a53da66..bdafc4b 100644 --- a/LLMS.md +++ b/LLMS.md @@ -22,7 +22,7 @@ install. If the person you're helping wants to *look at* or share the model rath its generation, point them there; the DBML guidance below applies to both. > **Two input formats.** Since 1.8, model2data reads a model as one YAML document, -> `.model2data.yml` ([spec 0.4.0](model2data/spec/README.md), with a JSON Schema at +> `.model2data.yml` ([spec 0.5.0](model2data/spec/README.md), with a JSON Schema at > `model2data/spec/model.schema.json` and a complete example at > `model2data/spec/examples/coffee_webshop.model2data.yml`), and DBML as supported input that it > converts to the same model. The DBML guidance below still holds; if you write the YAML form @@ -134,6 +134,7 @@ is_paid boolean [note: '{"true_rate": 0.9}'] ' fraction of non-nul shipping_city varchar [note: '{"distinct": 12}'] ' draw from a pool this size (not fk/pk/unique/enum) customer_id int [note: '{"skew": 0.9}'] ' per-column override of --skew (fk columns only) updated_at timestamp [note: '{"after": "created_at"}'] ' must fall after another date/timestamp column +order_date timestamp [note: '{"after": "customers.created_at"}'] ' ... or after a parent's column, or a list of both completed_at timestamp [note: '{"when": {"status": ["done"]}}'] ' set only where status is done, null elsewhere created_at timestamp [note: '{"business_hours": true}'] ' per-column override of --business-hours created_at timestamp [note: '{"growth": 0.4}'] ' per-column override of --growth (date/timestamp only) @@ -178,6 +179,24 @@ with the `after` to add. With `--hint-tests` each parent column writes the dbt t `model2data_not_before_parent`. Integer primary keys are 1..N and every output lists rows in key order. +*When to use it:* whenever a child row cannot exist before its parent: an order before its +customer signed up, a subscription before its organisation, a time entry before the employee's +hire date. Skip it when the child's date has no such tie (an event log that may predate the +account it is later linked to). + +*Acting on the warning:* it reads `tables.orders.columns.order_date: can fall before +customers.created_at (...); add `after: customers.created_at` to keep it after`. Add exactly +that `after` to `generate` on the column the path names (an existing `after` there becomes a +list: `after: [placed_at, customers.created_at]`), set `model2data: 0.5.0`, and run `validate` +again; the warning is gone and the file conforms either way. Over the Python API the warning's +`Issue.suggestion` (a `model2data.model.Suggestion`) carries `path`, `value` and `spec` for the +same edit. If the child may legitimately predate the parent, leave the warning. + +*Not built yet:* a table with two foreign keys to the same parent (`sender_id` and `receiver_id` +onto `users`) cannot use `after: users.created_at`: model2data cannot tell which row is meant, +so validation refuses it with an error naming both keys. There is no way yet to pick one; drop +the hint (the warning is not raised for such a table either). + **Tie lifecycle columns to the status they belong to with `when`.** Without it, every nullable column is filled or nulled independently of the others, so a `todo` task gets a `completed_at` and a `cancelled` subscription has no `cancelled_at`. `when` maps another column of the same @@ -304,7 +323,7 @@ model2data --file SCHEMA.dbml [OPTIONS] --adapter, -a TEXT duckdb (default) or postgres --unit-tests Also generate dbt unit test fixtures --hint-tests TEXT error, warn (default) or off: severity of the dbt tests written from - min/max, after, null_rate, distinct, when and grain hints + min/max, after (also a parent's column), null_rate, distinct, when and grain hints --test-tolerance FLOAT Absolute slack of the null_rate test (default 0.1) --defects TEXT clean (default), messy or training: break the data on purpose, so dbt tests are seen to fail; writes defects_report.json and From 6fe8a85a594e6d6bfb21577cc333d9fca630f95d Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Thu, 8 Oct 2026 22:12:22 +0200 Subject: [PATCH 10/14] Say how to act on the parent-date warning in the packaged guide Co-Authored-By: Claude Opus 5.5 --- model2data/guide/setup.md | 3 ++- model2data/guide/triage.md | 12 +++++++++++- model2data/guide/tune.md | 4 +++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/model2data/guide/setup.md b/model2data/guide/setup.md index 9c14152..2719c55 100644 --- a/model2data/guide/setup.md +++ b/model2data/guide/setup.md @@ -61,7 +61,8 @@ model2data --file .model2data.yml cd dbt_ && dbt build ``` -Done looks like: `✅ .model2data.yml conforms to spec 0.5.0.`, a summary with +If `validate` warns that a date `can fall before customers.created_at`, add the `after` it +names (`model2data guide triage`). Done looks like: `✅ .model2data.yml conforms to spec 0.5.0.`, a summary with `Columns using generic fallback text: 0` and no ⚠️ lines, and `dbt build` ending in `ERROR=0`. Anything else: `model2data guide triage`. diff --git a/model2data/guide/triage.md b/model2data/guide/triage.md index c3fbd66..d58259d 100644 --- a/model2data/guide/triage.md +++ b/model2data/guide/triage.md @@ -18,6 +18,13 @@ says it created the project. Fix the **model file**, never the generated project (`tables.orders.columns.status.generate.weights`). Fix that value. Exit 1 until all are gone. - `⚠️ : N warnings`: it conforms. A reference to a parent column that is not a key: make the parent `pk` or `unique`. A `grain` without a key: add the key, or drop `grain`. + A warning `can fall before customers.created_at (...); add `after: customers.created_at`` + on `tables.orders.columns.order_date`: a child's date can precede its parent's creation (an + order before its customer). Add exactly that `after` to the column's `generate` (a column with + an `after` already takes a list: `after: [placed_at, customers.created_at]`), set + `model2data: 0.5.0`, and validate again. If the child may truly predate the parent, leave it. + An error `reaches users through more than one foreign key (sender_id, receiver_id)` on an + `after`: choosing between two foreign keys to one parent is not built yet; drop that `after`. - A metrics file (`.metrics.yml`) is checked against the model beside it. An error naming two paths: the filter's table is reached twice (two foreign keys to one table); filter on a column of the table in between instead. `cannot be reached`: the column is on @@ -62,7 +69,10 @@ Take the first rule that applies to each line: seeds must come from the same run); if it still fails, report it with both files. Never edit `metric_values.json` or the test to make it pass. 5. A WARN from a hint test (`min`/`max`, `after`, `null_rate`, `distinct`, `grain`): the hint - and the data disagree. Fix the hint in the model. + and the data disagree. Fix the hint in the model. `model2data_not_before_parent` is the test + of an `after:
.`: a row dated before its parent row; the model is wrong only + if the parent column or the foreign key changed since the data was generated, so + regenerate with `--force` first. 6. Still failing after the model is fixed: report it with the model file and the output. Done looks like: `validate` exits 0, the summary has no ⚠️ lines, `dbt build` ends `ERROR=0` diff --git a/model2data/guide/tune.md b/model2data/guide/tune.md index 09205f8..95339e1 100644 --- a/model2data/guide/tune.md +++ b/model2data/guide/tune.md @@ -131,7 +131,9 @@ the `created_at` of the account its `account_id` points at (spec 0.5.0, so the m key onto its key; a list (`after: [started_at, accounts.created_at]`) follows each entry. A row dated before its parent takes another parent created by then, so the column keeps its growth and seasonality; with none (or a one-to-one) its date moves. A null foreign key -constrains nothing. `validate` warns where a child's first date can fall before its parent's +constrains nothing. Use it whenever a child cannot exist before its parent (an order before its +customer). Two foreign keys to one parent (`sender_id`, `receiver_id`) are an error: choosing +between them is not built yet. `validate` warns where a child's first date can fall before its parent's `created_at`-like date and names the `after` to add. It writes the dbt test `model2data_not_before_parent`. Integer primary keys number the rows 1..N, and every output lists the rows in key order. From a2483124a57d0f16d45702690a8ee32af6460bae Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Thu, 8 Oct 2026 22:12:23 +0200 Subject: [PATCH 11/14] Explain when to use a cross-table after and read its warning in the docs Co-Authored-By: Claude Opus 5.5 --- docs/generating.md | 6 ++++-- docs/model-file.md | 8 +++++++- docs/time-shapes.md | 16 +++++++++++++++- examples/README.md | 10 +++++++++- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/generating.md b/docs/generating.md index 1a7c468..857c6b4 100644 --- a/docs/generating.md +++ b/docs/generating.md @@ -38,8 +38,10 @@ The same model, seed and `--as-of` — with the same options (`--rows`, `--rows- - with the same model2data version, whenever you run it. A new release may change what a seed produces, to fix a bug or make the data more realistic. -When it does, the changelog says which models are affected and why. Pin the version -(`model2data==1.11.0`) when fixtures must stay the same across upgrades. +When it does, the changelog says which models are affected and why (1.15.0, for one, numbers +integer primary keys 1..N, which changes the bytes of every model that has one, and moves dates +only where a model uses a parent's column in `after`). Pin the version (`model2data==1.14.0`) +when fixtures must stay the same across upgrades. `tests/test_determinism.py` holds the file hashes of a spread of models, presets, multi-day runs and locales, and runs on every pull request — against the newest Faker and pandas too — so a diff --git a/docs/model-file.md b/docs/model-file.md index 49ac74d..53b3d94 100644 --- a/docs/model-file.md +++ b/docs/model-file.md @@ -1,7 +1,7 @@ # The model file A model is one YAML document, `.model2data.yml` — or the same document as JSON. Its -format is [spec 0.4.0](../model2data/spec/README.md), with a JSON Schema +format is [spec 0.5.0](../model2data/spec/README.md), with a JSON Schema ([`model.schema.json`](../model2data/spec/model.schema.json)) your editor can autocomplete and check against: @@ -60,6 +60,12 @@ Check a model without generating anything — every issue is printed with its pa model2data validate examples/ecommerce.model2data.yml ``` +A warning is not an error, and some name the fix. `can fall before customers.created_at ... add +`after: customers.created_at`` means a child's date can precede its parent's creation; the +`after` it names (see [When things happen](time-shapes.md)) is the edit, and spec 0.5.0 the +version to write. Integer primary keys number the rows 1..N, and every output lists them in that +order. + ## DBML is supported input `--file` also takes a `.dbml` file, which model2data converts to the same model before generating diff --git a/docs/time-shapes.md b/docs/time-shapes.md index eae62c6..271136f 100644 --- a/docs/time-shapes.md +++ b/docs/time-shapes.md @@ -32,11 +32,18 @@ review_date: generate: {after: [orders.ordered_at, products.launched_at]} ``` +Use it wherever a child cannot exist before its parent: an order before its customer signed up, +a subscription before its organisation, a time entry before the employee was hired. Leave it off +where the child's date has no such tie (an event that may predate the account it is later linked +to). + A list follows every entry, and may mix columns of the row itself (`after: [placed_at, customers.created_at]`). The table must reach each parent through exactly one one-column foreign key onto the parent's primary key or a unique column: a table with `sender_id` and `receiver_id` both onto `users` cannot say which user it means, and validation says so, naming -both. A date compared with a timestamp compares by day. A null foreign key constrains nothing, +both. Choosing one of several foreign keys to the same parent is not built yet: until it is, a +table like that cannot use a cross-table `after` on that parent, and keeps its own dates. A date +compared with a timestamp compares by day. A null foreign key constrains nothing, and a self-reference is not a parent: a column of the same row is named without its table. The child keeps its own shape. A row drawn before its parent takes another parent, one created @@ -57,6 +64,13 @@ names the `after` to add: it points at through customer_id); add `after: customers.created_at` to keep it after ``` +To act on it, put the `after` the warning names in `generate` on the column its path points at +(a column that already has an `after` gets a list: `after: [placed_at, customers.created_at]`), +write `model2data: 0.5.0`, and validate again. A tool reading the issues gets the same edit as +`Issue.suggestion` (the `path` to set, the `value`, and the `spec` version to write). The warning +only says the dates *can* disagree; if the child may legitimately predate its parent, leave it. +Generation itself is unchanged by the warning. + A column that only has a value in some states says which with `when` (spec 0.4.0: the model says `model2data: 0.4.0`): it holds a value on exactly the rows whose named column holds one of the listed values, and is null on every other diff --git a/examples/README.md b/examples/README.md index d34346b..d2a43b4 100644 --- a/examples/README.md +++ b/examples/README.md @@ -64,6 +64,10 @@ dbt seed && dbt run - Customer reviews and ratings with min/max constraints - Business constraints: unique emails, foreign key relationships - **Generation hints** (`generate`): Price ranges, stock levels, quantities, ratings + - **Dates that respect their parent**: `orders.order_date` has `after: customers.created_at`, + so no order predates its customer (spec 0.5.0). `ecommerce_daily` and `ecommerce_training` + do the same; `saas_platform` (subscriptions, invoices and audit events after their + organisation) and `advanced_features` (time entries after `employees.hire_date`) use it too **Tables**: 5 tables with 5 relationships - `customers` — Customer directory with realistic names, emails, phone numbers @@ -76,6 +80,9 @@ dbt seed && dbt run - **Faker data types**: Uses `email`, `first_name`, `last_name`, `phone_number`, `country`, `word`, `ean13` - **Generation hints**: `price: {type: numeric, not_null: true, generate: {min: 5.99, max: 999.99}}` - **Foreign keys**: Transactional relationships between customers, orders, and products +- **Cross-table `after`**: `order_date: {type: timestamp, generate: {after: customers.created_at}}` + keeps each order on or after the customer it points at. Add it where a child cannot exist + before its parent; `model2data validate` warns where it is missing - **Unique constraints**: Email uniqueness per customer - **Realistic data generation**: Names, emails, phone numbers generated via Faker library @@ -360,7 +367,8 @@ allocation: {type: numeric, generate: {min: 0.5, max: 8}} (A column with `generate` is usually written as a block mapping, as above; the one-line form is the same YAML.) Every other hint -- `null_rate`, `weights`, `true_rate`, `distinct`, `skew`, -`after`, `business_hours`, `growth`, `seasonality`, `distribution` -- is documented in the +`after` (a column of the row, a parent's `
.`, or a list), `business_hours`, +`growth`, `seasonality`, `distribution` -- is documented in the [spec's schema](../model2data/spec/model.schema.json). ### When to Use Constraints From fe258af2da8a43f5e378ec3aa30dab7edfa313d0 Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Thu, 8 Oct 2026 22:12:23 +0200 Subject: [PATCH 12/14] Mention a parent's column in the hint-tests and validate help Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++- model2data/cli.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02612df..f19aeed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). table the row has no foreign key to (or only a composite one, or one onto a column that is no key), the column's own table (a column of the row is named without its table; a self-reference is not followed), more than one foreign key to the parent (`messages` - reaching `users` through `sender_id` and `receiver_id` is an error naming both), two + reaching `users` through `sender_id` and `receiver_id` is an error naming both; picking one + of several foreign keys to a parent is not built yet), two tables whose `after`s name each other's columns, and a parent generated after the child (a cycle of foreign keys broken at the very key followed). - **The child keeps its own shape.** A row whose date falls before its parent's takes another diff --git a/model2data/cli.py b/model2data/cli.py index 65cf1fe..d3c21d0 100644 --- a/model2data/cli.py +++ b/model2data/cli.py @@ -401,8 +401,9 @@ def main( "warn", "--hint-tests", help=( - "Write the model's generation hints as dbt tests (min/max range, after, " - "null_rate, distinct, when, grain) at this severity: error, warn or off. They " + "Write the model's generation hints as dbt tests (min/max range, after " + "(also a parent's column), null_rate, distinct, when, grain) at this severity: " + "error, warn or off. They " "describe intent, so they warn by default rather than break a first dbt " "build on real data." ), @@ -939,7 +940,8 @@ def validate_command( Takes one or more files and/or --glob patterns. A metrics file (*.metrics.yml) is checked against metrics spec 0.1.0 and against its model. Exits 1 when any file has an error; - warnings are printed, and the file conforms. + warnings are printed, and the file conforms. A warning that a child's date `can fall + before` its parent's names the `after:
.` to add. """ if output_format not in ("text", "github"): raise typer.BadParameter("must be 'text' or 'github'", param_hint="--format") From 70d7a2a097663020a3392350c33128a5242b570d Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Tue, 13 Oct 2026 15:29:58 +0200 Subject: [PATCH 13/14] Release 1.16.0, with the fixes of the 1.15.0 release that never reached PyPI Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 ++++++- docs/generating.md | 2 +- model2data/generate/faker.py | 2 +- model2data/spec/PUBLISHING.md | 6 +++--- pyproject.toml | 2 +- tests/test_column_time_hints.py | 2 +- tests/test_defects_review.py | 2 +- tests/test_distributions.py | 2 +- tests/test_shaping.py | 2 +- tests/test_table_seeds.py | 2 +- tests/test_timeline.py | 4 ++-- uv.lock | 2 +- 12 files changed, 20 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1761d8..05ec710 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +## [1.16.0] - 2026-10-13 + +The v1.15.0 GitHub release never reached PyPI (its package still said 1.14.0), so its fixes +ship here, together with the changes below. + ### Added - **`after` can name a parent's column.** `after: customers.created_at` on `orders.order_date` keeps every order on or after the `created_at` of the customer its foreign key points at (in @@ -83,7 +88,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). from the table's random stream, so every column after it draws from a different point of it. Dates change only where a model uses a parent's column in `after` (the examples listed above now do). The same model, seed, `--as-of` and options still give the same bytes on any - machine and any day with 1.15.0. Pin `model2data==1.14.0` to keep fixtures of the old shape. + machine and any day with 1.16.0. Pin `model2data==1.14.0` to keep fixtures of the old shape. ### Fixed - **A `binary` column holds a short value.** `binary` is also the name of a Faker provider, and diff --git a/docs/generating.md b/docs/generating.md index 857c6b4..5777000 100644 --- a/docs/generating.md +++ b/docs/generating.md @@ -38,7 +38,7 @@ The same model, seed and `--as-of` — with the same options (`--rows`, `--rows- - with the same model2data version, whenever you run it. A new release may change what a seed produces, to fix a bug or make the data more realistic. -When it does, the changelog says which models are affected and why (1.15.0, for one, numbers +When it does, the changelog says which models are affected and why (1.16.0, for one, numbers integer primary keys 1..N, which changes the bytes of every model that has one, and moves dates only where a model uses a parent's column in `after`). Pin the version (`model2data==1.14.0`) when fixtures must stay the same across upgrades. diff --git a/model2data/generate/faker.py b/model2data/generate/faker.py index ddbfc53..c8b9be9 100644 --- a/model2data/generate/faker.py +++ b/model2data/generate/faker.py @@ -908,7 +908,7 @@ def generator() -> int: elif ensure_unique and "pk" in column.settings and not had_explicit_range: # A table's own integer primary key numbers its rows 1..N, as an # identity column does: the first row is 1 and the last is N. - # Before 1.15 it was a shuffled sample starting at 0. + # Before 1.16 it was a shuffled sample starting at 0. values = list(range(1, row_count + 1)) elif ensure_unique: if not had_explicit_range: diff --git a/model2data/spec/PUBLISHING.md b/model2data/spec/PUBLISHING.md index 650636f..2191c30 100644 --- a/model2data/spec/PUBLISHING.md +++ b/model2data/spec/PUBLISHING.md @@ -24,8 +24,8 @@ On the website (`JB-Analytica/jba-website`), serve this directory's files at the | --- | --- | | `https://www.jbanalytica.com/model2data/spec/metrics/0.1.0/metrics.schema.json` | `model2data/spec/metrics/metrics.schema.json` of engine 1.13.0 | | `https://www.jbanalytica.com/model2data/spec/metrics/0.1.0/README.md` (optional, or render it as a page) | `model2data/spec/metrics/README.md` of engine 1.13.0 | -| `https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json` | `model2data/spec/model.schema.json` of engine 1.15.0 | -| `https://www.jbanalytica.com/model2data/spec/0.5.0/README.md` (optional, or render it as a page) | `model2data/spec/README.md` of engine 1.15.0 | +| `https://www.jbanalytica.com/model2data/spec/0.5.0/model.schema.json` | `model2data/spec/model.schema.json` of engine 1.16.0 | +| `https://www.jbanalytica.com/model2data/spec/0.5.0/README.md` (optional, or render it as a page) | `model2data/spec/README.md` of engine 1.16.0 | | `https://www.jbanalytica.com/model2data/spec/0.4.0/model.schema.json` | `model2data/spec/model.schema.json` of engine 1.11.0 | | `https://www.jbanalytica.com/model2data/spec/0.4.0/README.md` (optional, or render it as a page) | `model2data/spec/README.md` of engine 1.11.0 | | `https://www.jbanalytica.com/model2data/spec/0.3.0/model.schema.json` | `model2data/spec/model.schema.json` of engine 1.9.0 | @@ -41,7 +41,7 @@ Requirements: Monaco playgrounds) can fetch it. - **`Content-Type: application/schema+json`** (or `application/json`). - Copy the file from the released engine version, not from a branch: the tag that ships each - version (0.2.0: engine 1.8.0; 0.3.0: engine 1.9.0; 0.4.0: engine 1.11.0; 0.5.0: engine 1.15.0; + version (0.2.0: engine 1.8.0; 0.3.0: engine 1.9.0; 0.4.0: engine 1.11.0; 0.5.0: engine 1.16.0; metrics 0.1.0: engine 1.13.0). - The metrics spec is versioned on its own, under `/spec/metrics//`, beside the model spec's versions; a metrics file's first line points at it the same way: diff --git a/pyproject.toml b/pyproject.toml index fa033d5..22b050f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "model2data" -version = "1.14.0" +version = "1.16.0" description = "Turn a data model into realistic synthetic data, a runnable dbt project and metrics with known values" readme = "README_PYPI.md" requires-python = ">=3.10" diff --git a/tests/test_column_time_hints.py b/tests/test_column_time_hints.py index 3029de0..ff23aff 100644 --- a/tests/test_column_time_hints.py +++ b/tests/test_column_time_hints.py @@ -196,7 +196,7 @@ def test_no_hint_and_uniform_profile_matches_the_pre_feature_pinned_frame(): } df = generate_data_from_dbml(tables, [], base_rows=8, seed=2024, as_of=ANCHOR_DATE)["events"] - # As re-pinned in 1.15.0 (integer keys numbered 1..N). + # As re-pinned in 1.16.0 (integer keys numbered 1..N). assert list(df["created_at"])[:2] == ["2025-12-07 15:29:52", "2025-05-20 12:00:46"] diff --git a/tests/test_defects_review.py b/tests/test_defects_review.py index 3874af2..938f94a 100644 --- a/tests/test_defects_review.py +++ b/tests/test_defects_review.py @@ -287,7 +287,7 @@ def test_an_undone_row_without_days(): def test_late_rows_a_later_defect_makes_on_time_are_not_counted(): model = load(SHOP) - # A seed whose late rows include one the nulls make on time (engine 1.15.0's + # A seed whose late rows include one the nulls make on time (engine 1.16.0's # parent dates changed which rows seed 11 picks). days = generate_days(model, 3, seed=2, as_of=AS_OF) plan = { diff --git a/tests/test_distributions.py b/tests/test_distributions.py index 2b649fa..85bf65c 100644 --- a/tests/test_distributions.py +++ b/tests/test_distributions.py @@ -38,7 +38,7 @@ def test_no_distribution_hint_reproduces_pre_1_7_frames(): ) } data = generate_data_from_dbml(tables, [], base_rows=8, seed=777) - # Re-pinned in 1.15.0: the primary key numbers the rows 1..N rather than + # Re-pinned in 1.16.0: the primary key numbers the rows 1..N rather than # drawing a shuffled sample, so the columns after it start elsewhere in the stream. assert data["t"]["amount"].tolist() == [ 7784.79, diff --git a/tests/test_shaping.py b/tests/test_shaping.py index 122b393..d81f204 100644 --- a/tests/test_shaping.py +++ b/tests/test_shaping.py @@ -83,7 +83,7 @@ def test_a_nullable_enum_column_takes_the_default_null_rate(): def test_uniform_generation_reproduces_pre_1_5_frames(): """A run that passes no shaping option at all must still produce exactly what 1.4.0 produced with the same seed -- this is the guard rail every - later change in this module works against. Re-pinned in 1.15.0, whose + later change in this module works against. Re-pinned in 1.16.0, whose primary keys number the rows 1..N instead of drawing a shuffled sample, so every column after the key draws from a different point of the stream.""" data = generate_data_from_dbml( diff --git a/tests/test_table_seeds.py b/tests/test_table_seeds.py index 81aca76..dc19278 100644 --- a/tests/test_table_seeds.py +++ b/tests/test_table_seeds.py @@ -116,7 +116,7 @@ def test_referential_integrity_holds_against_the_new_parent(self): assert set(rolled["order_items"]["order_id"]) <= set(rolled["orders"]["id"]) def test_the_re_rolled_parent_holds_new_rows_under_the_same_numbering(self): - """An integer key numbers the rows 1..N (1.15.0), so a re-rolled parent + """An integer key numbers the rows 1..N (1.16.0), so a re-rolled parent keeps its ids and every child still points at a row that exists; the rows behind those ids are new.""" baseline = _generate() diff --git a/tests/test_timeline.py b/tests/test_timeline.py index 1f11d28..f41ce2a 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -55,9 +55,9 @@ def _created_updated_schema() -> dict[str, TableDef]: def test_uniform_profile_reproduces_pre_1_5_frames(): """A table with no ordered pair is untouched by the ordering pass, so its frame under the uniform profile is byte-identical to what 1.4.0 generated -- - but for 1.15.0's primary key, which numbers the rows 1..N instead of drawing + but for 1.16.0's primary key, which numbers the rows 1..N instead of drawing a shuffled sample, so every column after it draws from a different point of - the table's stream (pinned again in 1.15.0).""" + the table's stream (pinned again in 1.16.0).""" df = generate_data_from_dbml( _single_timestamp_and_date_schema(), [], base_rows=8, seed=2024, as_of=ANCHOR )["events"] diff --git a/uv.lock b/uv.lock index 1151899..37b3235 100644 --- a/uv.lock +++ b/uv.lock @@ -785,7 +785,7 @@ wheels = [ [[package]] name = "model2data" -version = "1.14.0" +version = "1.16.0" source = { editable = "." } dependencies = [ { name = "dbt-core" }, From 5121f431226a311206f739d3db1f2931e247f699 Mon Sep 17 00:00:00 2001 From: Jarich Braeckevelt Date: Tue, 13 Oct 2026 15:36:23 +0200 Subject: [PATCH 14/14] Write the coffee webshop's order_date hints in schema order So a writer that orders hints the schema's way, such as the studio's, keeps the example byte for byte. Only the copied model file's digest changes; the data is the same. Co-Authored-By: Claude Opus 5.5 --- model2data/spec/examples/coffee_webshop.model2data.yml | 2 +- tests/fixtures/determinism.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/model2data/spec/examples/coffee_webshop.model2data.yml b/model2data/spec/examples/coffee_webshop.model2data.yml index 8c5acb4..d597791 100644 --- a/model2data/spec/examples/coffee_webshop.model2data.yml +++ b/model2data/spec/examples/coffee_webshop.model2data.yml @@ -80,7 +80,7 @@ tables: order_date: type: timestamp not_null: true - generate: {business_hours: true, seasonality: 0.5, after: customers.created_at} + generate: {after: customers.created_at, business_hours: true, seasonality: 0.5} shipped_at: type: timestamp generate: {after: order_date} diff --git a/tests/fixtures/determinism.json b/tests/fixtures/determinism.json index c1664ea..9271afa 100644 --- a/tests/fixtures/determinism.json +++ b/tests/fixtures/determinism.json @@ -550,7 +550,7 @@ }, "metrics": { "dbt_det/coffee_webshop.metrics.yml": "16981620dc46a34c01f2c6cfc36644522bf9c5a80defb49206cc2a82eca67c32", - "dbt_det/coffee_webshop.model2data.yml": "596c24c87f0fed5b4c18a32c3f14af162ccc8c0dc3a278df1f73cd3f4b434079", + "dbt_det/coffee_webshop.model2data.yml": "e5b116739df19fdfcfea9ecb2290d731b36eca203afe0d0925c0b42b83250e1b", "dbt_det/data-tests/metrics/metric_average_order_value.sql": "9292cf214a82e05011817057157da7eaa467baf5fda2443a80a826b2302c0bd6", "dbt_det/data-tests/metrics/metric_coffee_units_sold.sql": "6e9d8b6182348163d5311790f71c5b41875c26b8cd7e96aa064075ba1849d19f", "dbt_det/data-tests/metrics/metric_order_items_count.sql": "3d0e8d87ceb9c396106908800b18872c499f9927b634780868271d003df3a415",