From 374e0bc4c644310ff56cdf9c0fe81eccdec862b0 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 7 Aug 2026 20:33:57 -0700 Subject: [PATCH 01/34] mdcode: add Knowledge Catalog push for the semantic model Adds the Knowledge Catalog push leg for the semantic-model scope, the counterpart to the merged BigQuery Graph leg (#270). Emits semantic-model/semantic-entity/semantic-metric entries against built-in dataplex-types/global system types, publishes relationships as schema-join entry links, shares model loading across legs, and reconciles removed entities/metrics on re-push. --target selects destinations (bq|kc|all); --print dumps each destination's native artifact. BigQuery leg is live-validated; the KC write path is covered hermetically (gated on the semantic-model system types CL). --- toolbox/mdcode/src/libts/gcp/api.ts | 6 + toolbox/mdcode/src/libts/gcp/dataplex.ts | 50 ++ .../{deploy.ts => deploy_bigquery.ts} | 168 +++--- .../semantic/deploy_knowledge_catalog.ts | 462 ++++++++++++++++ .../src/libts/semantic/knowledge_catalog.ts | 523 ++++++++++++++++++ toolbox/mdcode/src/libts/semantic/loader.ts | 54 ++ toolbox/mdcode/src/tool/commands.ts | 204 ++++++- toolbox/mdcode/src/tool/main.ts | 2 + ...deploy.test.ts => deploy_bigquery.test.ts} | 67 ++- .../semantic/deploy_knowledge_catalog.test.ts | 379 +++++++++++++ ...graph_target.knowledge_catalog.golden.json | 88 +++ .../star_orders_customer.bigquery.golden.sql | 5 +- ...ers_customer.knowledge_catalog.golden.json | 217 ++++++++ .../fixtures/star_orders_customer.yaml | 10 +- ...ds_date_edge.knowledge_catalog.golden.json | 518 +++++++++++++++++ .../semantic/knowledge_catalog.e2e.test.ts | 88 +++ .../libts/semantic/knowledge_catalog.test.ts | 343 ++++++++++++ .../semantic/load_semantic_models.test.ts | 86 +++ .../tests/libts/semantic/loader.test.ts | 9 +- .../tests/libts/semantic/target.test.ts | 45 ++ .../tests/tool/init_semantic_model.test.ts | 100 ++++ 21 files changed, 3259 insertions(+), 165 deletions(-) rename toolbox/mdcode/src/libts/semantic/{deploy.ts => deploy_bigquery.ts} (70%) create mode 100644 toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts create mode 100644 toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts rename toolbox/mdcode/tests/libts/semantic/{deploy.test.ts => deploy_bigquery.test.ts} (90%) create mode 100644 toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.knowledge_catalog.golden.json create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json create mode 100644 toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/load_semantic_models.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/target.test.ts create mode 100644 toolbox/mdcode/tests/tool/init_semantic_model.test.ts diff --git a/toolbox/mdcode/src/libts/gcp/api.ts b/toolbox/mdcode/src/libts/gcp/api.ts index 596bca6e..ffb2f43f 100644 --- a/toolbox/mdcode/src/libts/gcp/api.ts +++ b/toolbox/mdcode/src/libts/gcp/api.ts @@ -46,6 +46,12 @@ export class ApiClient { return this._requestRetry('PATCH', url, queryParams, body); } + async _delete(resourceName: string, + queryParams?: Record): Promise> { + const url = `${this._endpoint}/${this._pathPrefix}/${resourceName}`; + return this._requestRetry('DELETE', url, queryParams); + } + private async _requestRetry(method: string, url: string, queryParams?: Record, diff --git a/toolbox/mdcode/src/libts/gcp/dataplex.ts b/toolbox/mdcode/src/libts/gcp/dataplex.ts index b86a048c..a501e6f4 100644 --- a/toolbox/mdcode/src/libts/gcp/dataplex.ts +++ b/toolbox/mdcode/src/libts/gcp/dataplex.ts @@ -49,6 +49,27 @@ export interface Entry { aspects?: Record; } +export interface EntryReference { + // Full resource name of the referenced entry. + name: string; + // Path within the entry that is referenced; empty means the entry itself. + path?: string; + // SOURCE / TARGET for a directed link; UNSPECIFIED for an undirected one. + type: 'UNSPECIFIED' | 'SOURCE' | 'TARGET'; +} + +export interface EntryLink { + // Server-assigned (output only); set by the emitter to the destination name so + // the publisher can address it for an in-place update. + name?: string; + entryLinkType: string; + // Exactly two references. + entryReferences: EntryReference[]; + // At most one Dataplex-owned aspect (e.g. schema-join), keyed as + // `project.location.aspectType`. + aspects?: Record; +} + interface EntryList { entries: Entry[]; nextPageToken?: string; @@ -193,6 +214,12 @@ export class CatalogClient extends api.ApiClient { return res; } + async deleteEntry(project: string, location: string, entryGroup: string, + entry: string): Promise> { + const name = `${catalogContainer(project, location, entryGroup)}/entries/${entry}`; + return await this._delete(name); + } + async createEntryGroup(project: string, location: string, entryGroupId: string, entryGroup?: EntryGroup): Promise> { const parent = catalogContainer(project, location); @@ -205,6 +232,29 @@ export class CatalogClient extends api.ApiClient { return res; } + async createEntryLink(project: string, location: string, entryGroup: string, + entryLinkId: string, + entryLink: EntryLink): Promise> { + const parent = catalogContainer(project, location, entryGroup); + const resourceName = `${parent}/entryLinks`; + + const params: Record = { entryLinkId }; + + return await this._post(resourceName, entryLink, params); + } + + // Patches an existing entry link. Only the aspects are mutable (the entry + // references and link type are immutable server-side), so callers pass the + // link name + aspects with updateMask ['aspects']. + async updateEntryLink(entryLink: EntryLink, + updateMask?: string[]): Promise> { + const params: Record = {}; + if (updateMask && updateMask.length) { + params.updateMask = updateMask.join(','); + } + return await this._patch(entryLink.name!, entryLink, params); + } + } diff --git a/toolbox/mdcode/src/libts/semantic/deploy.ts b/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts similarity index 70% rename from toolbox/mdcode/src/libts/semantic/deploy.ts rename to toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts index 5af49505..b920ef9b 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts @@ -1,12 +1,11 @@ // Deploys a semantic model's BigQuery Graph. // // This is the BigQuery leg of `kcmd push` for the semantic-model scope: it -// parses each authored Ossie document into the semantic IR (loader), lowers it -// to `CREATE OR REPLACE PROPERTY GRAPH` DDL (generator), and executes that DDL -// against the BigQuery project named by the model's GOOGLE deployment target. -// -// The Knowledge Catalog resource emit (entries + entryLinks) for the semantic -// model is a follow-on; `push` currently deploys only the BigQuery Graph. +// consumes models already parsed into the semantic IR (see loadSemanticModels, +// shared with the Knowledge Catalog leg so a `--target all` push parses each +// document once), lowers each to `CREATE OR REPLACE PROPERTY GRAPH` DDL +// (generator), and executes that DDL against the BigQuery project named by the +// model's GOOGLE deployment target. // // This is a library module: it emits no console output. The generated DDL and // any loader/generator warnings are returned in `DeployResult` for the CLI @@ -19,7 +18,7 @@ import * as context from '../gcp/context'; import {generatePropertyGraph} from './bigquery'; import {SemanticModel} from './ir'; -import {loadModels} from './loader'; +import {LoadedModel} from './loader'; // Our own vendor tag in the Ossie `custom_extensions` list. @@ -242,20 +241,14 @@ async function datasetLocation( } -// Deploys the BigQuery Graph for each authored model document. Emits no console -// output; the generated DDL and any warnings are returned for the caller to -// print. -// -// `defaultProject` qualifies a dataset `source` that omits its project. The -// caller should pass the scope's declared project (a deterministic, -// user-authored value) rather than relying on the ambient gcloud project, which -// can silently drift away from where the model's tables live. It cannot default -// to each deployment target's own project: a document is parsed once, before -// its targets are known, and may declare several graphs across projects. +// Deploys the BigQuery Graph for each pre-loaded model. Emits no console output; +// the generated DDL and any warnings are returned for the caller to print. The +// models are parsed once by the caller (loadSemanticModels) and shared with the +// Knowledge Catalog leg; each carries its originating document name for +// diagnostics. export async function deployBigQuery( - docs: {name: string; text: string}[], ctx: context.ApiContext, - options: DeployOptions = {}, - defaultProject?: string): Promise { + models: LoadedModel[], ctx: context.ApiContext, + options: DeployOptions = {}): Promise { const bigQuery = new BigQueryClient(ctx); const ddl: string[] = []; const warnings: string[] = []; @@ -282,93 +275,76 @@ export async function deployBigQuery( return loc; }; - for (const doc of docs) { - // A document that fails to parse (or violates the model schema) is an - // authoring error. Report it against the specific document rather than - // letting the loader's exception propagate as an uncaught stack trace, and - // name the document so a bad file in a multi-document push is identifiable. - let loaded; + for (const {document, model} of models) { + modelsSeen++; + let targets: BigQueryGraphTarget[]; + let malformed: string[]; try { - loaded = - loadModels(doc.text, {defaultProject: defaultProject ?? ctx.project}); + ({targets, malformed} = bigQueryGraphTargets(model)); } catch (err: any) { - return fail(`Model document '${doc.name}': ${err.message || err}`); + return fail( + `Model '${model.name}' (${document}): ${err.message || err}`); } - for (const w of loaded.warnings) { - warnings.push(`[${doc.name}] ${w}`); + if (!targets.length) { + // A malformed-but-present target must not be reported as "none + // declared" -- that sends the author hunting for an extension they + // already wrote. Name the URIs that failed to parse. + if (malformed.length) { + return fail( + `Model '${model.name}' (${document}) declares BigQuery Graph ` + + `deploymentTarget(s) that could not be parsed: ` + + `${malformed.join(', ')}. Expected //bigquery.googleapis.com/` + + `projects/

/datasets//propertyGraphs/.`); + } + return fail( + `Model '${model.name}' (${document}) declares no BigQuery Graph ` + + `deploymentTarget in a GOOGLE custom_extension; nothing to deploy.`); } + // Malformed targets alongside valid ones: surface them so a typo'd URI is + // not silently dropped when the deploy otherwise succeeds. + for (const bad of malformed) { + warnings.push(`[${ + model.name}] ignoring unparseable BigQuery Graph target: ${bad}`); + } + + for (const target of targets) { + const gen = generatePropertyGraph(model, { + project: target.project, + dataset: target.dataset, + graphName: target.graphName, + }); + for (const w of gen.warnings) { + warnings.push(`[${model.name} -> ${target.graphName}] ${w}`); + } + ddl.push(`-- ${target.uri}\n${gen.ddl}`); - for (const model of loaded.models) { - modelsSeen++; - let targets: BigQueryGraphTarget[]; - let malformed: string[]; + if (options.validateOnly) { + continue; + } + + let location: string|undefined; try { - ({targets, malformed} = bigQueryGraphTargets(model)); + location = await locationFor(target); } catch (err: any) { return fail( - `Model '${model.name}' (${doc.name}): ${err.message || err}`); + `Failed to deploy '${target.graphName}': ${err.message || err}`); } - if (!targets.length) { - // A malformed-but-present target must not be reported as "none - // declared" -- that sends the author hunting for an extension they - // already wrote. Name the URIs that failed to parse. - if (malformed.length) { - return fail( - `Model '${model.name}' (${doc.name}) declares BigQuery Graph ` + - `deploymentTarget(s) that could not be parsed: ` + - `${malformed.join(', ')}. Expected //bigquery.googleapis.com/` + - `projects/

/datasets//propertyGraphs/.`); - } - return fail( - `Model '${model.name}' (${doc.name}) declares no BigQuery Graph ` + - `deploymentTarget in a GOOGLE custom_extension; nothing to deploy.`); - } - // Malformed targets alongside valid ones: surface them so a typo'd URI is - // not silently dropped when the deploy otherwise succeeds. - for (const bad of malformed) { - warnings.push(`[${ - model.name}] ignoring unparseable BigQuery Graph target: ${bad}`); + const started = await bigQuery.query(target.project, gen.ddl, location); + const outcome = await awaitQueryDone( + bigQuery, target.project, started, maxPolls, backoffMs); + if (!outcome.ok) { + // These are CREATE OR REPLACE statements executed one at a time, so + // any graphs already deployed in this run have mutated production and + // are not rolled back. Surface that alongside the failing target. + const partial = deployed ? + ` (${deployed} graph(s) already deployed in this run; ` + + `CREATE OR REPLACE changes are not rolled back)` : + ''; + return fail(`Failed to deploy '${target.graphName}': ${ + outcome.error}${partial}`); } - for (const target of targets) { - const gen = generatePropertyGraph(model, { - project: target.project, - dataset: target.dataset, - graphName: target.graphName, - }); - for (const w of gen.warnings) { - warnings.push(`[${model.name} -> ${target.graphName}] ${w}`); - } - ddl.push(`-- ${target.uri}\n${gen.ddl}`); - - if (options.validateOnly) { - continue; - } - - let location: string|undefined; - try { - location = await locationFor(target); - } catch (err: any) { - return fail( - `Failed to deploy '${target.graphName}': ${err.message || err}`); - } - const started = await bigQuery.query(target.project, gen.ddl, location); - const outcome = await awaitQueryDone( - bigQuery, target.project, started, maxPolls, backoffMs); - if (!outcome.ok) { - // These are CREATE OR REPLACE statements executed one at a time, so - // any graphs already deployed in this run have mutated production and - // are not rolled back. Surface that alongside the failing target. - const partial = deployed ? - ` (${deployed} graph(s) already deployed in this run; ` + - `CREATE OR REPLACE changes are not rolled back)` : - ''; - return fail(`Failed to deploy '${target.graphName}': ${ - outcome.error}${partial}`); - } - - deployed++; - } + deployed++; } } diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts new file mode 100644 index 00000000..69bb391f --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -0,0 +1,462 @@ +// Deploys a semantic model's Knowledge Catalog resources. +// +// This is the Knowledge Catalog leg of `kcmd push` for the semantic-model +// scope, the counterpart to `deploy_bigquery.ts`. It consumes models already +// parsed into the semantic IR (see loadSemanticModels, shared with the BigQuery +// leg so a `--target all` push parses each document once), maps each to catalog +// Entries + Aspects (the pure emitter in knowledge_catalog.ts), and writes them +// through the Knowledge Catalog client. +// +// Types: the `semantic-model`/`semantic-entity`/`semantic-metric` entry and +// aspect types — and the built-in `schema` aspect — are built-in system types +// in `dataplex-types/global`. Push does NOT provision any type, nor the entry +// group (that is created at `init`); it only writes entries. The caller needs +// `dataplex.entryGroups.useSemanticModelAspect` on the destination entry group. +// +// Publish sequence (mirrors the BigQuery leg's structure): +// * Create each model's entries in array order: the semantic-model anchor +// first (it is the parentEntry of every entity/metric entry), then the +// children concurrently. No entry-group or type creation -- the entry +// group is provisioned at `init` and the system types are built-in. +// * A re-push upserts: an entry that already exists is updated in place. +// * Reconcile deletions: an entity or metric removed from a still-present +// model leaves an orphaned entry under its anchor; after writing, delete +// any entry this push owns (by entry-id prefix) that was not re-emitted. +// * Relationship edges are published as schema-join entry links between the +// two entity entries, written after that model's entries (both endpoints +// must exist first). A re-push upserts the link's aspect; a many-to-many +// (association) edge is not published yet (the emitter warns and skips it). +// The caller additionally needs `dataplex.entryGroups.useSchemaJoinEntryLink` +// and `useSchemaJoinAspect` on the destination entry group. +// +// This is a library module: it emits no console output. Warnings and the +// dry-run plan are returned in `KcDeployResult` for the CLI (commands.ts) to +// print. +// + +import {ApiResult} from '../gcp/api'; +import * as context from '../gcp/context'; +import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; + +import {generateCatalogResources, KcResources} from './knowledge_catalog'; +import {LoadedModel} from './loader'; + + +export interface KcDeployOptions { + // The resolved Knowledge Catalog destination (flag overrides applied over the + // catalog.yaml scope defaults). + project: string; + location: string; + entryGroup: string; + // Where the built-in `semantic-*` / `schema` system types are referenced. + // Default: `dataplex-types` / `global`. Overridable to reference them from a + // staging project; the emitted entries are otherwise unchanged. + systemTypeProject?: string; + systemTypeLocation?: string; + // Compile + report only; never writes. + validateOnly?: boolean; + // entries.create can briefly 404 on a just-created entry group; retry that + // window. Overridable so tests can exercise the path without burning + // wall-clock. + entryCreateTries?: number; + entryCreateRetryMs?: number; +} + +export interface KcDeployResult { + success: boolean; + details?: string; + // Loader and emitter warnings collected across all documents. + warnings: string[]; + // Entries created / updated-in-place / deleted (all 0 for validateOnly). + created: number; + updated: number; + deleted: number; + // Relationship (schema-join) entry links written -- created or upserted (0 for + // validateOnly). + linked: number; + // A human-readable plan of what would be written; populated for validateOnly. + plan: string[]; +} + + +// entries.create propagation retry: a just-created entry group can briefly 404. +const ENTRY_CREATE_TRIES = 3; +const ENTRY_CREATE_RETRY_MS = 3000; + +function sleep(ms: number): Promise { + return new Promise(r => setTimeout(r, ms)); +} + + +// Deploys the Knowledge Catalog resources for each authored model document. +// Emits no console output; warnings and the dry-run plan are returned for the +// caller to print. `defaultProject` qualifies a dataset `source` that omits its +// project (the scope's declared project, a deterministic user-authored value). +export async function deployKnowledgeCatalog( + models: LoadedModel[], ctx: context.ApiContext, + opts: KcDeployOptions): Promise { + const warnings: string[] = []; + const plan: string[] = []; + let created = 0; + let updated = 0; + let deleted = 0; + let linked = 0; + let modelsSeen = 0; + + const fail = (details: string): KcDeployResult => + ({success: false, details, warnings, created, updated, deleted, linked, + plan}); + + // Emit every model up front (pure): dry-run and warnings need no network, and + // a generation warning surfaces even if a later write fails. + const emitted: {model: string; resources: KcResources}[] = []; + for (const {document, model} of models) { + modelsSeen++; + // The emitter is pure but not infallible: bigQueryGraphTargets (reached + // via the semantic-model aspect) throws on a malformed GOOGLE + // custom_extension. Report it against the document -- as the BigQuery leg + // does -- rather than letting it escape as an uncaught stack trace. + let resources: KcResources; + try { + resources = generateCatalogResources(model, { + project: opts.project, + location: opts.location, + entryGroup: opts.entryGroup, + systemTypeProject: opts.systemTypeProject, + systemTypeLocation: opts.systemTypeLocation, + }); + } catch (err: any) { + return fail( + `Model '${model.name}' (${document}): ${err.message || err}`); + } + for (const w of resources.warnings) { + warnings.push(`[${model.name}] ${w}`); + } + emitted.push({model: model.name, resources}); + } + + // A parsed document always yields at least one model (the loader enforces + // `semantic_model` min 1), so modelsSeen is 0 only when no documents were + // found. validateOnly mutates nothing, so an empty workspace is a clean no-op + // there; a real push treats it as a configuration error worth flagging. + if (!modelsSeen) { + if (opts.validateOnly) { + warnings.push('No semantic model documents found; nothing to validate.'); + return {success: true, warnings, created, updated, deleted, linked, plan}; + } + return fail('No semantic model documents found; nothing to deploy.'); + } + + // Entry ids must be unique within the destination entry group. The emitter + // dedups within one model, but two models in a single push (two documents, or + // two `semantic_model`s in one document) whose names normalize to the same id + // generate colliding entry names; on publish the later one would 409 and + // silently upsert over the earlier. Catch that across models here and fail + // before any write, naming the entry so the author can rename one model. The + // owner is tracked by index, not model name, so two same-named models (the + // most common collision) are still distinguished. + const entryOwner = new Map(); + for (let i = 0; i < emitted.length; i++) { + for (const entry of emitted[i].resources.entries) { + const prev = entryOwner.get(entry.name); + if (prev !== undefined && prev !== i) { + return fail( + `models '${emitted[prev].model}' and '${emitted[i].model}' both ` + + `generate catalog entry '${idOf(entry.name)}'; entry ids must be ` + + `unique within entry group '${opts.entryGroup}' -- rename one ` + + `model.`); + } + entryOwner.set(entry.name, i); + } + } + + for (const {model, resources} of emitted) { + plan.push(...planSummary(model, resources, opts)); + } + if (opts.validateOnly) { + return {success: true, warnings, created, updated, deleted, linked, plan}; + } + + // The destination entry group is provisioned at `init`, not here: push writes + // only entries, matching how the standard layout's push operates (it creates + // entries, never the entry group). A missing group surfaces as a clear entry + // creation error, and createEntryWithRetry rides out the brief post-init + // propagation window. + const cat = new CatalogClient(ctx); + + for (const {model, resources} of emitted) { + const outcome = await createEntries(cat, opts, resources.entries); + if (outcome.error) { + return fail(`Model '${model}': ${outcome.error}`); + } + created += outcome.created; + updated += outcome.updated; + + // Links reference this model's entity entries, so they are written after the + // entries above (both endpoints must exist first). + const links = await createEntryLinks(cat, opts, resources.entryLinks); + if (links.error) { + return fail(`Model '${model}': ${links.error}`); + } + linked += links.linked; + } + + // Reconcile deletions: an entity or metric removed from a still-present model + // since the last push leaves an orphaned entry under the model's anchor. + // Delete any entry this push owns that was not re-emitted (see + // reconcileDeletions). Runs only on a real push -- validateOnly returned + // above and never lists remote state. + const recon = await reconcileDeletions(cat, opts, emitted); + if (recon.error) { + return fail(recon.error); + } + deleted = recon.deleted; + + return {success: true, warnings, created, updated, deleted, linked, plan}; +} + + +interface ReconcileOutcome { + deleted: number; + error?: string; +} + +// Deletes entries this push OWNS but did not re-emit -- the entities/metrics +// removed from a model since its last push. Ownership is scoped by entry id so +// reconciliation never touches entries outside the models in this push (a +// shared entry group may legitimately hold others): an existing entry is owned +// when its id is a pushed model's anchor id or is prefixed by that anchor's +// `.entities.` / `.metrics.` child namespace. An anchor is always +// re-emitted, so it is never deleted here. +// +// TODO: reconcile whole-model removals too. Deleting a model's document drops +// its anchor from this push, so its anchor + children are no longer owned and +// survive. Removing them safely needs a scope-level record of which models this +// entry group manages (follow-up). +// +// TODO: reconcile removed relationship links. There is no list API for entry +// links (only LookupEntryLinks, per referenced entry), so an orphaned link left +// by a dropped relationship is not deleted yet; it needs a lookup-per-entity +// sweep (follow-up). +async function reconcileDeletions( + cat: CatalogClient, opts: KcDeployOptions, + emitted: {resources: KcResources}[]): Promise { + const emittedIds = new Set(); + const anchorIds = new Set(); + const childPrefixes: string[] = []; + for (const {resources} of emitted) { + for (const e of resources.entries) emittedIds.add(idOf(e.name)); + // entries[0] is the model anchor (the emitter writes it first). + const anchorId = idOf(resources.entries[0].name); + anchorIds.add(anchorId); + childPrefixes.push(`${anchorId}.entities.`, `${anchorId}.metrics.`); + } + const owned = (id: string) => + anchorIds.has(id) || childPrefixes.some(p => id.startsWith(p)); + + const orphans: string[] = []; + try { + for await (const entry of cat.listEntries( + opts.project, opts.location, opts.entryGroup)) { + const id = idOf(entry.name); + if (owned(id) && !emittedIds.has(id)) orphans.push(id); + } + } catch (err: any) { + return {deleted: 0, error: `listing entries to reconcile deletions: ${ + err.message || err}`}; + } + + let deleted = 0; + for (const id of orphans) { + const res = + await cat.deleteEntry(opts.project, opts.location, opts.entryGroup, id); + // A 404 means it is already gone -- reconciliation's goal is met either way. + if (isOk(res) || res.status === 404) { + deleted++; + continue; + } + return {deleted, error: `deleting orphaned entry '${id}': ${errText(res)}`}; + } + return {deleted}; +} + + +interface EntriesOutcome { + created: number; + updated: number; + error?: string; +} + +// Creates a model's entries. The anchor (entries[0]) is the parent of every +// child and is written first. Entity entries are then written before metric +// entries -- a metric's aspect references its entity by name, so the entity must +// exist first -- and within each of those two waves the entries are independent +// and written concurrently. An entry that already exists is updated in place +// (idempotent re-push). +async function createEntries( + cat: CatalogClient, opts: KcDeployOptions, + entries: Entry[]): Promise { + if (!entries.length) return {created: 0, updated: 0}; + const [anchor, ...children] = entries; + + const anchorRes = await writeEntry(cat, opts, anchor); + if (anchorRes.error) return {created: 0, updated: 0, error: anchorRes.error}; + + let created = anchorRes.updated ? 0 : 1; + let updated = anchorRes.updated ? 1 : 0; + + // Entities first, then metrics (a metric references its entity); each wave is + // written concurrently. + const isMetric = (e: Entry) => (e.entryType ?? '').endsWith('/semantic-metric'); + for (const wave of [children.filter(e => !isMetric(e)), + children.filter(isMetric)]) { + const res = await Promise.all(wave.map(e => writeEntry(cat, opts, e))); + const firstErr = res.find(r => r.error); + if (firstErr) return {created, updated, error: firstErr.error}; + for (const r of res) { + if (r.updated) + updated++; + else + created++; + } + } + return {created, updated}; +} + + +interface LinksOutcome { + linked: number; + error?: string; +} + +// Writes a model's schema-join entry links. Both endpoint entries already exist +// (createEntries ran first for this model), and links are independent of each +// other, so they are written concurrently. A link that already exists is upserted +// (its aspect refreshed). +async function createEntryLinks( + cat: CatalogClient, opts: KcDeployOptions, + links: EntryLink[]): Promise { + if (!links.length) return {linked: 0}; + const res = await Promise.all(links.map(l => writeEntryLink(cat, opts, l))); + const firstErr = res.find(r => r.error); + if (firstErr) return {linked: 0, error: firstErr.error}; + return {linked: links.length}; +} + +// Writes one entry link: create, then fall back to an in-place aspect update if +// it already exists. A link's entry references and type are immutable, so a +// re-push only refreshes the aspect (the join detail); a relationship whose +// endpoints changed keeps its stale link -- an accepted limitation until link +// reconciliation lands (see reconcileDeletions TODO). +async function writeEntryLink( + cat: CatalogClient, opts: KcDeployOptions, + link: EntryLink): Promise<{error?: string}> { + const linkId = idOf(link.name ?? ''); + const res = await cat.createEntryLink( + opts.project, opts.location, opts.entryGroup, linkId, link); + if (isExists(res)) { + const upd = + await cat.updateEntryLink({name: link.name, aspects: link.aspects} as + EntryLink, + ['aspects']); + if (!isOk(upd)) return {error: `entry link '${linkId}': ${errText(upd)}`}; + return {}; + } + if (!isOk(res)) return {error: `entry link '${linkId}': ${errText(res)}`}; + return {}; +} + + +interface WriteOutcome { + updated?: boolean; // true when the entry already existed and was updated + error?: string; +} + +// Writes one entry: create (retrying the group-propagation window), then fall +// back to update-in-place if it already exists. +async function writeEntry( + cat: CatalogClient, opts: KcDeployOptions, + entry: Entry): Promise { + const entryId = idOf(entry.name); + let res = await createEntryWithRetry(cat, opts, entryId, entry); + if (isExists(res)) { + // Idempotent re-push: refresh the existing entry's source + aspects. + const upd = await cat.updateEntry( + entry, ['entry_source', 'aspects'], Object.keys(entry.aspects ?? {})); + if (!isOk(upd)) return {error: `entry '${entryId}': ${errText(upd)}`}; + return {updated: true}; + } + if (!isOk(res)) return {error: `entry '${entryId}': ${errText(res)}`}; + return {}; +} + +// entries.create can briefly 404 on a just-created entry group; retry that +// window. +async function createEntryWithRetry( + cat: CatalogClient, opts: KcDeployOptions, entryId: string, + entry: Entry): Promise> { + const tries = opts.entryCreateTries ?? ENTRY_CREATE_TRIES; + const retryMs = opts.entryCreateRetryMs ?? ENTRY_CREATE_RETRY_MS; + let res = await cat.createEntry( + opts.project, opts.location, opts.entryGroup, entryId, entry); + for (let attempt = 1; attempt < tries; attempt++) { + if (isOk(res) || isExists(res) || !isPropagating(res)) break; + await sleep(retryMs); + res = await cat.createEntry( + opts.project, opts.location, opts.entryGroup, entryId, entry); + } + return res; +} + + +// A human-readable summary of what a (dry-run) push would write for one model. +function planSummary( + model: string, resources: KcResources, opts: KcDeployOptions): string[] { + const dest = `${opts.project}.${opts.location}.${opts.entryGroup}`; + const lines = [ + `Knowledge Catalog plan for '${model}' (destination ${dest}):`, + ` ${resources.entries.length} entr${ + resources.entries.length === 1 ? 'y' : 'ies'}:`, + ...resources.entries.map( + e => ` - ${idOf(e.name)} (${idOf(e.entryType)})`), + ]; + if (resources.entryLinks.length) { + lines.push( + ` ${resources.entryLinks.length} schema-join link${ + resources.entryLinks.length === 1 ? '' : 's'}:`, + ...resources.entryLinks.map( + l => ` - ${idOf(l.name ?? '')}`)); + } + return lines; +} + + +// The id segment of a full entry/entryType resource name (after the last '/'). +function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + +function isOk(res: {status: number}): boolean { + return res.status === 200; +} + +// A create that failed because the resource already exists — treated as success +// for idempotent provisioning and re-push. The API layer preserves the HTTP +// status (gcp/api.ts), so the 409 ALREADY_EXISTS status is authoritative; no +// need to match the error text. +function isExists(res: {status: number}): boolean { + return res.status === 409; +} + +// A transient "not visible yet" error worth retrying, matching the propagation +// phrasing specifically rather than a bare "not found" (which also covers a +// genuinely missing aspect/entry type — a real, non-transient failure). +function isPropagating(res: {message?: string}): boolean { + const msg = res.message ?? ''; + return /may not exist/i.test(msg) || + /entry group .*(not found|does not exist)/i.test(msg); +} + +function errText(res: {status: number; message?: string}): string { + return res.message?.trim() || `HTTP ${res.status}`; +} diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts new file mode 100644 index 00000000..53f67709 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -0,0 +1,523 @@ +// Generates Knowledge Catalog resources from the Semantic Model IR. +// +// The IR (./ir) is pure semantics. This module is its Knowledge Catalog +// emitter, the counterpart to `bigquery.ts`: it maps the model to catalog +// Entries, each carrying the `semantic-*` Aspect(s) that describe it. Like +// `bigquery.ts` it is a PURE function of the IR: no GCP calls, no I/O. The +// orchestration layer (`deploy_knowledge_catalog.ts`, the counterpart to +// `deploy_bigquery.ts`) drives this emitter and writes the resulting resources +// via the Knowledge Catalog client. +// +// Target schema: the `semantic-model`, `semantic-entity`, and `semantic-metric` +// entry/aspect types are built-in system types under `dataplex-types/global`, +// alongside the built-in `schema` aspect. This emitter references them; it never +// provisions them. Each entry type declares `required_aspects`, so the emitted aspect set +// per entry is a hard contract: +// * semantic-model entry -> { semantic-model } +// * semantic-entity entry -> { semantic-entity, schema } +// * semantic-metric entry -> { semantic-metric } +// +// Aspect data shapes mirror the aspect types' CLOSED metadataTemplates exactly +// (a server aspect type rejects an undeclared data field): +// * semantic-model = { deploymentTargets: string[] } +// * semantic-entity = { source: { resources: string[], importedSystem?, +// importedResource? } } +// * semantic-metric = { entity?, dataType (required), expression?, +// importedExpression? } +// * schema = { fields: [{ name, dataType, metadataType, +// description?, semantics: { expression?, +// importedExpression?, role } }] } +// +// Relationships become `schema-join` entry links between the two entity entries. +// schema-join is a built-in, undirected entry link type in `dataplex-types/global` +// whose required `schema-join` aspect carries the join detail (the paired join +// columns, JOIN vs FOREIGN_KEY, USER inference). The join's direction -- which +// side holds the foreign key -- is preserved inside that aspect, not by the link. +// Many-to-many (association / junction-table) edges are not emitted yet: a +// junction is two joins through a third table, which schema-join's single +// source/target pair does not model; the emitter warns and skips them (the edge +// still lives in the BigQuery property graph, see bigquery.ts). +// + +import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; + +import {bigQueryGraphTargets} from './deploy_bigquery'; +import {DataType, Entity, Metric, Relationship, SemanticModel} from './ir'; + +// Where the `semantic-*` and `schema` system types live: built-in types in +// project `dataplex-types`, location `global`. Callers may override to reference +// them from a staging project. +const DEFAULT_TYPE_PROJECT = 'dataplex-types'; +const DEFAULT_TYPE_LOCATION = 'global'; + +export interface KcGenerateOptions { + project: string; // project the entries are created in (destination) + location: string; // destination location + entryGroup: string; // destination entry group + systemTypeProject?: string; // default 'dataplex-types' + systemTypeLocation?: string; // default 'global' +} + +export interface KcResources { + entries: Entry[]; + entryLinks: EntryLink[]; + warnings: string[]; +} + +/** + * Generates the Knowledge Catalog resources for a semantic model. + * + * Returns the entries (model anchor first, then entities and metrics), the + * schema-join entry links for the model's relationships, and any warnings + * collected while mapping the IR (missing keys, un-typed metrics, skipped M:N + * relationships). The resources reference the built-in system types; they do not + * create them. + */ +export function generateCatalogResources( + model: SemanticModel, opts: KcGenerateOptions): KcResources { + const warnings: string[] = []; + const entities = model.entities ?? []; + const metrics = model.metrics ?? []; + const relationships = model.relationships ?? []; + + if (!entities.length) { + warnings.push( + 'model has no entities; only the semantic-model entry will be generated'); + } + + const names = new Namer(opts); + + // Entry ids must be unique within the entry group. Two source names that + // normalize to the same id (see slug), or exact duplicates the loader did not + // reject, would otherwise emit two entries with the same name and have the + // later write silently overwrite the earlier. Track ids and skip a collision. + const seen = new Set(); + + // The model entry is the anchor and the parent of every entity/metric entry, + // so it is created first: it is entries[0] and the publisher writes in array + // order. Reserve its id up front so nothing else can claim it. + const modelId = names.modelId(model); + const modelEntryName = names.entry(modelId); + claim(seen, modelId, 'entry', `model '${model.name}'`, warnings); + + const entries: Entry[] = [{ + name: modelEntryName, + entryType: names.typeName('entry', 'semantic-model'), + entrySource: source(model.name, model.description), + aspects: aspectMap(names, { + 'semantic-model': modelAspectData(model), + }), + }]; + + // Maps an entity's model name to its published entry name, so a relationship + // can resolve its endpoints to the entries the link references. Only entities + // actually emitted (not skipped for a duplicate id) are recorded. + const entityEntryName = new Map(); + for (const entity of entities) { + const entityId = names.entityId(model, entity); + if (!claim(seen, entityId, 'entry', `entity '${entity.name}'`, warnings)) + continue; + entityEntryName.set(entity.name, names.entry(entityId)); + if (!entity.keys || !entity.keys.length) { + warnings.push( + `entity '${entity.name}': no keys declared in the source model`); + } + entries.push({ + name: names.entry(entityId), + entryType: names.typeName('entry', 'semantic-entity'), + parentEntry: modelEntryName, + entrySource: source(entity.name, entity.description), + // required_aspects: semantic-entity AND the built-in schema. + aspects: aspectMap(names, { + 'semantic-entity': entityAspectData(entity), + 'schema': schemaAspectData(entity), + }), + }); + } + + for (const metric of metrics) { + const metricId = names.metricId(model, metric); + if (!claim(seen, metricId, 'entry', `metric '${metric.name}'`, warnings)) + continue; + entries.push({ + name: names.entry(metricId), + entryType: names.typeName('entry', 'semantic-metric'), + parentEntry: modelEntryName, + entrySource: source(metric.name, metric.description), + aspects: aspectMap(names, { + 'semantic-metric': metricAspectData(metric, warnings), + }), + }); + } + + // Relationships map to schema-join entry links between their endpoint entries. + const entryLinks: EntryLink[] = []; + const seenLinks = new Set(); + for (const rel of relationships) { + const link = relationshipLink( + names, model, rel, entityEntryName, seenLinks, warnings); + if (link) entryLinks.push(link); + } + + return {entries, entryLinks, warnings: [...new Set(warnings)]}; +} + + +// Builds the schema-join entry link for one relationship, or undefined when it +// cannot be published. A many-to-many (association) edge is skipped -- a junction +// is two joins, which the single source/target schema-join does not model -- and +// so is an edge whose endpoint entity was not emitted (e.g. skipped for a +// duplicate id). Both cases warn; the BigQuery property graph still carries the +// edge. +function relationshipLink( + names: Namer, model: SemanticModel, rel: Relationship, + entityEntryName: Map, seenLinks: Set, + warnings: string[]): EntryLink|undefined { + if (rel.association) { + warnings.push( + `relationship '${rel.name}': many-to-many (association) edges are not ` + + `published to Knowledge Catalog yet; the edge lives in the BigQuery ` + + `property graph.`); + return undefined; + } + const src = entityEntryName.get(rel.source.entity); + const dst = entityEntryName.get(rel.destination.entity); + if (!src || !dst) { + const missing = !src ? rel.source.entity : rel.destination.entity; + warnings.push( + `relationship '${rel.name}': endpoint entity '${missing}' is not a ` + + `published entity; the relationship link is skipped.`); + return undefined; + } + const linkId = names.linkId(model, rel); + if (!claim( + seenLinks, linkId, 'entry link', `relationship '${rel.name}'`, + warnings)) + return undefined; + + return { + name: names.entryLink(linkId), + entryLinkType: names.typeName('entryLink', 'schema-join'), + // schema-join is undirected: both endpoints are UNSPECIFIED references. The + // join direction lives in the aspect (source is the foreign-key side). + entryReferences: [ + {name: src, type: 'UNSPECIFIED'}, + {name: dst, type: 'UNSPECIFIED'}, + ], + aspects: aspectMap(names, { + 'schema-join': schemaJoinAspectData(model, rel), + }), + }; +} + +// The schema-join aspect for a direct foreign key: a single join pairing the +// source (FK) columns to the destination (key) columns positionally. Each side's +// `name` is the entity's SQL table representation (its dataSource). `type` is +// FOREIGN_KEY (only direct FKs are emitted today); `inferenceSource` is USER +// because the join is author-declared, and `userManaged` stops Dataplex from +// overwriting it. Field names/nesting mirror the schema-join aspect type's +// metadataTemplate. +function schemaJoinAspectData( + model: SemanticModel, rel: Relationship): Record { + const srcEntity = entityByName(model, rel.source.entity); + const dstEntity = entityByName(model, rel.destination.entity); + return { + joins: [compact({ + source: { + name: srcEntity ? srcEntity.dataSource : rel.source.entity, + fields: rel.source.columns, + }, + target: { + name: dstEntity ? dstEntity.dataSource : rel.destination.entity, + fields: rel.destination.columns, + }, + description: rel.description, + type: 'FOREIGN_KEY', + inferenceSource: 'USER', + })], + userManaged: true, + }; +} + +function entityByName(model: SemanticModel, name: string): Entity|undefined { + return (model.entities ?? []).find(e => e.name === name); +} + + +// --------------------------------------------------------------------------- +// Aspect-data mappers. Field names/nesting mirror the server aspect types' +// CLOSED metadataTemplates exactly (see file header); the golden tests pin +// these shapes so any schema-driven change is a visible, reviewable diff. +// --------------------------------------------------------------------------- + +// semantic-model: the BigQuery Graph deployment target URIs this model deploys +// to (the same targets the BigQuery leg executes against). Empty data is valid; +// the aspect is still attached to satisfy the entry type's required_aspects. +function modelAspectData(model: SemanticModel): Record { + const {targets} = bigQueryGraphTargets(model); + return compact({ + deploymentTargets: targets.length ? targets.map(t => t.uri) : undefined, + }); +} + +// semantic-entity: the base table(s) backing the entity. `source` and its +// `resources` array are required by the aspect type. importedSystem/ +// importedResource have no IR source today and are left unset. +function entityAspectData(entity: Entity): Record { + return { + source: compact({ + resources: [resourcePath(entity.dataSource)], + }), + }; +} + +// The built-in schema aspect, carrying each field's column type plus the new +// per-field `semantics` block (expression / importedExpression / role). name, +// dataType, and metadataType are required per column. +function schemaAspectData(entity: Entity): Record { + return { + fields: (entity.fields ?? + []).map(f => compact({ + name: f.name, + dataType: columnDataType(f.type), + metadataType: columnMetadataType(f.type), + description: f.description, + semantics: compact({ + expression: f.expression, + importedExpression: f.importedExpression, + // A field with any dimension metadata is a dimension; + // otherwise DEFAULT. + role: f.dimension ? 'DIMENSION' : 'DEFAULT', + }), + })), + }; +} + +// semantic-metric: the model-level aggregate. `dataType` is required by the +// aspect type; when the model does not declare one, fall back to NUMERIC +// (decimal) and warn (metrics are aggregates, so an exact numeric is the +// sensible default; dimensions, in schemaAspectData, default to STRING) rather +// than emit an invalid aspect. +function metricAspectData( + metric: Metric, warnings: string[]): Record { + let dataType = metric.type ? columnDataType(metric.type) : undefined; + if (!dataType) { + warnings.push( + `metric '${ + metric.name}': no datatype in the source model; defaulting the ` + + `required semantic-metric.dataType to 'NUMERIC'`); + dataType = 'NUMERIC'; + } + return compact({ + entity: metric.entity, + dataType, + expression: metric.expression, + importedExpression: metric.importedExpression, + }); +} + + +// Maps the IR's logical DataType to a column data-type string for the schema +// aspect's required `dataType`. A conventional GoogleSQL type name; the field +// is a free string server-side. Unknown/undefined falls back to STRING. +function columnDataType(type: DataType|undefined): string { + switch (type) { + case 'Integer': + return 'INT64'; + case 'Decimal': + return 'NUMERIC'; + case 'Float': + return 'FLOAT64'; + case 'Boolean': + return 'BOOL'; + case 'Date': + return 'DATE'; + case 'Time': + return 'TIME'; + case 'DateTime': + return 'DATETIME'; + case 'DateTimeTz': + return 'TIMESTAMP'; + case 'String': + case 'Opaque': + default: + return 'STRING'; + } +} + +// Maps the IR's logical DataType to the schema aspect's required `metadataType` +// enum (BOOLEAN/NUMBER/STRING/BYTES/DATETIME/TIMESTAMP/GEOSPATIAL/STRUCT/RANGE/ +// OTHER). Numeric types collapse to NUMBER; unknown/undefined falls back to +// STRING (the loader leaves most fields un-typed). +function columnMetadataType(type: DataType|undefined): string { + switch (type) { + case 'Integer': + case 'Decimal': + case 'Float': + return 'NUMBER'; + case 'Boolean': + return 'BOOLEAN'; + case 'Date': + case 'Time': + case 'DateTime': + return 'DATETIME'; + case 'DateTimeTz': + return 'TIMESTAMP'; + case 'Opaque': + return 'OTHER'; + case 'String': + default: + return 'STRING'; + } +} + +// Maps the IR's opaque, canonical `dataSource` to a catalog resource string. A +// clean three-part `project.dataset.table` becomes the BigQuery linked-resource +// URI; anything else (a query, an under/over-qualified ref) is passed through +// verbatim so nothing is lost. +// TODO: settle the linked-resource form for Iceberg / BigLake tables, which may +// need a different URI shape than the BigQuery managed-table one below. +function resourcePath(dataSource: string): string { + const trimmed = (dataSource ?? '').trim(); + if (!trimmed || /\s/.test(trimmed)) return trimmed; + const parts = trimmed.split('.').map(unquote); + if (parts.length === 3 && parts.every(p => p.length)) { + return `//bigquery.googleapis.com/projects/${parts[0]}/datasets/${ + parts[1]}/tables/${parts[2]}`; + } + return trimmed; +} + + +// --------------------------------------------------------------------------- +// Naming and small helpers. +// --------------------------------------------------------------------------- + +// Builds the fully-qualified resource names for a destination. Kept in one +// place so entry/type name construction is consistent and the emitter body +// reads as pure mapping. +class Namer { + private readonly typeProj: string; + private readonly typeLoc: string; + constructor(private readonly opts: KcGenerateOptions) { + this.typeProj = opts.systemTypeProject ?? DEFAULT_TYPE_PROJECT; + this.typeLoc = opts.systemTypeLocation ?? DEFAULT_TYPE_LOCATION; + } + + // Full resource name of a system type. `kind` selects the collection. + typeName(kind: 'entry'|'aspect'|'entryLink', name: string): string { + return `projects/${this.typeProj}/locations/${this.typeLoc}/${kind}Types/${ + name}`; + } + + // Aspect-map key: the `project.location.type` reference form the client keys + // an entry's aspects by (see dataplex._nameToTypeRef / _fixEntry). + aspectRef(name: string): string { + return `${this.typeProj}.${this.typeLoc}.${name}`; + } + + entry(entryId: string): string { + return `${this.container()}/entries/${entryId}`; + } + + entryLink(linkId: string): string { + return `${this.container()}/entryLinks/${linkId}`; + } + + private container(): string { + return `projects/${this.opts.project}/locations/${ + this.opts.location}/entryGroups/${this.opts.entryGroup}`; + } + + modelId(model: SemanticModel): string { + return slug(model.name); + } + entityId(model: SemanticModel, entity: Entity): string { + return `${slug(model.name)}.entities.${slug(entity.name)}`; + } + metricId(model: SemanticModel, metric: Metric): string { + return `${slug(model.name)}.metrics.${slug(metric.name)}`; + } + // Entry link ids are more restricted than entry ids: lowercase letters, + // numbers and hyphens only, starting with a letter (see linkSlug). + linkId(model: SemanticModel, rel: Relationship): string { + return linkSlug(`${model.name}-${rel.name}`); + } +} + + +// Wraps aspect data (keyed by bare type id) as the client's aspect map: each +// key is the `project.location.type` reference form, each value the +// fully-qualified aspectType plus its data. +function aspectMap(names: Namer, byType: Record>): + Record { + const out: Record = {}; + for (const [type, data] of Object.entries(byType)) { + out[names.aspectRef(type)] = { + aspectType: names.typeName('aspect', type), + data + }; + } + return out; +} + +// The native catalog entry source (display name + description), separate from +// the semantic-* aspect copy that carries full model fidelity. +function source( + displayName: string, description?: string): Entry['entrySource'] { + return compact({displayName, description}) as Entry['entrySource']; +} + +// Drops undefined-valued keys so the emitted JSON (and its golden) only shows +// fields the model actually set — matching how the BigQuery emitter omits empty +// OPTIONS rather than rendering blanks. +function compact>(obj: T): T { + const out: Record = {}; + for (const [k, v] of Object.entries(obj)) { + if (v !== undefined) out[k] = v; + } + return out as T; +} + +// Records a generated entry id in `seen`, returning true when it is new. On a +// repeat it warns and returns false so the caller skips that resource: two +// entries sharing one name would otherwise have the later write silently +// overwrite the earlier one on publish. +function claim( + seen: Set, id: string, kind: string, label: string, + warnings: string[]): boolean { + if (seen.has(id)) { + warnings.push( + `${label}: generated ${kind} id '${id}' duplicates an earlier one; ` + + `skipped (rename to avoid overwriting it on publish)`); + return false; + } + seen.add(id); + return true; +} + +// Entry IDs allow letters, numbers, underscores, hyphens, and periods; map +// anything else to an underscore so a model/entity name with spaces or other +// characters still yields a valid, stable ID. +function slug(s: string): string { + return s.replace(/[^A-Za-z0-9_.-]/g, '_'); +} + +// Entry link ids allow only lowercase letters, numbers and hyphens, must start +// with a letter, must end with a letter or number, and are capped at 63 chars. +// Lowercase, map any other character to a hyphen, collapse runs, then trim +// leading non-letters and edge hyphens so the id satisfies the API contract. +function linkSlug(s: string): string { + let out = s.toLowerCase() + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-+/g, '-') + .replace(/^[^a-z]+/, '') + .replace(/^-+|-+$/g, '') + .slice(0, 63) + .replace(/-+$/, ''); + return out || 'link'; +} + +function unquote(part: string): string { + return part.replace(/^[`"]/, '').replace(/[`"]$/, ''); +} diff --git a/toolbox/mdcode/src/libts/semantic/loader.ts b/toolbox/mdcode/src/libts/semantic/loader.ts index 86ba27eb..5b61607b 100644 --- a/toolbox/mdcode/src/libts/semantic/loader.ts +++ b/toolbox/mdcode/src/libts/semantic/loader.ts @@ -478,3 +478,57 @@ function parseSource(source: string, opts: LoadOptions, function unquote(part: string): string { return part.replace(/^[`"]/, '').replace(/[`"]$/, ''); } + + +// A single model parsed from one authored model file, tagged with that file's +// name so a consumer (a deploy leg) can attribute warnings and errors back to +// the file the author wrote. +export interface LoadedModel { + // The model file this was parsed from: the `.yaml` basename the layout + // discovered (e.g. `sales` for `sales.yaml`), not a filesystem path. Used + // only to prefix this model's warnings/errors so they point at the author's + // file; not part of the deployed IR. + document: string; + model: SemanticModel; +} + +export interface LoadedModels { + models: LoadedModel[]; + // Loader warnings across all documents, each prefixed with its document name. + warnings: string[]; + // Set when a document failed to parse or violated the schema, naming the + // document. `models` then holds whatever parsed before the failure; callers + // should treat a set `error` as fatal and not deploy. + error?: string; +} + +/** + * Loads every authored document into the IR once, so a multi-destination push + * parses and validates each model a single time and fans the result out to each + * deploy leg (BigQuery, Knowledge Catalog) rather than re-parsing per leg. + * + * A parse/schema error is returned as `error` (naming the document) rather than + * thrown, mirroring how the deploy legs previously reported it; loader warnings + * are prefixed with their document name. + */ +export function loadSemanticModels( + docs: {name: string; text: string}[], + opts: LoadOptions = {}): LoadedModels { + const models: LoadedModel[] = []; + const warnings: string[] = []; + for (const doc of docs) { + let loaded: LoadResult; + try { + loaded = loadModels(doc.text, opts); + } catch (err: any) { + return { + models, + warnings, + error: `Model document '${doc.name}': ${err.message || err}`, + }; + } + for (const w of loaded.warnings) warnings.push(`[${doc.name}] ${w}`); + for (const model of loaded.models) models.push({document: doc.name, model}); + } + return {models, warnings}; +} diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index 23ef2ef2..9ad40413 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -10,7 +10,9 @@ import * as context from '../libts/gcp/context'; import { Sources } from '../libts/source'; import { SemanticModelLayout } from '../libts/layouts/semantic-model'; import { SemanticModelSource } from '../libts/sources/semantic-model'; -import * as deploy from '../libts/semantic/deploy'; +import * as deploy from '../libts/semantic/deploy_bigquery'; +import * as kc from '../libts/semantic/deploy_knowledge_catalog'; +import {LoadedModel, loadSemanticModels} from '../libts/semantic/loader'; export interface InitOptions { @@ -25,6 +27,58 @@ export interface InitOptions { export interface PushOptions { force?: boolean; validateOnly?: boolean; + // Semantic-model push destination(s): 'bq', 'kc', 'all' (default), or a + // comma-separated list (e.g. 'bq,kc'). Ignored for non-semantic-model scopes. + target?: string; + // Print each pushed destination's generated artifact in that destination's + // native format (BigQuery Graph -> SQL DDL, Knowledge Catalog -> the entry + // plan), each block labeled by destination. Scope which destinations run with + // --target. Works with or without --validate-only. Semantic-model push only. + print?: boolean; +} + + +export type PushTarget = 'bigquery' | 'kc'; + +// All known semantic-model push destinations, in canonical run order. `all` +// expands to this list, and resolveTargets always emits in this order so the +// run is deterministic and BigQuery-first fail-fast holds regardless of how the +// user ordered the flag. Append new destinations here as they land. +const DESTINATIONS: PushTarget[] = ['bigquery', 'kc']; + +// The default when --target is omitted: push to every destination. +const DEFAULT_TARGET = 'all'; + +// User-typeable aliases for a single destination. +const TARGET_ALIASES: Record = { + bq: 'bigquery', + bigquery: 'bigquery', + kc: 'kc', +}; + +// Resolves a --target flag value to its ordered, de-duplicated destinations, or +// undefined if any token is unrecognized (the caller reports the error). +// Accepts a comma-separated list ('bq,kc'), the keyword 'all' (every +// destination), and defaults to 'bq'. The result is always in canonical +// DESTINATIONS order. +export function resolveTargets(target?: string): PushTarget[] | undefined { + const tokens = (target ?? DEFAULT_TARGET) + .toLowerCase() + .split(',') + .map(t => t.trim()) + .filter(t => t.length); + if (!tokens.length) return undefined; + const selected = new Set(); + for (const tok of tokens) { + if (tok === 'all') { + DESTINATIONS.forEach(d => selected.add(d)); + continue; + } + const dest = TARGET_ALIASES[tok]; + if (!dest) return undefined; + selected.add(dest); + } + return DESTINATIONS.filter(d => selected.has(d)); } @@ -50,8 +104,23 @@ export async function init(options: InitOptions): Promise { } else if (options.semanticModel) { manifest = await kcmd.CatalogManifest.initWithSemanticModel(options.semanticModel, ctx); - const entryGroup = manifest.source.entryGroup!; - fs.mkdirSync(path.join('catalog', 'EntryGroups', entryGroup), { recursive: true }); + const source = manifest.source as SemanticModelSource; + // Provision the destination entry group now, at init, so push writes only + // entries -- matching how the standard layout operates (its push creates + // entries, never the entry group). Idempotent: an already-existing group + // (409) is success. + const catalog = new dataplex.CatalogClient(ctx); + const res = await catalog.createEntryGroup( + source.project, source.location, source.entryGroup); + if (res.status !== 200 && res.status !== 409) { + console.error( + `Error: failed to create entry group '${source.name}': ` + + `${res.message || res.status}`); + return 1; + } + fs.mkdirSync( + path.join('catalog', 'EntryGroups', source.entryGroup), + { recursive: true }); } else { console.error('Error: Must provide --entry-group, --bigquery-dataset, --kb, or --semantic-model'); @@ -106,32 +175,42 @@ export async function push(options: PushOptions): Promise { // (see createLayout), so this cast is safe. const layout = snapshot.layout as SemanticModelLayout; const source = snapshot.manifest.source as SemanticModelSource; - console.log('Pushing semantic model (BigQuery Graph)...'); - const result = await deploy.deployBigQuery( - layout.modelDocuments(), ctx, options, source.project); - for (const w of result.warnings) { - console.warn(`Warning: ${w}`); + const targets = resolveTargets(options.target); + if (!targets) { + console.error( + `Error: invalid --target '${options.target}'; expected bq, kc, all, ` + + `or a comma-separated list (e.g. bq,kc).`); + return 1; } - if (options.validateOnly) { - for (const block of result.ddl) { - console.log(`${block}\n`); - } + + // Load + validate every model ONCE, then fan the parsed models out to each + // destination leg. Both legs consume the same IR, so a `--target all` push + // parses each document a single time instead of once per leg. A parse error + // fails the whole push before any destination runs. defaultProject is the + // scope's declared project (deterministic) rather than the ambient gcloud + // project, which can drift from where the model's tables live. + const docs = layout.modelDocuments(); + const loaded = loadSemanticModels( + docs, {defaultProject: source.project ?? ctx.project}); + if (loaded.error) { + console.error('Error:', loaded.error); + return 1; + } + for (const w of loaded.warnings) { + console.warn(`Warning: ${w}`); } - if (result.success) { - if (options.validateOnly) { - console.log('Validation complete; no changes applied.'); - } - else { - console.log( - `Deployed ${result.deployed} BigQuery Graph(s). ` + - `Knowledge Catalog resource emit for the semantic model is not yet implemented.`); - } - return 0; + // Run the resolved destinations in canonical order (BigQuery first); the + // early return below fails fast, skipping later legs when an earlier one + // fails. + for (const target of targets) { + const code = target === 'bigquery' + ? await pushBigQuery(loaded.models, ctx, options) + : await pushKnowledgeCatalog(loaded.models, ctx, options, source); + if (code !== 0) return code; } - console.error('Error pushing semantic model:', result.details); - return 1; + return 0; } const catalog = new dataplex.CatalogClient(ctx); @@ -149,3 +228,80 @@ export async function push(options: PushOptions): Promise { return 1; } } + + +// Deploys the semantic model's BigQuery Graph leg (over the pre-loaded models) +// and prints the result. Returns a process exit code (0 on success). +async function pushBigQuery( + models: LoadedModel[], ctx: context.ApiContext, + options: PushOptions): Promise { + console.log(options.validateOnly + ? 'Validating semantic model for BigQuery Graph...' + : 'Pushing semantic model (BigQuery Graph)...'); + const result = await deploy.deployBigQuery(models, ctx, options); + + for (const w of result.warnings) { + console.warn(`Warning: ${w}`); + } + if (options.print) { + console.log('-- BigQuery Graph --'); + for (const block of result.ddl) { + console.log(`${block}\n`); + } + } + + if (!result.success) { + console.error('Error pushing semantic model to BigQuery:', result.details); + return 1; + } + console.log(options.validateOnly + ? 'Validation complete; no changes applied.' + : `Deployed ${result.deployed} BigQuery Graph(s).`); + return 0; +} + + +// Deploys the semantic model's Knowledge Catalog leg (over the pre-loaded +// models) and prints the result. The destination coordinates come from the +// scope (project.location.entryGroup). Returns a process exit code (0 on +// success). +async function pushKnowledgeCatalog( + models: LoadedModel[], ctx: context.ApiContext, + options: PushOptions, source: SemanticModelSource): Promise { + console.log(options.validateOnly + ? 'Validating semantic model for Knowledge Catalog...' + : 'Pushing semantic model (Knowledge Catalog)...'); + const result = await kc.deployKnowledgeCatalog(models, ctx, { + project: source.project, + location: source.location, + entryGroup: source.entryGroup, + validateOnly: options.validateOnly, + }); + + for (const w of result.warnings) { + console.warn(`Warning: ${w}`); + } + if (options.print) { + console.log('-- Knowledge Catalog --'); + for (const line of result.plan) { + console.log(line); + } + } + + if (!result.success) { + console.error( + 'Error pushing semantic model to Knowledge Catalog:', result.details); + return 1; + } + const n = result.created + result.updated; + const removed = + result.deleted ? `; removed ${result.deleted} orphaned` : ''; + const linked = result.linked + ? `; linked ${result.linked} relationship${result.linked === 1 ? '' : 's'}` + : ''; + console.log(options.validateOnly + ? 'Validation complete; no changes applied.' + : `Wrote ${result.created} new and ${result.updated} updated ` + + `Knowledge Catalog entr${n === 1 ? 'y' : 'ies'}${removed}${linked}.`); + return 0; +} diff --git a/toolbox/mdcode/src/tool/main.ts b/toolbox/mdcode/src/tool/main.ts index 75aa710e..c55c3d02 100644 --- a/toolbox/mdcode/src/tool/main.ts +++ b/toolbox/mdcode/src/tool/main.ts @@ -41,6 +41,8 @@ cli.command('pull', 'Pull catalog entries') cli.command('push', 'Push catalog entries') .option('--force', 'Force push changes') .option('--validate-only', 'Only validate changes without applying') + .option('--target ', 'Semantic-model push destination(s): bq, kc, all (default), or a comma-separated list (e.g. bq,kc)') + .option('--print', 'Print each pushed destination\'s generated artifact in its native format (BigQuery Graph SQL DDL, Knowledge Catalog entry plan); scope with --target (semantic-model push only)') .action(async (options) => { let exitCode = 1; try { diff --git a/toolbox/mdcode/tests/libts/semantic/deploy.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_bigquery.test.ts similarity index 90% rename from toolbox/mdcode/tests/libts/semantic/deploy.test.ts rename to toolbox/mdcode/tests/libts/semantic/deploy_bigquery.test.ts index 7d930e17..200d0930 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/deploy_bigquery.test.ts @@ -1,5 +1,5 @@ // Tests for the semantic-model BigQuery deploy leg -// (src/libts/semantic/deploy.ts). +// (src/libts/semantic/deploy_bigquery.ts). // // `bigQueryGraphTargets` is exercised as a pure function; `deployBigQuery` is // exercised end to end over an Ossie fixture (loader -> IR -> generator -> @@ -11,11 +11,22 @@ import * as path from 'node:path'; import * as bq from '../../../src/libts/gcp/bigquery'; import {ApiContext} from '../../../src/libts/gcp/context'; -import {bigQueryGraphTargets, deployBigQuery} from '../../../src/libts/semantic/deploy'; +import {bigQueryGraphTargets, deployBigQuery} from '../../../src/libts/semantic/deploy_bigquery'; import {SemanticModel} from '../../../src/libts/semantic/ir'; +import {loadSemanticModels} from '../../../src/libts/semantic/loader'; const CTX = new ApiContext('test-project', 'us', 'test-token'); +// The deploy leg now consumes models already parsed by loadSemanticModels +// (shared with the Knowledge Catalog leg). These tests still author documents, +// so this helper parses them the way commands.ts does. defaultProject mirrors +// the scope's declared project. +function models(docs: {name: string; text: string}[], defaultProject = 'test-project') { + const r = loadSemanticModels(docs, {defaultProject}); + if (r.error) throw new Error(r.error); + return r.models; +} + const FIXTURES = path.join(__dirname, 'fixtures'); const BQ_URI = @@ -146,7 +157,7 @@ describe('deployBigQuery', () => { const querySpy = spyOn(bq.BigQueryClient.prototype, 'query'); try { const res = await deployBigQuery( - [{name: 'sales', text: OSSIE}], CTX, {validateOnly: true}); + models([{name: 'sales', text: OSSIE}]), CTX, {validateOnly: true}); expect(res.success).toBe(true); expect(res.deployed).toBe(0); expect(querySpy).not.toHaveBeenCalled(); @@ -169,7 +180,7 @@ describe('deployBigQuery', () => { .mockImplementation( async () => ({status: 200, result: {jobComplete: true}})); try { - const res = await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + const res = await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(true); expect(res.deployed).toBe(1); expect(querySpy).toHaveBeenCalledTimes(1); @@ -201,7 +212,7 @@ describe('deployBigQuery', () => { .mockImplementation( async () => ({status: 200, result: {jobComplete: true}})); try { - const res = await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + const res = await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(true); expect(pollSpy).toHaveBeenCalledTimes(1); // getQueryResults(project, jobId, location) -- jobId is the second arg. @@ -225,7 +236,7 @@ describe('deployBigQuery', () => { async () => ({status: 200, result: {jobComplete: false}})); try { const res = - await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.details).toContain('no job reference to poll'); } finally { @@ -256,7 +267,7 @@ describe('deployBigQuery', () => { async () => ({status: 200, result: {status: {}}})); try { const res = - await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(true); expect(jobSpy).toHaveBeenCalledTimes(1); } finally { @@ -287,7 +298,7 @@ describe('deployBigQuery', () => { }, })); try { - const res = await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + const res = await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.details).toContain('Not found: table'); } finally { @@ -313,7 +324,7 @@ describe('deployBigQuery', () => { })); try { const res = - await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.details).toContain('graph already exists'); } finally { @@ -328,7 +339,7 @@ describe('deployBigQuery', () => { .mockImplementation( async () => ({status: 400, message: 'bad DDL'})); try { - const res = await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + const res = await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.details).toContain('bad DDL'); } finally { @@ -354,7 +365,7 @@ describe('deployBigQuery', () => { .mockImplementation(async () => stuck); try { const res = await deployBigQuery( - [{name: 'sales', text: OSSIE}], CTX, + models([{name: 'sales', text: OSSIE}]), CTX, {maxQueryPolls: 3, pollBackoffMs: 0}); expect(res.success).toBe(false); expect(res.details).toContain('did not complete after 3 polls'); @@ -373,7 +384,7 @@ describe('deployBigQuery', () => { .mockImplementation(async () => ({status: 404} as any)); const querySpy = spyOn(bq.BigQueryClient.prototype, 'query'); try { - const res = await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + const res = await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.details).toContain('demo.sales not found'); expect(querySpy).not.toHaveBeenCalled(); @@ -398,7 +409,7 @@ describe('deployBigQuery', () => { async () => ({status: 200, result: {jobComplete: true}})); try { const res = - await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(true); expect(querySpy.mock.calls[0][2]).toBeUndefined(); expect(res.warnings.some( @@ -422,7 +433,7 @@ describe('deployBigQuery', () => { async () => ({status: 200, result: {jobComplete: true}})); try { const res = await deployBigQuery( - [{name: 'a', text: OSSIE}, {name: 'b', text: OSSIE}], CTX, {}); + models([{name: 'a', text: OSSIE}, {name: 'b', text: OSSIE}]), CTX, {}); expect(res.success).toBe(true); expect(res.deployed).toBe(2); expect(dsSpy).toHaveBeenCalledTimes(1); @@ -438,7 +449,7 @@ describe('deployBigQuery', () => { // A typo'd URI that carries the BigQuery prefix must be reported as // unparseable, not as "declares no deploymentTarget". const res = await deployBigQuery( - [{name: 'sales', text: OSSIE_BAD_TARGET}], CTX, + models([{name: 'sales', text: OSSIE_BAD_TARGET}]), CTX, {validateOnly: true}); expect(res.success).toBe(false); expect(res.details).toContain('could not be parsed'); @@ -462,7 +473,7 @@ describe('deployBigQuery', () => { }); try { const res = await deployBigQuery( - [{name: 'a', text: OSSIE}, {name: 'b', text: OSSIE}], CTX, {}); + models([{name: 'a', text: OSSIE}, {name: 'b', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.deployed).toBe(1); expect(res.details).toContain('boom'); @@ -473,24 +484,10 @@ describe('deployBigQuery', () => { } }); - test( - 'fails with the document name when a document cannot be parsed', - async () => { - // A malformed doc must produce a clean, document-scoped failure rather - // than an uncaught loader exception. - const res = await deployBigQuery( - [ - {name: 'sales', text: OSSIE}, {name: 'broken', text: 'other: 1\n'} - ], - CTX, {validateOnly: true}); - expect(res.success).toBe(false); - expect(res.details).toContain('Model document \'broken\''); - }); - test( 'fails when a model declares no BigQuery deployment target', async () => { const res = await deployBigQuery( - [{name: 'sales', text: OSSIE_NO_TARGET}], CTX, + models([{name: 'sales', text: OSSIE_NO_TARGET}]), CTX, {validateOnly: true}); expect(res.success).toBe(false); expect(res.details).toContain('no BigQuery Graph'); @@ -500,7 +497,7 @@ describe('deployBigQuery', () => { 'fails when the only deployment target is a non-BigQuery destination', async () => { const res = await deployBigQuery( - [{name: 'sales', text: OSSIE_DATAPLEX_ONLY}], CTX, + models([{name: 'sales', text: OSSIE_DATAPLEX_ONLY}]), CTX, {validateOnly: true}); expect(res.success).toBe(false); expect(res.details).toContain('no BigQuery Graph'); @@ -513,8 +510,8 @@ describe('deployBigQuery', () => { // ('scope-proj'), not the ambient ctx.project ('test-project'): the // scope's declared project is deterministic where gcloud's is not. const res = await deployBigQuery( - [{name: 'sales', text: OSSIE_UNQUALIFIED_SOURCE}], CTX, - {validateOnly: true}, 'scope-proj'); + models([{name: 'sales', text: OSSIE_UNQUALIFIED_SOURCE}], 'scope-proj'), + CTX, {validateOnly: true}); expect(res.success).toBe(true); const ddl = res.ddl.join('\n'); expect(ddl).toContain('`scope-proj.sales.orders`'); @@ -547,7 +544,7 @@ describe('deployBigQuery', () => { spyOn(bq.BigQueryClient.prototype, 'query') .mockImplementation(async () => ({status: 200} as any)); try { - const res = await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + const res = await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.details).toContain('no response body'); } finally { diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts new file mode 100644 index 00000000..87950569 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts @@ -0,0 +1,379 @@ +// Tests for the semantic-model Knowledge Catalog deploy leg +// (src/libts/semantic/deploy_knowledge_catalog.ts). +// +// `deployKnowledgeCatalog` is exercised end to end over an Ossie fixture +// (loader -> IR -> emitter -> writes), with the catalog client stubbed so no +// network call is made. The focus is the publish SEQUENCE the emitter +// goldens cannot show: anchor-first entry writes (the entry group is +// provisioned at `init`, not here), idempotent upsert on re-push, delete +// reconciliation, and the dry-run plan. + +import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import {ApiResult} from '../../../src/libts/gcp/api'; +import {ApiContext} from '../../../src/libts/gcp/context'; +import {CatalogClient} from '../../../src/libts/gcp/dataplex'; +import {deployKnowledgeCatalog} from '../../../src/libts/semantic/deploy_knowledge_catalog'; +import {loadSemanticModels} from '../../../src/libts/semantic/loader'; + +const CTX = new ApiContext('test-project', 'us', 'test-token'); +const FIXTURES = path.join(__dirname, 'fixtures'); + +// A loader-valid model: one entity (orders) + one metric (total_revenue), so a +// push writes exactly three entries (model anchor + entity + metric). +const OSSIE = + fs.readFileSync(path.join(FIXTURES, 'sales_bq_graph_target.yaml'), 'utf8'); +const DOCS = [{name: 'sales.yaml', text: OSSIE}]; + +// A model with a direct-FK relationship: 5 entries (anchor + 2 entities + 2 +// metrics) plus 1 schema-join entry link (orders -> customer). Used to exercise +// the link write path the OSSIE fixture (no relationship) cannot. +const STAR = + fs.readFileSync(path.join(FIXTURES, 'star_orders_customer.yaml'), 'utf8'); +const STAR_DOCS = [{name: 'star.yaml', text: STAR}]; + +// The deploy leg now consumes models already parsed by loadSemanticModels +// (shared with the BigQuery leg). These tests author documents, so this helper +// parses them the way commands.ts does. +function models(docs: {name: string; text: string}[]) { + const r = loadSemanticModels(docs, {defaultProject: 'test-project'}); + if (r.error) throw new Error(r.error); + return r.models; +} + +// A destination entry name for a given entry id, as the catalog returns it from +// listEntries; reconciliation keys off the id (the last path segment). +function entryName(id: string): string { + return `projects/dest/locations/us/entryGroups/eg/entries/${id}`; +} + +// The same loader-valid model, but its GOOGLE custom_extension carries invalid +// JSON: the doc parses, yet the emitter (via bigQueryGraphTargets) throws while +// building the semantic-model aspect. Surgically swap only the `data:` value so +// the rest of the document stays valid. +const MALFORMED_EXTENSION = + OSSIE.replace(/data: '[^\n]*'/, 'data: \'not valid json\''); + +// entryCreateTries: 1 keeps the propagation-retry loop from sleeping in tests. +const OPTS = { + project: 'dest', + location: 'us', + entryGroup: 'eg', + entryCreateTries: 1 +}; + +// bun's spyOn accumulates calls across tests; restore originals after each so +// per-test call counts and ordering assertions are isolated. +afterEach(() => { + mock.restore(); +}); + +function ok(result?: T): ApiResult { + return {status: 200, result}; +} +function err(status: number, message: string): ApiResult { + return {status, message}; +} + +// Stubs createEntryGroup + createEntry (+ optionally updateEntry) and the entry- +// link writes, returning the spies so a test can assert call counts and +// ordering. +function stubClient(opts: { + group?: ApiResult, + create?: (entryId: string) => ApiResult, + update?: ApiResult, + // Entry ids the destination entry group already holds (yielded by + // listEntries during delete reconciliation). Default: none. + existing?: string[], + del?: (entryId: string) => ApiResult, + // Result of createEntryLink, keyed by the entry-link id. Default: 200. + createLink?: (linkId: string) => ApiResult, + updateLink?: ApiResult, +} = {}) { + const group = spyOn(CatalogClient.prototype, 'createEntryGroup') + .mockImplementation(async () => opts.group ?? ok({})); + const create = spyOn(CatalogClient.prototype, 'createEntry') + .mockImplementation( + async (_p, _l, _eg, entryId) => + (opts.create ?? (() => ok({})))(entryId)); + const update = spyOn(CatalogClient.prototype, 'updateEntry') + .mockImplementation(async () => opts.update ?? ok({})); + const list = spyOn(CatalogClient.prototype, 'listEntries') + .mockImplementation(async function*() { + for (const id of opts.existing ?? []) { + yield {name: entryName(id), entryType: 'semantic'} as any; + } + }); + const del = spyOn(CatalogClient.prototype, 'deleteEntry') + .mockImplementation( + async (_p, _l, _eg, entryId) => + (opts.del ?? (() => ok({})))(entryId)); + const createLink = spyOn(CatalogClient.prototype, 'createEntryLink') + .mockImplementation( + async (_p, _l, _eg, linkId) => + (opts.createLink ?? (() => ok({})))(linkId)); + const updateLink = spyOn(CatalogClient.prototype, 'updateEntryLink') + .mockImplementation( + async () => opts.updateLink ?? ok({})); + return {group, create, update, list, del, createLink, updateLink}; +} + + +describe('deployKnowledgeCatalog: happy path', () => { + test('writes entries anchor-first, provisioning nothing', async () => { + const {group, create, update} = stubClient(); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.created).toBe(3); + expect(result.updated).toBe(0); + + // Push provisions neither the entry group (created at `init`) nor any type + // (the semantic types are built-in): it only writes the three entries. + expect(group).not.toHaveBeenCalled(); + expect(create).toHaveBeenCalledTimes(3); + expect(update).not.toHaveBeenCalled(); + + // The model anchor is written before its children. + const firstEntryId = create.mock.calls[0][3]; + expect(firstEntryId).toBe('sales'); + }); +}); + + +describe('deployKnowledgeCatalog: re-push upserts', () => { + test('an entry that already exists is updated in place', async () => { + const {create, update} = stubClient({ + create: () => err(409, 'entry already exists'), + update: ok({}), + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.created).toBe(0); + expect(result.updated).toBe(3); + expect(create).toHaveBeenCalledTimes(3); + expect(update).toHaveBeenCalledTimes(3); + }); +}); + + +describe('deployKnowledgeCatalog: relationship entry links', () => { + test('a direct-FK relationship is written as one schema-join link', async () => { + const {create, createLink, updateLink} = stubClient(); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.created).toBe(5); // anchor + 2 entities + 2 metrics + expect(result.linked).toBe(1); + expect(create).toHaveBeenCalledTimes(5); + expect(createLink).toHaveBeenCalledTimes(1); + expect(updateLink).not.toHaveBeenCalled(); + // Links are written to the same destination the entries are. + const [project, location, entryGroup, linkId] = createLink.mock.calls[0]; + expect(project).toBe('dest'); + expect(location).toBe('us'); + expect(entryGroup).toBe('eg'); + expect(linkId).toBe('sales-orders-to-customer'); + }); + + test('a link that already exists is upserted via updateEntryLink', async () => { + const {createLink, updateLink} = stubClient({ + createLink: () => err(409, 'entry link already exists'), + updateLink: ok({}), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.linked).toBe(1); + expect(createLink).toHaveBeenCalledTimes(1); + expect(updateLink).toHaveBeenCalledTimes(1); + }); + + test('a failed link write fails the push, naming the link', async () => { + stubClient({ + createLink: () => err(500, 'boom'), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(false); + expect(result.details).toContain('sales-orders-to-customer'); + }); + + test('a model with no relationships writes no links', async () => { + const {createLink} = stubClient(); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.linked).toBe(0); + expect(createLink).not.toHaveBeenCalled(); + }); +}); + + +describe('deployKnowledgeCatalog: validateOnly', () => { + test('writes nothing and returns a plan', async () => { + const {group, create} = stubClient(); + + const result = + await deployKnowledgeCatalog(models(DOCS), CTX, {...OPTS, validateOnly: true}); + + expect(result.success).toBe(true); + expect(result.created).toBe(0); + expect(group).not.toHaveBeenCalled(); + expect(create).not.toHaveBeenCalled(); + expect(result.plan.join('\n')).toContain('Knowledge Catalog plan'); + expect(result.plan.join('\n')).toContain('sales'); + }); +}); + + +describe('deployKnowledgeCatalog: failures', () => { + test('a failed anchor write stops before its children', async () => { + const {create} = stubClient({ + create: (id) => id === 'sales' ? err(500, 'boom') : ok({}), + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(false); + expect(create).toHaveBeenCalledTimes(1); // anchor only; children skipped + }); + + test( + 'a model that throws during emit fails with the model + document named', + async () => { + // A parseable doc whose GOOGLE extension is invalid JSON makes the + // emitter throw; the publisher must report it, not crash the push. + const {create} = stubClient(); + const docs = [{name: 'broken.yaml', text: MALFORMED_EXTENSION}]; + + const result = await deployKnowledgeCatalog(models(docs), CTX, OPTS); + + expect(result.success).toBe(false); + expect(result.details).toContain('broken.yaml'); + expect(result.details).toContain('sales'); // the model name + expect(create).not.toHaveBeenCalled(); + }); + + test( + 'two models generating the same entry id fail before any write', + async () => { + // Both docs declare a model named 'sales', so their entry ids collide + // within the entry group; the second would silently upsert over the + // first. The publisher must catch the collision up front. + const {group, create} = stubClient(); + const docs = [ + {name: 'a.yaml', text: OSSIE}, + {name: 'b.yaml', text: OSSIE}, + ]; + + const result = await deployKnowledgeCatalog(models(docs), CTX, OPTS); + + expect(result.success).toBe(false); + expect(result.details).toContain('sales'); // the colliding entry id + expect(result.details).toContain('unique'); // the reason + expect(group).not.toHaveBeenCalled(); + expect(create).not.toHaveBeenCalled(); + }); + + test('no documents is a hard error for a real push', async () => { + stubClient(); + const result = await deployKnowledgeCatalog([], CTX, OPTS); + expect(result.success).toBe(false); + expect(result.details).toContain('No semantic model documents'); + }); + + test('no documents is a clean no-op for validateOnly', async () => { + stubClient(); + const result = + await deployKnowledgeCatalog([], CTX, {...OPTS, validateOnly: true}); + expect(result.success).toBe(true); + }); +}); + + +describe('deployKnowledgeCatalog: delete reconciliation', () => { + // The fixture model 'sales' emits exactly three ids: the anchor 'sales', the + // entity 'sales.entities.orders', and the metric 'sales.metrics.total_revenue'. + const EMITTED = ['sales', 'sales.entities.orders', 'sales.metrics.total_revenue']; + + test('deletes entities/metrics removed from the model, scoped by owner', async () => { + // The group also holds two orphans owned by the 'sales' anchor (an entity + // and a metric no longer in the model) plus two entries owned by a + // different model. Only the two orphans under 'sales' must be deleted. + const {del, list} = stubClient({ + existing: [ + ...EMITTED, + 'sales.entities.removed', + 'sales.metrics.removed', + 'other', + 'other.entities.x', + ], + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.deleted).toBe(2); + expect(list).toHaveBeenCalledTimes(1); + const deletedIds = del.mock.calls.map(c => c[3]).sort(); + expect(deletedIds).toEqual(['sales.entities.removed', 'sales.metrics.removed']); + }); + + test('deletes nothing when the model still emits every entry', async () => { + const {del} = stubClient({existing: EMITTED}); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.deleted).toBe(0); + expect(del).not.toHaveBeenCalled(); + }); + + test('a 404 on an orphan is tolerated (already gone)', async () => { + const {del} = stubClient({ + existing: [...EMITTED, 'sales.entities.removed'], + del: () => err(404, 'not found'), + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.deleted).toBe(1); + expect(del).toHaveBeenCalledTimes(1); + }); + + test('a failed delete fails the push, naming the entry', async () => { + const {del} = stubClient({ + existing: [...EMITTED, 'sales.metrics.removed'], + del: () => err(500, 'boom'), + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(false); + expect(result.details).toContain('sales.metrics.removed'); + expect(del).toHaveBeenCalledTimes(1); + }); + + test('validateOnly never lists or deletes (offline)', async () => { + const {list, del} = stubClient({existing: ['sales.entities.removed']}); + + const result = + await deployKnowledgeCatalog(models(DOCS), CTX, {...OPTS, validateOnly: true}); + + expect(result.success).toBe(true); + expect(result.deleted).toBe(0); + expect(list).not.toHaveBeenCalled(); + expect(del).not.toHaveBeenCalled(); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.knowledge_catalog.golden.json new file mode 100644 index 00000000..2f14e7b6 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.knowledge_catalog.golden.json @@ -0,0 +1,88 @@ +{ + "entries": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-model", + "entrySource": { + "displayName": "sales" + }, + "aspects": { + "dataplex-types.global.semantic-model": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-model", + "data": { + "deploymentTargets": [ + "//bigquery.googleapis.com/projects/demo/datasets/sales/propertyGraphs/sales_graph" + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.entities.orders", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "orders" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/demo/datasets/sales/tables/orders" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "o_orderkey", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "o_orderkey", + "role": "DEFAULT" + } + }, + { + "name": "o_totalprice", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "o_totalprice", + "role": "DEFAULT" + } + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.metrics.total_revenue", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-metric", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "total_revenue" + }, + "aspects": { + "dataplex-types.global.semantic-metric": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-metric", + "data": { + "entity": "orders", + "dataType": "NUMERIC", + "expression": "SUM(orders.o_totalprice)" + } + } + } + } + ], + "entryLinks": [], + "warnings": [ + "metric 'total_revenue': no datatype in the source model; defaulting the required semantic-metric.dataType to 'NUMERIC'" + ] +} diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.bigquery.golden.sql b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.bigquery.golden.sql index 8120bcb1..ff4bbb89 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.bigquery.golden.sql +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.bigquery.golden.sql @@ -8,7 +8,8 @@ NODE TABLES ( o_custkey, o_orderdate OPTIONS(description="Order Date\n\nTime dimension.\n\nSynonyms: order date, date"), o_totalprice, - MEASURE(SUM(o_totalprice)) AS total_revenue OPTIONS(description="Total order revenue\n\nSynonyms: revenue, sales") + MEASURE(SUM(o_totalprice)) AS total_revenue OPTIONS(description="Total order revenue\n\nSynonyms: revenue, sales"), + MEASURE(COUNT(o_orderkey)) AS order_count OPTIONS(description="Number of orders") ), `samples.tpch.customer` AS customer KEY(c_custkey) @@ -27,5 +28,3 @@ OPTIONS(description="Sales orders with customer attributes\n\nUse this model for -- warnings -- -- note: no 'BIGQUERY' dialect for one or more expressions; using the portable 'ANSI_SQL' dialect verbatim ('BIGQUERY' accepts the ANSI core subset — supply 'BIGQUERY' variants only for BIGQUERY-specific SQL) --- metric 'order_count': expression references no known entity; it may not be placeable downstream --- metric 'order_count' references no known entity; skipped (cannot be a single MEASURE) diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json new file mode 100644 index 00000000..3f4681df --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json @@ -0,0 +1,217 @@ +{ + "entries": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-model", + "entrySource": { + "displayName": "sales", + "description": "Sales orders with customer attributes" + }, + "aspects": { + "dataplex-types.global.semantic-model": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-model", + "data": { + "deploymentTargets": [ + "//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales" + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.entities.orders", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "orders", + "description": "One row per order" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/samples/datasets/tpch/tables/orders" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "o_orderkey", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Order identifier", + "semantics": { + "expression": "o_orderkey", + "role": "DEFAULT" + } + }, + { + "name": "o_custkey", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "o_custkey", + "role": "DEFAULT" + } + }, + { + "name": "o_orderdate", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "o_orderdate", + "role": "DIMENSION" + } + }, + { + "name": "o_totalprice", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "o_totalprice", + "role": "DEFAULT" + } + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.entities.customer", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "customer" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/samples/datasets/tpch/tables/customer" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "c_custkey", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "c_custkey", + "role": "DEFAULT" + } + }, + { + "name": "c_name", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Customer name", + "semantics": { + "expression": "c_name", + "role": "DEFAULT" + } + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.metrics.total_revenue", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-metric", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "total_revenue", + "description": "Total order revenue" + }, + "aspects": { + "dataplex-types.global.semantic-metric": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-metric", + "data": { + "entity": "orders", + "dataType": "NUMERIC", + "expression": "SUM(orders.o_totalprice)" + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.metrics.order_count", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-metric", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "order_count", + "description": "Number of orders" + }, + "aspects": { + "dataplex-types.global.semantic-metric": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-metric", + "data": { + "entity": "orders", + "dataType": "NUMERIC", + "expression": "COUNT(orders.o_orderkey)" + } + } + } + } + ], + "entryLinks": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entryLinks/sales-orders-to-customer", + "entryLinkType": "projects/dataplex-types/locations/global/entryLinkTypes/schema-join", + "entryReferences": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.entities.orders", + "type": "UNSPECIFIED" + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.entities.customer", + "type": "UNSPECIFIED" + } + ], + "aspects": { + "dataplex-types.global.schema-join": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema-join", + "data": { + "joins": [ + { + "source": { + "name": "samples.tpch.orders", + "fields": [ + "o_custkey" + ] + }, + "target": { + "name": "samples.tpch.customer", + "fields": [ + "c_custkey" + ] + }, + "type": "FOREIGN_KEY", + "inferenceSource": "USER" + } + ], + "userManaged": true + } + } + } + } + ], + "warnings": [ + "metric 'total_revenue': no datatype in the source model; defaulting the required semantic-metric.dataType to 'NUMERIC'", + "metric 'order_count': no datatype in the source model; defaulting the required semantic-metric.dataType to 'NUMERIC'" + ] +} diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.yaml index f9bf2958..15d4a5cf 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.yaml @@ -1,14 +1,18 @@ # Test fixture -- AI-first semantics format (v0.2.0.dev0). # -# Minimal star schema: orders (fact) -> customer (dimension). Exercises +# Minimal star schema: orders (fact) -> customer (dimension), with a GOOGLE +# BigQuery Graph deployment target so the model is executable. Exercises # ai_context at model / field / metric level, a time dimension (is_time), a -# field label, and a COUNT(*) metric that carries no entity qualifier. +# field label, and an entity-scoped COUNT metric. version: "0.2.0.dev0" semantic_model: - name: sales description: Sales orders with customer attributes + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": ["//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales"]}' ai_context: instructions: "Use this model for order analysis." datasets: @@ -77,5 +81,5 @@ semantic_model: expression: dialects: - dialect: ANSI_SQL - expression: COUNT(*) + expression: COUNT(orders.o_orderkey) description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json new file mode 100644 index 00000000..fce856be --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json @@ -0,0 +1,518 @@ +{ + "entries": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-model", + "entrySource": { + "displayName": "tpcds_model", + "description": "TPC-DS retail model" + }, + "aspects": { + "dataplex-types.global.semantic-model": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-model", + "data": {} + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store_sales", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "store_sales", + "description": "Fact table containing all store sales transactions" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/tpcds/datasets/public/tables/store_sales" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "ss_item_sk", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "ss_item_sk", + "importedExpression": "{label/store_sales.attr.store_sales.ss_item_sk}", + "role": "DIMENSION" + } + }, + { + "name": "ss_ticket_number", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "ss_ticket_number", + "importedExpression": "{label/store_sales.attr.store_sales.ss_ticket_number}", + "role": "DIMENSION" + } + }, + { + "name": "ss_customer_sk", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "ss_customer_sk", + "importedExpression": "{label/store_sales.attr.store_sales.ss_customer_sk}", + "role": "DIMENSION" + } + }, + { + "name": "ss_store_sk", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "ss_store_sk", + "importedExpression": "{label/store_sales.attr.store_sales.ss_store_sk}", + "role": "DIMENSION" + } + }, + { + "name": "ss_quantity", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Quantity of items sold", + "semantics": { + "expression": "ss_quantity", + "importedExpression": "{fact/store_sales.fact.store_sales.ss_quantity}", + "role": "DEFAULT" + } + }, + { + "name": "ss_sales_price", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Sales price per unit", + "semantics": { + "expression": "ss_sales_price", + "importedExpression": "{fact/store_sales.fact.store_sales.ss_sales_price}", + "role": "DEFAULT" + } + }, + { + "name": "ss_ext_sales_price", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Extended sales price (quantity * price)", + "semantics": { + "expression": "ss_ext_sales_price", + "importedExpression": "{fact/store_sales.fact.store_sales.ss_ext_sales_price}", + "role": "DEFAULT" + } + }, + { + "name": "ss_net_profit", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Net profit from the sale", + "semantics": { + "expression": "ss_net_profit", + "importedExpression": "{fact/store_sales.fact.store_sales.ss_net_profit}", + "role": "DEFAULT" + } + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.customer", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "customer", + "description": "Customer dimension with demographic information" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/tpcds/datasets/public/tables/customer" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "c_customer_sk", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "c_customer_sk", + "role": "DIMENSION" + } + }, + { + "name": "c_first_name", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Customer first name", + "semantics": { + "expression": "c_first_name", + "role": "DIMENSION" + } + }, + { + "name": "c_last_name", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Customer last name", + "semantics": { + "expression": "c_last_name", + "role": "DIMENSION" + } + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.item", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "item", + "description": "Item/Product dimension" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/tpcds/datasets/public/tables/item" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "i_item_sk", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "i_item_sk", + "role": "DIMENSION" + } + }, + { + "name": "i_brand", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "i_brand", + "role": "DIMENSION" + } + }, + { + "name": "i_category", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "i_category", + "role": "DIMENSION" + } + }, + { + "name": "i_current_price", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Current price of the item", + "semantics": { + "expression": "i_current_price", + "role": "DEFAULT" + } + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "store", + "description": "Store dimension with location attributes" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/tpcds/datasets/public/tables/store" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "s_store_sk", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "s_store_sk", + "role": "DIMENSION" + } + }, + { + "name": "s_store_name", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "s_store_name", + "role": "DIMENSION" + } + }, + { + "name": "s_city", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "s_city", + "role": "DIMENSION" + } + }, + { + "name": "s_state", + "dataType": "STRING", + "metadataType": "STRING", + "semantics": { + "expression": "s_state", + "role": "DIMENSION" + } + }, + { + "name": "s_number_employees", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Number of employees at the store", + "semantics": { + "expression": "s_number_employees", + "role": "DEFAULT" + } + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.date_dim", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "date_dim", + "description": "Date dimension with calendar attributes" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/tables/date_dim" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [] + } + } + } + } + ], + "entryLinks": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entryLinks/tpcds-model-store-sales-to-date-dim", + "entryLinkType": "projects/dataplex-types/locations/global/entryLinkTypes/schema-join", + "entryReferences": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store_sales", + "type": "UNSPECIFIED" + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.date_dim", + "type": "UNSPECIFIED" + } + ], + "aspects": { + "dataplex-types.global.schema-join": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema-join", + "data": { + "joins": [ + { + "source": { + "name": "tpcds.public.store_sales", + "fields": [ + "ss_sold_date_sk" + ] + }, + "target": { + "name": "sqlgen-testing.demo.date_dim", + "fields": [ + "ss_sold_date_sk" + ] + }, + "type": "FOREIGN_KEY", + "inferenceSource": "USER" + } + ], + "userManaged": true + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entryLinks/tpcds-model-store-sales-to-customer", + "entryLinkType": "projects/dataplex-types/locations/global/entryLinkTypes/schema-join", + "entryReferences": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store_sales", + "type": "UNSPECIFIED" + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.customer", + "type": "UNSPECIFIED" + } + ], + "aspects": { + "dataplex-types.global.schema-join": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema-join", + "data": { + "joins": [ + { + "source": { + "name": "tpcds.public.store_sales", + "fields": [ + "ss_customer_sk" + ] + }, + "target": { + "name": "tpcds.public.customer", + "fields": [ + "c_customer_sk" + ] + }, + "type": "FOREIGN_KEY", + "inferenceSource": "USER" + } + ], + "userManaged": true + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entryLinks/tpcds-model-store-sales-to-item", + "entryLinkType": "projects/dataplex-types/locations/global/entryLinkTypes/schema-join", + "entryReferences": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store_sales", + "type": "UNSPECIFIED" + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.item", + "type": "UNSPECIFIED" + } + ], + "aspects": { + "dataplex-types.global.schema-join": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema-join", + "data": { + "joins": [ + { + "source": { + "name": "tpcds.public.store_sales", + "fields": [ + "ss_item_sk" + ] + }, + "target": { + "name": "tpcds.public.item", + "fields": [ + "i_item_sk" + ] + }, + "type": "FOREIGN_KEY", + "inferenceSource": "USER" + } + ], + "userManaged": true + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entryLinks/tpcds-model-store-sales-to-store", + "entryLinkType": "projects/dataplex-types/locations/global/entryLinkTypes/schema-join", + "entryReferences": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store_sales", + "type": "UNSPECIFIED" + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store", + "type": "UNSPECIFIED" + } + ], + "aspects": { + "dataplex-types.global.schema-join": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema-join", + "data": { + "joins": [ + { + "source": { + "name": "tpcds.public.store_sales", + "fields": [ + "ss_store_sk" + ] + }, + "target": { + "name": "tpcds.public.store", + "fields": [ + "s_store_sk" + ] + }, + "type": "FOREIGN_KEY", + "inferenceSource": "USER" + } + ], + "userManaged": true + } + } + } + } + ], + "warnings": [ + "entity 'date_dim': no keys declared in the source model" + ] +} diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts new file mode 100644 index 00000000..5452ec55 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts @@ -0,0 +1,88 @@ +// End-to-end tests for the Knowledge Catalog destination: real-shaped fixtures +// run the full file -> IR -> catalog-resources path. +// +// The primary check is a GOLDEN test: for every fixture in the corpus, the +// generated Entries (with their `semantic-*` / `schema` aspects) plus the +// emitter warnings are compared byte-for-byte against a committed +// `.knowledge_catalog.golden.json`. The golden is the reviewable "big +// picture" — open a `.yaml` next to its `.knowledge_catalog.golden.json` to see +// the full input and the exact catalog resources it maps to; a changed aspect +// shape, dropped field, or reordered entry shows up as a diff. +// +// Regenerate goldens after an intentional emitter change: +// UPDATE_GOLDENS=1 npx bun test \ +// ./tests/libts/semantic/knowledge_catalog.e2e.test.ts +// then read the diff before committing. +// +// Focused behavior (warnings, schema-join link emission, metric dataType +// fallback) is asserted in knowledge_catalog.test.ts; the publisher's write +// sequence in deploy_knowledge_catalog.test.ts. + +import {describe, expect, test} from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; + +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {loadModels, LoadOptions} from '../../../src/libts/semantic/loader'; + +const FIXTURES = path.join(__dirname, 'fixtures'); + +// Fixtures that get a KC golden. Chosen to exercise the distinct mappings: +// sales_bq_graph_target -> model aspect deploymentTargets + un-typed metric +// (dataType fallback); star_orders_customer -> a direct-FK relationship +// (schema-join link) + multiple entities/metrics; tpcds_date_edge -> +// temporal field types. +const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', +]; + +// A fixed destination + default (dataplex-types/global) system types, so the +// golden pins the exact resource names the emitter produces. +const GEN_OPTS = { + project: 'sqlgen-testing', + location: 'us', + entryGroup: 'semantic' +}; + +function loadFixture(fixture: string, load: LoadOptions = {}) { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + return loadModels( + text, + {defaultProject: 'sqlgen-testing', defaultDataset: 'demo', ...load}); +} + +// The exact artifact a golden captures: the generated entries, then the +// relationship entry links, then the emitter warnings, as pretty JSON so a +// reviewer sees the full resource shapes. +function render(fixture: string): string { + const {models} = loadFixture(fixture); + const {entries, entryLinks, warnings} = + generateCatalogResources(models[0], GEN_OPTS); + return JSON.stringify({entries, entryLinks, warnings}, null, 2) + '\n'; +} + +const goldenPath = (fixture: string) => path.join( + FIXTURES, fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); + + +describe( + 'golden KC resources: each corpus fixture maps to its exact catalog entries', + () => { + for (const fixture of CORPUS) { + test(fixture, () => { + const actual = render(fixture); + const golden = goldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename(golden)} — run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts new file mode 100644 index 00000000..1bb108c4 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts @@ -0,0 +1,343 @@ +// Behavior specification for the Knowledge Catalog emitter +// (src/libts/semantic/knowledge_catalog.ts). +// +// The readable "big picture" goldens live in `knowledge_catalog.e2e.test.ts` (a +// corpus of `.yaml` inputs, each with a committed +// `.knowledge_catalog.golden.json`). This file holds what a loader +// fixture +// CANNOT express, because the open AI-first format the loader reads is a subset +// of the IR: +// - the logical DataType -> schema `dataType`/`metadataType` mapping and the +// DIMENSION role (the corpus fixtures declare no field `datatype`, so every +// field falls back to STRING/DEFAULT); +// - the `dataSource` -> resource-path forms (BigQuery URI vs. verbatim); +// - IR-contract cases the loader never produces (a metric with an explicit +// type, a cross-entity metric with no attach entity, duplicate ids). + +import {describe, expect, test} from 'bun:test'; + +import {DataType, Entity, SemanticModel} from '../../../src/libts/semantic/ir'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; + +const OPTS = { + project: 'dest-proj', + location: 'us', + entryGroup: 'eg' +}; + +// A one-entity model whose single field carries the given IR type + dimension, +// so a test can read back the emitted schema aspect for that field. +function modelWithField( + type: DataType|undefined, dimension = false): SemanticModel { + const entity: Entity = { + name: 'e', + dataSource: 'p.d.t', + keys: ['k'], + fields: [{ + name: 'f', + expression: 'e.f', + ...(type ? {type} : {}), + ...(dimension ? {dimension: {}} : {}), + }], + }; + return {name: 'm', entities: [entity], relationships: [], metrics: []}; +} + +// The schema-aspect field record for the sole field of modelWithField. +function schemaField(model: SemanticModel): Record { + const {entries} = generateCatalogResources(model, OPTS); + const entity = entries.find(e => e.entryType.endsWith('/semantic-entity'))!; + const schema = entity.aspects!['dataplex-types.global.schema'].data!; + return schema.fields[0]; +} + + +describe( + 'logical DataType maps to the schema aspect dataType + metadataType', + () => { + const cases: [DataType, string, string][] = [ + ['String', 'STRING', 'STRING'], + ['Integer', 'INT64', 'NUMBER'], + ['Decimal', 'NUMERIC', 'NUMBER'], + ['Float', 'FLOAT64', 'NUMBER'], + ['Boolean', 'BOOL', 'BOOLEAN'], + ['Date', 'DATE', 'DATETIME'], + ['Time', 'TIME', 'DATETIME'], + ['DateTime', 'DATETIME', 'DATETIME'], + ['DateTimeTz', 'TIMESTAMP', 'TIMESTAMP'], + ['Opaque', 'STRING', 'OTHER'], + ]; + for (const [type, dataType, metadataType] of cases) { + test(`${type} -> ${dataType} / ${metadataType}`, () => { + const f = schemaField(modelWithField(type)); + expect(f.dataType).toBe(dataType); + expect(f.metadataType).toBe(metadataType); + }); + } + + test('an un-typed field falls back to STRING / STRING', () => { + const f = schemaField(modelWithField(undefined)); + expect(f.dataType).toBe('STRING'); + expect(f.metadataType).toBe('STRING'); + }); + }); + + +describe( + 'a field with dimension metadata gets role DIMENSION, else DEFAULT', () => { + test('dimension -> DIMENSION', () => { + expect(schemaField(modelWithField('String', true)).semantics.role) + .toBe('DIMENSION'); + }); + test('no dimension -> DEFAULT', () => { + expect(schemaField(modelWithField('String', false)).semantics.role) + .toBe('DEFAULT'); + }); + }); + + +describe('the schema aspect carries expression fidelity in semantics', () => { + test('both target and imported expressions are preserved', () => { + const model: SemanticModel = { + name: 'm', + relationships: [], + metrics: [], + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: ['k'], + fields: [{ + name: 'f', + expression: 'CAST(e.f AS INT64)', + importedExpression: 'e.f::int', + importedDialect: 'SNOWFLAKE', + }], + }], + }; + const f = schemaField(model); + expect(f.semantics.expression).toBe('CAST(e.f AS INT64)'); + expect(f.semantics.importedExpression).toBe('e.f::int'); + }); +}); + + +describe('entity dataSource maps to a resource path', () => { + function resources(dataSource: string): string[] { + const model: SemanticModel = { + name: 'm', + relationships: [], + metrics: [], + entities: [{name: 'e', dataSource, keys: ['k'], fields: []}], + }; + const {entries} = generateCatalogResources(model, OPTS); + const entity = entries.find(e => e.entryType.endsWith('/semantic-entity'))!; + return entity.aspects!['dataplex-types.global.semantic-entity'] + .data!.source.resources; + } + + test( + 'a clean project.dataset.table becomes a BigQuery linked-resource URI', + () => { + expect(resources('proj.ds.tbl')).toEqual([ + '//bigquery.googleapis.com/projects/proj/datasets/ds/tables/tbl', + ]); + }); + test('a query-like source is kept verbatim', () => { + expect(resources('SELECT 1')).toEqual(['SELECT 1']); + }); + test('an under-qualified reference is kept verbatim', () => { + expect(resources('ds.tbl')).toEqual(['ds.tbl']); + }); +}); + + +describe('semantic-metric aspect', () => { + function metricData(model: SemanticModel) { + const {entries, warnings} = generateCatalogResources(model, OPTS); + const metric = entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + return { + data: metric.aspects!['dataplex-types.global.semantic-metric'].data!, + warnings + }; + } + + test( + 'a typed metric carries its mapped dataType and attach entity, no warning', + () => { + const model: SemanticModel = { + name: 'm', + entities: [], + relationships: [], + metrics: [ + {name: 'rev', expression: 'SUM(o.p)', entity: 'o', type: 'Decimal'} + ], + }; + const {data, warnings} = metricData(model); + expect(data).toEqual( + {entity: 'o', dataType: 'NUMERIC', expression: 'SUM(o.p)'}); + expect(warnings.some(w => w.includes('dataType'))).toBe(false); + }); + + test('an un-typed metric falls back to NUMERIC dataType and warns', () => { + const model: SemanticModel = { + name: 'm', + entities: [], + relationships: [], + metrics: [{name: 'rev', expression: 'COUNT(*)'}], + }; + const {data, warnings} = metricData(model); + expect(data.dataType).toBe('NUMERIC'); + expect(data.entity).toBeUndefined(); // cross-entity / unattached + expect(warnings.some( + w => w.includes('metric \'rev\'') && w.includes('NUMERIC'))) + .toBe(true); + }); +}); + + +describe('model-level structure', () => { + test( + 'a model with no BigQuery targets emits an empty semantic-model aspect', + () => { + const model: SemanticModel = + {name: 'm', entities: [], relationships: [], metrics: []}; + const {entries, warnings} = generateCatalogResources(model, OPTS); + const anchor = entries[0]; + expect(anchor.entryType.endsWith('/semantic-model')).toBe(true); + expect(anchor.aspects!['dataplex-types.global.semantic-model'].data) + .toEqual({}); + expect(warnings).toContain( + 'model has no entities; only the semantic-model entry will be generated'); + }); + + test('the anchor is emitted first and is the parent of every child', () => { + const model: SemanticModel = { + name: 'm', + relationships: [], + entities: [{name: 'e', dataSource: 'p.d.t', keys: ['k'], fields: []}], + metrics: + [{name: 'rev', expression: 'SUM(e.x)', entity: 'e', type: 'Integer'}], + }; + const {entries} = generateCatalogResources(model, OPTS); + expect(entries[0].entryType.endsWith('/semantic-model')).toBe(true); + const anchorName = entries[0].name; + for (const child of entries.slice(1)) { + expect(child.parentEntry).toBe(anchorName); + } + }); + + test( + 'two entities that normalize to the same id: the duplicate is skipped + warned', + () => { + // 'a b' and 'a-b' both slug to 'a_b' vs 'a-b'? '-' is allowed, space -> + // '_'. Use names that both map to 'a_b': 'a b' and 'a.b' would differ + // ('.' kept). 'a b' and 'a/b' both become 'a_b'. + const model: SemanticModel = { + name: 'm', + relationships: [], + metrics: [], + entities: [ + {name: 'a b', dataSource: 'p.d.t1', keys: ['k'], fields: []}, + {name: 'a/b', dataSource: 'p.d.t2', keys: ['k'], fields: []}, + ], + }; + const {entries, warnings} = generateCatalogResources(model, OPTS); + const entityEntries = + entries.filter(e => e.entryType.endsWith('/semantic-entity')); + expect(entityEntries.length).toBe(1); + expect(warnings.some(w => w.includes('duplicates an earlier one'))) + .toBe(true); + }); +}); + + +describe('relationships map to schema-join entry links', () => { + // A two-entity model joined by a direct foreign key (orders.custkey -> + // customer.custkey), the common case the loader produces from + // `relationships`. + function directFkModel(): SemanticModel { + return { + name: 'm', + metrics: [], + entities: [ + {name: 'orders', dataSource: 'p.d.orders', keys: ['o_key'], fields: []}, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: ['c_key'], + fields: [] + }, + ], + relationships: [{ + name: 'orders_to_customer', + source: {entity: 'orders', columns: ['custkey']}, + destination: {entity: 'customer', columns: ['c_key']}, + description: 'each order belongs to a customer', + }], + }; + } + + test('a direct FK becomes one FOREIGN_KEY schema-join link', () => { + const {entryLinks, warnings} = + generateCatalogResources(directFkModel(), OPTS); + expect(entryLinks.length).toBe(1); + const link = entryLinks[0]; + + // Link id is slugged and undirected: two UNSPECIFIED references naming the + // endpoint entities' entries, typed schema-join. + expect(link.name!.endsWith('/entryLinks/m-orders-to-customer')).toBe(true); + expect(link.entryLinkType.endsWith('/entryLinkTypes/schema-join')) + .toBe(true); + expect(link.entryReferences.map(r => r.type)).toEqual([ + 'UNSPECIFIED', 'UNSPECIFIED' + ]); + expect(link.entryReferences[0].name.endsWith('/entries/m.entities.orders')) + .toBe(true); + expect( + link.entryReferences[1].name.endsWith('/entries/m.entities.customer')) + .toBe(true); + + // The join direction + column pairing live in the aspect: source is the FK + // side, each side's `name` is the entity's dataSource, type is FOREIGN_KEY. + const aspect = link.aspects!['dataplex-types.global.schema-join'].data!; + expect(aspect.userManaged).toBe(true); + expect(aspect.joins.length).toBe(1); + const join = aspect.joins[0]; + expect(join.type).toBe('FOREIGN_KEY'); + expect(join.inferenceSource).toBe('USER'); + expect(join.source).toEqual({name: 'p.d.orders', fields: ['custkey']}); + expect(join.target).toEqual({name: 'p.d.customer', fields: ['c_key']}); + expect(join.description).toBe('each order belongs to a customer'); + expect(warnings.length).toBe(0); + }); + + test( + 'a many-to-many (association) edge is skipped and warned, no link', + () => { + const model = directFkModel(); + model.relationships[0].association = { + dataSource: 'p.d.order_customer', + keys: ['id'], + sourceColumns: ['o_key'], + destinationColumns: ['c_key'], + }; + const {entryLinks, warnings} = generateCatalogResources(model, OPTS); + expect(entryLinks.length).toBe(0); + expect(warnings.some( + w => w.includes('orders_to_customer') && + w.includes('many-to-many'))) + .toBe(true); + }); + + test('an edge to an unpublished entity is skipped and warned', () => { + const model = directFkModel(); + // Point the destination at an entity the model does not declare, so it is + // never emitted and the link has no endpoint entry to reference. + model.relationships[0].destination.entity = 'ghost'; + const {entryLinks, warnings} = generateCatalogResources(model, OPTS); + expect(entryLinks.length).toBe(0); + expect(warnings.some( + w => w.includes('orders_to_customer') && w.includes('ghost'))) + .toBe(true); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/load_semantic_models.test.ts b/toolbox/mdcode/tests/libts/semantic/load_semantic_models.test.ts new file mode 100644 index 00000000..02b6c6e3 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/load_semantic_models.test.ts @@ -0,0 +1,86 @@ +// Tests for the shared document loader (loadSemanticModels in +// src/libts/semantic/loader.ts). +// +// loadSemanticModels parses every authored document into the IR ONCE so a +// multi-destination push (BigQuery + Knowledge Catalog) validates each model a +// single time and fans the result out to both legs. It tags each model with its +// document, prefixes loader warnings with the document name, and returns a +// parse/schema error (naming the document) rather than throwing. + +import {describe, expect, test} from 'bun:test'; + +import {loadSemanticModels} from '../../../src/libts/semantic/loader'; + +// A minimal loader-valid document: one model, one dataset (with a primary key +// so it does not warn), no metrics. +function doc(modelName: string, source = 'proj.ds.tbl'): string { + return `semantic_model: + - name: ${modelName} + datasets: + - name: orders + source: ${source} + primary_key: [id] + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id +`; +} + +describe('loadSemanticModels', () => { + test( + 'flattens models across documents, tagging each with its document', + () => { + const res = loadSemanticModels([ + {name: 'a.yaml', text: doc('sales')}, + {name: 'b.yaml', text: doc('ops')}, + ]); + expect(res.error).toBeUndefined(); + expect(res.models.map(m => m.document)).toEqual(['a.yaml', 'b.yaml']); + expect(res.models.map(m => m.model.name)).toEqual(['sales', 'ops']); + }); + + test('prefixes loader warnings with the originating document name', () => { + // A dataset with no primary_key makes the loader warn; the warning must + // carry the document name so a multi-document push stays diagnosable. + const noKey = `semantic_model: + - name: sales + datasets: + - name: orders + source: proj.ds.tbl +`; + const res = loadSemanticModels([{name: 'sales.yaml', text: noKey}]); + expect(res.error).toBeUndefined(); + expect(res.warnings.some(w => w.startsWith('[sales.yaml] '))).toBe(true); + }); + + test('returns a document-named error on a parse/schema failure', () => { + // The second document is not a valid model; the error must name it and the + // models parsed before it are still returned. + const res = loadSemanticModels([ + {name: 'good.yaml', text: doc('sales')}, + {name: 'broken.yaml', text: 'semantic_model: [ this is: not valid'}, + ]); + expect(res.error).toBeDefined(); + expect(res.error).toContain('broken.yaml'); + expect(res.models.map(m => m.model.name)).toEqual(['sales']); + }); + + test('applies defaultProject when a source omits its project', () => { + const res = loadSemanticModels( + [{name: 'sales.yaml', text: doc('sales', 'ds.tbl')}], + {defaultProject: 'scope-proj'}); + expect(res.error).toBeUndefined(); + expect(res.models[0].model.entities[0].dataSource) + .toBe('scope-proj.ds.tbl'); + }); + + test('no documents yields no models and no error', () => { + const res = loadSemanticModels([]); + expect(res.error).toBeUndefined(); + expect(res.models).toEqual([]); + expect(res.warnings).toEqual([]); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/loader.test.ts b/toolbox/mdcode/tests/libts/semantic/loader.test.ts index 5f0eef71..12d9c91e 100644 --- a/toolbox/mdcode/tests/libts/semantic/loader.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/loader.test.ts @@ -720,7 +720,7 @@ function fixture(name: string): string { describe('gold fixtures parse from disk (real YAML files)', () => { test('star_orders_customer.yaml: happy path (ai_context, time dimension, metrics)', () => { - const { models, warnings } = loadModels(fixture('star_orders_customer.yaml')); + const { models } = loadModels(fixture('star_orders_customer.yaml')); expect(models).toHaveLength(1); const m = models[0]; expect(m.name).toBe('sales'); @@ -741,10 +741,11 @@ describe('gold fixtures parse from disk (real YAML files)', () => { expect(revenue.expression).toBe('SUM(orders.o_totalprice)'); expect(revenue.entity).toBe('orders'); expect(revenue.aiContext?.synonyms).toEqual(['revenue', 'sales']); - // COUNT(*) references no entity -> warned, attach entity omitted. + // order_count is entity-scoped (COUNT(orders.o_orderkey)) -> attach entity + // inferred from the qualifier, so it is a valid single-entity measure. const count = m.metrics.find(mt => mt.name === 'order_count')!; - expect(count.entity).toBeUndefined(); - expect(warnings.some(w => w.includes("metric 'order_count'"))).toBe(true); + expect(count.entity).toBe('orders'); + expect(count.expression).toBe('COUNT(orders.o_orderkey)'); }); test('vendor_dialects.yaml: non-target dialects kept as imported_expression', () => { diff --git a/toolbox/mdcode/tests/libts/semantic/target.test.ts b/toolbox/mdcode/tests/libts/semantic/target.test.ts new file mode 100644 index 00000000..476a2200 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/target.test.ts @@ -0,0 +1,45 @@ +// Tests --target resolution for the semantic-model push (src/tool/commands.ts). +// +// The two deploy legs are covered end to end elsewhere +// (deploy_bigquery.test.ts for BigQuery, deploy_knowledge_catalog.test.ts for +// Knowledge Catalog); this pins only the flag-to-destinations mapping and its +// ordering, which drives the dispatch in push(). + +import {describe, expect, test} from 'bun:test'; + +import {resolveTargets} from '../../../src/tool/commands'; + +describe('resolveTargets', () => { + test('defaults to every destination when no target is given', () => { + expect(resolveTargets(undefined)).toEqual(['bigquery', 'kc']); + }); + test('\'bq\' and \'bigquery\' both select BigQuery', () => { + expect(resolveTargets('bq')).toEqual(['bigquery']); + expect(resolveTargets('bigquery')).toEqual(['bigquery']); + }); + test('\'kc\' selects Knowledge Catalog', () => { + expect(resolveTargets('kc')).toEqual(['kc']); + }); + test('a comma-separated list selects each destination', () => { + expect(resolveTargets('bq,kc')).toEqual(['bigquery', 'kc']); + }); + test('\'all\' selects every destination', () => { + expect(resolveTargets('all')).toEqual(['bigquery', 'kc']); + }); + test('the result is always in canonical order (BigQuery first)', () => { + // Regardless of how the user orders the flag, fail-fast runs BigQuery + // first. + expect(resolveTargets('kc,bq')).toEqual(['bigquery', 'kc']); + }); + test('duplicate tokens are de-duplicated', () => { + expect(resolveTargets('bq,bq,kc')).toEqual(['bigquery', 'kc']); + expect(resolveTargets('all,kc')).toEqual(['bigquery', 'kc']); + }); + test('is case-insensitive and tolerates surrounding whitespace', () => { + expect(resolveTargets(' BQ , KC ')).toEqual(['bigquery', 'kc']); + }); + test('an unknown token anywhere in the list resolves to undefined', () => { + expect(resolveTargets('snowflake')).toBeUndefined(); + expect(resolveTargets('bq,snowflake')).toBeUndefined(); + }); +}); diff --git a/toolbox/mdcode/tests/tool/init_semantic_model.test.ts b/toolbox/mdcode/tests/tool/init_semantic_model.test.ts new file mode 100644 index 00000000..9191b7a3 --- /dev/null +++ b/toolbox/mdcode/tests/tool/init_semantic_model.test.ts @@ -0,0 +1,100 @@ +// Tests that `kcmd init --semantic-model` provisions the destination entry +// group (src/tool/commands.ts, init()). +// +// The entry group is created at init -- not on push -- so a semantic-model push +// writes only entries, matching how the standard layout operates (its push +// creates entries, never the entry group). These tests spy on the catalog +// client so no network call is made and run init inside a temp working +// directory (it writes catalog.yaml + the layout dirs relative to cwd). + +import {afterEach, beforeEach, describe, expect, mock, spyOn, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import {ApiResult} from '../../src/libts/gcp/api'; +import {ApiContext} from '../../src/libts/gcp/context'; +import {CatalogClient} from '../../src/libts/gcp/dataplex'; +import {init} from '../../src/tool/commands'; + +const CTX = new ApiContext('test-project', 'us', 'test-token'); + +function ok(result?: T): ApiResult { + return {status: 200, result}; +} +function err(status: number, message: string): ApiResult { + return {status, message}; +} + +let dir = ''; +let cwd = ''; + +beforeEach(() => { + cwd = process.cwd(); + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kcmd-init-')); + process.chdir(dir); + // init() resolves its context from gcloud; pin it to a hermetic value. + spyOn(ApiContext, 'default').mockReturnValue(CTX); + // init() prints the generated catalog.yaml; keep test output quiet. + spyOn(console, 'log').mockImplementation(() => {}); + spyOn(console, 'error').mockImplementation(() => {}); +}); + +afterEach(() => { + process.chdir(cwd); + if (dir) fs.rmSync(dir, {recursive: true, force: true}); + dir = ''; + mock.restore(); +}); + + +describe('init --semantic-model: entry-group provisioning', () => { + test( + 'creates the destination entry group and the local layout dir', + async () => { + const group = + spyOn(CatalogClient.prototype, 'createEntryGroup') + .mockImplementation(async () => ok({name: 'sales-group'})); + + const code = await init({semanticModel: 'proj.us.sales-group'}); + + expect(code).toBe(0); + expect(group).toHaveBeenCalledTimes(1); + const [project, location, entryGroupId] = group.mock.calls[0]; + expect(project).toBe('proj'); + expect(location).toBe('us'); + expect(entryGroupId).toBe('sales-group'); + expect( + fs.existsSync(path.join('catalog', 'EntryGroups', 'sales-group'))) + .toBe(true); + expect(fs.readFileSync('catalog.yaml', 'utf8')) + .toContain('scope: semantic-model.proj.us.sales-group'); + }); + + test('an already-existing entry group (409) is success', async () => { + const group = + spyOn(CatalogClient.prototype, 'createEntryGroup') + .mockImplementation(async () => err(409, 'already exists')); + + const code = await init({semanticModel: 'proj.us.sales-group'}); + + expect(code).toBe(0); + expect(group).toHaveBeenCalledTimes(1); + expect(fs.existsSync(path.join('catalog', 'EntryGroups', 'sales-group'))) + .toBe(true); + }); + + test('a fatal entry-group error fails init', async () => { + const group = + spyOn(CatalogClient.prototype, 'createEntryGroup') + .mockImplementation(async () => err(403, 'permission denied')); + + const code = await init({semanticModel: 'proj.us.sales-group'}); + + expect(code).toBe(1); + expect(group).toHaveBeenCalledTimes(1); + // The layout dir is not created when provisioning fails. + expect(fs.existsSync(path.join('catalog', 'EntryGroups', 'sales-group'))) + .toBe(false); + }); +}); From 7c087fd0a888ba13808a9759a8efb62f60b7fb04 Mon Sep 17 00:00:00 2001 From: "amir.hormati" Date: Thu, 13 Aug 2026 15:00:01 -0700 Subject: [PATCH 02/34] okf: stop demo cleanup from deleting the shared okf aspect type (#288) The okf aspect type is scoped to the project and location, while the entry group the demo creates is not, so every OKF bundle in the project carries its signal layer on the same type. Deleting it during cleanup stripped that signal from bundles the demo does not own, and the bare catch meant it happened silently. Delete only the entry group and print the command to remove the aspect type by hand. Co-authored-by: Claude Opus 4.7 --- toolbox/mdcode/demo/README.md | 5 ++++- toolbox/mdcode/demo/okf/cleanup.ts | 15 ++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/toolbox/mdcode/demo/README.md b/toolbox/mdcode/demo/README.md index b4da0fcb..4ffea73d 100644 --- a/toolbox/mdcode/demo/README.md +++ b/toolbox/mdcode/demo/README.md @@ -183,7 +183,10 @@ bun push.ts **Cleanup** -* Deletes the Dataplex EntryGroup and the custom `okf` aspect type. +* Deletes the Dataplex EntryGroup. The custom `okf` aspect type is left in + place: it is scoped to the project and location rather than to this demo, so + other OKF bundles in the same project carry their signal layer on it. The + command to remove it manually is printed at the end. ```bash bun cleanup.ts diff --git a/toolbox/mdcode/demo/okf/cleanup.ts b/toolbox/mdcode/demo/okf/cleanup.ts index 9fae7085..07f21190 100644 --- a/toolbox/mdcode/demo/okf/cleanup.ts +++ b/toolbox/mdcode/demo/okf/cleanup.ts @@ -14,10 +14,11 @@ function dataplex(cmd: string, data: string|null=null) { dataplex(`entry-groups delete ${entryGroup}`); console.log(`Deleted entry group ${entryGroup}`); -try { - dataplex(`aspect-types delete okf`); - console.log('Deleted custom aspect type okf'); -} -catch { - // Might not exist, or still referenced -} +// The okf aspect type is scoped to the project and location, not to this demo's +// entry group, so any other OKF bundle in the project attaches its signal layer +// to the same type. Deleting it here would strip that signal from bundles this +// demo does not own. +console.log( + `Left aspect type ${project}.${location}.okf in place (shared across OKF bundles). ` + + `To remove it: gcloud dataplex aspect-types delete okf --project ${project} --location ${location}` +); From 522e50edd325529a88733ad5266c2afc86584a51 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sat, 8 Aug 2026 03:54:29 +0000 Subject: [PATCH 03/34] mdcode: drop importedExpression from Knowledge Catalog emit The KC schema and semantic-metric aspects only need expression (our/ANSI form); importedExpression is the vendor/MAQL form and has no KC consumer. Stop emitting it from both the per-field schema semantics block and the semantic-metric aspect. --- toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts | 9 +++------ .../tpcds_date_edge.knowledge_catalog.golden.json | 8 -------- .../tests/libts/semantic/knowledge_catalog.test.ts | 7 ++++--- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index 53f67709..a9ba269a 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -22,11 +22,10 @@ // * semantic-model = { deploymentTargets: string[] } // * semantic-entity = { source: { resources: string[], importedSystem?, // importedResource? } } -// * semantic-metric = { entity?, dataType (required), expression?, -// importedExpression? } +// * semantic-metric = { entity?, dataType (required), expression? } // * schema = { fields: [{ name, dataType, metadataType, // description?, semantics: { expression?, -// importedExpression?, role } }] } +// role } }] } // // Relationships become `schema-join` entry links between the two entity entries. // schema-join is a built-in, undirected entry link type in `dataplex-types/global` @@ -272,7 +271,7 @@ function entityAspectData(entity: Entity): Record { } // The built-in schema aspect, carrying each field's column type plus the new -// per-field `semantics` block (expression / importedExpression / role). name, +// per-field `semantics` block (expression / role). name, // dataType, and metadataType are required per column. function schemaAspectData(entity: Entity): Record { return { @@ -284,7 +283,6 @@ function schemaAspectData(entity: Entity): Record { description: f.description, semantics: compact({ expression: f.expression, - importedExpression: f.importedExpression, // A field with any dimension metadata is a dimension; // otherwise DEFAULT. role: f.dimension ? 'DIMENSION' : 'DEFAULT', @@ -312,7 +310,6 @@ function metricAspectData( entity: metric.entity, dataType, expression: metric.expression, - importedExpression: metric.importedExpression, }); } diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json index fce856be..0d34bc05 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json @@ -43,7 +43,6 @@ "metadataType": "STRING", "semantics": { "expression": "ss_item_sk", - "importedExpression": "{label/store_sales.attr.store_sales.ss_item_sk}", "role": "DIMENSION" } }, @@ -53,7 +52,6 @@ "metadataType": "STRING", "semantics": { "expression": "ss_ticket_number", - "importedExpression": "{label/store_sales.attr.store_sales.ss_ticket_number}", "role": "DIMENSION" } }, @@ -63,7 +61,6 @@ "metadataType": "STRING", "semantics": { "expression": "ss_customer_sk", - "importedExpression": "{label/store_sales.attr.store_sales.ss_customer_sk}", "role": "DIMENSION" } }, @@ -73,7 +70,6 @@ "metadataType": "STRING", "semantics": { "expression": "ss_store_sk", - "importedExpression": "{label/store_sales.attr.store_sales.ss_store_sk}", "role": "DIMENSION" } }, @@ -84,7 +80,6 @@ "description": "Quantity of items sold", "semantics": { "expression": "ss_quantity", - "importedExpression": "{fact/store_sales.fact.store_sales.ss_quantity}", "role": "DEFAULT" } }, @@ -95,7 +90,6 @@ "description": "Sales price per unit", "semantics": { "expression": "ss_sales_price", - "importedExpression": "{fact/store_sales.fact.store_sales.ss_sales_price}", "role": "DEFAULT" } }, @@ -106,7 +100,6 @@ "description": "Extended sales price (quantity * price)", "semantics": { "expression": "ss_ext_sales_price", - "importedExpression": "{fact/store_sales.fact.store_sales.ss_ext_sales_price}", "role": "DEFAULT" } }, @@ -117,7 +110,6 @@ "description": "Net profit from the sale", "semantics": { "expression": "ss_net_profit", - "importedExpression": "{fact/store_sales.fact.store_sales.ss_net_profit}", "role": "DEFAULT" } } diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts index 1bb108c4..98182892 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts @@ -96,8 +96,8 @@ describe( }); -describe('the schema aspect carries expression fidelity in semantics', () => { - test('both target and imported expressions are preserved', () => { +describe('the schema aspect carries the target expression in semantics', () => { + test('the target expression is kept; imported vendor SQL is not emitted', () => { const model: SemanticModel = { name: 'm', relationships: [], @@ -116,7 +116,8 @@ describe('the schema aspect carries expression fidelity in semantics', () => { }; const f = schemaField(model); expect(f.semantics.expression).toBe('CAST(e.f AS INT64)'); - expect(f.semantics.importedExpression).toBe('e.f::int'); + // importedExpression is the vendor/MAQL form; KC has no consumer for it. + expect(f.semantics.importedExpression).toBeUndefined(); }); }); From 35511c45505bdb9c80fcb63c72b5f47d3eddd659 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sat, 8 Aug 2026 04:14:44 +0000 Subject: [PATCH 04/34] mdcode: gate push on deployment targets and metric entities Add a shared push-time validation step run once over the loaded models, before any destination leg and on the --validate-only path: every model must declare at least one deploymentTarget, and a model that targets a BigQuery graph must have each metric resolve to a single entity (else it cannot lower to a MEASURE and would be silently dropped). Promotes what was a warn-and-skip into a hard, up-front failure. --- .../src/libts/semantic/deploy_bigquery.ts | 33 ++++++++ toolbox/mdcode/src/libts/semantic/validate.ts | 57 +++++++++++++ toolbox/mdcode/src/tool/commands.ts | 14 ++++ .../tests/libts/semantic/validate.test.ts | 84 +++++++++++++++++++ 4 files changed, 188 insertions(+) create mode 100644 toolbox/mdcode/src/libts/semantic/validate.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/validate.test.ts diff --git a/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts b/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts index b920ef9b..1b89ba5c 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts @@ -124,6 +124,39 @@ export function bigQueryGraphTargets(model: SemanticModel): } +// Every deploymentTarget URI a model declares in its GOOGLE custom extension(s), +// across all of them, regardless of whether each parses as a BigQuery Graph +// target. Validation uses this to require that a model declares at least one +// deployment target; bigQueryGraphTargets narrows to the ones that are BigQuery +// graphs. Throws on malformed extension JSON (same contract as +// bigQueryGraphTargets). +export function deploymentTargetUris(model: SemanticModel): string[] { + const uris: string[] = []; + for (const ext of model.customExtensions ?? []) { + if (ext.vendorName !== GOOGLE_VENDOR) { + continue; + } + let data: any; + try { + data = JSON.parse(ext.data); + } catch { + throw new Error(`Model '${ + model.name}': GOOGLE custom_extension 'data' is not valid JSON.`); + } + const list = data?.deploymentTargets; + if (!Array.isArray(list)) { + continue; + } + for (const uri of list) { + if (typeof uri === 'string') { + uris.push(uri); + } + } + } + return uris; +} + + // Upper bound on getQueryResults polls before we give up on a job that never // reports completion (each poll long-polls the server for up to 10s). const MAX_QUERY_POLLS = 30; diff --git a/toolbox/mdcode/src/libts/semantic/validate.ts b/toolbox/mdcode/src/libts/semantic/validate.ts new file mode 100644 index 00000000..b2384368 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/validate.ts @@ -0,0 +1,57 @@ +// Push-time validation gate for a semantic model. +// +// Runs once over the shared, already-parsed models (see loadSemanticModels) +// before any destination leg, so a real `kcmd push` AND a `--validate-only` dry +// run enforce the same requirements. Returns one message per violation (an +// empty array means valid); the caller (commands.ts) prints them and aborts the +// push. Kept separate from the loader -- which validates a document against the +// schema -- because these are deployment requirements, not schema rules, and +// they read the GOOGLE deployment-target extension the BigQuery leg owns. + +import {bigQueryGraphTargets, deploymentTargetUris} from './deploy_bigquery'; +import {LoadedModel} from './loader'; + +// Checks every model against the push requirements and returns the collected +// error messages (empty when all models pass), each tagged with the model's +// source document so the author can find it. +export function validatePushRequirements(models: LoadedModel[]): string[] { + const errors: string[] = []; + for (const {document, model} of models) { + let uris: string[]; + let bqTargetCount: number; + try { + uris = deploymentTargetUris(model); + bqTargetCount = bigQueryGraphTargets(model).targets.length; + } catch (err: any) { + // Malformed GOOGLE extension JSON: surface it as a validation error here + // rather than letting it throw out of a later leg as an uncaught stack. + errors.push(`${err.message || err} (${document})`); + continue; + } + + // Every model must declare at least one deployment target. + if (!uris.length) { + errors.push( + `model '${model.name}' (${document}) declares no deploymentTargets; ` + + `add at least one under its GOOGLE custom_extension.`); + } + + // A model that targets a BigQuery graph must have every metric resolve to a + // single entity, or the metric cannot lower to a MEASURE and would be + // silently dropped from the graph. The loader sets metric.entity only when + // the expression resolves to exactly one entity, so an unset entity is the + // "references zero or multiple entities" case. + if (bqTargetCount > 0) { + for (const metric of model.metrics ?? []) { + if (!metric.entity) { + errors.push( + `metric '${metric.name}' in model '${model.name}' (${ + document}) targets a BigQuery graph but does not resolve to a ` + + `single entity; set its attach entity or scope its expression to ` + + `one entity.`); + } + } + } + } + return errors; +} diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index 9ad40413..da5bd90e 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -13,6 +13,7 @@ import { SemanticModelSource } from '../libts/sources/semantic-model'; import * as deploy from '../libts/semantic/deploy_bigquery'; import * as kc from '../libts/semantic/deploy_knowledge_catalog'; import {LoadedModel, loadSemanticModels} from '../libts/semantic/loader'; +import {validatePushRequirements} from '../libts/semantic/validate'; export interface InitOptions { @@ -201,6 +202,19 @@ export async function push(options: PushOptions): Promise { console.warn(`Warning: ${w}`); } + // Enforce push-time requirements once over the shared models, before any + // destination runs: every model must declare a deployment target, and a + // BigQuery-graph-targeting model's metrics must each resolve to one entity. + // This is also the --validate-only path, so a dry run reports the same + // failures. + const validationErrors = validatePushRequirements(loaded.models); + if (validationErrors.length) { + for (const e of validationErrors) { + console.error(`Error: ${e}`); + } + return 1; + } + // Run the resolved destinations in canonical order (BigQuery first); the // early return below fails fast, skipping later legs when an earlier one // fails. diff --git a/toolbox/mdcode/tests/libts/semantic/validate.test.ts b/toolbox/mdcode/tests/libts/semantic/validate.test.ts new file mode 100644 index 00000000..15899cd8 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/validate.test.ts @@ -0,0 +1,84 @@ +// Behavior spec for the push-time validation gate +// (src/libts/semantic/validate.ts). + +import {describe, expect, test} from 'bun:test'; + +import {CustomExtension, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; +import {LoadedModel} from '../../../src/libts/semantic/loader'; +import {validatePushRequirements} from '../../../src/libts/semantic/validate'; + +// A parsed BigQuery Graph deployment target the strict matcher accepts. +const BQ_TARGET = + '//bigquery.googleapis.com/projects/p/datasets/d/propertyGraphs/g'; + +function googleExt(targets: string[]): CustomExtension { + return {vendorName: 'GOOGLE', data: JSON.stringify({deploymentTargets: targets})}; +} + +function loaded(model: SemanticModel, document = 'doc'): LoadedModel { + return {document, model}; +} + +function model( + over: Partial = {}, + exts?: CustomExtension[]): SemanticModel { + return { + name: 'm', + entities: [], + relationships: [], + metrics: [], + ...(exts ? {customExtensions: exts} : {}), + ...over, + }; +} + +describe('validatePushRequirements', () => { + test('a model with a deployment target and resolved metrics passes', () => { + const m = model( + {metrics: [{name: 'rev', expression: 'SUM(o.p)', entity: 'o'} as Metric]}, + [googleExt([BQ_TARGET])]); + expect(validatePushRequirements([loaded(m)])).toEqual([]); + }); + + test('a model with no deployment targets is rejected', () => { + const errs = validatePushRequirements([loaded(model())]); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('no deploymentTargets'); + expect(errs[0]).toContain('doc'); + }); + + test('a BigQuery-target metric that resolves to no entity is rejected', () => { + const m = model( + {metrics: [{name: 'cnt', expression: 'COUNT(*)'} as Metric]}, + [googleExt([BQ_TARGET])]); + const errs = validatePushRequirements([loaded(m)]); + expect(errs.length).toBe(1); + expect(errs[0]).toContain("metric 'cnt'"); + expect(errs[0]).toContain('single entity'); + }); + + test('an entity-less metric is allowed when no BigQuery graph is targeted', + () => { + // A deployment target that is not a BigQuery Graph URI: the model still + // declares a target (passes the first check) but does not target a + // graph, so the metric-entity rule does not apply. + const m = model( + {metrics: [{name: 'cnt', expression: 'COUNT(*)'} as Metric]}, + [googleExt(['//dataplex.googleapis.com/projects/p/locations/us'])]); + expect(validatePushRequirements([loaded(m)])).toEqual([]); + }); + + test('malformed GOOGLE extension JSON is reported, not thrown', () => { + const m = model({}, [{vendorName: 'GOOGLE', data: '{not json'}]); + const errs = validatePushRequirements([loaded(m)]); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('not valid JSON'); + }); + + test('errors accumulate across models', () => { + const a = loaded(model({name: 'a'}), 'a'); + const b = loaded(model({name: 'b'}), 'b'); + const errs = validatePushRequirements([a, b]); + expect(errs.length).toBe(2); + }); +}); From 164ebbc4e689ccd8ab95d25a0a7fe388359dd03d Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sat, 8 Aug 2026 04:38:24 +0000 Subject: [PATCH 05/34] mdcode: add lookupEntryLinks and deleteEntryLink catalog client methods Entry links have no list-collection API; the server only exposes them per referenced entry via the location-scoped :lookupEntryLinks custom verb. lookupEntryLinks drains its 10-per-page results into a flat list; deleteEntryLink addresses a link by its entry-group-scoped resource name. Both are prerequisites for reconciling orphaned schema-join links and removed models on push. --- toolbox/mdcode/src/libts/gcp/dataplex.ts | 50 ++++++++ .../mdcode/tests/libts/gcp/dataplex.test.ts | 116 ++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 toolbox/mdcode/tests/libts/gcp/dataplex.test.ts diff --git a/toolbox/mdcode/src/libts/gcp/dataplex.ts b/toolbox/mdcode/src/libts/gcp/dataplex.ts index a501e6f4..58c4ca7a 100644 --- a/toolbox/mdcode/src/libts/gcp/dataplex.ts +++ b/toolbox/mdcode/src/libts/gcp/dataplex.ts @@ -75,6 +75,12 @@ interface EntryList { nextPageToken?: string; } +interface EntryLinkList { + // Absent when the referenced entry has no links of the requested type(s). + entryLinks?: EntryLink[]; + nextPageToken?: string; +} + export class CatalogClient extends api.ApiClient { @@ -255,6 +261,50 @@ export class CatalogClient extends api.ApiClient { return await this._patch(entryLink.name!, entryLink, params); } + async deleteEntryLink(project: string, location: string, entryGroup: string, + entryLinkId: string): Promise> { + const name = `${ + catalogContainer(project, location, entryGroup)}/entryLinks/${ + entryLinkId}`; + return await this._delete(name); + } + + // Returns every entry link that references `entry` (its full resource name), + // draining all pages. There is no list-entry-links collection API; the server + // only exposes links per referenced entry, via the location-scoped + // :lookupEntryLinks custom verb (mirroring :lookupEntry). `entryLinkTypes` + // optionally filters to specific link types (server caps it at 10); `entryMode` + // filters by the entry's role in the link (SOURCE/TARGET). The server caps a + // page at 10 links, so this follows nextPageToken until exhausted and returns + // the flat list. A non-200 on any page aborts and is returned as-is. + async lookupEntryLinks( + project: string, location: string, + opts: {entry: string; entryLinkTypes?: string[]; + entryMode?: 'UNSPECIFIED' | 'SOURCE' | 'TARGET'}): + Promise> { + const container = `${catalogContainer(project, location)}:lookupEntryLinks`; + const links: EntryLink[] = []; + let pageToken: string | undefined = undefined; + do { + const params: Record = { + entry: opts.entry, + entryLinkTypes: opts.entryLinkTypes, + entryMode: opts.entryMode, + pageSize: 10, + pageToken, + }; + const res = await this._get(container, params); + if (res.status != 200) { + return { status: res.status, message: res.message }; + } + for (const link of res.result?.entryLinks ?? []) { + links.push(link); + } + pageToken = res.result?.nextPageToken; + } while (pageToken); + return { status: 200, result: links }; + } + } diff --git a/toolbox/mdcode/tests/libts/gcp/dataplex.test.ts b/toolbox/mdcode/tests/libts/gcp/dataplex.test.ts new file mode 100644 index 00000000..6aa54e80 --- /dev/null +++ b/toolbox/mdcode/tests/libts/gcp/dataplex.test.ts @@ -0,0 +1,116 @@ +// Behavior spec for the entry-link client methods added for KC push +// reconciliation: CatalogClient.lookupEntryLinks (paginated, location-scoped +// custom verb) and CatalogClient.deleteEntryLink. Spies on the low-level +// _get/_delete so it pins the exact URL + query-param shape without a live +// Dataplex. + +import {describe, expect, spyOn, test} from 'bun:test'; + +import {ApiContext} from '../../../src/libts/gcp/context'; +import {CatalogClient, EntryLink} from '../../../src/libts/gcp/dataplex'; + +const CTX = new ApiContext('test-project', 'us', 'test-token'); +const SCHEMA_JOIN = + 'projects/dataplex-types/locations/global/entryLinkTypes/schema-join'; +const ENTRY = 'projects/proj/locations/us/entryGroups/g/entries/e1'; + +function link(id: string): EntryLink { + return { + name: `projects/proj/locations/us/entryGroups/g/entryLinks/${id}`, + entryLinkType: SCHEMA_JOIN, + entryReferences: [], + }; +} + +function idOf(l: EntryLink): string { + return l.name!.split('/').pop()!; +} + +describe('CatalogClient.lookupEntryLinks', () => { + test('issues a location-scoped :lookupEntryLinks GET with entry + filters', + async () => { + const client = new CatalogClient(CTX); + const get = spyOn(client as any, '_get') + .mockImplementation( + async () => + ({status: 200, result: {entryLinks: [link('a')]}})); + + const res = await client.lookupEntryLinks('proj', 'us', { + entry: ENTRY, + entryLinkTypes: [SCHEMA_JOIN], + entryMode: 'SOURCE', + }); + + expect(res.status).toBe(200); + expect(res.result?.map(idOf)).toEqual(['a']); + expect(get).toHaveBeenCalledTimes(1); + const [url, params] = get.mock.calls[0] as [string, any]; + expect(url).toBe('projects/proj/locations/us:lookupEntryLinks'); + expect(params.entry).toBe(ENTRY); + expect(params.entryLinkTypes).toEqual([SCHEMA_JOIN]); + expect(params.entryMode).toBe('SOURCE'); + expect(params.pageSize).toBe(10); + }); + + test('drains every page and flattens the links', async () => { + const client = new CatalogClient(CTX); + let call = 0; + const get = spyOn(client as any, '_get').mockImplementation(async () => { + call++; + if (call === 1) { + return { + status: 200, + result: {entryLinks: [link('a'), link('b')], nextPageToken: 'T'}, + }; + } + return {status: 200, result: {entryLinks: [link('c')]}}; + }); + + const res = await client.lookupEntryLinks('proj', 'us', {entry: ENTRY}); + + expect(get).toHaveBeenCalledTimes(2); + // The second page must carry the token the first page returned. + expect((get.mock.calls[1][1] as any).pageToken).toBe('T'); + expect(res.result?.map(idOf)).toEqual(['a', 'b', 'c']); + }); + + test('a non-200 page aborts and is returned as-is', async () => { + const client = new CatalogClient(CTX); + const get = + spyOn(client as any, '_get') + .mockImplementation(async () => ({status: 403, message: 'denied'})); + + const res = await client.lookupEntryLinks('proj', 'us', {entry: ENTRY}); + + expect(res.status).toBe(403); + expect(res.message).toBe('denied'); + expect(res.result).toBeUndefined(); + expect(get).toHaveBeenCalledTimes(1); + }); + + test('an entry with no links returns an empty list, not undefined', + async () => { + const client = new CatalogClient(CTX); + spyOn(client as any, '_get') + .mockImplementation(async () => ({status: 200, result: {}})); + + const res = await client.lookupEntryLinks('proj', 'us', {entry: ENTRY}); + + expect(res.status).toBe(200); + expect(res.result).toEqual([]); + }); +}); + +describe('CatalogClient.deleteEntryLink', () => { + test('DELETEs the fully-qualified entry-link resource', async () => { + const client = new CatalogClient(CTX); + const del = spyOn(client as any, '_delete') + .mockImplementation(async () => ({status: 200})); + + await client.deleteEntryLink('proj', 'us', 'grp', 'my-link'); + + expect(del).toHaveBeenCalledTimes(1); + expect(del.mock.calls[0][0]) + .toBe('projects/proj/locations/us/entryGroups/grp/entryLinks/my-link'); + }); +}); From d7344b9e4c602f1002706f509c72632c0c27da7a Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sat, 8 Aug 2026 04:49:59 +0000 Subject: [PATCH 06/34] mdcode: reconcile removed models and relationship links on KC push Push now snapshots the destination entry group once before writing and uses that listing for the full removal lifecycle: - Whole-model removal: a semantic-model anchor already in the group that this push does not re-emit (a removed or renamed model) is a hard error, unless --force-remove authorizes deleting its links and entries first. - Relationship links: after a model's current schema-join links are written, any link it owns (both endpoints under its entities namespace) that it no longer emits is deleted, covering dropped or renamed relationships. Links are looked up per entity via lookupEntryLinks and deduped. Deletion reconciliation now consumes the pre-write snapshot instead of listing again. KcDeployResult reports unlinked; the push summary surfaces it. --- .../semantic/deploy_knowledge_catalog.ts | 273 +++++++++++++++--- toolbox/mdcode/src/tool/commands.ts | 18 +- toolbox/mdcode/src/tool/main.ts | 1 + .../semantic/deploy_knowledge_catalog.test.ts | 213 +++++++++++++- 4 files changed, 462 insertions(+), 43 deletions(-) diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 69bb391f..1c8acac7 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -55,6 +55,10 @@ export interface KcDeployOptions { systemTypeLocation?: string; // Compile + report only; never writes. validateOnly?: boolean; + // Delete models already in the entry group that this push does not re-emit -- + // a removed or renamed model's entries and links. Without it, an unrecognized + // model in the group is a hard error rather than a silent orphan. + forceRemove?: boolean; // entries.create can briefly 404 on a just-created entry group; retry that // window. Overridable so tests can exercise the path without burning // wall-clock. @@ -74,6 +78,9 @@ export interface KcDeployResult { // Relationship (schema-join) entry links written -- created or upserted (0 for // validateOnly). linked: number; + // Orphaned schema-join links deleted -- from relationships dropped/renamed on a + // still-present model and from force-removed models (0 for validateOnly). + unlinked: number; // A human-readable plan of what would be written; populated for validateOnly. plan: string[]; } @@ -101,11 +108,12 @@ export async function deployKnowledgeCatalog( let updated = 0; let deleted = 0; let linked = 0; + let unlinked = 0; let modelsSeen = 0; const fail = (details: string): KcDeployResult => ({success: false, details, warnings, created, updated, deleted, linked, - plan}); + unlinked, plan}); // Emit every model up front (pure): dry-run and warnings need no network, and // a generation warning surfaces even if a later write fails. @@ -142,7 +150,8 @@ export async function deployKnowledgeCatalog( if (!modelsSeen) { if (opts.validateOnly) { warnings.push('No semantic model documents found; nothing to validate.'); - return {success: true, warnings, created, updated, deleted, linked, plan}; + return {success: true, warnings, created, updated, deleted, linked, + unlinked, plan}; } return fail('No semantic model documents found; nothing to deploy.'); } @@ -174,7 +183,8 @@ export async function deployKnowledgeCatalog( plan.push(...planSummary(model, resources, opts)); } if (opts.validateOnly) { - return {success: true, warnings, created, updated, deleted, linked, plan}; + return {success: true, warnings, created, updated, deleted, linked, unlinked, + plan}; } // The destination entry group is provisioned at `init`, not here: push writes @@ -184,6 +194,43 @@ export async function deployKnowledgeCatalog( // propagation window. const cat = new CatalogClient(ctx); + // Snapshot the destination entry group once, before any write. The same + // listing feeds the foreign-model guard below and deletion reconciliation at + // the end; a re-emitted entry is never a deletion candidate, so a pre-write + // snapshot stays correct there. + const listing = await listEntryGroup(cat, opts); + if (listing.error) { + return fail(listing.error); + } + const existing = listing.entries; + + // Whole-model lifecycle: a `semantic-model` anchor already in the group whose + // id this push does not re-emit belongs to a model whose document is gone -- a + // removed or renamed model. Leaving it silently accumulates stale models in + // the group, so refuse the push, unless --force-remove authorizes deleting + // each such model's links and entries first. + const pushedAnchors = + new Set(emitted.map(e => idOf(e.resources.entries[0].name))); + const foreignAnchors = existing + .filter(e => (e.entryType ?? '').endsWith('/semantic-model')) + .map(e => idOf(e.name)) + .filter(id => !pushedAnchors.has(id)); + if (foreignAnchors.length && !opts.forceRemove) { + return fail( + `entry group '${opts.entryGroup}' already contains model(s) this push ` + + `does not include: ${foreignAnchors.join(', ')}. Re-run with ` + + `--force-remove to delete them, or add their documents to this push.`); + } + if (foreignAnchors.length) { + const removed = + await removeForeignModels(cat, opts, existing, foreignAnchors); + if (removed.error) { + return fail(removed.error); + } + deleted += removed.deleted; + unlinked += removed.unlinked; + } + for (const {model, resources} of emitted) { const outcome = await createEntries(cat, opts, resources.entries); if (outcome.error) { @@ -199,20 +246,30 @@ export async function deployKnowledgeCatalog( return fail(`Model '${model}': ${links.error}`); } linked += links.linked; + + // Then drop any schema-join link this model owns but no longer emits (a + // relationship dropped or renamed). Runs after its current links are + // written, so a rename never leaves the pair with no link between them. + const relLinks = await reconcileLinks(cat, opts, resources); + if (relLinks.error) { + return fail(`Model '${model}': ${relLinks.error}`); + } + unlinked += relLinks.unlinked; } // Reconcile deletions: an entity or metric removed from a still-present model // since the last push leaves an orphaned entry under the model's anchor. - // Delete any entry this push owns that was not re-emitted (see - // reconcileDeletions). Runs only on a real push -- validateOnly returned - // above and never lists remote state. - const recon = await reconcileDeletions(cat, opts, emitted); + // Delete any entry this push owns that was not re-emitted, from the pre-write + // listing (see reconcileDeletions). Runs only on a real push -- validateOnly + // returned above and never lists remote state. + const recon = await reconcileDeletions(cat, opts, emitted, existing); if (recon.error) { return fail(recon.error); } - deleted = recon.deleted; + deleted += recon.deleted; - return {success: true, warnings, created, updated, deleted, linked, plan}; + return {success: true, warnings, created, updated, deleted, linked, unlinked, + plan}; } @@ -229,18 +286,16 @@ interface ReconcileOutcome { // `.entities.` / `.metrics.` child namespace. An anchor is always // re-emitted, so it is never deleted here. // -// TODO: reconcile whole-model removals too. Deleting a model's document drops -// its anchor from this push, so its anchor + children are no longer owned and -// survive. Removing them safely needs a scope-level record of which models this -// entry group manages (follow-up). -// -// TODO: reconcile removed relationship links. There is no list API for entry -// links (only LookupEntryLinks, per referenced entry), so an orphaned link left -// by a dropped relationship is not deleted yet; it needs a lookup-per-entity -// sweep (follow-up). -async function reconcileDeletions( +// Whole-model removal (an anchor no longer in this push) is handled separately by +// the --force-remove guard in deployKnowledgeCatalog; orphaned relationship links +// are handled by reconcileLinks. This step operates on `existing` -- the +// entry-group listing captured once before writes -- so it never issues its own +// list call; a re-emitted entry is never a deletion candidate, so the pre-write +// snapshot stays correct. +function reconcileDeletions( cat: CatalogClient, opts: KcDeployOptions, - emitted: {resources: KcResources}[]): Promise { + emitted: {resources: KcResources}[], + existing: Entry[]): Promise { const emittedIds = new Set(); const anchorIds = new Set(); const childPrefixes: string[] = []; @@ -254,18 +309,15 @@ async function reconcileDeletions( const owned = (id: string) => anchorIds.has(id) || childPrefixes.some(p => id.startsWith(p)); - const orphans: string[] = []; - try { - for await (const entry of cat.listEntries( - opts.project, opts.location, opts.entryGroup)) { - const id = idOf(entry.name); - if (owned(id) && !emittedIds.has(id)) orphans.push(id); - } - } catch (err: any) { - return {deleted: 0, error: `listing entries to reconcile deletions: ${ - err.message || err}`}; - } + const orphans = existing.map(e => idOf(e.name)) + .filter(id => owned(id) && !emittedIds.has(id)); + + return deleteOrphanEntries(cat, opts, orphans); +} +async function deleteOrphanEntries( + cat: CatalogClient, opts: KcDeployOptions, + orphans: string[]): Promise { let deleted = 0; for (const id of orphans) { const res = @@ -281,6 +333,159 @@ async function reconcileDeletions( } +// The schema-join entry-link type name, matching what the emitter stamps on each +// link (Namer.typeName('entryLink', 'schema-join')). Used to filter +// lookupEntryLinks to the links this leg owns. +function schemaJoinLinkType(opts: KcDeployOptions): string { + const proj = opts.systemTypeProject ?? 'dataplex-types'; + const loc = opts.systemTypeLocation ?? 'global'; + return `projects/${proj}/locations/${loc}/entryLinkTypes/schema-join`; +} + + +// Snapshots the destination entry group's entries once, before any write. A +// brand-new entry group can briefly fail to list its entries collection (the +// same propagation window createEntryWithRetry rides out); treat an unlistable +// group as empty -- there is nothing to guard against or reconcile, and the +// create path retries the window. Any other listing failure is real and +// surfaced. +async function listEntryGroup( + cat: CatalogClient, + opts: KcDeployOptions): Promise<{entries: Entry[]; error?: string}> { + const entries: Entry[] = []; + try { + for await (const entry of cat.listEntries( + opts.project, opts.location, opts.entryGroup)) { + entries.push(entry); + } + } catch (err: any) { + const msg = err.message || String(err); + if (/not found|does not exist|may not exist/i.test(msg)) { + return {entries: []}; + } + return {entries: [], error: `listing entries in entry group '${ + opts.entryGroup}': ${msg}`}; + } + return {entries}; +} + + +interface LinkReconcileOutcome { + unlinked: number; + error?: string; +} + +// Deletes the schema-join links referencing any of `entityNames` (full entry +// resource names) for which `shouldDelete` returns true. Links are looked up per +// referenced entry -- the only server-side access path -- so a link between two +// of the entities is returned twice; a `seen` set dedups it. A 404 on delete +// counts as success (already gone). +async function deleteOwnedLinks( + cat: CatalogClient, opts: KcDeployOptions, entityNames: string[], + shouldDelete: (link: EntryLink) => boolean): + Promise { + const linkType = schemaJoinLinkType(opts); + const seen = new Set(); + let unlinked = 0; + for (const entry of entityNames) { + const res = await cat.lookupEntryLinks( + opts.project, opts.location, {entry, entryLinkTypes: [linkType]}); + if (!isOk(res)) { + return {unlinked, error: `looking up entry links for '${idOf(entry)}': ${ + errText(res)}`}; + } + for (const link of res.result ?? []) { + const id = idOf(link.name ?? ''); + if (seen.has(id)) continue; + seen.add(id); + if (!shouldDelete(link)) continue; + const del = await cat.deleteEntryLink( + opts.project, opts.location, opts.entryGroup, id); + if (isOk(del) || del.status === 404) { + unlinked++; + continue; + } + return {unlinked, error: `deleting entry link '${id}': ${errText(del)}`}; + } + } + return {unlinked}; +} + + +// Reconciles a still-present model's schema-join links: deletes any link this +// model OWNS (both endpoints under its `.entities.` namespace) that the +// model no longer emits -- a relationship dropped or renamed since the last +// push. A link touching an entry outside this model is never treated as owned, +// so a shared entry group is safe. +function reconcileLinks( + cat: CatalogClient, opts: KcDeployOptions, + resources: KcResources): Promise { + const anchorId = idOf(resources.entries[0].name); + const entityPrefix = `${anchorId}.entities.`; + const entityNames = resources.entries + .filter(e => (e.entryType ?? '').endsWith('/semantic-entity')) + .map(e => e.name); + if (!entityNames.length) return Promise.resolve({unlinked: 0}); + + const emittedLinkIds = + new Set(resources.entryLinks.map(l => idOf(l.name ?? ''))); + const ownedByModel = (link: EntryLink) => + link.entryReferences.length === 2 && + link.entryReferences.every(r => idOf(r.name).startsWith(entityPrefix)); + + return deleteOwnedLinks( + cat, opts, entityNames, + link => + ownedByModel(link) && !emittedLinkIds.has(idOf(link.name ?? ''))); +} + + +// Deletes models already in the entry group that this push does not re-emit +// (--force-remove). For each foreign anchor: remove its schema-join links first +// (they reference entries about to be deleted), then its entries -- the anchor +// and its `.entities.` / `.metrics.` children present in the +// pre-write listing `existing`. +async function removeForeignModels( + cat: CatalogClient, opts: KcDeployOptions, existing: Entry[], + foreignAnchors: string[]): + Promise<{deleted: number; unlinked: number; error?: string}> { + let deleted = 0; + let unlinked = 0; + for (const anchor of foreignAnchors) { + const entityPrefix = `${anchor}.entities.`; + const metricPrefix = `${anchor}.metrics.`; + const owned = existing.filter(e => { + const id = idOf(e.name); + return id === anchor || id.startsWith(entityPrefix) || + id.startsWith(metricPrefix); + }); + const entityNames = owned + .filter(e => (e.entryType ?? '').endsWith('/semantic-entity')) + .map(e => e.name); + + // The whole model is going away, so every schema-join link referencing one + // of its entities is orphaned -- delete them all. + const links = await deleteOwnedLinks(cat, opts, entityNames, () => true); + if (links.error) return {deleted, unlinked, error: links.error}; + unlinked += links.unlinked; + + for (const e of owned) { + const id = idOf(e.name); + const res = await cat.deleteEntry( + opts.project, opts.location, opts.entryGroup, id); + if (isOk(res) || res.status === 404) { + deleted++; + continue; + } + return {deleted, unlinked, + error: `deleting entry '${id}' of removed model '${anchor}': ${ + errText(res)}`}; + } + } + return {deleted, unlinked}; +} + + interface EntriesOutcome { created: number; updated: number; @@ -345,9 +550,9 @@ async function createEntryLinks( // Writes one entry link: create, then fall back to an in-place aspect update if // it already exists. A link's entry references and type are immutable, so a -// re-push only refreshes the aspect (the join detail); a relationship whose -// endpoints changed keeps its stale link -- an accepted limitation until link -// reconciliation lands (see reconcileDeletions TODO). +// re-push only refreshes the aspect (the join detail). A relationship whose id +// changed writes a new link and leaves the old one; reconcileLinks deletes such +// orphaned links after this model's links are written. async function writeEntryLink( cat: CatalogClient, opts: KcDeployOptions, link: EntryLink): Promise<{error?: string}> { diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index da5bd90e..e42e256d 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -28,6 +28,10 @@ export interface InitOptions { export interface PushOptions { force?: boolean; validateOnly?: boolean; + // Delete Knowledge Catalog models already in the entry group that this push + // does not include (a removed or renamed model). Without it, an unrecognized + // model in the group fails the push. Semantic-model KC push only. + forceRemove?: boolean; // Semantic-model push destination(s): 'bq', 'kc', 'all' (default), or a // comma-separated list (e.g. 'bq,kc'). Ignored for non-semantic-model scopes. target?: string; @@ -290,6 +294,7 @@ async function pushKnowledgeCatalog( location: source.location, entryGroup: source.entryGroup, validateOnly: options.validateOnly, + forceRemove: options.forceRemove, }); for (const w of result.warnings) { @@ -308,14 +313,21 @@ async function pushKnowledgeCatalog( return 1; } const n = result.created + result.updated; - const removed = - result.deleted ? `; removed ${result.deleted} orphaned` : ''; + const removed = result.deleted + ? `; removed ${result.deleted} orphaned entr${ + result.deleted === 1 ? 'y' : 'ies'}` + : ''; const linked = result.linked ? `; linked ${result.linked} relationship${result.linked === 1 ? '' : 's'}` : ''; + const unlinked = result.unlinked + ? `; unlinked ${result.unlinked} orphaned link${ + result.unlinked === 1 ? '' : 's'}` + : ''; console.log(options.validateOnly ? 'Validation complete; no changes applied.' : `Wrote ${result.created} new and ${result.updated} updated ` + - `Knowledge Catalog entr${n === 1 ? 'y' : 'ies'}${removed}${linked}.`); + `Knowledge Catalog entr${n === 1 ? 'y' : 'ies'}${removed}${linked}${ + unlinked}.`); return 0; } diff --git a/toolbox/mdcode/src/tool/main.ts b/toolbox/mdcode/src/tool/main.ts index c55c3d02..ecc057a3 100644 --- a/toolbox/mdcode/src/tool/main.ts +++ b/toolbox/mdcode/src/tool/main.ts @@ -40,6 +40,7 @@ cli.command('pull', 'Pull catalog entries') cli.command('push', 'Push catalog entries') .option('--force', 'Force push changes') + .option('--force-remove', 'Delete Knowledge Catalog models in the entry group that this push does not include (removed/renamed models); semantic-model push only') .option('--validate-only', 'Only validate changes without applying') .option('--target ', 'Semantic-model push destination(s): bq, kc, all (default), or a comma-separated list (e.g. bq,kc)') .option('--print', 'Print each pushed destination\'s generated artifact in its native format (BigQuery Graph SQL DDL, Knowledge Catalog entry plan); scope with --target (semantic-model push only)') diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts index 87950569..c6288ed6 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts @@ -84,13 +84,19 @@ function stubClient(opts: { group?: ApiResult, create?: (entryId: string) => ApiResult, update?: ApiResult, - // Entry ids the destination entry group already holds (yielded by - // listEntries during delete reconciliation). Default: none. - existing?: string[], + // Entries the destination entry group already holds (yielded by listEntries). + // A bare id defaults to a generic entryType; pass {id, type} to stage a + // specific one (e.g. a foreign semantic-model anchor). Default: none. + existing?: (string | {id: string; type: string})[], del?: (entryId: string) => ApiResult, // Result of createEntryLink, keyed by the entry-link id. Default: 200. createLink?: (linkId: string) => ApiResult, updateLink?: ApiResult, + // Entry links the destination returns from lookupEntryLinks, keyed by the + // referenced entry's full resource name. Default: none (ok, empty list). + links?: (entry: string) => ApiResult, + // Result of deleteEntryLink, keyed by the entry-link id. Default: 200. + delLink?: (linkId: string) => ApiResult, } = {}) { const group = spyOn(CatalogClient.prototype, 'createEntryGroup') .mockImplementation(async () => opts.group ?? ok({})); @@ -102,8 +108,11 @@ function stubClient(opts: { .mockImplementation(async () => opts.update ?? ok({})); const list = spyOn(CatalogClient.prototype, 'listEntries') .mockImplementation(async function*() { - for (const id of opts.existing ?? []) { - yield {name: entryName(id), entryType: 'semantic'} as any; + for (const e of opts.existing ?? []) { + const id = typeof e === 'string' ? e : e.id; + const entryType = + typeof e === 'string' ? 'semantic' : e.type; + yield {name: entryName(id), entryType} as any; } }); const del = spyOn(CatalogClient.prototype, 'deleteEntry') @@ -117,7 +126,17 @@ function stubClient(opts: { const updateLink = spyOn(CatalogClient.prototype, 'updateEntryLink') .mockImplementation( async () => opts.updateLink ?? ok({})); - return {group, create, update, list, del, createLink, updateLink}; + const lookupLinks = + spyOn(CatalogClient.prototype, 'lookupEntryLinks') + .mockImplementation( + async (_p, _l, o: any) => (opts.links ?? (() => ok([])))(o.entry)); + const delLink = spyOn(CatalogClient.prototype, 'deleteEntryLink') + .mockImplementation( + async (_p, _l, _eg, linkId) => + (opts.delLink ?? (() => ok({})))(linkId)); + return {group, create, update, list, + del, createLink, updateLink, + lookupLinks, delLink}; } @@ -377,3 +396,185 @@ describe('deployKnowledgeCatalog: delete reconciliation', () => { expect(del).not.toHaveBeenCalled(); }); }); + + +// Built-in system entry types, as the catalog reports them on existing entries. +const MODEL_TYPE = + 'projects/dataplex-types/locations/global/entryTypes/semantic-model'; +const ENTITY_TYPE = + 'projects/dataplex-types/locations/global/entryTypes/semantic-entity'; +const METRIC_TYPE = + 'projects/dataplex-types/locations/global/entryTypes/semantic-metric'; + +// A schema-join entry link as lookupEntryLinks returns it: `refIds` are the two +// endpoint entry ids (undirected, both UNSPECIFIED). +function linkEntry(id: string, refIds: string[]): any { + return { + name: `projects/dest/locations/us/entryGroups/eg/entryLinks/${id}`, + entryLinkType: + 'projects/dataplex-types/locations/global/entryLinkTypes/schema-join', + entryReferences: refIds.map(r => ({name: entryName(r), type: 'UNSPECIFIED'})), + }; +} + + +describe('deployKnowledgeCatalog: link reconciliation', () => { + // The STAR model 'sales' has entities orders + customer and emits exactly one + // schema-join link, 'sales-orders-to-customer'. + const KEPT = linkEntry( + 'sales-orders-to-customer', + ['sales.entities.orders', 'sales.entities.customer']); + + test('deletes an owned schema-join link the model no longer emits', + async () => { + // The server also still holds a link to a dropped relationship (both + // endpoints under this model). It is returned from both endpoints it + // touches (orders + customer), so the dedup path is exercised too. + const orphan = linkEntry( + 'sales-orders-to-supplier', + ['sales.entities.orders', 'sales.entities.supplier']); + const {delLink} = stubClient({ + links: (entry: string) => entry.endsWith('sales.entities.orders') + ? ok([orphan, KEPT]) + : ok([KEPT]), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.unlinked).toBe(1); + expect(delLink).toHaveBeenCalledTimes(1); + expect(delLink.mock.calls[0][3]).toBe('sales-orders-to-supplier'); + }); + + test('keeps a link the model still emits', async () => { + const {delLink} = stubClient({ + links: (entry: string) => + entry.endsWith('sales.entities.orders') ? ok([KEPT]) : ok([KEPT]), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.unlinked).toBe(0); + expect(delLink).not.toHaveBeenCalled(); + }); + + test('never deletes a link that touches an entry outside the model', + async () => { + // One endpoint is another model's entity: not owned, so not this model's + // to reconcile even though it references one of our entities. + const foreign = linkEntry( + 'sales-orders-to-external', + ['sales.entities.orders', 'other.entities.x']); + const {delLink} = stubClient({ + links: (entry: string) => + entry.endsWith('sales.entities.orders') ? ok([foreign]) : ok([]), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.unlinked).toBe(0); + expect(delLink).not.toHaveBeenCalled(); + }); + + test('a failed link delete fails the push, naming the link', async () => { + const orphan = linkEntry( + 'sales-orders-to-supplier', + ['sales.entities.orders', 'sales.entities.supplier']); + const {delLink} = stubClient({ + links: (entry: string) => + entry.endsWith('sales.entities.orders') ? ok([orphan]) : ok([]), + delLink: () => err(500, 'boom'), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(false); + expect(result.details).toContain('sales-orders-to-supplier'); + expect(delLink).toHaveBeenCalledTimes(1); + }); + + test('validateOnly never looks up or deletes links (offline)', async () => { + const {lookupLinks, delLink} = stubClient({ + links: () => ok([linkEntry( + 'sales-orders-to-supplier', + ['sales.entities.orders', 'sales.entities.supplier'])]), + }); + + const result = await deployKnowledgeCatalog( + models(STAR_DOCS), CTX, {...OPTS, validateOnly: true}); + + expect(result.success).toBe(true); + expect(lookupLinks).not.toHaveBeenCalled(); + expect(delLink).not.toHaveBeenCalled(); + }); +}); + + +describe('deployKnowledgeCatalog: whole-model removal (--force-remove)', () => { + test('a model already in the group that this push omits fails without the flag', + async () => { + const {create, del} = stubClient({ + existing: [{id: 'old', type: MODEL_TYPE}], + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(false); + expect(result.details).toContain('old'); + expect(result.details).toContain('--force-remove'); + // Nothing is written or deleted -- the guard runs before any mutation. + expect(create).not.toHaveBeenCalled(); + expect(del).not.toHaveBeenCalled(); + }); + + test('--force-remove drops the omitted model (entries + links), then writes', + async () => { + const orphanLink = linkEntry( + 'old-a-to-b', ['old.entities.a', 'old.entities.b']); + const {create, del, delLink} = stubClient({ + existing: [ + {id: 'old', type: MODEL_TYPE}, + {id: 'old.entities.a', type: ENTITY_TYPE}, + {id: 'old.entities.b', type: ENTITY_TYPE}, + {id: 'old.metrics.m', type: METRIC_TYPE}, + ], + links: (entry: string) => + entry.endsWith('old.entities.a') ? ok([orphanLink]) : ok([]), + }); + + const result = await deployKnowledgeCatalog( + models(DOCS), CTX, {...OPTS, forceRemove: true}); + + expect(result.success).toBe(true); + // The foreign model's 4 entries and its 1 link are removed... + expect(result.deleted).toBe(4); + expect(result.unlinked).toBe(1); + expect(delLink.mock.calls.map(c => c[3])).toEqual(['old-a-to-b']); + expect(del.mock.calls.map(c => c[3]).sort()).toEqual( + ['old', 'old.entities.a', 'old.entities.b', 'old.metrics.m']); + // ...and the current model is still written (3 entries). + expect(create).toHaveBeenCalledTimes(3); + }); + + test('an existing anchor this push re-emits is not treated as foreign', + async () => { + // The group already holds the same model being pushed: a normal re-push, + // no --force-remove required, nothing removed. + const {del} = stubClient({ + existing: [ + {id: 'sales', type: MODEL_TYPE}, + {id: 'sales.entities.orders', type: ENTITY_TYPE}, + {id: 'sales.metrics.total_revenue', type: METRIC_TYPE}, + ], + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.deleted).toBe(0); + expect(del).not.toHaveBeenCalled(); + }); +}); From 431ab6fa6d6936fd300b005d3c1482bb10b01548 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sat, 8 Aug 2026 05:12:09 +0000 Subject: [PATCH 07/34] mdcode: address KC-push review findings Follow-up fixes from code review of the KC-push reconcile work: - reconcileLinks now enumerates a model's schema-join links via its entity entries in the pre-write snapshot rather than the emitted set. This finds a link both of whose endpoints were removed in the same push (previously unreachable and leaked as a dangling link referencing deleted entries), and, because a brand-new model has no server-side entries, a first push issues no link lookups. - listEntryGroup only tolerates the entry-group not-yet-visible propagation error as empty (mirroring isPropagating); any other listing failure is surfaced instead of masked, which had silently bypassed the foreign-model guard and deletion reconciliation. - Collapse the duplicated GOOGLE deployment-target parsing into a single googleDeploymentTargets pass; bigQueryGraphTargets/deploymentTargetUris are thin views and the validate gate calls it once per model. Regression tests added for the both-endpoints-removed link and the no-lookup first push. 232 pass, tsc clean. --- .../src/libts/semantic/deploy_bigquery.ts | 71 +++++++++---------- .../semantic/deploy_knowledge_catalog.ts | 30 +++++--- toolbox/mdcode/src/libts/semantic/validate.ts | 14 ++-- .../semantic/deploy_knowledge_catalog.test.ts | 54 ++++++++++++++ 4 files changed, 112 insertions(+), 57 deletions(-) diff --git a/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts b/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts index 1b89ba5c..033eef35 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts @@ -76,14 +76,20 @@ export interface DeployResult { } -// Extracts the BigQuery Graph deployment targets from a model's GOOGLE -// custom extension. The extension `data` is an opaque, vendor-serialized JSON -// string (the loader keeps it verbatim); we own its `deploymentTargets` shape. -export function bigQueryGraphTargets(model: SemanticModel): - {targets: BigQueryGraphTarget[]; malformed: string[]} { +// Reads a model's GOOGLE custom_extension(s) in a single pass and returns the +// deployment-target facts every caller derives from them: every declared +// deploymentTarget URI (`uris`), the subset that parse as BigQuery Graph targets +// (`targets`), and the BigQuery-prefixed URIs that failed the strict match +// (`malformed`, kept so a caller can name a typo instead of silently skipping). +// The extension `data` is an opaque, vendor-serialized JSON string (the loader +// keeps it verbatim); we own its `deploymentTargets` shape. Throws on malformed +// extension JSON. bigQueryGraphTargets and deploymentTargetUris are thin views +// over this, and validation calls it directly so a push parses each extension +// once rather than once per reader. +export function googleDeploymentTargets(model: SemanticModel): + {uris: string[]; targets: BigQueryGraphTarget[]; malformed: string[]} { + const uris: string[] = []; const targets: BigQueryGraphTarget[] = []; - // BigQuery-prefixed URIs that failed the strict match (typo, bad identifier - // char). Kept so the caller can name them instead of silently skipping. const malformed: string[] = []; for (const ext of model.customExtensions ?? []) { @@ -99,15 +105,16 @@ export function bigQueryGraphTargets(model: SemanticModel): model.name}': GOOGLE custom_extension 'data' is not valid JSON.`); } - const uris = data?.deploymentTargets; - if (!Array.isArray(uris)) { + const list = data?.deploymentTargets; + if (!Array.isArray(list)) { continue; } - for (const uri of uris) { + for (const uri of list) { if (typeof uri !== 'string') { continue; } + uris.push(uri); const m = uri.match(BQ_GRAPH_TARGET); if (m) { targets.push({project: m[1], dataset: m[2], graphName: m[3], uri}); @@ -120,40 +127,26 @@ export function bigQueryGraphTargets(model: SemanticModel): } } + return {uris, targets, malformed}; +} + + +// The BigQuery Graph deployment targets a model declares in its GOOGLE custom +// extension, plus any BigQuery-prefixed URIs that failed to parse. A view over +// googleDeploymentTargets. +export function bigQueryGraphTargets(model: SemanticModel): + {targets: BigQueryGraphTarget[]; malformed: string[]} { + const {targets, malformed} = googleDeploymentTargets(model); return {targets, malformed}; } -// Every deploymentTarget URI a model declares in its GOOGLE custom extension(s), -// across all of them, regardless of whether each parses as a BigQuery Graph -// target. Validation uses this to require that a model declares at least one -// deployment target; bigQueryGraphTargets narrows to the ones that are BigQuery -// graphs. Throws on malformed extension JSON (same contract as -// bigQueryGraphTargets). +// Every deploymentTarget URI a model declares, regardless of whether each parses +// as a BigQuery Graph target. Validation uses this to require that a model +// declares at least one deployment target; bigQueryGraphTargets narrows to the +// BigQuery graphs. Throws on malformed extension JSON. export function deploymentTargetUris(model: SemanticModel): string[] { - const uris: string[] = []; - for (const ext of model.customExtensions ?? []) { - if (ext.vendorName !== GOOGLE_VENDOR) { - continue; - } - let data: any; - try { - data = JSON.parse(ext.data); - } catch { - throw new Error(`Model '${ - model.name}': GOOGLE custom_extension 'data' is not valid JSON.`); - } - const list = data?.deploymentTargets; - if (!Array.isArray(list)) { - continue; - } - for (const uri of list) { - if (typeof uri === 'string') { - uris.push(uri); - } - } - } - return uris; + return googleDeploymentTargets(model).uris; } diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 1c8acac7..b0ec7e18 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -250,7 +250,7 @@ export async function deployKnowledgeCatalog( // Then drop any schema-join link this model owns but no longer emits (a // relationship dropped or renamed). Runs after its current links are // written, so a rename never leaves the pair with no link between them. - const relLinks = await reconcileLinks(cat, opts, resources); + const relLinks = await reconcileLinks(cat, opts, resources, existing); if (relLinks.error) { return fail(`Model '${model}': ${relLinks.error}`); } @@ -345,10 +345,11 @@ function schemaJoinLinkType(opts: KcDeployOptions): string { // Snapshots the destination entry group's entries once, before any write. A // brand-new entry group can briefly fail to list its entries collection (the -// same propagation window createEntryWithRetry rides out); treat an unlistable -// group as empty -- there is nothing to guard against or reconcile, and the -// create path retries the window. Any other listing failure is real and -// surfaced. +// same propagation window createEntryWithRetry rides out); treat only that +// not-yet-visible error as empty -- there is nothing to guard against or +// reconcile, and the create path retries the window. The match mirrors +// isPropagating (entry-group-scoped) so any OTHER listing failure -- a real +// backend error, a permission problem -- is surfaced rather than masked. async function listEntryGroup( cat: CatalogClient, opts: KcDeployOptions): Promise<{entries: Entry[]; error?: string}> { @@ -360,7 +361,8 @@ async function listEntryGroup( } } catch (err: any) { const msg = err.message || String(err); - if (/not found|does not exist|may not exist/i.test(msg)) { + if (/may not exist/i.test(msg) || + /entry group .*(not found|does not exist)/i.test(msg)) { return {entries: []}; } return {entries: [], error: `listing entries in entry group '${ @@ -418,13 +420,19 @@ async function deleteOwnedLinks( // push. A link touching an entry outside this model is never treated as owned, // so a shared entry group is safe. function reconcileLinks( - cat: CatalogClient, opts: KcDeployOptions, - resources: KcResources): Promise { + cat: CatalogClient, opts: KcDeployOptions, resources: KcResources, + existing: Entry[]): Promise { const anchorId = idOf(resources.entries[0].name); const entityPrefix = `${anchorId}.entities.`; - const entityNames = resources.entries - .filter(e => (e.entryType ?? '').endsWith('/semantic-entity')) - .map(e => e.name); + // Look up links via the model's entity entries KNOWN TO THE SERVER (the + // pre-write snapshot), not the ones this push emits. Only a server-side entry + // can already carry a link, and enumerating the snapshot also reaches a link + // both of whose endpoints were removed in this push -- neither is re-emitted, + // but both entries are still present in `existing` until reconcileDeletions + // deletes them at the end. A brand-new model has no such entries, so it issues + // no lookups at all. + const entityNames = existing.map(e => e.name) + .filter(name => idOf(name).startsWith(entityPrefix)); if (!entityNames.length) return Promise.resolve({unlinked: 0}); const emittedLinkIds = diff --git a/toolbox/mdcode/src/libts/semantic/validate.ts b/toolbox/mdcode/src/libts/semantic/validate.ts index b2384368..09007ca9 100644 --- a/toolbox/mdcode/src/libts/semantic/validate.ts +++ b/toolbox/mdcode/src/libts/semantic/validate.ts @@ -8,7 +8,7 @@ // schema -- because these are deployment requirements, not schema rules, and // they read the GOOGLE deployment-target extension the BigQuery leg owns. -import {bigQueryGraphTargets, deploymentTargetUris} from './deploy_bigquery'; +import {googleDeploymentTargets} from './deploy_bigquery'; import {LoadedModel} from './loader'; // Checks every model against the push requirements and returns the collected @@ -17,11 +17,11 @@ import {LoadedModel} from './loader'; export function validatePushRequirements(models: LoadedModel[]): string[] { const errors: string[] = []; for (const {document, model} of models) { - let uris: string[]; - let bqTargetCount: number; + let deployInfo: ReturnType; try { - uris = deploymentTargetUris(model); - bqTargetCount = bigQueryGraphTargets(model).targets.length; + // One pass over the model's GOOGLE extension(s): both checks below read + // the same parse rather than re-parsing the JSON per reader. + deployInfo = googleDeploymentTargets(model); } catch (err: any) { // Malformed GOOGLE extension JSON: surface it as a validation error here // rather than letting it throw out of a later leg as an uncaught stack. @@ -30,7 +30,7 @@ export function validatePushRequirements(models: LoadedModel[]): string[] { } // Every model must declare at least one deployment target. - if (!uris.length) { + if (!deployInfo.uris.length) { errors.push( `model '${model.name}' (${document}) declares no deploymentTargets; ` + `add at least one under its GOOGLE custom_extension.`); @@ -41,7 +41,7 @@ export function validatePushRequirements(models: LoadedModel[]): string[] { // silently dropped from the graph. The loader sets metric.entity only when // the expression resolves to exactly one entity, so an unset entity is the // "references zero or multiple entities" case. - if (bqTargetCount > 0) { + if (deployInfo.targets.length > 0) { for (const metric of model.metrics ?? []) { if (!metric.entity) { errors.push( diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts index c6288ed6..1d7fcc0b 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts @@ -425,6 +425,14 @@ describe('deployKnowledgeCatalog: link reconciliation', () => { 'sales-orders-to-customer', ['sales.entities.orders', 'sales.entities.customer']); + // The model's entity entries as the destination already holds them (a + // re-push). reconcileLinks looks up links via these server-known entities, so + // they must be in the pre-write listing for a link to be discoverable. + const STAR_ENTITIES = [ + {id: 'sales.entities.orders', type: ENTITY_TYPE}, + {id: 'sales.entities.customer', type: ENTITY_TYPE}, + ]; + test('deletes an owned schema-join link the model no longer emits', async () => { // The server also still holds a link to a dropped relationship (both @@ -434,6 +442,7 @@ describe('deployKnowledgeCatalog: link reconciliation', () => { 'sales-orders-to-supplier', ['sales.entities.orders', 'sales.entities.supplier']); const {delLink} = stubClient({ + existing: STAR_ENTITIES, links: (entry: string) => entry.endsWith('sales.entities.orders') ? ok([orphan, KEPT]) : ok([KEPT]), @@ -449,6 +458,7 @@ describe('deployKnowledgeCatalog: link reconciliation', () => { test('keeps a link the model still emits', async () => { const {delLink} = stubClient({ + existing: STAR_ENTITIES, links: (entry: string) => entry.endsWith('sales.entities.orders') ? ok([KEPT]) : ok([KEPT]), }); @@ -468,6 +478,7 @@ describe('deployKnowledgeCatalog: link reconciliation', () => { 'sales-orders-to-external', ['sales.entities.orders', 'other.entities.x']); const {delLink} = stubClient({ + existing: STAR_ENTITIES, links: (entry: string) => entry.endsWith('sales.entities.orders') ? ok([foreign]) : ok([]), }); @@ -484,6 +495,7 @@ describe('deployKnowledgeCatalog: link reconciliation', () => { 'sales-orders-to-supplier', ['sales.entities.orders', 'sales.entities.supplier']); const {delLink} = stubClient({ + existing: STAR_ENTITIES, links: (entry: string) => entry.endsWith('sales.entities.orders') ? ok([orphan]) : ok([]), delLink: () => err(500, 'boom'), @@ -510,6 +522,48 @@ describe('deployKnowledgeCatalog: link reconciliation', () => { expect(lookupLinks).not.toHaveBeenCalled(); expect(delLink).not.toHaveBeenCalled(); }); + + test('deletes a link whose BOTH endpoints were removed in this push', + async () => { + // The dropped relationship's two endpoints are both entities removed from + // the model, so the orphan link is reachable ONLY via those removed + // entries in the pre-write snapshot -- never via a still-emitted entity. + // Enumerating the snapshot (not the emitted set) is what finds it. + const orphan = linkEntry( + 'sales-supplier-to-warehouse', + ['sales.entities.supplier', 'sales.entities.warehouse']); + const {delLink} = stubClient({ + existing: [ + ...STAR_ENTITIES, + {id: 'sales.entities.supplier', type: ENTITY_TYPE}, + {id: 'sales.entities.warehouse', type: ENTITY_TYPE}, + ], + links: (entry: string) => + entry.endsWith('sales.entities.supplier') || + entry.endsWith('sales.entities.warehouse') + ? ok([orphan]) + : ok([]), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.unlinked).toBe(1); + expect(delLink.mock.calls.map(c => c[3])) + .toEqual(['sales-supplier-to-warehouse']); + }); + + test('a first push (empty group) issues no link lookups', async () => { + // No entity of the model exists on the server yet, so there is nothing to + // reconcile and reconcileLinks makes no lookupEntryLinks call. + const {lookupLinks, delLink} = stubClient(); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(lookupLinks).not.toHaveBeenCalled(); + expect(delLink).not.toHaveBeenCalled(); + }); }); From 96335011b6fc4f67325cd953e2d12d351cf74c59 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 03:30:52 +0000 Subject: [PATCH 08/34] mdcode: document semantic-model push for users Add a user guide (docs/semantic-model.md) covering the semantic-model push: authoring and deployment targets, --target/--validate-only/--print, what the BigQuery and Knowledge Catalog legs write, the push-time validation gate, and how a re-push reconciles removed entities, metrics, relationships, and whole models (--force-remove). Link it from the README. --- toolbox/mdcode/README.md | 8 ++ toolbox/mdcode/docs/semantic-model.md | 148 ++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 toolbox/mdcode/docs/semantic-model.md diff --git a/toolbox/mdcode/README.md b/toolbox/mdcode/README.md index e69065aa..bc9718f4 100644 --- a/toolbox/mdcode/README.md +++ b/toolbox/mdcode/README.md @@ -183,6 +183,14 @@ declared project (the `` in the init scope), not the ambient `gcloud` project; write a fully-qualified `project.dataset.table` source when the tables live elsewhere. +`kcmd push` also deploys the model to Knowledge Catalog (entries for the model, +its entities and metrics, and `schema-join` links for its relationships). Use +`--target bq|kc|all` (default `all`) to choose destinations, `--print` to dump +each destination's generated artifact, and `--force-remove` to delete models +left in the entry group that the push no longer includes. See +[docs/semantic-model.md](docs/semantic-model.md) for the full guide, including +how re-push reconciles removed entities, metrics, relationships, and models. + NOTE: The CLI uses `gcloud` to obtain authentication tokens, so ensure you are authenticated via `gcloud auth application-default login`. ### MCP Server diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md new file mode 100644 index 00000000..e6499df1 --- /dev/null +++ b/toolbox/mdcode/docs/semantic-model.md @@ -0,0 +1,148 @@ +# Semantic model push + +A *semantic model* describes entities (tables), the metrics computed over them, +and the relationships between them, authored as a single +[Apache Ossie](https://ossie.dev) document in your catalog snapshot. `kcmd push` +deploys that model to two destinations: + +* **BigQuery** — a `CREATE OR REPLACE PROPERTY GRAPH` over the model's tables. +* **Knowledge Catalog (Dataplex)** — catalog entries and links that describe the + model as metadata. + +Both destinations are generated from the same model; you never author them +separately. + +## Author a model + +Create the local layout for a model. The scope names the Dataplex EntryGroup the +model belongs to: + +```bash +kcmd init --semantic-model .. +``` + +The model is authored at +`catalog/EntryGroups//.yaml`. `init` also provisions the +destination EntryGroup (idempotent — an existing group is fine); `push` only ever +writes entries into it. + +### Deployment targets (required) + +Every model must declare at least one **deployment target** in its GOOGLE +`custom_extension`. A BigQuery Graph target is a URI of the form: + +``` +//bigquery.googleapis.com/projects//datasets//propertyGraphs/ +``` + +The target's project and dataset are where the property graph is created, and the +same URIs are recorded on the Knowledge Catalog `semantic-model` entry. A model +with no deployment target is rejected at push time (see **Validation** below). + +A table `dataSource` that omits its project is qualified with the scope's +declared project (the `` in the `init` scope), not your ambient +`gcloud` project. Write a fully-qualified `project.dataset.table` when the tables +live elsewhere. + +## Push + +```bash +# Deploy to every destination (the default). +kcmd push + +# Deploy to one or a subset. +kcmd push --target bq +kcmd push --target kc +kcmd push --target bq,kc + +# Dry run: validate and show what would be written, without touching anything. +kcmd push --validate-only + +# Print each destination's generated artifact in its native format +# (BigQuery Graph SQL DDL, Knowledge Catalog entry plan). Scoped by --target; +# works with or without --validate-only. +kcmd push --print +``` + +`--target` is the single control for destination selection; its default is `all`. +Targets are always deployed BigQuery-first and fail fast, so a bad model never +half-deploys. + +### What the BigQuery leg does + +It executes `CREATE OR REPLACE PROPERTY GRAPH` against the project and dataset +named by each BigQuery deployment target. It also reads the target dataset's +metadata (`bigquery.datasets.get`) to pin the query's processing location; this +degrades gracefully when the permission is absent, falling back to BigQuery's own +location inference. `--validate-only` prints the DDL without executing it. + +### What the Knowledge Catalog leg writes + +The model becomes catalog **entries**, all built-in system types under +`dataplex-types/global` (push references them; it never creates types): + +| Entry | Represents | +|-------------------|-------------------------------------------------------| +| `semantic-model` | the model (the anchor / parent of the others) | +| `semantic-entity` | one entity — carries its `schema` (fields + semantics) | +| `semantic-metric` | one metric — its data type and expression | + +Entry ids are derived from names: ``, `.entities.`, +`.metrics.`. + +Each **relationship** becomes a `schema-join` **entry link** between the two +entity entries. The join detail (paired columns, foreign-key direction) lives in +the link's aspect. Many-to-many (association / junction-table) relationships are +**not** published to the catalog yet — the push warns and skips them; the edge +still exists in the BigQuery property graph. + +Only the canonical `expression` (GoogleSQL/ANSI) is published to the catalog. The +original vendor SQL (`importedExpression`) is kept in your source model and used +as a fallback when generating BigQuery SQL, but it is not written to the catalog, +which has no consumer for it. + +Push requires `dataplex.entryGroups.useSemanticModelAspect` on the destination +entry group, plus `useSchemaJoinEntryLink` / `useSchemaJoinAspect` when the model +has relationships. + +## Validation + +The same checks gate a real push and `--validate-only`: + +* **Every model must declare at least one deployment target.** +* **On a BigQuery-graph model, every metric must resolve to exactly one entity** + — otherwise it cannot lower to a MEASURE and would be dropped from the graph. + Set the metric's attach entity, or scope its expression to a single entity. + +These are static checks (no network). Whether the underlying tables exist and are +readable in BigQuery is *not* checked here; that surfaces when the BigQuery leg +runs the DDL. + +## Re-push, updates, and cleanup + +Push is idempotent and reconciles the catalog to match your model: + +* **Re-push upserts.** An entry or link that already exists is updated in place. +* **Removed entity or metric.** If you delete an entity or metric from a model + and push again, its orphaned entry is deleted. Reported as + `removed N orphaned entr(y|ies)`. +* **Removed or renamed relationship.** The stale `schema-join` link is deleted + after the current links are written (a rename never leaves the pair with no + link). Reported as `unlinked N orphaned link(s)`. Links belonging to other + models in a shared entry group are never touched. +* **Removed or renamed whole model.** If the entry group already contains a model + your push does not include, push **refuses** with an error naming it, rather + than leaving a stale model or silently deleting one. Re-run with + **`--force-remove`** to delete that model's links and entries first, then write + your current models. + +A typical push summary reads: + +``` +Pushed 7 Knowledge Catalog entries; removed 1 orphaned entry; linked 2 relationships; unlinked 1 orphaned link. +``` + +## Authentication + +The CLI uses `gcloud` for auth tokens; run +`gcloud auth application-default login` first. From 147d4dc8eeb23d478f65a80e64ea838ed711a359 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 03:42:53 +0000 Subject: [PATCH 09/34] mdcode: validate BigQuery data-source access before push Add a live pre-flight to the push-time validate gate: before either the BigQuery or Knowledge Catalog leg runs, every entity's BigQuery source table (a plain project.dataset.table) is probed via a new getTable client method, and a missing or inaccessible table fails the push, naming the table and entity. This confirms a model can deploy up front rather than surfacing a bad table only when the BigQuery leg executes its DDL. Runs for every --target and for --validate-only; query / non-table sources are skipped. Static requirement checks (deployment target, metric-to-entity) are unchanged. Adds validateBigQueryDataSources + parseTableRef, BigQueryClient.getTable and its mock override, five unit tests, and updates the user guide. --- toolbox/mdcode/docs/semantic-model.md | 17 ++-- toolbox/mdcode/src/libts/gcp/bigquery.ts | 11 +++ toolbox/mdcode/src/libts/semantic/validate.ts | 65 +++++++++++++++ toolbox/mdcode/src/tool/commands.ts | 16 +++- toolbox/mdcode/tests/libts/mocks.ts | 9 ++ .../tests/libts/semantic/validate.test.ts | 82 ++++++++++++++++++- 6 files changed, 191 insertions(+), 9 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index e6499df1..8129c180 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -107,16 +107,21 @@ has relationships. ## Validation -The same checks gate a real push and `--validate-only`: +The same checks gate a real push and `--validate-only`, and run **before** either +destination is touched, so a model that cannot deploy fails fast: -* **Every model must declare at least one deployment target.** +* **Every model must declare at least one deployment target.** (static) * **On a BigQuery-graph model, every metric must resolve to exactly one entity** — otherwise it cannot lower to a MEASURE and would be dropped from the graph. Set the metric's attach entity, or scope its expression to a single entity. - -These are static checks (no network). Whether the underlying tables exist and are -readable in BigQuery is *not* checked here; that surfaces when the BigQuery leg -runs the DDL. + (static) +* **Every entity's BigQuery source table must be reachable.** Each entity's + `dataSource` that is a plain `project.dataset.table` is probed against BigQuery; + a table that does not exist or that you cannot access fails the push, naming the + table and the entity. This runs for every `--target` (the entity tables back + both legs) and confirms the model can deploy before any write to BigQuery *or* + Knowledge Catalog. Sources that are queries or are not plain table references + are skipped. (live — needs BigQuery read access) ## Re-push, updates, and cleanup diff --git a/toolbox/mdcode/src/libts/gcp/bigquery.ts b/toolbox/mdcode/src/libts/gcp/bigquery.ts index e684a922..7e0be1a8 100644 --- a/toolbox/mdcode/src/libts/gcp/bigquery.ts +++ b/toolbox/mdcode/src/libts/gcp/bigquery.ts @@ -60,6 +60,17 @@ export class BigQueryClient extends api.ApiClient { return await this._get(name, params); } + // Fetches a single table's metadata. Used as a cheap existence/access probe + // before deploy: a 200 means the table is reachable, a 404 that it does not + // exist, a 403 that the caller cannot see it. `selectedFields` trims the + // response to the reference alone, since only the status is consulted. + async getTable(project: string, dataset: string, table: string): Promise> { + const name = `projects/${project}/datasets/${dataset}/tables/${table}`; + const params: Record = { selectedFields: 'tableReference' }; + + return await this._get(name, params); + } + async *listTables(project: string, dataset: string): AsyncGenerator
{ const name = `projects/${project}/datasets/${dataset}/tables`; diff --git a/toolbox/mdcode/src/libts/semantic/validate.ts b/toolbox/mdcode/src/libts/semantic/validate.ts index 09007ca9..0e0928f9 100644 --- a/toolbox/mdcode/src/libts/semantic/validate.ts +++ b/toolbox/mdcode/src/libts/semantic/validate.ts @@ -8,6 +8,7 @@ // schema -- because these are deployment requirements, not schema rules, and // they read the GOOGLE deployment-target extension the BigQuery leg owns. +import {BigQueryClient} from '../gcp/bigquery'; import {googleDeploymentTargets} from './deploy_bigquery'; import {LoadedModel} from './loader'; @@ -55,3 +56,67 @@ export function validatePushRequirements(models: LoadedModel[]): string[] { } return errors; } + + +// Live pre-flight over the same models: confirms every entity's BigQuery source +// table is reachable BEFORE any destination leg runs, so a push -- to BigQuery +// or to Knowledge Catalog -- fails fast when the model could not deploy, rather +// than surfacing a missing table only once the BigQuery leg executes its DDL. +// The entity sources are BigQuery tables regardless of --target, so this runs +// for every destination and for --validate-only. +// +// Only a clean three-part `project.dataset.table` source is probed; a source +// the loader kept verbatim because it is a query or is not a plain table +// reference cannot be resolved to a tables.get and is skipped. Each distinct +// table is checked once. Returns one message per unreachable table (empty when +// all pass). +export async function validateBigQueryDataSources( + models: LoadedModel[], bq: BigQueryClient): Promise { + // Dedup by fully-qualified table so a table shared across entities/models is + // probed once; keep the first reference for a locatable error message. + const refs = + new Map(); + for (const {document, model} of models) { + for (const entity of model.entities ?? []) { + const parsed = parseTableRef(entity.dataSource); + if (!parsed) continue; + const key = `${parsed.project}.${parsed.dataset}.${parsed.table}`; + if (!refs.has(key)) { + refs.set(key, {document, model: model.name, entity: entity.name}); + } + } + } + + const errors: string[] = []; + for (const [key, where] of refs) { + const {project, dataset, table} = parseTableRef(key)!; + const res = await bq.getTable(project, dataset, table); + if (res.status === 200) continue; + const why = res.status === 404 ? + 'does not exist' : + res.status === 403 ? + 'is not accessible (permission denied)' : + `could not be verified (${res.message?.trim() || `HTTP ${res.status}`})`; + errors.push( + `entity '${where.entity}' in model '${where.model}' (${ + where.document}) references BigQuery table '${key}', which ${why}; ` + + `the model cannot be deployed. Create the table or grant access to it, ` + + `or fix the entity's source.`); + } + return errors; +} + + +// Parses a canonical entity dataSource into a BigQuery table reference, or null +// when it is not a plain three-part `project.dataset.table` name -- a query +// (contains whitespace) or an under/over-qualified reference the loader passed +// through verbatim -- and so cannot be probed with tables.get. +function parseTableRef(dataSource: string|undefined): + {project: string; dataset: string; table: string}|null { + const trimmed = (dataSource ?? '').trim(); + if (!trimmed || /\s/.test(trimmed)) return null; + const parts = trimmed.split('.').map( + p => p.replace(/^[`"]/, '').replace(/[`"]$/, '')); + if (parts.length !== 3 || parts.some(p => !p.length)) return null; + return {project: parts[0], dataset: parts[1], table: parts[2]}; +} diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index e42e256d..50854f61 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -12,8 +12,9 @@ import { SemanticModelLayout } from '../libts/layouts/semantic-model'; import { SemanticModelSource } from '../libts/sources/semantic-model'; import * as deploy from '../libts/semantic/deploy_bigquery'; import * as kc from '../libts/semantic/deploy_knowledge_catalog'; +import {BigQueryClient} from '../libts/gcp/bigquery'; import {LoadedModel, loadSemanticModels} from '../libts/semantic/loader'; -import {validatePushRequirements} from '../libts/semantic/validate'; +import {validateBigQueryDataSources, validatePushRequirements} from '../libts/semantic/validate'; export interface InitOptions { @@ -219,6 +220,19 @@ export async function push(options: PushOptions): Promise { return 1; } + // Live pre-flight over the same models, before any destination runs: every + // entity's BigQuery source table must be reachable, so a push to either + // BigQuery or Knowledge Catalog fails fast when the model could not deploy. + // Runs for every --target and for --validate-only. + const accessErrors = + await validateBigQueryDataSources(loaded.models, new BigQueryClient(ctx)); + if (accessErrors.length) { + for (const e of accessErrors) { + console.error(`Error: ${e}`); + } + return 1; + } + // Run the resolved destinations in canonical order (BigQuery first); the // early return below fails fast, skipping later legs when an earlier one // fails. diff --git a/toolbox/mdcode/tests/libts/mocks.ts b/toolbox/mdcode/tests/libts/mocks.ts index 3c3cc20a..05597cd0 100644 --- a/toolbox/mdcode/tests/libts/mocks.ts +++ b/toolbox/mdcode/tests/libts/mocks.ts @@ -169,6 +169,15 @@ export class BigQueryClientMock extends bigquery.BigQueryClient { return { status: 404, message: 'Not found' }; } + async getTable(project: string, dataset: string, table: string): Promise> { + const name = `projects/${project}/datasets/${dataset}/tables/${table}`; + const resource = this.mockTables.get(name); + if (resource) { + return { status: 200, result: resource }; + } + return { status: 404, message: 'Not found' }; + } + async *listTables(project: string, dataset: string): AsyncGenerator { for (const table of this.mockTables.values()) { if (table.tableReference.projectId === project && table.tableReference.datasetId === dataset) { diff --git a/toolbox/mdcode/tests/libts/semantic/validate.test.ts b/toolbox/mdcode/tests/libts/semantic/validate.test.ts index 15899cd8..cf917ade 100644 --- a/toolbox/mdcode/tests/libts/semantic/validate.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/validate.test.ts @@ -3,9 +3,10 @@ import {describe, expect, test} from 'bun:test'; -import {CustomExtension, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; +import {CustomExtension, Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; import {LoadedModel} from '../../../src/libts/semantic/loader'; -import {validatePushRequirements} from '../../../src/libts/semantic/validate'; +import {validateBigQueryDataSources, validatePushRequirements} from '../../../src/libts/semantic/validate'; +import {BigQueryClientMock} from '../mocks'; // A parsed BigQuery Graph deployment target the strict matcher accepts. const BQ_TARGET = @@ -82,3 +83,80 @@ describe('validatePushRequirements', () => { expect(errs.length).toBe(2); }); }); + + +// Helpers for the live data-source check. +function entity(name: string, dataSource: string): Entity { + return {name, dataSource, keys: [], fields: []} as Entity; +} + +function mockTable(project: string, dataset: string, tableId: string) { + return {id: tableId, tableReference: {projectId: project, datasetId: dataset, tableId}}; +} + +describe('validateBigQueryDataSources', () => { + test('passes when every entity source table is reachable', async () => { + const bq = new BigQueryClientMock(); + bq.addMockTable(mockTable('p', 'd', 'orders')); + bq.addMockTable(mockTable('p', 'd', 'customer')); + const m = model( + {entities: [entity('orders', 'p.d.orders'), + entity('customer', 'p.d.customer')]}); + expect(await validateBigQueryDataSources([loaded(m)], bq)).toEqual([]); + }); + + test('reports a missing source table, naming the entity/model/document', + async () => { + const bq = new BigQueryClientMock(); + bq.addMockTable(mockTable('p', 'd', 'orders')); + const m = model({ + entities: [entity('orders', 'p.d.orders'), + entity('gone', 'p.d.ghost')], + }); + const errs = await validateBigQueryDataSources([loaded(m)], bq); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('p.d.ghost'); + expect(errs[0]).toContain('does not exist'); + expect(errs[0]).toContain("entity 'gone'"); + expect(errs[0]).toContain('doc'); + }); + + test('reports a permission-denied (403) source table', async () => { + const bq = new BigQueryClientMock(); + bq.getTable = (async () => ({status: 403, message: 'denied'})) as any; + const m = model({entities: [entity('o', 'p.d.o')]}); + const errs = await validateBigQueryDataSources([loaded(m)], bq); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('permission denied'); + }); + + test('skips sources that are not a plain three-part table reference', + async () => { + // A query source and an under-qualified ref cannot be probed with + // tables.get, so neither is checked (and no table is mocked). + const bq = new BigQueryClientMock(); + const m = model({ + entities: [entity('q', 'SELECT * FROM x'), + entity('short', 'd.t')], + }); + expect(await validateBigQueryDataSources([loaded(m)], bq)).toEqual([]); + }); + + test('probes each distinct table once across entities and models', + async () => { + const bq = new BigQueryClientMock(); + bq.addMockTable(mockTable('p', 'd', 'shared')); + let calls = 0; + const orig = bq.getTable.bind(bq); + bq.getTable = ((p: string, d: string, t: string) => { + calls++; + return orig(p, d, t); + }) as any; + const m1 = model({name: 'm1', entities: [entity('a', 'p.d.shared')]}); + const m2 = model({name: 'm2', entities: [entity('b', 'p.d.shared')]}); + const errs = await validateBigQueryDataSources( + [loaded(m1, 'd1'), loaded(m2, 'd2')], bq); + expect(errs).toEqual([]); + expect(calls).toBe(1); + }); +}); From a49de8c2630e83d7101a06384b116d949ae1bba0 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 03:47:47 +0000 Subject: [PATCH 10/34] mdcode: correct semantic-model guide (--validate-only vs --print) --- toolbox/mdcode/docs/semantic-model.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 8129c180..4859a273 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -55,7 +55,9 @@ kcmd push --target bq kcmd push --target kc kcmd push --target bq,kc -# Dry run: validate and show what would be written, without touching anything. +# Dry run: run every validation check and report pass/fail, touching +# nothing. This does NOT print the generated artifacts on its own -- add +# --print to also see what would be written. kcmd push --validate-only # Print each destination's generated artifact in its native format @@ -74,7 +76,8 @@ It executes `CREATE OR REPLACE PROPERTY GRAPH` against the project and dataset named by each BigQuery deployment target. It also reads the target dataset's metadata (`bigquery.datasets.get`) to pin the query's processing location; this degrades gracefully when the permission is absent, falling back to BigQuery's own -location inference. `--validate-only` prints the DDL without executing it. +location inference. `--validate-only` generates and checks the DDL but does not +execute it; add `--print` (with or without `--validate-only`) to see the DDL. ### What the Knowledge Catalog leg writes @@ -141,10 +144,11 @@ Push is idempotent and reconciles the catalog to match your model: **`--force-remove`** to delete that model's links and entries first, then write your current models. -A typical push summary reads: +A `--target all` push logs one summary line per destination, for example: ``` -Pushed 7 Knowledge Catalog entries; removed 1 orphaned entry; linked 2 relationships; unlinked 1 orphaned link. +Deployed 1 BigQuery Graph(s). +Wrote 5 new and 2 updated Knowledge Catalog entries; removed 1 orphaned entry; linked 2 relationships; unlinked 1 orphaned link. ``` ## Authentication From 6fc96d8160c62f1153d17e37fc6e694d2f3d4692 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 03:55:07 +0000 Subject: [PATCH 11/34] mdcode: restructure semantic-model guide for readability --- toolbox/mdcode/docs/semantic-model.md | 265 +++++++++++++++----------- 1 file changed, 157 insertions(+), 108 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 4859a273..b5cd22c3 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -1,157 +1,206 @@ -# Semantic model push +# Deploying a semantic model -A *semantic model* describes entities (tables), the metrics computed over them, -and the relationships between them, authored as a single -[Apache Ossie](https://ossie.dev) document in your catalog snapshot. `kcmd push` -deploys that model to two destinations: +A *semantic model* describes your entities (tables), the metrics computed over +them, and the relationships between them, authored as a single +[Apache Ossie](https://ossie.dev) document. `kcmd push` deploys one model to two +destinations at once: -* **BigQuery** — a `CREATE OR REPLACE PROPERTY GRAPH` over the model's tables. -* **Knowledge Catalog (Dataplex)** — catalog entries and links that describe the - model as metadata. +* **BigQuery** — a queryable `CREATE OR REPLACE PROPERTY GRAPH` over the model's + tables, so the model can be traversed and its metrics computed in SQL. +* **Knowledge Catalog (Dataplex)** — catalog entries and links that make the + model discoverable as metadata. -Both destinations are generated from the same model; you never author them -separately. +Both are generated from the same source document — you never author them +separately, and a single `push` keeps them in sync. -## Author a model +This guide covers authoring, deploying, and updating a model. For the Ossie +document format itself, see [ossie.dev](https://ossie.dev). -Create the local layout for a model. The scope names the Dataplex EntryGroup the -model belongs to: +## Prerequisites + +`kcmd` authenticates with your `gcloud` credentials. Log in once before pushing: ```bash -kcmd init --semantic-model .. +gcloud auth application-default login ``` -The model is authored at -`catalog/EntryGroups//.yaml`. `init` also provisions the -destination EntryGroup (idempotent — an existing group is fine); `push` only ever -writes entries into it. +You also need read/write access to whichever destinations you deploy to — see +[Permissions](#permissions). + +## 1. Author a model + +Create the local layout. The scope is the Dataplex EntryGroup the model will be +published to, written as `..`: + +```bash +kcmd init --semantic-model my-project.us.my_models +``` + +`init` provisions that EntryGroup (idempotent — an existing group is fine) and +creates its local directory. Author the model at +`catalog/EntryGroups//.yaml`: + +```yaml +version: "0.2.0.dev0" + +semantic_model: + - name: sales + # Required: at least one deployment target, in a GOOGLE custom extension. + # `data` is a JSON string whose deploymentTargets is a list of graph URIs. + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": ["//bigquery.googleapis.com/projects/my-project/datasets/sales/propertyGraphs/sales_graph"]}' + datasets: # each dataset becomes an entity + - name: orders + source: my-project.sales.orders # the backing BigQuery table + primary_key: [o_orderkey] + fields: + - name: o_orderkey + expression: {dialects: [{dialect: ANSI_SQL, expression: o_orderkey}]} + - name: o_totalprice + expression: {dialects: [{dialect: ANSI_SQL, expression: o_totalprice}]} + metrics: + - name: total_revenue + expression: {dialects: [{dialect: ANSI_SQL, expression: SUM(orders.o_totalprice)}]} +``` ### Deployment targets (required) -Every model must declare at least one **deployment target** in its GOOGLE -`custom_extension`. A BigQuery Graph target is a URI of the form: +Every model must declare at least one **deployment target** — the BigQuery +property graph it deploys to — in a `GOOGLE` custom extension, as shown above. A +target is a URI of the form: ``` //bigquery.googleapis.com/projects//datasets//propertyGraphs/ ``` -The target's project and dataset are where the property graph is created, and the -same URIs are recorded on the Knowledge Catalog `semantic-model` entry. A model -with no deployment target is rejected at push time (see **Validation** below). +The target's project and dataset are where the property graph is created; the +same URIs are also recorded on the model's Knowledge Catalog entry. A model with +no deployment target is rejected at push time (see [Validation](#validation)). + +### Table sources -A table `dataSource` that omits its project is qualified with the scope's -declared project (the `` in the `init` scope), not your ambient -`gcloud` project. Write a fully-qualified `project.dataset.table` when the tables -live elsewhere. +Each entity's `source` is its backing BigQuery table. A `source` written as +`dataset.table` (two parts) is qualified with the scope's project — the +`` from `init`, *not* your ambient `gcloud` project. Write the full +`project.dataset.table` when a table lives in another project. -## Push +## 2. Push ```bash -# Deploy to every destination (the default). kcmd push +``` -# Deploy to one or a subset. -kcmd push --target bq -kcmd push --target kc -kcmd push --target bq,kc - -# Dry run: run every validation check and report pass/fail, touching -# nothing. This does NOT print the generated artifacts on its own -- add -# --print to also see what would be written. -kcmd push --validate-only +With no flags this deploys to **every** destination, BigQuery first. The flags +below select destinations, dry-run, and preview: -# Print each destination's generated artifact in its native format -# (BigQuery Graph SQL DDL, Knowledge Catalog entry plan). Scoped by --target; -# works with or without --validate-only. -kcmd push --print +```bash +kcmd push --target bq # BigQuery only +kcmd push --validate-only # run all checks, write nothing +kcmd push --print # also print the generated DDL / entry plan ``` -`--target` is the single control for destination selection; its default is `all`. -Targets are always deployed BigQuery-first and fail fast, so a bad model never -half-deploys. +| Flag | Effect | +|------|--------| +| `--target ` | Which destination(s) to deploy to; accepts a comma-separated list (`bq,kc`). Default `all`. | +| `--validate-only` | Run every validation check and report pass/fail, but write nothing. | +| `--print` | Print each destination's generated artifact (BigQuery DDL, Knowledge Catalog entry plan). Combine with `--validate-only` to preview without deploying. | +| `--force-remove` | Delete models in the entry group that this push no longer includes (see [Updating and removing models](#updating-and-removing-models)). | -### What the BigQuery leg does +Destinations always deploy BigQuery-first and fail fast, so a rejected model +never half-deploys. -It executes `CREATE OR REPLACE PROPERTY GRAPH` against the project and dataset -named by each BigQuery deployment target. It also reads the target dataset's -metadata (`bigquery.datasets.get`) to pin the query's processing location; this -degrades gracefully when the permission is absent, falling back to BigQuery's own -location inference. `--validate-only` generates and checks the DDL but does not -execute it; add `--print` (with or without `--validate-only`) to see the DDL. +### What gets created in BigQuery -### What the Knowledge Catalog leg writes +`push` executes `CREATE OR REPLACE PROPERTY GRAPH` in the project and dataset +named by each deployment target. It reads the target dataset's location +(`bigquery.datasets.get`) so the statement runs in the right region; if it can't +(missing permission), it falls back to BigQuery's own location inference and +warns. Nothing runs under `--validate-only`; add `--print` to see the DDL. -The model becomes catalog **entries**, all built-in system types under -`dataplex-types/global` (push references them; it never creates types): +### What gets created in Knowledge Catalog -| Entry | Represents | -|-------------------|-------------------------------------------------------| -| `semantic-model` | the model (the anchor / parent of the others) | +The model becomes catalog **entries** of the built-in system types under +`dataplex-types/global` (push references these types; it never creates them): + +| Entry | Represents | +|-------|-----------| +| `semantic-model` | the model — the anchor / parent of the others | | `semantic-entity` | one entity — carries its `schema` (fields + semantics) | -| `semantic-metric` | one metric — its data type and expression | +| `semantic-metric` | one metric — its data type and expression | -Entry ids are derived from names: ``, `.entities.`, +Entry ids are derived from names: ``, `.entities.`, and `.metrics.`. Each **relationship** becomes a `schema-join` **entry link** between the two -entity entries. The join detail (paired columns, foreign-key direction) lives in +entity entries; the join detail (paired columns, foreign-key direction) lives in the link's aspect. Many-to-many (association / junction-table) relationships are -**not** published to the catalog yet — the push warns and skips them; the edge +**not** published to the catalog yet — push warns and skips them, though the edge still exists in the BigQuery property graph. -Only the canonical `expression` (GoogleSQL/ANSI) is published to the catalog. The -original vendor SQL (`importedExpression`) is kept in your source model and used -as a fallback when generating BigQuery SQL, but it is not written to the catalog, -which has no consumer for it. - -Push requires `dataplex.entryGroups.useSemanticModelAspect` on the destination -entry group, plus `useSchemaJoinEntryLink` / `useSchemaJoinAspect` when the model -has relationships. +> **Note:** only the canonical `expression` (GoogleSQL/ANSI) is written to the +> catalog. The original vendor SQL (`importedExpression`) stays in your source +> model — it is still used as a fallback when generating BigQuery SQL — but the +> catalog has no consumer for it. ## Validation -The same checks gate a real push and `--validate-only`, and run **before** either -destination is touched, so a model that cannot deploy fails fast: - -* **Every model must declare at least one deployment target.** (static) -* **On a BigQuery-graph model, every metric must resolve to exactly one entity** - — otherwise it cannot lower to a MEASURE and would be dropped from the graph. - Set the metric's attach entity, or scope its expression to a single entity. - (static) -* **Every entity's BigQuery source table must be reachable.** Each entity's - `dataSource` that is a plain `project.dataset.table` is probed against BigQuery; - a table that does not exist or that you cannot access fails the push, naming the - table and the entity. This runs for every `--target` (the entity tables back - both legs) and confirms the model can deploy before any write to BigQuery *or* - Knowledge Catalog. Sources that are queries or are not plain table references - are skipped. (live — needs BigQuery read access) - -## Re-push, updates, and cleanup - -Push is idempotent and reconciles the catalog to match your model: - -* **Re-push upserts.** An entry or link that already exists is updated in place. -* **Removed entity or metric.** If you delete an entity or metric from a model - and push again, its orphaned entry is deleted. Reported as - `removed N orphaned entr(y|ies)`. -* **Removed or renamed relationship.** The stale `schema-join` link is deleted - after the current links are written (a rename never leaves the pair with no - link). Reported as `unlinked N orphaned link(s)`. Links belonging to other - models in a shared entry group are never touched. -* **Removed or renamed whole model.** If the entry group already contains a model - your push does not include, push **refuses** with an error naming it, rather - than leaving a stale model or silently deleting one. Re-run with - **`--force-remove`** to delete that model's links and entries first, then write - your current models. - -A `--target all` push logs one summary line per destination, for example: +`push` and `--validate-only` run the same checks, **before either destination is +touched**, so a model that cannot deploy fails fast instead of half-deploying: + +* **At least one deployment target per model.** *(static)* +* **Every metric on a BigQuery-graph model resolves to exactly one entity** — + otherwise it cannot lower to a MEASURE and would be dropped from the graph. Set + the metric's attach entity, or scope its expression to a single entity. + *(static)* +* **Every entity's source table is reachable.** Each `source` that is a plain + `project.dataset.table` is probed against BigQuery; a table that does not exist + or that you cannot access fails the push, naming the table and the entity. + Sources that are queries or non-table references are skipped. *(live — needs + BigQuery read access; see [Permissions](#permissions))* + +The live table check runs for **every** `--target`, because the same tables back +both destinations — so even a Knowledge-Catalog-only push confirms the model +could deploy to BigQuery. + +## Updating and removing models + +`push` is idempotent: re-running reconciles each destination to match your +current model. + +* **Re-push updates in place.** An existing entry or link is upserted. +* **Removed entity or metric** — its now-orphaned catalog entry is deleted. + (`removed N orphaned entr(y|ies)`) +* **Removed or renamed relationship** — the stale `schema-join` link is deleted + after the current links are written, so a rename never leaves the pair + unlinked. (`unlinked N orphaned link(s)`) Links owned by other models in a + shared entry group are never touched. +* **Removed or renamed whole model** — if the entry group still contains a model + your push no longer includes, push **refuses** and names it, rather than + leaving a stale model or silently deleting one. Re-run with **`--force-remove`** + to delete that model's entries and links first. + +Each destination prints a one-line summary. For a `--target all` push: ``` Deployed 1 BigQuery Graph(s). Wrote 5 new and 2 updated Knowledge Catalog entries; removed 1 orphaned entry; linked 2 relationships; unlinked 1 orphaned link. ``` -## Authentication +## Permissions + +`push` needs access to whichever destinations you deploy to. + +**BigQuery** — for `--target bq` or `all`, and for the validation table check: + +* permission to run `CREATE OR REPLACE PROPERTY GRAPH` in the target dataset + (create/replace tables and run jobs) +* `bigquery.tables.get` on each entity's source table (validation pre-flight) +* `bigquery.datasets.get` on the target dataset (region detection; optional — + push degrades gracefully without it) + +**Knowledge Catalog / Dataplex** — for `--target kc` or `all`: -The CLI uses `gcloud` for auth tokens; run -`gcloud auth application-default login` first. +* `dataplex.entryGroups.useSemanticModelAspect` on the destination entry group +* `dataplex.entryGroups.useSchemaJoinEntryLink` and + `dataplex.entryGroups.useSchemaJoinAspect` when the model has relationships From 12fe65ddc5bbdd38c83afdc27994f8eb96d9a73f Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 04:07:56 +0000 Subject: [PATCH 12/34] mdcode: validate BigQuery sources via dry-run to cover multi-part (REST catalog) names --- toolbox/mdcode/src/libts/gcp/bigquery.ts | 9 +- toolbox/mdcode/src/libts/semantic/validate.ts | 96 ++++++++++++------- toolbox/mdcode/src/tool/commands.ts | 4 +- toolbox/mdcode/tests/libts/mocks.ts | 34 +++++++ .../tests/libts/semantic/validate.test.ts | 55 +++++++---- 5 files changed, 140 insertions(+), 58 deletions(-) diff --git a/toolbox/mdcode/src/libts/gcp/bigquery.ts b/toolbox/mdcode/src/libts/gcp/bigquery.ts index 7e0be1a8..e8670d0e 100644 --- a/toolbox/mdcode/src/libts/gcp/bigquery.ts +++ b/toolbox/mdcode/src/libts/gcp/bigquery.ts @@ -98,13 +98,18 @@ export class BigQueryClient extends api.ApiClient { // Executes a SQL statement (including DDL) synchronously via jobs.query. // When `location` is set it pins the job's processing location so it agrees // with getQueryResults/getJob; otherwise BigQuery infers it from the - // referenced tables. - async query(project: string, sql: string, location?: string): Promise> { + // referenced tables. With `dryRun` the statement is validated -- name + // resolution, table existence and access -- but not executed, so it serves as + // a cheap pre-flight probe over any reference form BigQuery can resolve. + async query(project: string, sql: string, location?: string, dryRun?: boolean): Promise> { const name = `projects/${project}/queries`; const body: Record = { query: sql, useLegacySql: false }; if (location) { body.location = location; } + if (dryRun) { + body.dryRun = true; + } return await this._post(name, body); } diff --git a/toolbox/mdcode/src/libts/semantic/validate.ts b/toolbox/mdcode/src/libts/semantic/validate.ts index 0e0928f9..39d620af 100644 --- a/toolbox/mdcode/src/libts/semantic/validate.ts +++ b/toolbox/mdcode/src/libts/semantic/validate.ts @@ -10,6 +10,7 @@ import {BigQueryClient} from '../gcp/bigquery'; import {googleDeploymentTargets} from './deploy_bigquery'; +import {SemanticModel} from './ir'; import {LoadedModel} from './loader'; // Checks every model against the push requirements and returns the collected @@ -65,58 +66,85 @@ export function validatePushRequirements(models: LoadedModel[]): string[] { // The entity sources are BigQuery tables regardless of --target, so this runs // for every destination and for --validate-only. // -// Only a clean three-part `project.dataset.table` source is probed; a source -// the loader kept verbatim because it is a query or is not a plain table -// reference cannot be resolved to a tables.get and is skipped. Each distinct -// table is checked once. Returns one message per unreachable table (empty when -// all pass). +// Each distinct source is probed with a dry-run query (`SELECT 1 FROM `), +// so BigQuery resolves the reference exactly as the generated DDL will. That +// covers every reference form the generator emits -- a three-part +// `project.dataset.table`, a four-part federated REST-catalog / Lakehouse name +// (e.g. an Apache Iceberg table via BigLake), and quoted identifiers -- rather +// than only a three-part name. A source the loader kept verbatim because it is a +// query (contains whitespace) is not a table and is skipped. The dry-run is +// billed to the model's BigQuery deployment-target project (the same project the +// deploy runs against), falling back to `defaultProject`. Each distinct (billing +// project, reference) pair is probed once. Returns one message per unreachable +// table (empty when all pass). export async function validateBigQueryDataSources( - models: LoadedModel[], bq: BigQueryClient): Promise { - // Dedup by fully-qualified table so a table shared across entities/models is - // probed once; keep the first reference for a locatable error message. - const refs = - new Map(); + models: LoadedModel[], bq: BigQueryClient, + defaultProject: string): Promise { + // Dedup by billing project + reference so a table shared across + // entities/models is probed once; keep the first reference for a locatable + // error message. + const refs = new Map(); for (const {document, model} of models) { + const project = billingProject(model, defaultProject); for (const entity of model.entities ?? []) { - const parsed = parseTableRef(entity.dataSource); - if (!parsed) continue; - const key = `${parsed.project}.${parsed.dataset}.${parsed.table}`; + const ref = probeableRef(entity.dataSource); + if (!ref) continue; + const key = `${project}\u0000${ref}`; if (!refs.has(key)) { - refs.set(key, {document, model: model.name, entity: entity.name}); + refs.set(key, { + project, ref, document, model: model.name, entity: entity.name, + }); } } } const errors: string[] = []; - for (const [key, where] of refs) { - const {project, dataset, table} = parseTableRef(key)!; - const res = await bq.getTable(project, dataset, table); + for (const {project, ref, document, model, entity} of refs.values()) { + const res = + await bq.query(project, `SELECT 1 FROM \`${ref}\``, undefined, true); if (res.status === 200) continue; - const why = res.status === 404 ? + const msg = res.message?.trim() || `HTTP ${res.status}`; + const why = /not found/i.test(msg) ? 'does not exist' : - res.status === 403 ? + /access denied|permission denied|not authorized|does not have permission/i + .test(msg) ? 'is not accessible (permission denied)' : - `could not be verified (${res.message?.trim() || `HTTP ${res.status}`})`; + `could not be verified (${msg})`; errors.push( - `entity '${where.entity}' in model '${where.model}' (${ - where.document}) references BigQuery table '${key}', which ${why}; ` + - `the model cannot be deployed. Create the table or grant access to it, ` + - `or fix the entity's source.`); + `entity '${entity}' in model '${model}' (${document}) references ` + + `BigQuery table '${ref}', which ${why}; the model cannot be deployed. ` + + `Create the table or grant access to it, or fix the entity's source.`); } return errors; } -// Parses a canonical entity dataSource into a BigQuery table reference, or null -// when it is not a plain three-part `project.dataset.table` name -- a query -// (contains whitespace) or an under/over-qualified reference the loader passed -// through verbatim -- and so cannot be probed with tables.get. -function parseTableRef(dataSource: string|undefined): - {project: string; dataset: string; table: string}|null { +// The BigQuery project a model's deploy -- and thus its dry-run pre-flight -- +// bills to: the project of the model's first BigQuery Graph deployment target +// (where the CREATE PROPERTY GRAPH runs), falling back to the scope's default +// project when the model declares no parseable BigQuery Graph target. +// googleDeploymentTargets is safe here: validatePushRequirements ran first and +// already rejected a malformed GOOGLE extension. +function billingProject(model: SemanticModel, defaultProject: string): string { + try { + return googleDeploymentTargets(model).targets[0]?.project ?? defaultProject; + } catch { + return defaultProject; + } +} + + +// A source that can be probed as a BigQuery table: the canonical `dataSource`, +// trimmed, or null when it is not a table reference -- empty, or a query the +// loader kept verbatim (contains whitespace). Unlike a tables.get probe this +// imposes no part-count limit, so a three-part `project.dataset.table` and a +// four-part REST-catalog / Lakehouse name are both returned for the dry-run to +// resolve. +function probeableRef(dataSource: string|undefined): string|null { const trimmed = (dataSource ?? '').trim(); if (!trimmed || /\s/.test(trimmed)) return null; - const parts = trimmed.split('.').map( - p => p.replace(/^[`"]/, '').replace(/[`"]$/, '')); - if (parts.length !== 3 || parts.some(p => !p.length)) return null; - return {project: parts[0], dataset: parts[1], table: parts[2]}; + return trimmed; } diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index 50854f61..0f4a964b 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -224,8 +224,8 @@ export async function push(options: PushOptions): Promise { // entity's BigQuery source table must be reachable, so a push to either // BigQuery or Knowledge Catalog fails fast when the model could not deploy. // Runs for every --target and for --validate-only. - const accessErrors = - await validateBigQueryDataSources(loaded.models, new BigQueryClient(ctx)); + const accessErrors = await validateBigQueryDataSources( + loaded.models, new BigQueryClient(ctx), source.project ?? ctx.project); if (accessErrors.length) { for (const e of accessErrors) { console.error(`Error: ${e}`); diff --git a/toolbox/mdcode/tests/libts/mocks.ts b/toolbox/mdcode/tests/libts/mocks.ts index 05597cd0..6518e450 100644 --- a/toolbox/mdcode/tests/libts/mocks.ts +++ b/toolbox/mdcode/tests/libts/mocks.ts @@ -185,4 +185,38 @@ export class BigQueryClientMock extends bigquery.BigQueryClient { } } } + + // Reachable sources for the dry-run probe (validateBigQueryDataSources). Use + // for reference forms tables.get cannot address, e.g. a four-part REST-catalog + // name; a three-part table added via addMockTable is reachable too. + public mockSources: Set = new Set(); + + addMockSource(ref: string) { + this.mockSources.add(ref); + } + + // Dry-run table probe: a query of the form `SELECT 1 FROM \`\`` resolves + // against the reachable set (mockSources, plus any three-part table added via + // addMockTable). An unknown reference returns a BigQuery-style "Not found" so + // the validator reports it; a non-probe query is a benign success. + async query(project: string, sql: string, location?: string, dryRun?: boolean): + Promise> { + const m = /FROM `([^`]+)`/.exec(sql); + if (dryRun && m) { + return this.isReachableSource(m[1]) ? + {status: 200, result: {}} : + {status: 400, message: `Not found: Table ${m[1]} was not found`}; + } + return {status: 200, result: {}}; + } + + private isReachableSource(ref: string): boolean { + if (this.mockSources.has(ref)) return true; + const parts = ref.split('.'); + if (parts.length === 3) { + return this.mockTables.has( + `projects/${parts[0]}/datasets/${parts[1]}/tables/${parts[2]}`); + } + return false; + } } diff --git a/toolbox/mdcode/tests/libts/semantic/validate.test.ts b/toolbox/mdcode/tests/libts/semantic/validate.test.ts index cf917ade..34b6ba1e 100644 --- a/toolbox/mdcode/tests/libts/semantic/validate.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/validate.test.ts @@ -102,7 +102,7 @@ describe('validateBigQueryDataSources', () => { const m = model( {entities: [entity('orders', 'p.d.orders'), entity('customer', 'p.d.customer')]}); - expect(await validateBigQueryDataSources([loaded(m)], bq)).toEqual([]); + expect(await validateBigQueryDataSources([loaded(m)], bq, 'p')).toEqual([]); }); test('reports a missing source table, naming the entity/model/document', @@ -113,7 +113,7 @@ describe('validateBigQueryDataSources', () => { entities: [entity('orders', 'p.d.orders'), entity('gone', 'p.d.ghost')], }); - const errs = await validateBigQueryDataSources([loaded(m)], bq); + const errs = await validateBigQueryDataSources([loaded(m)], bq, 'p'); expect(errs.length).toBe(1); expect(errs[0]).toContain('p.d.ghost'); expect(errs[0]).toContain('does not exist'); @@ -121,41 +121,56 @@ describe('validateBigQueryDataSources', () => { expect(errs[0]).toContain('doc'); }); - test('reports a permission-denied (403) source table', async () => { + test('covers a four-part REST-catalog / Iceberg source', async () => { + // A federated REST-catalog table (e.g. Iceberg via BigLake) is a four-part + // name that tables.get cannot address; the dry-run resolves it as the deploy + // will. A reachable one passes; a missing one is reported by name. const bq = new BigQueryClientMock(); - bq.getTable = (async () => ({status: 403, message: 'denied'})) as any; + bq.addMockSource('ice_cat.db.sales.orders'); + const ok = model({entities: [entity('orders', 'ice_cat.db.sales.orders')]}); + expect(await validateBigQueryDataSources([loaded(ok)], bq, 'p')).toEqual([]); + + const missing = + model({entities: [entity('lost', 'ice_cat.db.sales.ghost')]}); + const errs = await validateBigQueryDataSources([loaded(missing)], bq, 'p'); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('ice_cat.db.sales.ghost'); + expect(errs[0]).toContain('does not exist'); + }); + + test('reports a permission-denied source table', async () => { + const bq = new BigQueryClientMock(); + bq.query = + (async () => ({status: 403, message: 'Access Denied: no permission'})) as + any; const m = model({entities: [entity('o', 'p.d.o')]}); - const errs = await validateBigQueryDataSources([loaded(m)], bq); + const errs = await validateBigQueryDataSources([loaded(m)], bq, 'p'); expect(errs.length).toBe(1); expect(errs[0]).toContain('permission denied'); }); - test('skips sources that are not a plain three-part table reference', - async () => { - // A query source and an under-qualified ref cannot be probed with - // tables.get, so neither is checked (and no table is mocked). - const bq = new BigQueryClientMock(); - const m = model({ - entities: [entity('q', 'SELECT * FROM x'), - entity('short', 'd.t')], - }); - expect(await validateBigQueryDataSources([loaded(m)], bq)).toEqual([]); - }); + test('skips a source that is a query, not a table', async () => { + // A query source (contains whitespace) is not a table and cannot be probed, + // so it is not checked (nothing is mocked for it). + const bq = new BigQueryClientMock(); + const m = model({entities: [entity('q', 'SELECT * FROM x')]}); + expect(await validateBigQueryDataSources([loaded(m)], bq, 'p')).toEqual([]); + }); test('probes each distinct table once across entities and models', async () => { const bq = new BigQueryClientMock(); bq.addMockTable(mockTable('p', 'd', 'shared')); let calls = 0; - const orig = bq.getTable.bind(bq); - bq.getTable = ((p: string, d: string, t: string) => { + const orig = bq.query.bind(bq); + bq.query = ((p: string, sql: string, loc?: string, dry?: boolean) => { calls++; - return orig(p, d, t); + return orig(p, sql, loc, dry); }) as any; const m1 = model({name: 'm1', entities: [entity('a', 'p.d.shared')]}); const m2 = model({name: 'm2', entities: [entity('b', 'p.d.shared')]}); const errs = await validateBigQueryDataSources( - [loaded(m1, 'd1'), loaded(m2, 'd2')], bq); + [loaded(m1, 'd1'), loaded(m2, 'd2')], bq, 'p'); expect(errs).toEqual([]); expect(calls).toBe(1); }); From 32ae888c05ccd0f85de8c60246696ecb05901aff Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 04:07:56 +0000 Subject: [PATCH 13/34] mdcode: document REST-catalog sources; clarify Knowledge Catalog mapping --- toolbox/mdcode/docs/semantic-model.md | 60 ++++++++++++++++----------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index b5cd22c3..b82c3f65 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -85,6 +85,14 @@ Each entity's `source` is its backing BigQuery table. A `source` written as `` from `init`, *not* your ambient `gcloud` project. Write the full `project.dataset.table` when a table lives in another project. +Sources are not limited to native BigQuery tables. A name with more than +three parts — for example a four-part `catalog.database.schema.table` +reference — points at a table in a **federated REST catalog**, such as an +Apache Iceberg table exposed through BigLake. Write it exactly as BigQuery +resolves the name; it is emitted into the property graph verbatim, and +validation resolves it the same way the deploy does (see +[Validation](#validation)). + ## 2. Push ```bash @@ -120,22 +128,22 @@ warns. Nothing runs under `--validate-only`; add `--print` to see the DDL. ### What gets created in Knowledge Catalog -The model becomes catalog **entries** of the built-in system types under -`dataplex-types/global` (push references these types; it never creates them): - -| Entry | Represents | -|-------|-----------| -| `semantic-model` | the model — the anchor / parent of the others | -| `semantic-entity` | one entity — carries its `schema` (fields + semantics) | -| `semantic-metric` | one metric — its data type and expression | - -Entry ids are derived from names: ``, `.entities.`, and -`.metrics.`. - -Each **relationship** becomes a `schema-join` **entry link** between the two -entity entries; the join detail (paired columns, foreign-key direction) lives in -the link's aspect. Many-to-many (association / junction-table) relationships are -**not** published to the catalog yet — push warns and skips them, though the edge +Each element of your model maps to one catalog resource. Every resource type +below is a built-in system type under `dataplex-types/global` — push references +them, it never creates them. + +| Model element | Catalog resource | Kind | Id | +|---|---|---|---| +| the model | `semantic-model` | entry — anchor / parent of the rest | `` | +| an entity | `semantic-entity` (+ built-in `schema` aspect) | entry | `.entities.` | +| a metric | `semantic-metric` | entry | `.metrics.` | +| a relationship (1:1, 1:N) | `schema-join` | entry link between the two entity entries | derived from the relationship name | +| a relationship (M:N) | *(none yet)* | not published — push warns and skips | — | + +An entity entry carries its columns and per-field semantics in the `schema` +aspect; a `schema-join` link carries the join detail — the paired columns and +foreign-key direction — in its aspect. A many-to-many (association / +junction-table) relationship is not published to the catalog yet, but the edge still exists in the BigQuery property graph. > **Note:** only the canonical `expression` (GoogleSQL/ANSI) is written to the @@ -153,11 +161,13 @@ touched**, so a model that cannot deploy fails fast instead of half-deploying: otherwise it cannot lower to a MEASURE and would be dropped from the graph. Set the metric's attach entity, or scope its expression to a single entity. *(static)* -* **Every entity's source table is reachable.** Each `source` that is a plain - `project.dataset.table` is probed against BigQuery; a table that does not exist - or that you cannot access fails the push, naming the table and the entity. - Sources that are queries or non-table references are skipped. *(live — needs - BigQuery read access; see [Permissions](#permissions))* +* **Every entity's source table is reachable.** Each `source` is probed with a + dry-run query, so BigQuery resolves it exactly as the deploy will — a + three-part `project.dataset.table`, a four-part federated REST-catalog name + (e.g. an Iceberg table via BigLake), or a quoted identifier all work. A table + that does not exist or that you cannot access fails the push, naming the table + and the entity; a `source` that is a query (not a table) is skipped. *(live — + needs BigQuery access; see [Permissions](#permissions))* The live table check runs for **every** `--target`, because the same tables back both destinations — so even a Knowledge-Catalog-only push confirms the model @@ -191,11 +201,11 @@ Wrote 5 new and 2 updated Knowledge Catalog entries; removed 1 orphaned entry; l `push` needs access to whichever destinations you deploy to. -**BigQuery** — for `--target bq` or `all`, and for the validation table check: +**BigQuery** — for `--target bq` or `all`, and for the validation pre-flight: -* permission to run `CREATE OR REPLACE PROPERTY GRAPH` in the target dataset - (create/replace tables and run jobs) -* `bigquery.tables.get` on each entity's source table (validation pre-flight) +* `bigquery.jobs.create` in the deployment-target project — to run the deploy's + `CREATE OR REPLACE PROPERTY GRAPH` and the validation dry-run query +* read access on each entity's source table, so the dry-run can resolve it * `bigquery.datasets.get` on the target dataset (region detection; optional — push degrades gracefully without it) From 5495d4a88fc5e613b977e45c1228ae8ea2308a30 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 04:19:50 +0000 Subject: [PATCH 14/34] mdcode: address KC-push guide/options review comments - PushOptions: comment every field; clarify --force (generic CatalogSync push) vs --force-remove (semantic-model KC deletion authorization). - Guide: add a BigQuery mapping table mirroring the Knowledge Catalog one; drop articles from the mapping tables' Model element column. - Guide: make the importedExpression note explicit that the catalog is not a full copy of the model (vendor SQL is not stored; document is source of truth). - Guide: rewrite 'Updating and removing models' from the user's perspective. --- toolbox/mdcode/docs/semantic-model.md | 84 +++++++++++++++++---------- toolbox/mdcode/src/tool/commands.ts | 9 +++ 2 files changed, 63 insertions(+), 30 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index b82c3f65..e3280e92 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -120,11 +120,21 @@ never half-deploys. ### What gets created in BigQuery -`push` executes `CREATE OR REPLACE PROPERTY GRAPH` in the project and dataset -named by each deployment target. It reads the target dataset's location -(`bigquery.datasets.get`) so the statement runs in the right region; if it can't -(missing permission), it falls back to BigQuery's own location inference and -warns. Nothing runs under `--validate-only`; add `--print` to see the DDL. +`push` executes a single `CREATE OR REPLACE PROPERTY GRAPH` per deployment +target, in the project and dataset the target names. Each part of your model +becomes one part of that graph: + +| Model element | BigQuery construct | Notes | +|---|---|---| +| Model | one `PROPERTY GRAPH` | named by the deployment-target URI | +| Entity | a `NODE TABLE` | backed by the entity's `source` table, keyed by its primary key | +| Relationship | an `EDGE TABLE` | connects the two entities' node tables | +| Metric | a `MEASURE` on a node table | must reduce to one aggregate over one entity, or it is skipped with a warning | + +`push` reads the target dataset's location (`bigquery.datasets.get`) so the +statement runs in the right region; without that permission it falls back to +BigQuery's own location inference and warns. Nothing runs under +`--validate-only`; add `--print` to see the DDL. ### What gets created in Knowledge Catalog @@ -134,11 +144,11 @@ them, it never creates them. | Model element | Catalog resource | Kind | Id | |---|---|---|---| -| the model | `semantic-model` | entry — anchor / parent of the rest | `` | -| an entity | `semantic-entity` (+ built-in `schema` aspect) | entry | `.entities.` | -| a metric | `semantic-metric` | entry | `.metrics.` | -| a relationship (1:1, 1:N) | `schema-join` | entry link between the two entity entries | derived from the relationship name | -| a relationship (M:N) | *(none yet)* | not published — push warns and skips | — | +| Model | `semantic-model` | entry — anchor / parent of the rest | `` | +| Entity | `semantic-entity` (+ built-in `schema` aspect) | entry | `.entities.` | +| Metric | `semantic-metric` | entry | `.metrics.` | +| Relationship (1:1, 1:N) | `schema-join` | entry link between the two entity entries | derived from the relationship name | +| Relationship (M:N) | *(none yet)* | not published — push warns and skips | — | An entity entry carries its columns and per-field semantics in the `schema` aspect; a `schema-join` link carries the join detail — the paired columns and @@ -146,10 +156,13 @@ foreign-key direction — in its aspect. A many-to-many (association / junction-table) relationship is not published to the catalog yet, but the edge still exists in the BigQuery property graph. -> **Note:** only the canonical `expression` (GoogleSQL/ANSI) is written to the -> catalog. The original vendor SQL (`importedExpression`) stays in your source -> model — it is still used as a fallback when generating BigQuery SQL — but the -> catalog has no consumer for it. +> **Note — the catalog is not a full copy of your model.** Only the canonical +> `expression` (GoogleSQL/ANSI) is written to Knowledge Catalog; the original +> vendor SQL (`importedExpression` — e.g. the MAQL or Snowflake form a metric +> was imported from) is **not**. It stays in your authored document, and is still +> used when generating BigQuery SQL, but nothing in the catalog stores it. Keep +> your model document as the source of truth: a model reconstructed only from +> the catalog would come back without its vendor SQL. ## Validation @@ -175,22 +188,33 @@ could deploy to BigQuery. ## Updating and removing models -`push` is idempotent: re-running reconciles each destination to match your -current model. - -* **Re-push updates in place.** An existing entry or link is upserted. -* **Removed entity or metric** — its now-orphaned catalog entry is deleted. - (`removed N orphaned entr(y|ies)`) -* **Removed or renamed relationship** — the stale `schema-join` link is deleted - after the current links are written, so a rename never leaves the pair - unlinked. (`unlinked N orphaned link(s)`) Links owned by other models in a - shared entry group are never touched. -* **Removed or renamed whole model** — if the entry group still contains a model - your push no longer includes, push **refuses** and names it, rather than - leaving a stale model or silently deleting one. Re-run with **`--force-remove`** - to delete that model's entries and links first. - -Each destination prints a one-line summary. For a `--target all` push: +Your model document is the source of truth. To change what is deployed, edit +the document and run `kcmd push` again — you never edit the catalog or the +BigQuery graph by hand. Re-running is safe: each push makes the destinations +match the document as it stands now. + +**When you edit an entity, metric, or relationship** — push overwrites the +existing one in place; you don't get duplicates. + +**When you delete an entity or metric** — remove it from the document and +push. Push deletes the leftover catalog entry for you (the summary shows +`removed N orphaned entries`). + +**When you rename or delete a relationship** — push removes the old +`schema-join` link after writing the new ones, so a rename never leaves the +two entities disconnected (the summary shows `unlinked N orphaned links`). +Relationships owned by other models that share the entry group are left +untouched. + +**When you remove a whole model** — deleting its document does *not* remove it +from the catalog on the next push. Instead, push stops and names the model the +catalog still has that you no longer push, so you can't wipe out a model by +accident or by pushing from the wrong directory. When you do mean to remove it, +run push again with `--force-remove` and its entries and links are deleted +first. + +Every push prints one line per destination summarizing what it did. For a +`--target all` push: ``` Deployed 1 BigQuery Graph(s). diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index 0f4a964b..f2dbabf8 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -27,11 +27,20 @@ export interface InitOptions { export interface PushOptions { + // Generic push flag for non-semantic-model (CatalogSync) scopes; + // forwarded to CatalogSync.push. The semantic-model legs ignore it. + // The catch-all "force the push" toggle -- distinct from + // `forceRemove` below, which specifically authorizes deleting models + // this push no longer includes. force?: boolean; + // Run every validation check and report pass/fail, but write nothing + // to any destination (a dry run). Applies to both push paths. validateOnly?: boolean; // Delete Knowledge Catalog models already in the entry group that this push // does not include (a removed or renamed model). Without it, an unrecognized // model in the group fails the push. Semantic-model KC push only. + // Unlike `force` above, this authorizes a destructive delete rather + // than overriding a conflict. forceRemove?: boolean; // Semantic-model push destination(s): 'bq', 'kc', 'all' (default), or a // comma-separated list (e.g. 'bq,kc'). Ignored for non-semantic-model scopes. From 97fd0a69571e3032701f913cd027ac9d90af2f3c Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 04:26:40 +0000 Subject: [PATCH 15/34] mdcode: refactor deployKnowledgeCatalog into phase helpers; comment KC deploy fields - Split deployKnowledgeCatalog into named phase helpers (emitModels, checkEntryIdCollisions, buildPlan, guardForeignModels, writeModels) so the function body reads as an explicit sequence of steps; behavior unchanged. - KcDeployOptions and KcDeployResult: a comment on every field. - Rewrite reconcileDeletions' doc comment from the user's perspective. - Fix a stale doc comment that referenced a non-existent defaultProject param. --- .../semantic/deploy_knowledge_catalog.ts | 364 ++++++++++-------- 1 file changed, 212 insertions(+), 152 deletions(-) diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index b0ec7e18..0f9596ef 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -43,49 +43,77 @@ import {LoadedModel} from './loader'; export interface KcDeployOptions { - // The resolved Knowledge Catalog destination (flag overrides applied over the - // catalog.yaml scope defaults). + // Project that owns the destination entry group. Flag overrides are applied + // over the catalog.yaml scope defaults before this is set. project: string; + // Location (region) of the destination entry group, e.g. `global` or `us`. location: string; + // Id of the destination entry group (provisioned at `init`, not by push). entryGroup: string; - // Where the built-in `semantic-*` / `schema` system types are referenced. - // Default: `dataplex-types` / `global`. Overridable to reference them from a - // staging project; the emitted entries are otherwise unchanged. + // Project the built-in `semantic-*` / `schema` system types are referenced + // from. Default `dataplex-types`. Override to reference them from a staging + // project; the emitted entries are otherwise unchanged. systemTypeProject?: string; + // Location the built-in system types are referenced from. Default `global`. systemTypeLocation?: string; - // Compile + report only; never writes. + // Compile and report only; never writes to the catalog (a dry run). validateOnly?: boolean; // Delete models already in the entry group that this push does not re-emit -- // a removed or renamed model's entries and links. Without it, an unrecognized // model in the group is a hard error rather than a silent orphan. forceRemove?: boolean; - // entries.create can briefly 404 on a just-created entry group; retry that - // window. Overridable so tests can exercise the path without burning - // wall-clock. + // How many times to try entries.create before giving up: a just-created entry + // group can briefly 404, and the create path retries that window. Overridable + // so tests exercise the retry without burning wall-clock. Default + // ENTRY_CREATE_TRIES. entryCreateTries?: number; + // Delay between entries.create retries, in ms. Default ENTRY_CREATE_RETRY_MS. entryCreateRetryMs?: number; } export interface KcDeployResult { + // Whether the push (or --validate-only run) completed without error. success: boolean; + // On failure, a human-readable reason; unset on success. details?: string; - // Loader and emitter warnings collected across all documents. + // Loader and emitter warnings collected across all documents (e.g. a skipped + // many-to-many relationship, or a metric that could not be lowered). warnings: string[]; - // Entries created / updated-in-place / deleted (all 0 for validateOnly). + // New entries created in the entry group (0 for validateOnly). created: number; + // Existing entries updated in place by an idempotent re-push (0 for + // validateOnly). updated: number; + // Entries deleted: those orphaned by removed entities/metrics, plus every + // entry of a --force-remove'd model (0 for validateOnly). deleted: number; // Relationship (schema-join) entry links written -- created or upserted (0 for // validateOnly). linked: number; - // Orphaned schema-join links deleted -- from relationships dropped/renamed on a - // still-present model and from force-removed models (0 for validateOnly). + // Orphaned schema-join links deleted -- from relationships dropped or renamed + // on a still-present model, and from force-removed models (0 for validateOnly). unlinked: number; - // A human-readable plan of what would be written; populated for validateOnly. + // A human-readable plan of what would be written: the sole output of a + // validateOnly run, and also returned (for --print) on a real push. plan: string[]; } +// Running tallies threaded through the write phase so a partial failure still +// reports what had been done. Field names match KcDeployResult so this spreads +// straight into the result. +interface Counts { + created: number; + updated: number; + deleted: number; + linked: number; + unlinked: number; +} + +// One authored model paired with the catalog resources it emitted. +type EmittedModel = {model: string; resources: KcResources}; + + // entries.create propagation retry: a just-created entry group can briefly 404. const ENTRY_CREATE_TRIES = 3; const ENTRY_CREATE_RETRY_MS = 3000; @@ -95,35 +123,99 @@ function sleep(ms: number): Promise { } -// Deploys the Knowledge Catalog resources for each authored model document. -// Emits no console output; warnings and the dry-run plan are returned for the -// caller to print. `defaultProject` qualifies a dataset `source` that omits its -// project (the scope's declared project, a deterministic user-authored value). +// Deploys every authored model's Knowledge Catalog resources. The body is the +// sequence of phases, each a helper below: +// emitModels -- turn every model into catalog resources (pure) +// checkEntryIdCollisions -- fail if two models want the same entry id +// buildPlan -- the dry-run plan (and stop here for --validate-only) +// listEntryGroup -- snapshot the group once, before any write +// guardForeignModels -- refuse (or --force-remove) models no longer pushed +// writeModels -- create/upsert each model's entries and links +// reconcileDeletions -- delete entries orphaned by removed entities/metrics +// Emits no console output; warnings and the plan are returned in KcDeployResult +// for the caller (commands.ts) to print. export async function deployKnowledgeCatalog( models: LoadedModel[], ctx: context.ApiContext, opts: KcDeployOptions): Promise { const warnings: string[] = []; - const plan: string[] = []; - let created = 0; - let updated = 0; - let deleted = 0; - let linked = 0; - let unlinked = 0; - let modelsSeen = 0; + const counts: Counts = + {created: 0, updated: 0, deleted: 0, linked: 0, unlinked: 0}; + let plan: string[] = []; + // Builds the return value from the running state; details is set only on a + // failure, and counts/plan reflect whatever had been done when called. + const result = (success: boolean, details?: string): KcDeployResult => + ({success, warnings, ...counts, plan, ...(details ? {details} : {})}); + + // Emit every model to catalog resources up front (pure -- no network). + const emit = emitModels(models, opts); + warnings.push(...emit.warnings); + if (emit.error) return result(false, emit.error); + const emitted = emit.emitted; + + // A parsed document always yields at least one model (the loader enforces + // `semantic_model` min 1), so an empty `emitted` means no documents were + // found. validateOnly mutates nothing, so that is a clean no-op; a real push + // treats an empty workspace as a configuration error worth flagging. + if (!emitted.length) { + if (opts.validateOnly) { + warnings.push('No semantic model documents found; nothing to validate.'); + return result(true); + } + return result( + false, 'No semantic model documents found; nothing to deploy.'); + } + + // Fail before any write if two models generate the same entry id. + const collision = checkEntryIdCollisions(emitted, opts); + if (collision) return result(false, collision); + + // The plan is built for every push (printed with --print) and is the only + // output of a --validate-only run, which writes nothing. + plan = buildPlan(emitted, opts); + if (opts.validateOnly) return result(true); + + // From here on we write. The entry group is provisioned at `init`, not here; + // a missing group surfaces as a clear entry-creation error, and createEntries + // rides out the brief post-init propagation window. + const cat = new CatalogClient(ctx); + + // Snapshot the entry group once, before any write: the same listing feeds the + // foreign-model guard, link reconciliation, and deletion reconciliation. A + // re-emitted entry is never a deletion candidate, so a pre-write snapshot is + // correct for all three. + const listing = await listEntryGroup(cat, opts); + if (listing.error) return result(false, listing.error); + const existing = listing.entries; + + // Whole-model lifecycle: refuse (or, with --force-remove, delete) any model + // the group still holds that this push no longer includes. + const guard = await guardForeignModels(cat, opts, emitted, existing, counts); + if (guard.error) return result(false, guard.error); + + // Write each model's entries and relationship links. + const written = await writeModels(cat, opts, emitted, existing, counts); + if (written.error) return result(false, written.error); + + // Finally, delete entries orphaned by entities/metrics removed from a + // still-present model since its last push. + const recon = await reconcileDeletions(cat, opts, emitted, existing); + if (recon.error) return result(false, recon.error); + counts.deleted += recon.deleted; + + return result(true); +} - const fail = (details: string): KcDeployResult => - ({success: false, details, warnings, created, updated, deleted, linked, - unlinked, plan}); - // Emit every model up front (pure): dry-run and warnings need no network, and - // a generation warning surfaces even if a later write fails. - const emitted: {model: string; resources: KcResources}[] = []; +// Turns every authored model into its catalog resources. Pure -- no network I/O +// -- so the dry-run plan and any generation warnings are produced even when a +// later write fails. A malformed GOOGLE custom_extension (reached via the +// semantic-model aspect) throws; report it against its document, as the BigQuery +// leg does, rather than letting it escape as an uncaught stack trace. +function emitModels(models: LoadedModel[], opts: KcDeployOptions): + {emitted: EmittedModel[]; warnings: string[]; error?: string} { + const emitted: EmittedModel[] = []; + const warnings: string[] = []; for (const {document, model} of models) { - modelsSeen++; - // The emitter is pure but not infallible: bigQueryGraphTargets (reached - // via the semantic-model aspect) throws on a malformed GOOGLE - // custom_extension. Report it against the document -- as the BigQuery leg - // does -- rather than letting it escape as an uncaught stack trace. let resources: KcResources; try { resources = generateCatalogResources(model, { @@ -134,142 +226,113 @@ export async function deployKnowledgeCatalog( systemTypeLocation: opts.systemTypeLocation, }); } catch (err: any) { - return fail( - `Model '${model.name}' (${document}): ${err.message || err}`); - } - for (const w of resources.warnings) { - warnings.push(`[${model.name}] ${w}`); + return { + emitted, warnings, + error: `Model '${model.name}' (${document}): ${err.message || err}`, + }; } + for (const w of resources.warnings) warnings.push(`[${model.name}] ${w}`); emitted.push({model: model.name, resources}); } + return {emitted, warnings}; +} - // A parsed document always yields at least one model (the loader enforces - // `semantic_model` min 1), so modelsSeen is 0 only when no documents were - // found. validateOnly mutates nothing, so an empty workspace is a clean no-op - // there; a real push treats it as a configuration error worth flagging. - if (!modelsSeen) { - if (opts.validateOnly) { - warnings.push('No semantic model documents found; nothing to validate.'); - return {success: true, warnings, created, updated, deleted, linked, - unlinked, plan}; - } - return fail('No semantic model documents found; nothing to deploy.'); - } - // Entry ids must be unique within the destination entry group. The emitter - // dedups within one model, but two models in a single push (two documents, or - // two `semantic_model`s in one document) whose names normalize to the same id - // generate colliding entry names; on publish the later one would 409 and - // silently upsert over the earlier. Catch that across models here and fail - // before any write, naming the entry so the author can rename one model. The - // owner is tracked by index, not model name, so two same-named models (the - // most common collision) are still distinguished. +// Fails (returns the error message) if two models in this push generate the same +// catalog entry id. Entry ids must be unique within the entry group; the emitter +// dedups within one model, but two models (two documents, or two `semantic_model`s +// in one document) whose names normalize to the same id would collide -- on +// publish the later one 409s and silently upserts over the earlier. Catch it here +// and name the entry so the author can rename a model. Ownership is tracked by +// index, not model name, so two same-named models (the common collision) are +// still distinguished. +function checkEntryIdCollisions( + emitted: EmittedModel[], opts: KcDeployOptions): string|undefined { const entryOwner = new Map(); for (let i = 0; i < emitted.length; i++) { for (const entry of emitted[i].resources.entries) { const prev = entryOwner.get(entry.name); if (prev !== undefined && prev !== i) { - return fail( - `models '${emitted[prev].model}' and '${emitted[i].model}' both ` + - `generate catalog entry '${idOf(entry.name)}'; entry ids must be ` + - `unique within entry group '${opts.entryGroup}' -- rename one ` + - `model.`); + return `models '${emitted[prev].model}' and '${emitted[i].model}' ` + + `both generate catalog entry '${idOf(entry.name)}'; entry ids must ` + + `be unique within entry group '${opts.entryGroup}' -- rename one ` + + `model.`; } entryOwner.set(entry.name, i); } } + return undefined; +} + +// The full dry-run plan across all models (one block per model; see planSummary). +function buildPlan(emitted: EmittedModel[], opts: KcDeployOptions): string[] { + const plan: string[] = []; for (const {model, resources} of emitted) { plan.push(...planSummary(model, resources, opts)); } - if (opts.validateOnly) { - return {success: true, warnings, created, updated, deleted, linked, unlinked, - plan}; - } - - // The destination entry group is provisioned at `init`, not here: push writes - // only entries, matching how the standard layout's push operates (it creates - // entries, never the entry group). A missing group surfaces as a clear entry - // creation error, and createEntryWithRetry rides out the brief post-init - // propagation window. - const cat = new CatalogClient(ctx); + return plan; +} - // Snapshot the destination entry group once, before any write. The same - // listing feeds the foreign-model guard below and deletion reconciliation at - // the end; a re-emitted entry is never a deletion candidate, so a pre-write - // snapshot stays correct there. - const listing = await listEntryGroup(cat, opts); - if (listing.error) { - return fail(listing.error); - } - const existing = listing.entries; - // Whole-model lifecycle: a `semantic-model` anchor already in the group whose - // id this push does not re-emit belongs to a model whose document is gone -- a - // removed or renamed model. Leaving it silently accumulates stale models in - // the group, so refuse the push, unless --force-remove authorizes deleting - // each such model's links and entries first. +// Whole-model lifecycle guard. A `semantic-model` anchor already in the group +// whose id this push does not re-emit belongs to a model whose document is gone +// (removed or renamed). Refuse the push and name them -- unless --force-remove, +// which deletes each such model's links and entries first (tallied into counts). +async function guardForeignModels( + cat: CatalogClient, opts: KcDeployOptions, emitted: EmittedModel[], + existing: Entry[], counts: Counts): Promise<{error?: string}> { const pushedAnchors = new Set(emitted.map(e => idOf(e.resources.entries[0].name))); const foreignAnchors = existing .filter(e => (e.entryType ?? '').endsWith('/semantic-model')) .map(e => idOf(e.name)) .filter(id => !pushedAnchors.has(id)); - if (foreignAnchors.length && !opts.forceRemove) { - return fail( - `entry group '${opts.entryGroup}' already contains model(s) this push ` + - `does not include: ${foreignAnchors.join(', ')}. Re-run with ` + - `--force-remove to delete them, or add their documents to this push.`); - } - if (foreignAnchors.length) { - const removed = - await removeForeignModels(cat, opts, existing, foreignAnchors); - if (removed.error) { - return fail(removed.error); - } - deleted += removed.deleted; - unlinked += removed.unlinked; + if (!foreignAnchors.length) return {}; + if (!opts.forceRemove) { + return { + error: + `entry group '${opts.entryGroup}' already contains model(s) this ` + + `push does not include: ${foreignAnchors.join(', ')}. Re-run with ` + + `--force-remove to delete them, or add their documents to this push.`, + }; } + const removed = await removeForeignModels(cat, opts, existing, foreignAnchors); + if (removed.error) return {error: removed.error}; + counts.deleted += removed.deleted; + counts.unlinked += removed.unlinked; + return {}; +} + +// Writes every model's entries and relationship links, in model order. For each +// model: create/upsert its entries (anchor first), write its schema-join links +// (both endpoints must exist first), then drop any link it owns but no longer +// emits (a dropped or renamed relationship). Progress accumulates into counts, so +// a mid-way failure still reports what had been written. +async function writeModels( + cat: CatalogClient, opts: KcDeployOptions, emitted: EmittedModel[], + existing: Entry[], counts: Counts): Promise<{error?: string}> { for (const {model, resources} of emitted) { - const outcome = await createEntries(cat, opts, resources.entries); - if (outcome.error) { - return fail(`Model '${model}': ${outcome.error}`); - } - created += outcome.created; - updated += outcome.updated; + const entries = await createEntries(cat, opts, resources.entries); + if (entries.error) return {error: `Model '${model}': ${entries.error}`}; + counts.created += entries.created; + counts.updated += entries.updated; - // Links reference this model's entity entries, so they are written after the - // entries above (both endpoints must exist first). + // Links reference this model's entity entries, so they follow the entries + // above (both endpoints must exist first). const links = await createEntryLinks(cat, opts, resources.entryLinks); - if (links.error) { - return fail(`Model '${model}': ${links.error}`); - } - linked += links.linked; + if (links.error) return {error: `Model '${model}': ${links.error}`}; + counts.linked += links.linked; // Then drop any schema-join link this model owns but no longer emits (a - // relationship dropped or renamed). Runs after its current links are - // written, so a rename never leaves the pair with no link between them. + // relationship dropped or renamed), after its current links are written so a + // rename never leaves the pair with no link between them. const relLinks = await reconcileLinks(cat, opts, resources, existing); - if (relLinks.error) { - return fail(`Model '${model}': ${relLinks.error}`); - } - unlinked += relLinks.unlinked; - } - - // Reconcile deletions: an entity or metric removed from a still-present model - // since the last push leaves an orphaned entry under the model's anchor. - // Delete any entry this push owns that was not re-emitted, from the pre-write - // listing (see reconcileDeletions). Runs only on a real push -- validateOnly - // returned above and never lists remote state. - const recon = await reconcileDeletions(cat, opts, emitted, existing); - if (recon.error) { - return fail(recon.error); + if (relLinks.error) return {error: `Model '${model}': ${relLinks.error}`}; + counts.unlinked += relLinks.unlinked; } - deleted += recon.deleted; - - return {success: true, warnings, created, updated, deleted, linked, unlinked, - plan}; + return {}; } @@ -278,23 +341,20 @@ interface ReconcileOutcome { error?: string; } -// Deletes entries this push OWNS but did not re-emit -- the entities/metrics -// removed from a model since its last push. Ownership is scoped by entry id so -// reconciliation never touches entries outside the models in this push (a -// shared entry group may legitimately hold others): an existing entry is owned -// when its id is a pushed model's anchor id or is prefixed by that anchor's -// `.entities.` / `.metrics.` child namespace. An anchor is always -// re-emitted, so it is never deleted here. +// Removes the catalog entries left behind when you delete an entity or metric +// from a model and push again -- the entries the model no longer emits. Only +// entries this push OWNS are ever touched: an entry whose id is a pushed model's +// anchor, or that lives under that anchor's `.entities.` / +// `.metrics.` namespace. Entries belonging to other models that share the +// entry group are left alone, and an anchor (always re-emitted) is never deleted +// here. // -// Whole-model removal (an anchor no longer in this push) is handled separately by -// the --force-remove guard in deployKnowledgeCatalog; orphaned relationship links -// are handled by reconcileLinks. This step operates on `existing` -- the -// entry-group listing captured once before writes -- so it never issues its own -// list call; a re-emitted entry is never a deletion candidate, so the pre-write -// snapshot stays correct. +// Deleting a whole model is handled by the --force-remove guard, and orphaned +// relationship links by reconcileLinks. This step reads the pre-write snapshot +// `existing`, so it issues no list call of its own (a re-emitted entry is never a +// deletion candidate, so the snapshot stays correct). function reconcileDeletions( - cat: CatalogClient, opts: KcDeployOptions, - emitted: {resources: KcResources}[], + cat: CatalogClient, opts: KcDeployOptions, emitted: EmittedModel[], existing: Entry[]): Promise { const emittedIds = new Set(); const anchorIds = new Set(); From 6c1076d82d14477910226170c827eb837f421fe4 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 04:29:31 +0000 Subject: [PATCH 16/34] mdcode: capitalize BigQuery Graph; drop articles from BigQuery construct column --- toolbox/mdcode/docs/semantic-model.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index e3280e92..646890e5 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -126,10 +126,10 @@ becomes one part of that graph: | Model element | BigQuery construct | Notes | |---|---|---| -| Model | one `PROPERTY GRAPH` | named by the deployment-target URI | -| Entity | a `NODE TABLE` | backed by the entity's `source` table, keyed by its primary key | -| Relationship | an `EDGE TABLE` | connects the two entities' node tables | -| Metric | a `MEASURE` on a node table | must reduce to one aggregate over one entity, or it is skipped with a warning | +| Model | `PROPERTY GRAPH` | named by the deployment-target URI | +| Entity | `NODE TABLE` | backed by the entity's `source` table, keyed by its primary key | +| Relationship | `EDGE TABLE` | connects the two entities' node tables | +| Metric | `MEASURE` on a node table | must reduce to one aggregate over one entity, or it is skipped with a warning | `push` reads the target dataset's location (`bigquery.datasets.get`) so the statement runs in the right region; without that permission it falls back to @@ -170,8 +170,8 @@ still exists in the BigQuery property graph. touched**, so a model that cannot deploy fails fast instead of half-deploying: * **At least one deployment target per model.** *(static)* -* **Every metric on a BigQuery-graph model resolves to exactly one entity** — - otherwise it cannot lower to a MEASURE and would be dropped from the graph. Set +* **Every metric on a BigQuery Graph model resolves to exactly one entity** — + otherwise it cannot lower to a MEASURE and would be dropped from the BigQuery Graph. Set the metric's attach entity, or scope its expression to a single entity. *(static)* * **Every entity's source table is reachable.** Each `source` is probed with a @@ -190,7 +190,7 @@ could deploy to BigQuery. Your model document is the source of truth. To change what is deployed, edit the document and run `kcmd push` again — you never edit the catalog or the -BigQuery graph by hand. Re-running is safe: each push makes the destinations +BigQuery Graph by hand. Re-running is safe: each push makes the destinations match the document as it stands now. **When you edit an entity, metric, or relationship** — push overwrites the From 25a0b496193807b1804841ff3c3905dac7aef65f Mon Sep 17 00:00:00 2001 From: Bei Li Date: Tue, 11 Aug 2026 21:01:39 +0000 Subject: [PATCH 17/34] mdcode: address semantic-model guide review comments - Use 'Knowledge Catalog' throughout (drop 'Dataplex'); fix Ossie link to ossie.apache.org. - Author section: 1 model per entry group (my-project.us-central1.my_model); model name matches .yaml; single deployment target for now; dialect: BIGQUERY (1P yaml); drop redundant 'ambient gcloud project' and 'emitted verbatim' asides. - Drop many-to-many from the Knowledge Catalog mapping (Ossie has no M:N yet); 'schema-join link carries the relationship detail'; tighten the metric validation note. - Remove the Permissions section for now (add back once the list is final); drop its inbound links. --- toolbox/mdcode/docs/semantic-model.md | 73 +++++++++------------------ 1 file changed, 25 insertions(+), 48 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 646890e5..8ab2cdf1 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -2,19 +2,19 @@ A *semantic model* describes your entities (tables), the metrics computed over them, and the relationships between them, authored as a single -[Apache Ossie](https://ossie.dev) document. `kcmd push` deploys one model to two -destinations at once: +[Apache Ossie](https://ossie.apache.org/) document. `kcmd push` deploys one +model to two destinations at once: * **BigQuery** — a queryable `CREATE OR REPLACE PROPERTY GRAPH` over the model's tables, so the model can be traversed and its metrics computed in SQL. -* **Knowledge Catalog (Dataplex)** — catalog entries and links that make the +* **Knowledge Catalog** — catalog entries and links that make the model discoverable as metadata. Both are generated from the same source document — you never author them separately, and a single `push` keeps them in sync. This guide covers authoring, deploying, and updating a model. For the Ossie -document format itself, see [ossie.dev](https://ossie.dev). +document format itself, see [ossie.apache.org](https://ossie.apache.org/). ## Prerequisites @@ -24,19 +24,18 @@ document format itself, see [ossie.dev](https://ossie.dev). gcloud auth application-default login ``` -You also need read/write access to whichever destinations you deploy to — see -[Permissions](#permissions). +You also need read/write access to whichever destinations you deploy to. ## 1. Author a model -Create the local layout. The scope is the Dataplex EntryGroup the model will be -published to, written as `..`: +Create the local layout. The scope is the Knowledge Catalog entry group the +model will be published to, written as `..`: ```bash -kcmd init --semantic-model my-project.us.my_models +kcmd init --semantic-model my-project.us-central1.my_model ``` -`init` provisions that EntryGroup (idempotent — an existing group is fine) and +`init` provisions that entry group (idempotent — an existing group is fine) and creates its local directory. Author the model at `catalog/EntryGroups//.yaml`: @@ -44,9 +43,10 @@ creates its local directory. Author the model at version: "0.2.0.dev0" semantic_model: - - name: sales - # Required: at least one deployment target, in a GOOGLE custom extension. - # `data` is a JSON string whose deploymentTargets is a list of graph URIs. + - name: sales # must match the .yaml filename + # Required: the deployment target, in a GOOGLE custom extension. `data` is a + # JSON string whose deploymentTargets holds the target graph URI (for now, + # exactly one). custom_extensions: - vendor_name: GOOGLE data: '{"deploymentTargets": ["//bigquery.googleapis.com/projects/my-project/datasets/sales/propertyGraphs/sales_graph"]}' @@ -56,12 +56,12 @@ semantic_model: primary_key: [o_orderkey] fields: - name: o_orderkey - expression: {dialects: [{dialect: ANSI_SQL, expression: o_orderkey}]} + expression: {dialects: [{dialect: BIGQUERY, expression: o_orderkey}]} - name: o_totalprice - expression: {dialects: [{dialect: ANSI_SQL, expression: o_totalprice}]} + expression: {dialects: [{dialect: BIGQUERY, expression: o_totalprice}]} metrics: - name: total_revenue - expression: {dialects: [{dialect: ANSI_SQL, expression: SUM(orders.o_totalprice)}]} + expression: {dialects: [{dialect: BIGQUERY, expression: SUM(orders.o_totalprice)}]} ``` ### Deployment targets (required) @@ -82,15 +82,14 @@ no deployment target is rejected at push time (see [Validation](#validation)). Each entity's `source` is its backing BigQuery table. A `source` written as `dataset.table` (two parts) is qualified with the scope's project — the -`` from `init`, *not* your ambient `gcloud` project. Write the full -`project.dataset.table` when a table lives in another project. +`` from `init`. Write the full `project.dataset.table` when a table +lives in another project. Sources are not limited to native BigQuery tables. A name with more than three parts — for example a four-part `catalog.database.schema.table` reference — points at a table in a **federated REST catalog**, such as an Apache Iceberg table exposed through BigLake. Write it exactly as BigQuery -resolves the name; it is emitted into the property graph verbatim, and -validation resolves it the same way the deploy does (see +resolves the name, and validation resolves it the same way the deploy does (see [Validation](#validation)). ## 2. Push @@ -147,14 +146,11 @@ them, it never creates them. | Model | `semantic-model` | entry — anchor / parent of the rest | `` | | Entity | `semantic-entity` (+ built-in `schema` aspect) | entry | `.entities.` | | Metric | `semantic-metric` | entry | `.metrics.` | -| Relationship (1:1, 1:N) | `schema-join` | entry link between the two entity entries | derived from the relationship name | -| Relationship (M:N) | *(none yet)* | not published — push warns and skips | — | +| Relationship | `schema-join` | entry link between the two entity entries | derived from the relationship name | An entity entry carries its columns and per-field semantics in the `schema` -aspect; a `schema-join` link carries the join detail — the paired columns and -foreign-key direction — in its aspect. A many-to-many (association / -junction-table) relationship is not published to the catalog yet, but the edge -still exists in the BigQuery property graph. +aspect; a `schema-join` link carries the relationship detail — the paired columns +and foreign-key direction — in its aspect. > **Note — the catalog is not a full copy of your model.** Only the canonical > `expression` (GoogleSQL/ANSI) is written to Knowledge Catalog; the original @@ -171,16 +167,15 @@ touched**, so a model that cannot deploy fails fast instead of half-deploying: * **At least one deployment target per model.** *(static)* * **Every metric on a BigQuery Graph model resolves to exactly one entity** — - otherwise it cannot lower to a MEASURE and would be dropped from the BigQuery Graph. Set - the metric's attach entity, or scope its expression to a single entity. - *(static)* + otherwise it would be dropped from the BigQuery Graph. Set the metric's attach + entity, or scope its expression to a single entity. *(static)* * **Every entity's source table is reachable.** Each `source` is probed with a dry-run query, so BigQuery resolves it exactly as the deploy will — a three-part `project.dataset.table`, a four-part federated REST-catalog name (e.g. an Iceberg table via BigLake), or a quoted identifier all work. A table that does not exist or that you cannot access fails the push, naming the table and the entity; a `source` that is a query (not a table) is skipped. *(live — - needs BigQuery access; see [Permissions](#permissions))* + needs BigQuery access)* The live table check runs for **every** `--target`, because the same tables back both destinations — so even a Knowledge-Catalog-only push confirms the model @@ -220,21 +215,3 @@ Every push prints one line per destination summarizing what it did. For a Deployed 1 BigQuery Graph(s). Wrote 5 new and 2 updated Knowledge Catalog entries; removed 1 orphaned entry; linked 2 relationships; unlinked 1 orphaned link. ``` - -## Permissions - -`push` needs access to whichever destinations you deploy to. - -**BigQuery** — for `--target bq` or `all`, and for the validation pre-flight: - -* `bigquery.jobs.create` in the deployment-target project — to run the deploy's - `CREATE OR REPLACE PROPERTY GRAPH` and the validation dry-run query -* read access on each entity's source table, so the dry-run can resolve it -* `bigquery.datasets.get` on the target dataset (region detection; optional — - push degrades gracefully without it) - -**Knowledge Catalog / Dataplex** — for `--target kc` or `all`: - -* `dataplex.entryGroups.useSemanticModelAspect` on the destination entry group -* `dataplex.entryGroups.useSchemaJoinEntryLink` and - `dataplex.entryGroups.useSchemaJoinAspect` when the model has relationships From ea7f00d081f2d7b292e49459c06c0a26e5cffbab Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 00:22:09 +0000 Subject: [PATCH 18/34] mdcode: patch entry links with aspectKeys, not a nonexistent updateMask Dataplex UpdateEntryLink has no update_mask field; it takes aspect_keys (each project.location.aspectType). writeEntryLink was sending ?updateMask=aspects on the re-push upsert path, which the server rejects with 400 'Unknown name updateMask'. Send the link's aspect-map keys instead. Verified against a live re-push: the schema-join link now PATCHes 200 and the push is idempotent. --- toolbox/mdcode/src/libts/gcp/dataplex.ts | 13 +++++++------ .../src/libts/semantic/deploy_knowledge_catalog.ts | 7 +++---- .../libts/semantic/deploy_knowledge_catalog.test.ts | 4 ++++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/toolbox/mdcode/src/libts/gcp/dataplex.ts b/toolbox/mdcode/src/libts/gcp/dataplex.ts index 58c4ca7a..8aa92daa 100644 --- a/toolbox/mdcode/src/libts/gcp/dataplex.ts +++ b/toolbox/mdcode/src/libts/gcp/dataplex.ts @@ -249,14 +249,15 @@ export class CatalogClient extends api.ApiClient { return await this._post(resourceName, entryLink, params); } - // Patches an existing entry link. Only the aspects are mutable (the entry - // references and link type are immutable server-side), so callers pass the - // link name + aspects with updateMask ['aspects']. + // Patches an existing entry link. UpdateEntryLink has no update mask: the + // aspects present in the request body are the ones written, narrowed to + // `aspectKeys` (each `project.location.aspectType`) when given. The entry + // references and link type are immutable server-side. async updateEntryLink(entryLink: EntryLink, - updateMask?: string[]): Promise> { + aspectKeys?: string[]): Promise> { const params: Record = {}; - if (updateMask && updateMask.length) { - params.updateMask = updateMask.join(','); + if (aspectKeys && aspectKeys.length) { + params.aspectKeys = aspectKeys; } return await this._patch(entryLink.name!, entryLink, params); } diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 0f9596ef..3daa6612 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -628,10 +628,9 @@ async function writeEntryLink( const res = await cat.createEntryLink( opts.project, opts.location, opts.entryGroup, linkId, link); if (isExists(res)) { - const upd = - await cat.updateEntryLink({name: link.name, aspects: link.aspects} as - EntryLink, - ['aspects']); + const upd = await cat.updateEntryLink( + {name: link.name, aspects: link.aspects} as EntryLink, + Object.keys(link.aspects ?? {})); if (!isOk(upd)) return {error: `entry link '${linkId}': ${errText(upd)}`}; return {}; } diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts index 1d7fcc0b..b8c9dc37 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts @@ -213,6 +213,10 @@ describe('deployKnowledgeCatalog: relationship entry links', () => { expect(result.linked).toBe(1); expect(createLink).toHaveBeenCalledTimes(1); expect(updateLink).toHaveBeenCalledTimes(1); + // The upsert narrows the patch to the link's aspect keys (schema-join); + // UpdateEntryLink has no update mask, so a stale ['aspects'] mask would 400. + const aspectKeys = updateLink.mock.calls[0][1] ?? []; + expect(aspectKeys.some((k: string) => k.endsWith('schema-join'))).toBe(true); }); test('a failed link write fails the push, naming the link', async () => { From 0a2fccfe122f9021608cb00e184195c74555a147 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 00:22:25 +0000 Subject: [PATCH 19/34] mdcode: gate KC SQL-expression fields behind --emit-expressions (off) Live validation against Dataplex showed the published schema and semantic-metric system types do not yet carry the SQL expression fields the emitter was writing: per-field schema.semantics{expression,role} and semantic-metric.expression. Emitting them fails the create with an aspect-parsing error. Put both behind KcGenerateOptions.emitExpressions (default off), threaded through deploy -> push -> the new --emit-expressions CLI flag, so the default push matches the live types and the fields can be flipped on once the templates gain them, without a re-add. Regenerated the affected goldens to the default-off shape. --- .../semantic/deploy_knowledge_catalog.ts | 6 + .../src/libts/semantic/knowledge_catalog.ts | 41 ++++-- toolbox/mdcode/src/tool/commands.ts | 6 + toolbox/mdcode/src/tool/main.ts | 1 + ...graph_target.knowledge_catalog.golden.json | 15 +-- ...ers_customer.knowledge_catalog.golden.json | 42 ++---- ...ds_date_edge.knowledge_catalog.golden.json | 120 +++--------------- .../libts/semantic/knowledge_catalog.test.ts | 48 +++++-- 8 files changed, 115 insertions(+), 164 deletions(-) diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 3daa6612..71512f63 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -56,6 +56,11 @@ export interface KcDeployOptions { systemTypeProject?: string; // Location the built-in system types are referenced from. Default `global`. systemTypeLocation?: string; + // Emit the SQL-expression fields not yet in the published system-type + // templates (per-field `schema.semantics` and `semantic-metric.expression`). + // Off by default so the push matches the live types; see + // KcGenerateOptions.emitExpressions. + emitExpressions?: boolean; // Compile and report only; never writes to the catalog (a dry run). validateOnly?: boolean; // Delete models already in the entry group that this push does not re-emit -- @@ -224,6 +229,7 @@ function emitModels(models: LoadedModel[], opts: KcDeployOptions): entryGroup: opts.entryGroup, systemTypeProject: opts.systemTypeProject, systemTypeLocation: opts.systemTypeLocation, + emitExpressions: opts.emitExpressions, }); } catch (err: any) { return { diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index a9ba269a..a3051b22 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -22,10 +22,14 @@ // * semantic-model = { deploymentTargets: string[] } // * semantic-entity = { source: { resources: string[], importedSystem?, // importedResource? } } -// * semantic-metric = { entity?, dataType (required), expression? } +// * semantic-metric = { entity?, dataType (required) } // * schema = { fields: [{ name, dataType, metadataType, -// description?, semantics: { expression?, -// role } }] } +// description? }] } +// +// The SQL-expression fields -- `semantic-metric.expression` and the per-field +// `schema.semantics` { expression, role } block -- are NOT in the published +// system-type templates yet, so they are gated behind +// KcGenerateOptions.emitExpressions (off by default) and omitted above. // // Relationships become `schema-join` entry links between the two entity entries. // schema-join is a built-in, undirected entry link type in `dataplex-types/global` @@ -55,6 +59,13 @@ export interface KcGenerateOptions { entryGroup: string; // destination entry group systemTypeProject?: string; // default 'dataplex-types' systemTypeLocation?: string; // default 'global' + // Emit the SQL-expression fields the published Dataplex system-type templates + // do not carry yet: the per-field `schema.semantics` block (expression + + // role) and `semantic-metric.expression`. Off by default so a push matches + // the live types; flip on once the templates gain these fields. Enabling it + // against today's types fails the push with an ASPECT_*_PARSING_FAILURE on + // the unknown property. + emitExpressions?: boolean; } export interface KcResources { @@ -85,6 +96,7 @@ export function generateCatalogResources( } const names = new Namer(opts); + const emitExpr = opts.emitExpressions ?? false; // Entry ids must be unique within the entry group. Two source names that // normalize to the same id (see slug), or exact duplicates the loader did not @@ -129,7 +141,7 @@ export function generateCatalogResources( // required_aspects: semantic-entity AND the built-in schema. aspects: aspectMap(names, { 'semantic-entity': entityAspectData(entity), - 'schema': schemaAspectData(entity), + 'schema': schemaAspectData(entity, emitExpr), }), }); } @@ -144,7 +156,7 @@ export function generateCatalogResources( parentEntry: modelEntryName, entrySource: source(metric.name, metric.description), aspects: aspectMap(names, { - 'semantic-metric': metricAspectData(metric, warnings), + 'semantic-metric': metricAspectData(metric, warnings, emitExpr), }), }); } @@ -273,7 +285,8 @@ function entityAspectData(entity: Entity): Record { // The built-in schema aspect, carrying each field's column type plus the new // per-field `semantics` block (expression / role). name, // dataType, and metadataType are required per column. -function schemaAspectData(entity: Entity): Record { +function schemaAspectData( + entity: Entity, emitExpressions: boolean): Record { return { fields: (entity.fields ?? []).map(f => compact({ @@ -281,12 +294,17 @@ function schemaAspectData(entity: Entity): Record { dataType: columnDataType(f.type), metadataType: columnMetadataType(f.type), description: f.description, - semantics: compact({ + // The per-field `semantics` block (expression + role) is + // not in the published `schema` aspect template yet, so + // it is gated off by default (see + // KcGenerateOptions.emitExpressions); compact() then + // drops the undefined key. + semantics: emitExpressions ? compact({ expression: f.expression, // A field with any dimension metadata is a dimension; // otherwise DEFAULT. role: f.dimension ? 'DIMENSION' : 'DEFAULT', - }), + }) : undefined, })), }; } @@ -297,7 +315,8 @@ function schemaAspectData(entity: Entity): Record { // sensible default; dimensions, in schemaAspectData, default to STRING) rather // than emit an invalid aspect. function metricAspectData( - metric: Metric, warnings: string[]): Record { + metric: Metric, warnings: string[], + emitExpressions: boolean): Record { let dataType = metric.type ? columnDataType(metric.type) : undefined; if (!dataType) { warnings.push( @@ -309,7 +328,9 @@ function metricAspectData( return compact({ entity: metric.entity, dataType, - expression: metric.expression, + // `expression` is not in the published semantic-metric aspect template yet; + // gated off by default (see KcGenerateOptions.emitExpressions). + expression: emitExpressions ? metric.expression : undefined, }); } diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index f2dbabf8..bd8cf696 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -50,6 +50,11 @@ export interface PushOptions { // plan), each block labeled by destination. Scope which destinations run with // --target. Works with or without --validate-only. Semantic-model push only. print?: boolean; + // Emit the SQL-expression fields not yet supported by the published Knowledge + // Catalog system-type templates (per-field schema semantics and the metric + // expression). Off by default so a push matches the live types; enable once + // the templates gain these fields. Semantic-model KC push only. + emitExpressions?: boolean; } @@ -318,6 +323,7 @@ async function pushKnowledgeCatalog( entryGroup: source.entryGroup, validateOnly: options.validateOnly, forceRemove: options.forceRemove, + emitExpressions: options.emitExpressions, }); for (const w of result.warnings) { diff --git a/toolbox/mdcode/src/tool/main.ts b/toolbox/mdcode/src/tool/main.ts index ecc057a3..cbc67e95 100644 --- a/toolbox/mdcode/src/tool/main.ts +++ b/toolbox/mdcode/src/tool/main.ts @@ -41,6 +41,7 @@ cli.command('pull', 'Pull catalog entries') cli.command('push', 'Push catalog entries') .option('--force', 'Force push changes') .option('--force-remove', 'Delete Knowledge Catalog models in the entry group that this push does not include (removed/renamed models); semantic-model push only') + .option('--emit-expressions', 'Emit SQL-expression fields not yet in the published Knowledge Catalog system-type templates (per-field schema semantics, metric expression); off by default, enable once the templates support them; semantic-model push only') .option('--validate-only', 'Only validate changes without applying') .option('--target ', 'Semantic-model push destination(s): bq, kc, all (default), or a comma-separated list (e.g. bq,kc)') .option('--print', 'Print each pushed destination\'s generated artifact in its native format (BigQuery Graph SQL DDL, Knowledge Catalog entry plan); scope with --target (semantic-model push only)') diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.knowledge_catalog.golden.json index 2f14e7b6..7c56b706 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.knowledge_catalog.golden.json +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.knowledge_catalog.golden.json @@ -42,20 +42,12 @@ { "name": "o_orderkey", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "o_orderkey", - "role": "DEFAULT" - } + "metadataType": "STRING" }, { "name": "o_totalprice", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "o_totalprice", - "role": "DEFAULT" - } + "metadataType": "STRING" } ] } @@ -74,8 +66,7 @@ "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-metric", "data": { "entity": "orders", - "dataType": "NUMERIC", - "expression": "SUM(orders.o_totalprice)" + "dataType": "NUMERIC" } } } diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json index 3f4681df..91354101 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json @@ -45,38 +45,22 @@ "name": "o_orderkey", "dataType": "STRING", "metadataType": "STRING", - "description": "Order identifier", - "semantics": { - "expression": "o_orderkey", - "role": "DEFAULT" - } + "description": "Order identifier" }, { "name": "o_custkey", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "o_custkey", - "role": "DEFAULT" - } + "metadataType": "STRING" }, { "name": "o_orderdate", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "o_orderdate", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "o_totalprice", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "o_totalprice", - "role": "DEFAULT" - } + "metadataType": "STRING" } ] } @@ -108,21 +92,13 @@ { "name": "c_custkey", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "c_custkey", - "role": "DEFAULT" - } + "metadataType": "STRING" }, { "name": "c_name", "dataType": "STRING", "metadataType": "STRING", - "description": "Customer name", - "semantics": { - "expression": "c_name", - "role": "DEFAULT" - } + "description": "Customer name" } ] } @@ -142,8 +118,7 @@ "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-metric", "data": { "entity": "orders", - "dataType": "NUMERIC", - "expression": "SUM(orders.o_totalprice)" + "dataType": "NUMERIC" } } } @@ -161,8 +136,7 @@ "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-metric", "data": { "entity": "orders", - "dataType": "NUMERIC", - "expression": "COUNT(orders.o_orderkey)" + "dataType": "NUMERIC" } } } diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json index 0d34bc05..cc872725 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json @@ -40,78 +40,46 @@ { "name": "ss_item_sk", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "ss_item_sk", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "ss_ticket_number", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "ss_ticket_number", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "ss_customer_sk", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "ss_customer_sk", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "ss_store_sk", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "ss_store_sk", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "ss_quantity", "dataType": "STRING", "metadataType": "STRING", - "description": "Quantity of items sold", - "semantics": { - "expression": "ss_quantity", - "role": "DEFAULT" - } + "description": "Quantity of items sold" }, { "name": "ss_sales_price", "dataType": "STRING", "metadataType": "STRING", - "description": "Sales price per unit", - "semantics": { - "expression": "ss_sales_price", - "role": "DEFAULT" - } + "description": "Sales price per unit" }, { "name": "ss_ext_sales_price", "dataType": "STRING", "metadataType": "STRING", - "description": "Extended sales price (quantity * price)", - "semantics": { - "expression": "ss_ext_sales_price", - "role": "DEFAULT" - } + "description": "Extended sales price (quantity * price)" }, { "name": "ss_net_profit", "dataType": "STRING", "metadataType": "STRING", - "description": "Net profit from the sale", - "semantics": { - "expression": "ss_net_profit", - "role": "DEFAULT" - } + "description": "Net profit from the sale" } ] } @@ -144,31 +112,19 @@ { "name": "c_customer_sk", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "c_customer_sk", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "c_first_name", "dataType": "STRING", "metadataType": "STRING", - "description": "Customer first name", - "semantics": { - "expression": "c_first_name", - "role": "DIMENSION" - } + "description": "Customer first name" }, { "name": "c_last_name", "dataType": "STRING", "metadataType": "STRING", - "description": "Customer last name", - "semantics": { - "expression": "c_last_name", - "role": "DIMENSION" - } + "description": "Customer last name" } ] } @@ -201,39 +157,23 @@ { "name": "i_item_sk", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "i_item_sk", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "i_brand", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "i_brand", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "i_category", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "i_category", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "i_current_price", "dataType": "STRING", "metadataType": "STRING", - "description": "Current price of the item", - "semantics": { - "expression": "i_current_price", - "role": "DEFAULT" - } + "description": "Current price of the item" } ] } @@ -266,48 +206,28 @@ { "name": "s_store_sk", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "s_store_sk", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "s_store_name", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "s_store_name", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "s_city", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "s_city", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "s_state", "dataType": "STRING", - "metadataType": "STRING", - "semantics": { - "expression": "s_state", - "role": "DIMENSION" - } + "metadataType": "STRING" }, { "name": "s_number_employees", "dataType": "STRING", "metadataType": "STRING", - "description": "Number of employees at the store", - "semantics": { - "expression": "s_number_employees", - "role": "DEFAULT" - } + "description": "Number of employees at the store" } ] } diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts index 98182892..621f402a 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts @@ -24,6 +24,9 @@ const OPTS = { location: 'us', entryGroup: 'eg' }; +// The expression fields (per-field schema semantics, metric expression) are +// gated off by default; this turns them on to assert their content. +const OPTS_EXPR = {...OPTS, emitExpressions: true}; // A one-entity model whose single field carries the given IR type + dimension, // so a test can read back the emitted schema aspect for that field. @@ -44,8 +47,9 @@ function modelWithField( } // The schema-aspect field record for the sole field of modelWithField. -function schemaField(model: SemanticModel): Record { - const {entries} = generateCatalogResources(model, OPTS); +function schemaField( + model: SemanticModel, opts = OPTS): Record { + const {entries} = generateCatalogResources(model, opts); const entity = entries.find(e => e.entryType.endsWith('/semantic-entity'))!; const schema = entity.aspects!['dataplex-types.global.schema'].data!; return schema.fields[0]; @@ -86,11 +90,11 @@ describe( describe( 'a field with dimension metadata gets role DIMENSION, else DEFAULT', () => { test('dimension -> DIMENSION', () => { - expect(schemaField(modelWithField('String', true)).semantics.role) + expect(schemaField(modelWithField('String', true), OPTS_EXPR).semantics.role) .toBe('DIMENSION'); }); test('no dimension -> DEFAULT', () => { - expect(schemaField(modelWithField('String', false)).semantics.role) + expect(schemaField(modelWithField('String', false), OPTS_EXPR).semantics.role) .toBe('DEFAULT'); }); }); @@ -114,7 +118,7 @@ describe('the schema aspect carries the target expression in semantics', () => { }], }], }; - const f = schemaField(model); + const f = schemaField(model, OPTS_EXPR); expect(f.semantics.expression).toBe('CAST(e.f AS INT64)'); // importedExpression is the vendor/MAQL form; KC has no consumer for it. expect(f.semantics.importedExpression).toBeUndefined(); @@ -122,6 +126,20 @@ describe('the schema aspect carries the target expression in semantics', () => { }); +describe('the schema semantics block is gated behind emitExpressions', () => { + test('omitted by default; the non-gated columns are still emitted', () => { + const f = schemaField(modelWithField('String', true)); + expect(f.semantics).toBeUndefined(); + expect(f.dataType).toBe('STRING'); + expect(f.metadataType).toBe('STRING'); + }); + test('emitExpressions re-adds the semantics block (expression + role)', () => { + const f = schemaField(modelWithField('String', true), OPTS_EXPR); + expect(f.semantics).toEqual({expression: 'e.f', role: 'DIMENSION'}); + }); +}); + + describe('entity dataSource maps to a resource path', () => { function resources(dataSource: string): string[] { const model: SemanticModel = { @@ -153,8 +171,8 @@ describe('entity dataSource maps to a resource path', () => { describe('semantic-metric aspect', () => { - function metricData(model: SemanticModel) { - const {entries, warnings} = generateCatalogResources(model, OPTS); + function metricData(model: SemanticModel, opts = OPTS) { + const {entries, warnings} = generateCatalogResources(model, opts); const metric = entries.find(e => e.entryType.endsWith('/semantic-metric'))!; return { data: metric.aspects!['dataplex-types.global.semantic-metric'].data!, @@ -173,7 +191,7 @@ describe('semantic-metric aspect', () => { {name: 'rev', expression: 'SUM(o.p)', entity: 'o', type: 'Decimal'} ], }; - const {data, warnings} = metricData(model); + const {data, warnings} = metricData(model, OPTS_EXPR); expect(data).toEqual( {entity: 'o', dataType: 'NUMERIC', expression: 'SUM(o.p)'}); expect(warnings.some(w => w.includes('dataType'))).toBe(false); @@ -193,6 +211,20 @@ describe('semantic-metric aspect', () => { w => w.includes('metric \'rev\'') && w.includes('NUMERIC'))) .toBe(true); }); + + test('the expression is gated: omitted by default, kept with emitExpressions', + () => { + const model: SemanticModel = { + name: 'm', + entities: [], + relationships: [], + metrics: [{ + name: 'rev', expression: 'SUM(o.p)', entity: 'o', type: 'Decimal' + }], + }; + expect(metricData(model).data).toEqual({entity: 'o', dataType: 'NUMERIC'}); + expect(metricData(model, OPTS_EXPR).data.expression).toBe('SUM(o.p)'); + }); }); From 1620706fef3d97bc76e2443c8bd9217604718900 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 00:24:52 +0000 Subject: [PATCH 20/34] mdcode: document --emit-expressions and the gated-off SQL fields in the guide The default KC push no longer writes per-field schema.semantics or semantic-metric.expression (the published system-type templates lack them). Update the guide's 'not a full copy of your model' note and add --emit-expressions to the push flags table. --- toolbox/mdcode/docs/semantic-model.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 8ab2cdf1..46b379c2 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -113,6 +113,7 @@ kcmd push --print # also print the generated DDL / entry plan | `--validate-only` | Run every validation check and report pass/fail, but write nothing. | | `--print` | Print each destination's generated artifact (BigQuery DDL, Knowledge Catalog entry plan). Combine with `--validate-only` to preview without deploying. | | `--force-remove` | Delete models in the entry group that this push no longer includes (see [Updating and removing models](#updating-and-removing-models)). | +| `--emit-expressions` | Also write the SQL-expression fields (per-field `schema.semantics` and `semantic-metric.expression`) to Knowledge Catalog. Off by default: the published system-type templates do not carry them yet. Knowledge Catalog push only. | Destinations always deploy BigQuery-first and fail fast, so a rejected model never half-deploys. @@ -148,17 +149,20 @@ them, it never creates them. | Metric | `semantic-metric` | entry | `.metrics.` | | Relationship | `schema-join` | entry link between the two entity entries | derived from the relationship name | -An entity entry carries its columns and per-field semantics in the `schema` -aspect; a `schema-join` link carries the relationship detail — the paired columns -and foreign-key direction — in its aspect. - -> **Note — the catalog is not a full copy of your model.** Only the canonical -> `expression` (GoogleSQL/ANSI) is written to Knowledge Catalog; the original -> vendor SQL (`importedExpression` — e.g. the MAQL or Snowflake form a metric -> was imported from) is **not**. It stays in your authored document, and is still -> used when generating BigQuery SQL, but nothing in the catalog stores it. Keep -> your model document as the source of truth: a model reconstructed only from -> the catalog would come back without its vendor SQL. +An entity entry carries its columns in the `schema` aspect (name, data type, and +description per field); a `schema-join` link carries the relationship detail — the +paired columns and foreign-key direction — in its aspect. + +> **Note — the catalog is not a full copy of your model.** By default the SQL +> expressions are **not** written to Knowledge Catalog: the published system-type +> templates do not yet carry a per-field `semantics` block or a +> `semantic-metric.expression` field, so the default push omits them (pass +> `--emit-expressions` to write them once the templates gain the fields). The +> original vendor SQL (`importedExpression` — e.g. the MAQL or Snowflake form a +> metric was imported from) is never written either. All of it stays in your +> authored document and is still used when generating BigQuery SQL. Keep your +> model document as the source of truth: a model reconstructed only from the +> catalog would come back without its SQL. ## Validation From f292d806a541b006a0c75495d08c24a7383ce54d Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 00:54:10 +0000 Subject: [PATCH 21/34] mdcode: enforce one model per entry group on KC push Only one model per entry group is supported for now, so a push emitting a different count is rejected up front: an empty workspace stays a clean no-op under --validate-only and a configuration error on a real push, and more than one model is always rejected. That guarantees two models can never race for the same entry id, so the now-redundant checkEntryIdCollisions pass is removed. Also narrows the systemTypeProject doc comment (the override exists only so hermetic tests can point at a fixture types project). --- .../semantic/deploy_knowledge_catalog.ts | 65 ++++++------------- .../semantic/deploy_knowledge_catalog.test.ts | 11 ++-- 2 files changed, 26 insertions(+), 50 deletions(-) diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 71512f63..5676ec86 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -51,8 +51,9 @@ export interface KcDeployOptions { // Id of the destination entry group (provisioned at `init`, not by push). entryGroup: string; // Project the built-in `semantic-*` / `schema` system types are referenced - // from. Default `dataplex-types`. Override to reference them from a staging - // project; the emitted entries are otherwise unchanged. + // from. Defaults to `dataplex-types`, where these types live; overridable + // only so hermetic tests can point at a fixture types project. The emitted + // entries are otherwise unchanged. systemTypeProject?: string; // Location the built-in system types are referenced from. Default `global`. systemTypeLocation?: string; @@ -130,8 +131,7 @@ function sleep(ms: number): Promise { // Deploys every authored model's Knowledge Catalog resources. The body is the // sequence of phases, each a helper below: -// emitModels -- turn every model into catalog resources (pure) -// checkEntryIdCollisions -- fail if two models want the same entry id +// emitModels -- turn the model into catalog resources (pure) // buildPlan -- the dry-run plan (and stop here for --validate-only) // listEntryGroup -- snapshot the group once, before any write // guardForeignModels -- refuse (or --force-remove) models no longer pushed @@ -157,23 +157,27 @@ export async function deployKnowledgeCatalog( if (emit.error) return result(false, emit.error); const emitted = emit.emitted; - // A parsed document always yields at least one model (the loader enforces - // `semantic_model` min 1), so an empty `emitted` means no documents were - // found. validateOnly mutates nothing, so that is a clean no-op; a real push - // treats an empty workspace as a configuration error worth flagging. - if (!emitted.length) { - if (opts.validateOnly) { - warnings.push('No semantic model documents found; nothing to validate.'); - return result(true); + // Exactly one model per entry group is supported for now. An empty workspace + // is a clean no-op under --validate-only and a configuration error on a real + // push; more than one model in a single push is always rejected (which also + // means two models can never race for the same entry id). + if (emitted.length !== 1) { + if (!emitted.length) { + if (opts.validateOnly) { + warnings.push( + 'No semantic model documents found; nothing to validate.'); + return result(true); + } + return result( + false, 'No semantic model documents found; nothing to deploy.'); } return result( - false, 'No semantic model documents found; nothing to deploy.'); + false, + `entry group '${opts.entryGroup}' would receive ${ + emitted.length} models, but only one model per entry group is ` + + `supported; split them into separate entry groups.`); } - // Fail before any write if two models generate the same entry id. - const collision = checkEntryIdCollisions(emitted, opts); - if (collision) return result(false, collision); - // The plan is built for every push (printed with --print) and is the only // output of a --validate-only run, which writes nothing. plan = buildPlan(emitted, opts); @@ -244,33 +248,6 @@ function emitModels(models: LoadedModel[], opts: KcDeployOptions): } -// Fails (returns the error message) if two models in this push generate the same -// catalog entry id. Entry ids must be unique within the entry group; the emitter -// dedups within one model, but two models (two documents, or two `semantic_model`s -// in one document) whose names normalize to the same id would collide -- on -// publish the later one 409s and silently upserts over the earlier. Catch it here -// and name the entry so the author can rename a model. Ownership is tracked by -// index, not model name, so two same-named models (the common collision) are -// still distinguished. -function checkEntryIdCollisions( - emitted: EmittedModel[], opts: KcDeployOptions): string|undefined { - const entryOwner = new Map(); - for (let i = 0; i < emitted.length; i++) { - for (const entry of emitted[i].resources.entries) { - const prev = entryOwner.get(entry.name); - if (prev !== undefined && prev !== i) { - return `models '${emitted[prev].model}' and '${emitted[i].model}' ` + - `both generate catalog entry '${idOf(entry.name)}'; entry ids must ` + - `be unique within entry group '${opts.entryGroup}' -- rename one ` + - `model.`; - } - entryOwner.set(entry.name, i); - } - } - return undefined; -} - - // The full dry-run plan across all models (one block per model; see planSummary). function buildPlan(emitted: EmittedModel[], opts: KcDeployOptions): string[] { const plan: string[] = []; diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts index b8c9dc37..1d637505 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts @@ -288,11 +288,11 @@ describe('deployKnowledgeCatalog: failures', () => { }); test( - 'two models generating the same entry id fail before any write', + 'more than one model in a single push is rejected before any write', async () => { - // Both docs declare a model named 'sales', so their entry ids collide - // within the entry group; the second would silently upsert over the - // first. The publisher must catch the collision up front. + // Only one model per entry group is supported, so a push carrying two + // documents is rejected up front -- which also means two models can + // never race for the same entry id. const {group, create} = stubClient(); const docs = [ {name: 'a.yaml', text: OSSIE}, @@ -302,8 +302,7 @@ describe('deployKnowledgeCatalog: failures', () => { const result = await deployKnowledgeCatalog(models(docs), CTX, OPTS); expect(result.success).toBe(false); - expect(result.details).toContain('sales'); // the colliding entry id - expect(result.details).toContain('unique'); // the reason + expect(result.details).toContain('one model per entry group'); expect(group).not.toHaveBeenCalled(); expect(create).not.toHaveBeenCalled(); }); From 82419d051f2c0299082a0aa76b32b762efd6bd3e Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 00:54:17 +0000 Subject: [PATCH 22/34] mdcode: reject invalid deployment targets; narrow source probe A model must declare exactly one deployment target, and it must be a BigQuery Graph URI. Any deploymentTarget that does not parse as one is now collected as malformed and rejected (the validate gate names it, so a --target kc push is covered too) rather than silently ignored, which removes the redundant "looks-like-a-graph" hint heuristic. The shared gate also rejects a model that declares zero or several targets. The data-source pre-flight probe gains a WHERE FALSE so the dry-run scans no data while still resolving multi-part (federated REST-catalog) names as the deploy will. Guide updated to say exactly one deployment target. --- toolbox/mdcode/docs/semantic-model.md | 7 ++-- .../src/libts/semantic/deploy_bigquery.ts | 27 ++++++---------- toolbox/mdcode/src/libts/semantic/validate.ts | 23 +++++++++---- .../libts/semantic/deploy_bigquery.test.ts | 20 ++++++------ .../tests/libts/semantic/validate.test.ts | 32 ++++++++++++------- 5 files changed, 61 insertions(+), 48 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 46b379c2..19d192ba 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -66,7 +66,7 @@ semantic_model: ### Deployment targets (required) -Every model must declare at least one **deployment target** — the BigQuery +Every model must declare exactly one **deployment target** — the BigQuery property graph it deploys to — in a `GOOGLE` custom extension, as shown above. A target is a URI of the form: @@ -76,7 +76,8 @@ target is a URI of the form: The target's project and dataset are where the property graph is created; the same URIs are also recorded on the model's Knowledge Catalog entry. A model with -no deployment target is rejected at push time (see [Validation](#validation)). +no deployment target — or with more than one — is rejected at push time (see +[Validation](#validation)). ### Table sources @@ -169,7 +170,7 @@ paired columns and foreign-key direction — in its aspect. `push` and `--validate-only` run the same checks, **before either destination is touched**, so a model that cannot deploy fails fast instead of half-deploying: -* **At least one deployment target per model.** *(static)* +* **Exactly one deployment target per model.** *(static)* * **Every metric on a BigQuery Graph model resolves to exactly one entity** — otherwise it would be dropped from the BigQuery Graph. Set the metric's attach entity, or scope its expression to a single entity. *(static)* diff --git a/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts b/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts index 033eef35..a5d35679 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts @@ -29,22 +29,12 @@ const GOOGLE_VENDOR = 'GOOGLE'; // The capture groups are restricted to valid BigQuery identifier characters: // the components are interpolated into DDL unescaped (see qualifyGraph), so a // permissive `[^/]+` would let backticks or semicolons through. A URI that -// carries the BigQuery Graph prefix but fails this full match is collected as -// `malformed` (see bigQueryGraphTargets) and named in the deploy error, rather -// than being silently skipped and later misreported as "no target declared". +// fails this full match is collected as `malformed` (see +// googleDeploymentTargets) and named in the deploy/validation error, rather than +// being silently skipped and later misreported as "no target declared". const BQ_GRAPH_TARGET = /^\/\/bigquery\.googleapis\.com\/projects\/([A-Za-z0-9_-]+)\/datasets\/([A-Za-z0-9_-]+)\/propertyGraphs\/([A-Za-z0-9_-]+)$/; -// A deployment target that "looks like" a BigQuery Graph URI but fails the -// strict match above: it carries the bigquery.googleapis host (even under a -// typo'd scheme such as `https://`, or a truncated host missing `.com`) or the -// graph-specific `/propertyGraph(s)/` path segment. Such a URI is reported as -// malformed (see bigQueryGraphTargets) rather than silently dropped, so a -// host/scheme/segment/identifier typo surfaces instead of being misreported as -// "no target declared". An unrelated destination (e.g. a Dataplex URI) matches -// neither this hint nor the strict form and is left alone. -const BQ_GRAPH_TARGET_HINT = /bigquery\.googleapis|\/propertyGraphs?\//; - export interface BigQueryGraphTarget { project: string; @@ -79,7 +69,7 @@ export interface DeployResult { // Reads a model's GOOGLE custom_extension(s) in a single pass and returns the // deployment-target facts every caller derives from them: every declared // deploymentTarget URI (`uris`), the subset that parse as BigQuery Graph targets -// (`targets`), and the BigQuery-prefixed URIs that failed the strict match +// (`targets`), and the URIs that did not parse as a BigQuery Graph target // (`malformed`, kept so a caller can name a typo instead of silently skipping). // The extension `data` is an opaque, vendor-serialized JSON string (the loader // keeps it verbatim); we own its `deploymentTargets` shape. Throws on malformed @@ -118,10 +108,11 @@ export function googleDeploymentTargets(model: SemanticModel): const m = uri.match(BQ_GRAPH_TARGET); if (m) { targets.push({project: m[1], dataset: m[2], graphName: m[3], uri}); - } else if (BQ_GRAPH_TARGET_HINT.test(uri)) { - // Looks like a BigQuery Graph target but doesn't parse (host, scheme, - // path-segment, or identifier-char typo); a plain Dataplex/other URI is - // not our concern and is left alone. + } else { + // We only support BigQuery Graph deployment targets, so any URI that + // does not parse as one -- a host/scheme/segment/identifier typo, or an + // unrelated destination -- is collected as malformed and rejected + // (validate.ts, and the deploy leg) rather than silently ignored. malformed.push(uri); } } diff --git a/toolbox/mdcode/src/libts/semantic/validate.ts b/toolbox/mdcode/src/libts/semantic/validate.ts index 39d620af..8c9248a7 100644 --- a/toolbox/mdcode/src/libts/semantic/validate.ts +++ b/toolbox/mdcode/src/libts/semantic/validate.ts @@ -31,11 +31,20 @@ export function validatePushRequirements(models: LoadedModel[]): string[] { continue; } - // Every model must declare at least one deployment target. - if (!deployInfo.uris.length) { + // Every model must declare exactly one deployment target -- a single + // BigQuery Graph URI (we do not support zero or several graphs per model). + if (deployInfo.uris.length !== 1) { errors.push( - `model '${model.name}' (${document}) declares no deploymentTargets; ` + - `add at least one under its GOOGLE custom_extension.`); + `model '${model.name}' (${document}) declares ${ + deployInfo.uris.length} deploymentTargets; exactly one BigQuery ` + + `Graph target is required under its GOOGLE custom_extension.`); + } else if (deployInfo.malformed.length) { + // The single target is present but is not a valid BigQuery Graph URI. + errors.push( + `model '${model.name}' (${document}) deploymentTarget '${ + deployInfo.malformed[0]}' is not a valid BigQuery Graph URI; ` + + `expected //bigquery.googleapis.com/projects/

/datasets//` + + `propertyGraphs/.`); } // A model that targets a BigQuery graph must have every metric resolve to a @@ -66,7 +75,8 @@ export function validatePushRequirements(models: LoadedModel[]): string[] { // The entity sources are BigQuery tables regardless of --target, so this runs // for every destination and for --validate-only. // -// Each distinct source is probed with a dry-run query (`SELECT 1 FROM `), +// Each distinct source is probed with a dry-run query (`SELECT 1 FROM `, +// suffixed `WHERE FALSE` so it scans no data), // so BigQuery resolves the reference exactly as the generated DDL will. That // covers every reference form the generator emits -- a three-part // `project.dataset.table`, a four-part federated REST-catalog / Lakehouse name @@ -104,7 +114,8 @@ export async function validateBigQueryDataSources( const errors: string[] = []; for (const {project, ref, document, model, entity} of refs.values()) { const res = - await bq.query(project, `SELECT 1 FROM \`${ref}\``, undefined, true); + await bq.query( + project, `SELECT 1 FROM \`${ref}\` WHERE FALSE`, undefined, true); if (res.status === 200) continue; const msg = res.message?.trim() || `HTTP ${res.status}`; const why = /not found/i.test(msg) ? diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_bigquery.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_bigquery.test.ts index 200d0930..b24595ec 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy_bigquery.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/deploy_bigquery.test.ts @@ -39,7 +39,7 @@ const OSSIE = const OSSIE_NO_TARGET = fs.readFileSync(path.join(FIXTURES, 'sales_no_target.yaml'), 'utf8'); // An existing fixture whose only GOOGLE deployment target is a (non-BigQuery) -// Dataplex URI; exercises the deploy leg's "no BigQuery Graph target" path. +// Dataplex URI; exercises the deploy leg's unparseable-target path. const OSSIE_DATAPLEX_ONLY = fs.readFileSync(path.join(FIXTURES, 'sales_google_ext.yaml'), 'utf8'); // A model whose only GOOGLE target carries the BigQuery prefix but fails the @@ -84,14 +84,15 @@ describe('bigQueryGraphTargets', () => { expect(targets).toEqual([]); }); - test('ignores deployment targets that are not BigQuery Graphs', () => { - // A non-BigQuery URI is not our concern: neither a target nor malformed. + test('reports a non-BigQuery deployment target as malformed', () => { + // We only support BigQuery Graph targets, so any other URI does not parse + // as one and is collected as malformed (rejected), not silently ignored. const dataplexUri = 'projects/demo/locations/us/entryGroups/@bigquery/entries/sales_graph'; const {targets, malformed} = bigQueryGraphTargets( modelWithExtension(JSON.stringify({deploymentTargets: [dataplexUri]}))); expect(targets).toEqual([]); - expect(malformed).toEqual([]); + expect(malformed).toEqual([dataplexUri]); }); test('returns [] for a model with no custom extensions', () => { @@ -128,9 +129,8 @@ describe('bigQueryGraphTargets', () => { test('reports a host/scheme-typo target as malformed, not silently dropped', () => { // A truncated host (`.co` not `.com`) and an `https://` scheme both - // fail the strict match, yet clearly intend a BigQuery Graph target. - // They must be reported (via the /propertyGraphs?/ + host hint) rather - // than dropped and later misreported as "no target declared". + // fail the strict match; both are reported as malformed rather than + // dropped and later misreported as "no target declared". const hostTypo = '//bigquery.googleapis.co/projects/demo/datasets/sales/propertyGraphs/g'; const scheme = @@ -494,13 +494,15 @@ describe('deployBigQuery', () => { }); test( - 'fails when the only deployment target is a non-BigQuery destination', + 'fails when the only deployment target is not a BigQuery Graph URI', async () => { + // A non-BigQuery destination does not parse as a BigQuery Graph target, + // so it is reported as unparseable rather than "none declared". const res = await deployBigQuery( models([{name: 'sales', text: OSSIE_DATAPLEX_ONLY}]), CTX, {validateOnly: true}); expect(res.success).toBe(false); - expect(res.details).toContain('no BigQuery Graph'); + expect(res.details).toContain('could not be parsed'); }); test( diff --git a/toolbox/mdcode/tests/libts/semantic/validate.test.ts b/toolbox/mdcode/tests/libts/semantic/validate.test.ts index 34b6ba1e..048482c2 100644 --- a/toolbox/mdcode/tests/libts/semantic/validate.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/validate.test.ts @@ -41,10 +41,10 @@ describe('validatePushRequirements', () => { expect(validatePushRequirements([loaded(m)])).toEqual([]); }); - test('a model with no deployment targets is rejected', () => { + test('a model with no deployment target is rejected', () => { const errs = validatePushRequirements([loaded(model())]); expect(errs.length).toBe(1); - expect(errs[0]).toContain('no deploymentTargets'); + expect(errs[0]).toContain('exactly one'); expect(errs[0]).toContain('doc'); }); @@ -58,16 +58,24 @@ describe('validatePushRequirements', () => { expect(errs[0]).toContain('single entity'); }); - test('an entity-less metric is allowed when no BigQuery graph is targeted', - () => { - // A deployment target that is not a BigQuery Graph URI: the model still - // declares a target (passes the first check) but does not target a - // graph, so the metric-entity rule does not apply. - const m = model( - {metrics: [{name: 'cnt', expression: 'COUNT(*)'} as Metric]}, - [googleExt(['//dataplex.googleapis.com/projects/p/locations/us'])]); - expect(validatePushRequirements([loaded(m)])).toEqual([]); - }); + test('a non-BigQuery Graph deployment target is rejected as malformed', () => { + // We only support BigQuery Graph targets; a single, otherwise well-formed + // non-BigQuery URI still fails because it does not parse as one. + const m = model( + {}, [googleExt(['//dataplex.googleapis.com/projects/p/locations/us'])]); + const errs = validatePushRequirements([loaded(m)]); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('not a valid BigQuery Graph URI'); + }); + + test('more than one deployment target is rejected', () => { + const other = + '//bigquery.googleapis.com/projects/p/datasets/d/propertyGraphs/g2'; + const m = model({}, [googleExt([BQ_TARGET, other])]); + const errs = validatePushRequirements([loaded(m)]); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('exactly one'); + }); test('malformed GOOGLE extension JSON is reported, not thrown', () => { const m = model({}, [{vendorName: 'GOOGLE', data: '{not json'}]); From d8a4c633100bd39045a04afe21cd58278581bdb3 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 04:44:39 +0000 Subject: [PATCH 23/34] mdcode: add semantic-model pull from Knowledge Catalog Add the inverse of the KC emitter: read semantic-model / -entity / -metric entries and their aspects back into the IR, serialize to YAML, and wire a 'pull' command (with --dry-run and --model) for the semantic-model scope. Re-stacked onto the KC-push follow-ups: the emitter no longer writes importedExpression, so the reader no longer recovers it; idOf is shared from knowledge_catalog.ts; and push entry/link writes use the same bounded mapConcurrent pool as pull hydration. --- .../src/libts/layouts/semantic-model.ts | 30 ++ .../semantic/deploy_knowledge_catalog.ts | 199 +++++++++++- .../src/libts/semantic/knowledge_catalog.ts | 263 +++++++++++++++- .../mdcode/src/libts/semantic/serialize.ts | 256 ++++++++++++++++ toolbox/mdcode/src/tool/commands.ts | 90 +++++- toolbox/mdcode/src/tool/main.ts | 6 +- .../deploy_knowledge_catalog.pull.test.ts | 244 +++++++++++++++ .../semantic/knowledge_catalog.read.test.ts | 258 ++++++++++++++++ .../semantic/semantic_model_layout.test.ts | 75 +++++ .../tests/libts/semantic/serialize.test.ts | 290 ++++++++++++++++++ 10 files changed, 1695 insertions(+), 16 deletions(-) create mode 100644 toolbox/mdcode/src/libts/semantic/serialize.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/serialize.test.ts diff --git a/toolbox/mdcode/src/libts/layouts/semantic-model.ts b/toolbox/mdcode/src/libts/layouts/semantic-model.ts index 214fe930..42730af7 100644 --- a/toolbox/mdcode/src/libts/layouts/semantic-model.ts +++ b/toolbox/mdcode/src/libts/layouts/semantic-model.ts @@ -91,6 +91,36 @@ export class SemanticModelLayout implements CatalogLayout { return docs; } + // True when a model document with this handle already exists on disk. `pull` + // uses it to report which files it would overwrite vs. create. + hasModel(name: string): boolean { + return fs.existsSync(this.modelPath(name)); + } + + // The absolute path a model document with this handle maps to: + // `/EntryGroups//.yaml`. Path separators in the + // model name are replaced so a name still yields a single flat file. Requires + // the layout to be scoped to an entry group (the semantic-model source always + // is). + modelPath(name: string): string { + if (!this._entryGroup) { + throw new Error( + 'SemanticModel layout has no entry group; cannot resolve a model path.'); + } + const file = `${name.replace(/[/\\]/g, '_')}.yaml`; + return path.join(this._catalogPath, 'EntryGroups', this._entryGroup, file); + } + + // Writes a model's serialized document to its path, creating the EntryGroup + // directory if needed, and indexes it so a later modelDocuments() sees it. + // This is the sink `pull` writes reconstructed models to. + writeModelDocument(name: string, text: string): void { + const localPath = this.modelPath(name); + fs.mkdirSync(path.dirname(localPath), {recursive: true}); + fs.writeFileSync(localPath, text); + this._index.set(name, localPath); + } + // The Knowledge Catalog entry-level members are not applicable to this // push-only layout; the model is authored as a single Ossie document, not as // per-entry Knowledge Catalog files. These are wired when KC-resource emit diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 5676ec86..0bf69527 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -38,7 +38,8 @@ import {ApiResult} from '../gcp/api'; import * as context from '../gcp/context'; import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; -import {generateCatalogResources, KcResources} from './knowledge_catalog'; +import {SemanticModel} from './ir'; +import {generateCatalogResources, idOf, KcResources, modelsFromCatalogResources} from './knowledge_catalog'; import {LoadedModel} from './loader'; @@ -120,6 +121,10 @@ interface Counts { type EmittedModel = {model: string; resources: KcResources}; +// Upper bound on concurrent entry / entry-link writes within one model's wave +// (mirrors HYDRATE_CONCURRENCY on the pull side). +const WRITE_CONCURRENCY = 8; + // entries.create propagation retry: a just-created entry group can briefly 404. const ENTRY_CREATE_TRIES = 3; const ENTRY_CREATE_RETRY_MS = 3000; @@ -566,7 +571,8 @@ async function createEntries( const isMetric = (e: Entry) => (e.entryType ?? '').endsWith('/semantic-metric'); for (const wave of [children.filter(e => !isMetric(e)), children.filter(isMetric)]) { - const res = await Promise.all(wave.map(e => writeEntry(cat, opts, e))); + const res = await mapConcurrent( + wave, WRITE_CONCURRENCY, e => writeEntry(cat, opts, e)); const firstErr = res.find(r => r.error); if (firstErr) return {created, updated, error: firstErr.error}; for (const r of res) { @@ -593,7 +599,8 @@ async function createEntryLinks( cat: CatalogClient, opts: KcDeployOptions, links: EntryLink[]): Promise { if (!links.length) return {linked: 0}; - const res = await Promise.all(links.map(l => writeEntryLink(cat, opts, l))); + const res = await mapConcurrent( + links, WRITE_CONCURRENCY, l => writeEntryLink(cat, opts, l)); const firstErr = res.find(r => r.error); if (firstErr) return {linked: 0, error: firstErr.error}; return {linked: links.length}; @@ -686,11 +693,6 @@ function planSummary( } -// The id segment of a full entry/entryType resource name (after the last '/'). -function idOf(name: string): string { - return name.split('/').pop() ?? name; -} - function isOk(res: {status: number}): boolean { return res.status === 200; } @@ -715,3 +717,184 @@ function isPropagating(res: {message?: string}): boolean { function errText(res: {status: number; message?: string}): string { return res.message?.trim() || `HTTP ${res.status}`; } + + +// --------------------------------------------------------------------------- +// Pull: Knowledge Catalog -> Semantic Model IR. +// +// The read counterpart of deployKnowledgeCatalog and the inverse of push. +// Unlike a write, a pull needs no server-side type provisioning -- only that +// the `semantic-*` entries exist. It enumerates the entry group, keeps the +// semantic entries, hydrates each one's aspect data (a BASIC list omits aspect +// data, so each entry is re-fetched with its aspect types -- an entity needs +// BOTH its `semantic-entity` aspect and the built-in `schema` aspect), and +// hands the hydrated entries to the pure reader (modelsFromCatalogResources). +// --------------------------------------------------------------------------- + +export interface KcPullOptions { + project: string; + location: string; + entryGroup: string; + model?: string; // limit to a single model by name (default: all) +} + +export interface KcPullResult { + models: SemanticModel[]; + warnings: string[]; +} + +// Upper bound on in-flight aspect-hydration fetches during a pull. +const HYDRATE_CONCURRENCY = 8; + +// Reads the semantic models back from a Knowledge Catalog entry group. Emits no +// console output; warnings (skipped entries, no match for --model, reader +// warnings) are returned for the caller to print. +export async function pullKnowledgeCatalog( + cat: CatalogClient, opts: KcPullOptions): Promise { + const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`; + const warnings: string[] = []; + + // Enumerate the group (paging is inherently sequential) and pick the semantic + // entries, then hydrate their aspects concurrently: a BASIC list omits aspect + // data, so each entry needs its own lookupEntry, and those fetches are + // independent. The pool preserves input order so warnings stay deterministic. + const targets: {entry: Entry; aspectTypes: string[]}[] = []; + for await (const entry of cat.listEntries( + opts.project, opts.location, opts.entryGroup)) { + const aspectTypes = semanticAspectTypes(entry.entryType); + if (aspectTypes) targets.push({entry, aspectTypes}); + // else: not part of a semantic model; ignore it. + } + + // When scoped to one model, hydrate only that model's entries -- its anchor + // (matched by name) plus the children pointing at it. A list already carries + // entrySource + parentEntry, so this avoids fetching every other model's + // aspects. No match short-circuits with just the not-found warning. + let scoped = targets; + if (opts.model) { + scoped = scopeToModel(targets, opts.model); + if (!scoped.length) { + return { + models: [], + warnings: [ + `no semantic model named '${opts.model}' found in ${destination}` + ], + }; + } + } + + const fetched = await mapConcurrent( + scoped, HYDRATE_CONCURRENCY, async ({entry, aspectTypes}) => { + const res = await cat.lookupEntry( + opts.project, opts.location, entry.name, aspectTypes); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry '${entry.name}' (status ${ + res.status}); skipped` + }; + } + return {entry: res.result}; + }); + + const hydrated: Entry[] = []; + for (const r of fetched) { + if (r.entry) + hydrated.push(r.entry); + else if (r.warning) + warnings.push(r.warning); + } + + const read = modelsFromCatalogResources(hydrated); + warnings.push(...read.warnings); + + // Defense in depth: keep only the requested model even if the reader surfaced + // another anchor (e.g. a child whose parentEntry pointed outside the scope). + let models = read.models; + if (opts.model) { + models = models.filter(m => m.name === opts.model); + if (!models.length) { + warnings.push( + `no semantic model named '${opts.model}' found in ${destination}`); + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// The aspect type resource names to hydrate for a semantic entry, derived from +// its entryType (the aspect types are the parallel resources in the same +// project/location). An entity carries two aspects: its `semantic-entity` +// aspect and the built-in `schema` aspect that holds its fields. Returns +// undefined for entries that are not part of a semantic model. +function semanticAspectTypes(entryType: string): string[]|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + const typeBase = entryType.slice(0, idx); + const t = entryType.slice(idx + marker.length); + const aspectType = (name: string) => `${typeBase}/aspectTypes/${name}`; + switch (t) { + case 'semantic-model': + return [aspectType('semantic-model')]; + case 'semantic-entity': + return [aspectType('semantic-entity'), aspectType('schema')]; + case 'semantic-metric': + return [aspectType('semantic-metric')]; + default: + return undefined; + } +} + + +// Restricts hydration targets to a single model: the semantic-model anchor +// whose name (entrySource.displayName, else the entry id) matches `model`, plus +// every child entry whose parentEntry is that anchor. Uses only list-level +// fields (no aspect data), so it runs before hydration and avoids fetching +// unrelated models' aspects. Returns [] when no anchor matches. +function scopeToModel( + targets: {entry: Entry; aspectTypes: string[]}[], + model: string): {entry: Entry; aspectTypes: string[]}[] { + const isAnchor = (t: {entry: Entry}) => + !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); + const allAnchorNames = new Set(targets.filter(isAnchor).map(t => t.entry.name)); + const matchedAnchorNames = new Set( + targets.filter(isAnchor) + .filter( + t => (t.entry.entrySource?.displayName ?? idOf(t.entry.name)) === + model) + .map(t => t.entry.name)); + if (!matchedAnchorNames.size) return []; + // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds + // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a + // project-id normalization mismatch) still belongs to it. Without this a + // scoped pull would drop children a full pull keeps. + const soleAnchor = + allAnchorNames.size === 1 ? [...allAnchorNames][0] : undefined; + const soleMatched = + soleAnchor !== undefined && matchedAnchorNames.has(soleAnchor); + return targets.filter( + t => matchedAnchorNames.has(t.entry.name) || + matchedAnchorNames.has(t.entry.parentEntry ?? '') || + (soleMatched && !isAnchor(t) && + !allAnchorNames.has(t.entry.parentEntry ?? ''))); +} + + +// Maps `items` through `fn` with at most `limit` calls in flight, returning +// results in input order (so downstream ordering stays deterministic). +async function mapConcurrent( + items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results: R[] = new Array(items.length); + let next = 0; + async function worker(): Promise { + while (next < items.length) { + const i = next++; + results[i] = await fn(items[i]); + } + } + const workers = + Array.from({length: Math.min(limit, items.length)}, () => worker()); + await Promise.all(workers); + return results; +} diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index a3051b22..5e5860c9 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -45,7 +45,8 @@ import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; import {bigQueryGraphTargets} from './deploy_bigquery'; -import {DataType, Entity, Metric, Relationship, SemanticModel} from './ir'; +import {DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; +import {referencedEntityNames} from './sql_expr_utils'; // Where the `semantic-*` and `schema` system types live: built-in types in // project `dataplex-types`, location `global`. Callers may override to reference @@ -539,3 +540,263 @@ function linkSlug(s: string): string { function unquote(part: string): string { return part.replace(/^[`"]/, '').replace(/[`"]$/, ''); } + + +// --------------------------------------------------------------------------- +// Reader: Knowledge Catalog entries -> Semantic Model IR. +// +// The inverse of generateCatalogResources: it reconstructs the IR from the +// entries a pull hydrated (see deploy_knowledge_catalog.pullKnowledgeCatalog). +// `semantic-entity` / `semantic-metric` entries are grouped under their +// `semantic-model` anchor via `parentEntry`; entries of other types are +// ignored. Resources are matched by type-name SUFFIX, so a reader need not know +// which system-type project/location the emitter used. +// +// Fidelity is bounded by what the emitter persisted, so this read is the +// inverse of the WRITE, not of the authored document. It recovers names, +// descriptions, data sources, field datatypes (via the schema aspect) and +// DIMENSION roles, field/metric expressions, and each +// metric's attach entity (re-derived from its expression, as the loader does). +// It cannot recover what the emitter does not write: entity keys/unique keys, +// `ai_context`, field labels, `importedDialect`, `custom_extensions`, and +// relationships (the graph edges live in the BigQuery property graph, not the +// catalog). +// --------------------------------------------------------------------------- + +export interface ReadResult { + models: SemanticModel[]; + warnings: string[]; +} + +/** + * Reconstructs the Semantic Model IR from Knowledge Catalog entries. + * + * Returns one model per `semantic-model` anchor plus any warnings (no anchor, + * an orphaned child, an entry missing its aspect data). Entries must already be + * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a + * BASIC list omits aspect data, so the puller re-fetches each entry first. + */ +export function modelsFromCatalogResources(entries: Entry[]): ReadResult { + const warnings: string[] = []; + + const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); + const entityEntries = + entries.filter(e => semanticType(e) === 'semantic-entity'); + const metricEntries = + entries.filter(e => semanticType(e) === 'semantic-metric'); + + if (!anchors.length) { + warnings.push('no semantic-model entry found; nothing to reconstruct'); + return {models: [], warnings: [...new Set(warnings)]}; + } + + // A child belongs to its anchor by parentEntry. When there is exactly one + // anchor, children whose parentEntry does not resolve (e.g. a project-id + // normalization mismatch) are still attached to it rather than dropped. + const anchorNames = new Set(anchors.map(a => a.name)); + const soleAnchor = anchors.length === 1 ? anchors[0].name : undefined; + const childrenOf = (anchorName: string, pool: Entry[]) => pool.filter( + e => e.parentEntry === anchorName || + (soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? ''))); + + const models = anchors.map(anchor => { + const name = anchor.entrySource?.displayName ?? idOf(anchor.name); + + const entities = childrenOf(anchor.name, entityEntries) + .map(e => readEntity(e, warnings)); + const entityNames = entities.map(e => e.name); + const metrics = childrenOf(anchor.name, metricEntries) + .map(e => readMetric(e, entityNames, warnings)); + + // Relationships are not published to the catalog (see the file header), so + // a reconstructed model always has an empty edge set. + const model: SemanticModel = {name, entities, relationships: [], metrics}; + const description = anchor.entrySource?.description; + if (description !== undefined) model.description = description; + return model; + }); + + // Flag children that resolved to no anchor at all (only possible with + // multiple anchors, where the sole-anchor fallback does not apply). + if (!soleAnchor) { + for (const child of [...entityEntries, ...metricEntries]) { + if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { + warnings.push(`entry '${ + child.name}' has no resolvable parent semantic-model; omitted`); + } + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// Reconstructs an entity from its `semantic-entity` aspect (the backing source) +// and the built-in `schema` aspect (its fields). Keys are not persisted by the +// emitter and so come back empty. +function readEntity(entry: Entry, warnings: string[]): Entity { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const semantic = aspectData(entry, 'semantic-entity'); + const schema = aspectData(entry, 'schema'); + if (!Object.keys(semantic).length) { + warnings.push(`entity '${ + name}': no semantic-entity aspect data (fetch with the aspect type)`); + } + + const dataSource = dataSourceFromResource(semantic?.source?.resources?.[0]); + if (!dataSource) { + warnings.push( + `entity '${name}': no backing data source in the semantic-entity ` + + `aspect; 'source' will be empty and the entity may not load`); + } + const entity: Entity = { + name, + dataSource, + keys: [], // not persisted by the emitter; unrecoverable on read + fields: asArray(schema.fields).map(fd => readField(fd, name, warnings)), + }; + const description = entry.entrySource?.description; + if (description !== undefined) entity.description = description; + return entity; +} + + +// Reconstructs a field from one `schema` aspect field record, inverting +// schemaAspectData: the datatype from dataType/metadataType, expressions from +// the nested `semantics` block, and the DIMENSION role back to a dimension +// marker. +function readField(fd: any, entityName: string, warnings: string[]): Field { + const name = fd?.name; + if (name === undefined || name === '') { + warnings.push( + `entity '${entityName}': a schema field is missing its name; the ` + + `field may not load`); + } + const field: Field = {name}; + const sem = fd?.semantics ?? {}; + if (sem.expression !== undefined) field.expression = sem.expression; + const type = irDataType(fd?.dataType, fd?.metadataType); + if (type !== undefined) field.type = type; + if (sem.role === 'DIMENSION') field.dimension = {}; + if (fd?.description !== undefined) field.description = fd.description; + return field; +} + + +// Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` +// is re-derived from the expression (as the loader does) rather than read from +// the aspect, so it stays consistent with the reconstructed entity set. +function readMetric( + entry: Entry, entityNames: string[], warnings: string[]): Metric { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const data = aspectData(entry, 'semantic-metric'); + if (data.expression === undefined) { + warnings.push(`metric '${name}': no expression in semantic-metric aspect`); + } + + const metric: Metric = {name}; + if (data.expression !== undefined) metric.expression = data.expression; + const exprForRefs = data.expression ?? ''; + const referenced = referencedEntityNames(exprForRefs, entityNames); + if (referenced.length === 1) { + metric.entity = referenced[0]; + } else if (exprForRefs && !referenced.length) { + // Parity with the loader's convertMetric: an expression that qualifies no + // known entity is flagged as potentially unplaceable downstream. + warnings.push( + `metric '${name}': expression references no known entity; it may not ` + + `be placeable downstream`); + } + // The emitter writes a required dataType, defaulting a typeless metric to + // NUMERIC (see metricAspectData); NUMERIC maps back to Decimal, so a metric + // authored without a datatype round-trips as an explicit Decimal rather than + // un-typed. (Dimensions differ: their STRING default reads back as un-typed.) + const type = irDataType(data.dataType, undefined); + if (type !== undefined) metric.type = type; + const description = entry.entrySource?.description; + if (description !== undefined) metric.description = description; + return metric; +} + + +// The inverse of columnDataType/columnMetadataType: maps the schema aspect's +// dataType (disambiguated by metadataType only for the STRING family) back to +// the IR's logical DataType. STRING + OTHER is Opaque; a plain STRING is read +// as un-typed (undefined) -- the loader's default -- since the emitter cannot +// distinguish an authored `String` from an un-typed field (both emit STRING). +function irDataType(dataType: string|undefined, metadataType: string|undefined): + DataType|undefined { + switch (dataType) { + case 'INT64': + return 'Integer'; + case 'NUMERIC': + return 'Decimal'; + case 'FLOAT64': + return 'Float'; + case 'BOOL': + return 'Boolean'; + case 'DATE': + return 'Date'; + case 'TIME': + return 'Time'; + case 'DATETIME': + return 'DateTime'; + case 'TIMESTAMP': + return 'DateTimeTz'; + case 'STRING': + return metadataType === 'OTHER' ? 'Opaque' : undefined; + default: + return undefined; + } +} + + +// The inverse of resourcePath: a BigQuery linked-resource URI becomes the +// canonical `project.dataset.table` string; anything else (a verbatim query or +// a passthrough reference) is returned unchanged. +function dataSourceFromResource(resource: string|undefined): string { + const value = (resource ?? '').trim(); + const m = value.match( + /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/); + return m ? `${m[1]}.${m[2]}.${m[3]}` : value; +} + + +// The bare `semantic-*` type of an entry, matched by entryType suffix so the +// system-type project/location need not be known. Returns undefined for entries +// that are not part of a semantic model. +function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| + 'semantic-metric'|undefined { + for (const t of ['semantic-model', 'semantic-entity', 'semantic-metric'] as + const) { + if (entry.entryType?.endsWith(`/entryTypes/${t}`)) return t; + } + return undefined; +} + + +// The `data` payload of an entry's aspect of the given bare type, matched by +// the aspect key's `.` suffix or the aspectType's `/aspectTypes/` +// suffix (robust to whichever system-type project/location the emitter used). +// Returns an empty object when the aspect is absent. +function aspectData(entry: Entry, type: string): Record { + const aspects = entry.aspects ?? {}; + for (const [key, aspect] of Object.entries(aspects)) { + if (key.endsWith(`.${type}`) || + aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { + return aspect.data ?? {}; + } + } + return {}; +} + + +// The id segment of a full entry resource name (after the last '/'). +export function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + + +function asArray(value: any): any[] { + return Array.isArray(value) ? value : []; +} diff --git a/toolbox/mdcode/src/libts/semantic/serialize.ts b/toolbox/mdcode/src/libts/semantic/serialize.ts new file mode 100644 index 00000000..dd215100 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/serialize.ts @@ -0,0 +1,256 @@ +// Serializes the Semantic Model IR (./ir) back to the open AI-first semantics +// format (YAML). This is the inverse of `loader.ts`: `loader` reads authored +// YAML into the IR; this module writes the IR out as a YAML document the loader +// can read back. It is the local-workspace sink for `pull` (Knowledge Catalog +// -> IR -> YAML), the counterpart to how `push` compiles YAML -> IR -> a +// destination. +// +// Fidelity is at the IR level, not byte-for-byte with a hand-authored file. The +// loader normalizes several authoring conveniences into the IR at load time, so +// they are already gone before serialization and cannot be reproduced here: +// * per-dialect `expression.dialects[]` variants collapse to at most two +// forms +// (a target/canonical `expression` + an `importedExpression`); only those +// are re-emitted, each under a single dialect label. +// * comments and key ordering are not preserved. +// What the loader DOES keep on the IR round-trips here: names, descriptions, +// `ai_context` (instructions / synonyms / examples), `custom_extensions` +// (verbatim), keys and unique keys, data sources, field datatypes / labels / +// dimension flags, expressions, and relationship join columns. See +// serialize.test.ts. +// +// An `association` (junction-table) relationship has no open-format syntax (the +// loader cannot produce one), so only its direct foreign-key view (from/to + +// columns) is serialized; the junction detail is dropped with a note. + +import * as yaml from 'yaml'; + +import {AiContext, CustomExtension, Entity, Field, Metric, Relationship, SemanticModel,} from './ir'; + +// The schema version the loader was written against; re-emitted verbatim so a +// serialized document loads without a version-mismatch warning. Mirrors +// loader.SUPPORTED_VERSION. +const SERIALIZED_VERSION = '0.2.0.dev0'; + +// The dialect label for the IR's target/canonical `expression`. The IR does not +// record which authored dialect that string came from (the loader picked it +// from the target dialect or the ANSI_SQL fallback and discarded the label), +// and its contract is "GoogleSQL-valid". BIGQUERY is the loader's default +// target dialect, so labeling the canonical form BIGQUERY makes the loader +// re-pick it exactly on reload -- a clean round trip with no dialect-fallback +// note. +const CANONICAL_DIALECT = 'BIGQUERY'; + +// The dialect label used for an `importedExpression` whose `importedDialect` +// was lost (e.g. read back from a Knowledge Catalog aspect that does not +// persist the dialect). A non-target, non-canonical label so the loader +// re-reads it as the imported vendor form rather than the canonical expression. +const UNKNOWN_IMPORTED_DIALECT = 'IMPORTED'; + +export interface SerializeResult { + yaml: string; + warnings: string[]; +} + +/** + * Serializes a single semantic model to a YAML document string in the open + * AI-first semantics format. The document contains exactly this one model; + * `pull` writes one file per model + * (catalog/EntryGroups//.yaml). + * + * Warnings flag IR content that has no loadable representation (an association + * relationship's junction detail), so the caller can surface the lossy edge. + */ +export function serializeModel(model: SemanticModel): SerializeResult { + const warnings: string[] = []; + const text = yaml.stringify(modelDocument(model, warnings)); + return {yaml: text, warnings: [...new Set(warnings)]}; +} + +// Builds the plain document object (version + one model) that yaml.stringify +// renders. Kept separate so tests can assert the structure without parsing +// YAML. +export function modelDocument( + model: SemanticModel, warnings: string[] = []): Record { + return { + version: SERIALIZED_VERSION, + semantic_model: [modelDoc(model, warnings)], + }; +} + +function modelDoc( + model: SemanticModel, warnings: string[]): Record { + // `datasets` is required (min 1) by the loader. A reconstructed model with no + // entities (e.g. every entity fetch failed during a pull) would serialize to + // a document the loader rejects; emit the (empty) array but flag it so the + // lossy edge is visible rather than surfacing later as an opaque load error. + const datasets = (model.entities ?? []).map(e => datasetDoc(e, warnings)); + if (!datasets.length) { + warnings.push( + `model '${model.name}': no datasets (entities); the document requires ` + + `at least one and will not load until an entity is present.`); + } + return compact({ + name: model.name, + description: model.description, + ai_context: aiContextDoc(model.aiContext), + custom_extensions: customExtensionsDoc(model.customExtensions), + datasets, + relationships: nonEmpty( + (model.relationships ?? []).map(r => relationshipDoc(r, warnings))), + metrics: nonEmpty((model.metrics ?? []).map(m => metricDoc(m, warnings))), + }); +} + +function datasetDoc( + entity: Entity, warnings: string[]): Record { + return compact({ + name: entity.name, + source: entity.dataSource, + primary_key: nonEmpty(entity.keys), + unique_keys: nonEmpty(entity.uniqueKeys), + description: entity.description, + ai_context: aiContextDoc(entity.aiContext), + fields: nonEmpty((entity.fields ?? []).map(f => fieldDoc(f, warnings))), + custom_extensions: customExtensionsDoc(entity.customExtensions), + }); +} + +function fieldDoc(field: Field, warnings: string[]): Record { + const expression = expressionDoc( + field.expression, field.importedExpression, field.importedDialect); + if (!expression) { + warnings.push( + `field '${field.name}': no expression; the loader requires one per ` + + `field and the document will not load until it is set.`); + } + return compact({ + name: field.name, + expression, + datatype: field.type, + label: field.label, + dimension: dimensionDoc(field), + description: field.description, + ai_context: aiContextDoc(field.aiContext), + custom_extensions: customExtensionsDoc(field.customExtensions), + }); +} + +function metricDoc(metric: Metric, warnings: string[]): Record { + // `entity` is derived by the loader from the expression's entity qualifiers, + // so it is intentionally not emitted: the loader recomputes it on reload. + const expression = expressionDoc( + metric.expression, metric.importedExpression, metric.importedDialect); + if (!expression) { + warnings.push( + `metric '${metric.name}': no expression; the loader requires one per ` + + `metric and the document will not load until it is set.`); + } + return compact({ + name: metric.name, + expression, + datatype: metric.type, + description: metric.description, + ai_context: aiContextDoc(metric.aiContext), + custom_extensions: customExtensionsDoc(metric.customExtensions), + }); +} + +// Inverts loader.convertRelationship: `from`/`to` are the endpoint entities and +// `from_columns`/`to_columns` are their positional join columns. An association +// (junction-table) edge has no open-format syntax, so only this direct-FK view +// is emitted and the junction detail is flagged. +function relationshipDoc( + rel: Relationship, warnings: string[]): Record { + if (rel.association) { + warnings.push( + `relationship '${ + rel.name}': association (junction-table) detail has no ` + + `open-format representation and is not serialized; only its foreign-key ` + + `endpoints are written.`); + } + return compact({ + name: rel.name, + from: rel.source.entity, + to: rel.destination.entity, + from_columns: nonEmpty(rel.source.columns), + to_columns: nonEmpty(rel.destination.columns), + description: rel.description, + ai_context: aiContextDoc(rel.aiContext), + custom_extensions: customExtensionsDoc(rel.customExtensions), + }); +} + +// Renders the `expression` object matching the loader's schema (a `dialects` +// array of {dialect, expression}). Emits the target/canonical form under +// CANONICAL_DIALECT and the imported vendor form under its own dialect, so the +// loader re-picks each into the same IR field. Returns undefined when neither +// form is present (the loader requires an expression, but a pathological field +// with none is dropped rather than fabricated). +function expressionDoc( + expression: string|undefined, importedExpression: string|undefined, + importedDialect: string|undefined): Record|undefined { + const dialects: {dialect: string; expression: string}[] = []; + if (expression !== undefined) { + dialects.push({dialect: CANONICAL_DIALECT, expression}); + } + if (importedExpression !== undefined) { + let label = importedDialect ?? UNKNOWN_IMPORTED_DIALECT; + // Never emit two dialect entries under the same label: the loader would + // pick between them non-deterministically. If the imported form's dialect + // collides with the canonical label already pushed, fall back to the + // imported placeholder. + if (expression !== undefined && label === CANONICAL_DIALECT) { + label = UNKNOWN_IMPORTED_DIALECT; + } + dialects.push({dialect: label, expression: importedExpression}); + } + return dialects.length ? {dialects} : undefined; +} + +// Emits the field's `dimension` block when present, inverting +// loader.convertField (which sets `dimension = {}` for a bare marker and copies +// `is_time`). The key is emitted whenever the IR carries dimension metadata, +// even for an empty marker, so a dimension field reloads as a dimension. +function dimensionDoc(field: Field): Record|undefined { + if (!field.dimension) return undefined; + return compact({is_time: field.dimension.isTime}); +} + +// Emits the structured `ai_context` (the only authoring path the loader reads +// synonyms/instructions/examples from), inverting loader.normalizeAiContext. +// Returns undefined when the context carries nothing, so `compact` drops the +// key. +function aiContextDoc(ai: AiContext|undefined): Record|undefined { + if (!ai) return undefined; + const doc = compact({ + instructions: ai.instructions, + synonyms: nonEmpty(ai.synonyms), + examples: nonEmpty(ai.examples), + }); + return Object.keys(doc).length ? doc : undefined; +} + +// Emits vendor `custom_extensions` verbatim (`vendorName` -> `vendor_name`), +// inverting loader.toCustomExtensions. Returns undefined when there are none. +function customExtensionsDoc(exts: CustomExtension[]|undefined): + Record[]|undefined { + if (!exts || !exts.length) return undefined; + return exts.map(e => ({vendor_name: e.vendorName, data: e.data})); +} + +// Drops undefined-valued keys so the emitted YAML only shows fields the model +// actually set (matching the emitters' `compact` convention). +function compact>(obj: T): T { + const out: Record = {}; + for (const [k, v] of Object.entries(obj)) { + if (v !== undefined) out[k] = v; + } + return out as T; +} + +// Returns the array, or undefined when empty/absent, so an empty list is +// omitted rather than rendered as `[]`. +function nonEmpty(items: T[]|undefined): T[]|undefined { + return items && items.length ? items : undefined; +} diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index bd8cf696..59ce84e1 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -14,6 +14,7 @@ import * as deploy from '../libts/semantic/deploy_bigquery'; import * as kc from '../libts/semantic/deploy_knowledge_catalog'; import {BigQueryClient} from '../libts/gcp/bigquery'; import {LoadedModel, loadSemanticModels} from '../libts/semantic/loader'; +import { serializeModel } from '../libts/semantic/serialize'; import {validateBigQueryDataSources, validatePushRequirements} from '../libts/semantic/validate'; @@ -158,15 +159,20 @@ export async function init(options: InitOptions): Promise { } -export async function pull(): Promise { +export interface PullOptions { + // Reconstruct + report only; never writes a file. Mirrors push --validate-only. + dryRun?: boolean; + // Limit the pull to a single model by name (default: all in the entry group). + model?: string; +} + + +export async function pull(options: PullOptions = {}): Promise { const ctx = context.ApiContext.default(); const snapshot = await kcmd.CatalogSnapshot.fromPath('.', ctx); if (snapshot.manifest.source.type === Sources.SEMANTIC_MODEL) { - console.log( - 'Semantic-model scope: nothing to pull. Knowledge Catalog resource ' + - 'pull for the semantic model is not yet implemented.'); - return 0; + return await pullSemanticModel(ctx, snapshot, options); } const catalog = new dataplex.CatalogClient(ctx); @@ -360,3 +366,77 @@ async function pushKnowledgeCatalog( unlinked}.`); return 0; } + + +// Pulls the semantic model's Knowledge Catalog entries back into local model +// documents (catalog/EntryGroups//.yaml) and prints the +// result. The destination coordinates come from the scope +// (project.location.entryGroup). Overwrite policy matches the core pull: +// last-write-wins, local-only documents are left untouched (never deleted). +// Returns a process exit code (0 on success). +async function pullSemanticModel( + ctx: context.ApiContext, snapshot: kcmd.CatalogSnapshot, + options: PullOptions): Promise { + // The semantic-model source always resolves to the SemanticModel layout + // (see createLayout), so these casts are safe. + const layout = snapshot.layout as SemanticModelLayout; + const source = snapshot.manifest.source as SemanticModelSource; + + console.log(options.dryRun + ? 'Reconstructing semantic model from Knowledge Catalog (dry run)...' + : 'Pulling semantic model from Knowledge Catalog...'); + + const catalog = new dataplex.CatalogClient(ctx); + const result = await kc.pullKnowledgeCatalog(catalog, { + project: source.project, + location: source.location, + entryGroup: source.entryGroup, + model: options.model, + }); + + for (const w of result.warnings) { + console.warn(`Warning: ${w}`); + } + + if (!result.models.length) { + console.log('No semantic models found; nothing to pull.'); + return 0; + } + + let created = 0; + let updated = 0; + // Guard against two reconstructed models whose names map to the same file + // (path-separator sanitizing, or two anchors sharing a display name): the + // later write would silently clobber the earlier. Track written paths so the + // collision is reported and the dry-run/real counts agree on the repeat. + const writtenBy = new Map(); + for (const model of result.models) { + const serialized = serializeModel(model); + for (const w of serialized.warnings) { + console.warn(`Warning: [${model.name}] ${w}`); + } + const target = layout.modelPath(model.name); + const prior = writtenBy.get(target); + if (prior !== undefined && prior !== model.name) { + console.warn( + `Warning: models '${prior}' and '${model.name}' both map to ` + + `${target}; the later overwrites the earlier -- rename one model.`); + } + const existed = writtenBy.has(target) || layout.hasModel(model.name); + writtenBy.set(target, model.name); + if (options.dryRun) { + console.log(` would ${existed ? 'update' : 'create'} ${target}`); + } + else { + layout.writeModelDocument(model.name, serialized.yaml); + console.log(` ${existed ? 'updated' : 'created'} ${target}`); + } + if (existed) updated++; + else created++; + } + + console.log(options.dryRun + ? `Dry run: would write ${created} new and ${updated} updated model document(s).` + : `Wrote ${created} new and ${updated} updated model document(s).`); + return 0; +} diff --git a/toolbox/mdcode/src/tool/main.ts b/toolbox/mdcode/src/tool/main.ts index cbc67e95..84cceff9 100644 --- a/toolbox/mdcode/src/tool/main.ts +++ b/toolbox/mdcode/src/tool/main.ts @@ -25,10 +25,12 @@ cli.command('init', 'Initialize a new catalog snapshot') cli.command('pull', 'Pull catalog entries') - .action(async () => { + .option('--dry-run', 'Reconstruct and report only; do not write files (semantic-model scope)') + .option('--model ', 'Limit the pull to a single model by name (semantic-model scope)') + .action(async (options) => { let exitCode = 1; try { - exitCode = await commands.pull(); + exitCode = await commands.pull(options); } catch (err: any) { console.error('Error:', err.message || err); diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts new file mode 100644 index 00000000..e31a2f94 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts @@ -0,0 +1,244 @@ +// Tests for the semantic-model Knowledge Catalog pull leg +// (pullKnowledgeCatalog in src/libts/semantic/deploy_knowledge_catalog.ts). +// +// pullKnowledgeCatalog is the orchestration around the pure reader: enumerate +// the entry group, hydrate each semantic entry's aspect data, and reconstruct +// the IR. The catalog client is stubbed so no network call is made. The entries +// the fake serves are produced by the real emitter, so this exercises the true +// list -> hydrate -> read path end to end (an entity is re-fetched with BOTH +// its semantic-entity and schema aspects). The reader's own mapping is covered +// in knowledge_catalog.read.test.ts; the focus here is the fetch SEQUENCE: +// aspect hydration, the --model filter, skipped entries, and ignoring foreign +// entries. + +import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; + +import {ApiResult} from '../../../src/libts/gcp/api'; +import {CatalogClient, Entry} from '../../../src/libts/gcp/dataplex'; +import {pullKnowledgeCatalog} from '../../../src/libts/semantic/deploy_knowledge_catalog'; +import {SemanticModel} from '../../../src/libts/semantic/ir'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; + +const OPTS = { + project: 'dest', + location: 'us', + entryGroup: 'eg' +}; + +const SALES: SemanticModel = { + name: 'sales', + entities: [{ + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], + fields: [ + {name: 'o_totalprice', expression: 'orders.o_totalprice', type: 'Decimal'} + ], + }], + relationships: [], + metrics: [{ + name: 'total_revenue', + expression: 'SUM(orders.o_totalprice)', + entity: 'orders', + // Metrics carry a datatype through KC (semantic-metric.dataType is + // required); Decimal -> NUMERIC on emit, NUMERIC -> Decimal on read. + type: 'Decimal' + }], +}; + +// The entries the emitter would have written for a model. +function entriesFor(model: SemanticModel): Entry[] { + return generateCatalogResources(model, OPTS).entries; +} + +function ok(result?: T): ApiResult { + return {status: 200, result}; +} +function err(status: number, message: string): ApiResult { + return {status, message}; +} + +// Stubs listEntries (yields `listed`) and lookupEntry (serves `served` by name, +// or 404s an unknown name). `lookupFail` forces a failure for one entry name. +function stubClient(listed: Entry[], served: Entry[], lookupFail?: string): + {list: any; lookup: any} { + const byName = new Map(served.map(e => [e.name, e])); + const list = spyOn(CatalogClient.prototype, 'listEntries') + .mockImplementation(async function*() { + for (const e of listed) yield e; + } as any); + const lookup = spyOn(CatalogClient.prototype, 'lookupEntry') + .mockImplementation(async (_p, _l, name) => { + if (name === lookupFail) return err(500, 'boom'); + const e = byName.get(name); + return e ? ok(e) : err(404, 'not found'); + }); + return {list, lookup}; +} + +afterEach(() => { + mock.restore(); +}); + + +describe('pullKnowledgeCatalog: happy path', () => { + test('reconstructs the model from listed + hydrated entries', async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models).toHaveLength(1); + expect(models[0]).toEqual(SALES); + expect(warnings).toHaveLength(0); + // Every listed semantic entry (model + entity + metric) was hydrated. + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test( + 'an entity is hydrated with BOTH its semantic-entity and schema aspects', + async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + await pullKnowledgeCatalog(cat, OPTS); + + // Find the lookup call for the entity entry and inspect its requested + // aspect types (the 4th arg). + const entityEntry = + entries.find(e => e.entryType.endsWith('/semantic-entity'))!; + const call = + lookup.mock.calls.find((c: any[]) => c[2] === entityEntry.name)!; + const aspectTypes: string[] = call[3]; + expect( + aspectTypes.some(t => t.endsWith('/aspectTypes/semantic-entity'))) + .toBe(true); + expect(aspectTypes.some(t => t.endsWith('/aspectTypes/schema'))) + .toBe(true); + }); +}); + + +describe('pullKnowledgeCatalog: filtering and robustness', () => { + test('--model keeps only the named model', async () => { + const other: SemanticModel = { + name: 'inventory', + entities: + [{name: 'items', dataSource: 'p.d.items', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + const entries = [...entriesFor(SALES), ...entriesFor(other)]; + stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + expect(models.map(m => m.name)).toEqual(['sales']); + }); + + test('--model with no match returns nothing and warns', async () => { + const entries = entriesFor(SALES); + stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'nope'}); + expect(models).toHaveLength(0); + expect(warnings.some(w => /no semantic model named 'nope'/i.test(w))) + .toBe(true); + }); + + test( + 'a non-semantic entry in the group is ignored, not fetched', async () => { + const entries = entriesFor(SALES); + const foreign: Entry = { + name: `projects/dest/locations/us/entryGroups/eg/entries/foreign`, + entryType: 'projects/x/locations/us/entryTypes/some-other-type', + }; + const {lookup} = stubClient([...entries, foreign], entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, OPTS); + expect(models).toHaveLength(1); + // The foreign entry is never hydrated (only the 3 semantic entries + // are). + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test('a failed hydration is skipped with a warning', async () => { + const entries = entriesFor(SALES); + const metricEntry = + entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + stubClient(entries, entries, metricEntry.name); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + // The model + entity still reconstruct; only the metric is dropped. + expect(models).toHaveLength(1); + expect(models[0].metrics).toHaveLength(0); + expect(warnings.some( + w => /failed to fetch/i.test(w) && w.includes(metricEntry.name))) + .toBe(true); + }); + + test('--model hydrates only the target model\'s entries', async () => { + const other: SemanticModel = { + name: 'inventory', + entities: + [{name: 'items', dataSource: 'p.d.items', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + const entries = [...entriesFor(SALES), ...entriesFor(other)]; + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + expect(models.map(m => m.name)).toEqual(['sales']); + // SALES has 3 entries (model + entity + metric); inventory's are never + // fetched -- the flag scopes hydration, not just the final result. + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test( + '--model keeps a child whose parentEntry does not resolve (sole-anchor ' + + 'fallback)', + async () => { + const entries = entriesFor(SALES); + // Simulate a project-id normalization mismatch: the metric points at an + // anchor name that differs from the emitted one. With a single model in + // the group a full pull still attaches it via the reader's sole-anchor + // fallback, so a scoped pull must keep it too (else --model silently + // drops a child a full pull returns). + const metricEntry = + entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + metricEntry.parentEntry = + metricEntry.parentEntry!.replace('projects/dest/', 'projects/12345/'); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + + expect(models).toHaveLength(1); + expect(models[0].metrics.map(m => m.name)).toEqual(['total_revenue']); + // The metric was hydrated despite the parent mismatch (3 entries). + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test('--model with no match fetches nothing and warns', async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'nope'}); + expect(models).toHaveLength(0); + expect(lookup).not.toHaveBeenCalled(); + expect(warnings.some(w => /no semantic model named 'nope'/i.test(w))) + .toBe(true); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts new file mode 100644 index 00000000..a8d5e447 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts @@ -0,0 +1,258 @@ +// Behavior specification for the Knowledge Catalog reader +// (modelsFromCatalogResources in src/libts/semantic/knowledge_catalog.ts). +// +// The reader is the inverse of the emitter (generateCatalogResources). The +// central guarantee is an emitter -> reader round trip: emit a model's entries, +// read them back, and get an IR equal to the source WHERE the emitter is +// lossless. The write drops content by design (entity keys, ai_context, field +// labels, importedDialect, relationships -- see the emitter header), so the +// expected read-back is the source model with exactly those fields cleared. +// Targeted tests pin the mapping details a round trip cannot isolate (the +// dataType inverse, the DIMENSION role, resource-URI parsing, metric attach +// re-derivation, and parent/anchor grouping). + +import {describe, expect, test} from 'bun:test'; + +import {Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; +import {generateCatalogResources, modelsFromCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; + +const OPTS = { + project: 'dest', + location: 'us', + entryGroup: 'eg' +}; + +// Emits a model to entries and reads it straight back. +function roundTrip(model: SemanticModel): + {models: SemanticModel[]; warnings: string[]} { + const {entries} = generateCatalogResources(model, OPTS); + return modelsFromCatalogResources(entries); +} + + +describe('emitter -> reader round trip (lossless slice)', () => { + // A model using only round-trippable content: no keys/ai_context/labels/ + // relationships (all dropped by the write), and datatypes that invert + // cleanly. + const source: SemanticModel = { + name: 'sales', + description: 'the sales model', + entities: [{ + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], // keys are not persisted; keep empty so the round trip matches + fields: [ + {name: 'o_orderkey', expression: 'orders.o_orderkey', type: 'Integer'}, + { + name: 'o_orderdate', + expression: 'orders.o_orderdate', + type: 'Date', + dimension: {}, + description: 'order date', + }, + ], + }], + relationships: [], + metrics: [{ + name: 'total_revenue', + expression: 'SUM(orders.o_totalprice)', + entity: 'orders', + type: 'Decimal', + }], + }; + + test('reconstructs an IR equal to the source', () => { + const {models} = roundTrip(source); + expect(models).toHaveLength(1); + expect(models[0]).toEqual(source); + }); +}); + + +describe('dataType inverse (schema aspect -> IR type)', () => { + // Emit a one-field model of each IR type, read it back, and check the field's + // reconstructed type. String and Opaque both emit dataType STRING; String + // (indistinguishable from an un-typed field) reads back as undefined, while + // Opaque is disambiguated by metadataType OTHER. + const cases: [Metric['type']|undefined, Metric['type']|undefined][] = [ + ['Integer', 'Integer'], + ['Decimal', 'Decimal'], + ['Float', 'Float'], + ['Boolean', 'Boolean'], + ['Date', 'Date'], + ['Time', 'Time'], + ['DateTime', 'DateTime'], + ['DateTimeTz', 'DateTimeTz'], + ['Opaque', 'Opaque'], + ['String', undefined], // collapses to un-typed + [undefined, undefined], // un-typed stays un-typed + ]; + + for (const [type, expected] of cases) { + test(`${type ?? 'un-typed'} -> ${expected ?? 'un-typed'}`, () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'f', expression: 'e.f', ...(type ? {type} : {})}], + }], + relationships: [], + metrics: [], + }; + const back = roundTrip(model).models[0].entities[0].fields[0]; + expect(back.type).toBe(expected as any); + }); + } +}); + + +describe('field mapping details', () => { + function readField(field: Entity['fields'][number]) { + const model: SemanticModel = { + name: 'm', + entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: [field]}], + relationships: [], + metrics: [], + }; + return roundTrip(model).models[0].entities[0].fields[0]; + } + + test('a DIMENSION role reads back as a dimension marker', () => { + const back = readField({name: 'd', expression: 'e.d', dimension: {}}); + expect(back.dimension).toEqual({}); + }); + + test('a non-dimension field has no dimension marker', () => { + const back = readField({name: 'f', expression: 'e.f'}); + expect(back.dimension).toBeUndefined(); + }); + + test( + 'an imported expression is not persisted to or recovered from the catalog', + () => { + const back = readField({ + name: 'amt', + expression: 'e.amt', + importedExpression: 'e.amt::NUMBER', + importedDialect: 'SNOWFLAKE', + }); + expect(back.expression).toBe('e.amt'); + // The emitter no longer writes importedExpression/importedDialect, so + // neither survives the round trip. + expect(back.importedExpression).toBeUndefined(); + expect(back.importedDialect).toBeUndefined(); + }); +}); + + +describe('data source resource-path parsing', () => { + function readDataSource(dataSource: string): string { + const model: SemanticModel = { + name: 'm', + entities: [{name: 'e', dataSource, keys: [], fields: []}], + relationships: [], + metrics: [], + }; + return roundTrip(model).models[0].entities[0].dataSource; + } + + test( + 'a three-part BigQuery reference round-trips through the resource URI', + () => { + expect(readDataSource('proj.ds.tbl')).toBe('proj.ds.tbl'); + }); + + test('a verbatim query source is preserved unchanged', () => { + const query = 'SELECT * FROM t'; + expect(readDataSource(query)).toBe(query); + }); +}); + + +describe('metric attach entity is re-derived from the expression', () => { + test( + 'a single-entity metric attaches; a cross-entity metric does not', () => { + const model: SemanticModel = { + name: 'm', + entities: [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'amt', expression: 'orders.amt'}] + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'region', expression: 'customer.region'}] + }, + ], + relationships: [], + metrics: [ + {name: 'revenue', expression: 'SUM(orders.amt)', entity: 'orders'}, + { + name: 'mix', + expression: 'SUM(orders.amt) / COUNT(customer.region)' + }, + ], + }; + const {models} = roundTrip(model); + const byName = new Map(models[0].metrics.map(m => [m.name, m])); + expect(byName.get('revenue')!.entity).toBe('orders'); + expect(byName.get('mix')!.entity).toBeUndefined(); + }); +}); + + +describe('anchor / parent grouping', () => { + test('no semantic-model entry yields no models and a warning', () => { + const {models, warnings} = modelsFromCatalogResources([]); + expect(models).toHaveLength(0); + expect(warnings.some(w => /no semantic-model entry/i.test(w))).toBe(true); + }); + + test('two models keep their own children by parentEntry', () => { + const a = generateCatalogResources( + { + name: 'a', + entities: [{name: 'ea', dataSource: 'p.d.a', keys: [], fields: []}], + relationships: [], + metrics: [], + }, + OPTS); + const b = generateCatalogResources( + { + name: 'b', + entities: [{name: 'eb', dataSource: 'p.d.b', keys: [], fields: []}], + relationships: [], + metrics: [], + }, + OPTS); + const {models} = modelsFromCatalogResources([...a.entries, ...b.entries]); + const byName = new Map(models.map(m => [m.name, m])); + expect(byName.get('a')!.entities.map(e => e.name)).toEqual(['ea']); + expect(byName.get('b')!.entities.map(e => e.name)).toEqual(['eb']); + }); +}); + + +describe('metric expression referencing no known entity', () => { + test('warns that the metric may be unplaceable and leaves it unattached', + () => { + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'orders', dataSource: 'p.d.o', keys: [], fields: []}], + relationships: [], + // References `widgets`, which is not an entity of this model. + metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], + }; + const {models, warnings} = roundTrip(model); + expect(models[0].metrics[0].entity).toBeUndefined(); + expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) + .toBe(true); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts new file mode 100644 index 00000000..ac4c7bc0 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts @@ -0,0 +1,75 @@ +// Tests for the SemanticModel layout's pull write-path +// (src/libts/layouts/semantic-model.ts): modelPath / hasModel / +// writeModelDocument. These are the sink `pull` writes reconstructed models to; +// the push-side discovery (modelDocuments) is exercised via the deploy tests. + +import {afterEach, beforeEach, describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import {SemanticModelLayout} from '../../../src/libts/layouts/semantic-model'; + +let root: string; +let catalogPath: string; + +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'kcmd-layout-')); + catalogPath = path.join(root, 'catalog'); +}); + +afterEach(() => { + fs.rmSync(root, {recursive: true, force: true}); +}); + +async function layout(entryGroup?: string): Promise { + const l = new SemanticModelLayout(catalogPath, entryGroup); + await l.init(); + return l; +} + + +describe('SemanticModelLayout write path', () => { + test('modelPath maps to EntryGroups//.yaml', async () => { + const l = await layout('eg'); + expect(l.modelPath('sales')) + .toBe(path.join(catalogPath, 'EntryGroups', 'eg', 'sales.yaml')); + }); + + test('modelPath sanitizes path separators in the model name', async () => { + const l = await layout('eg'); + expect(l.modelPath('a/b')) + .toBe(path.join(catalogPath, 'EntryGroups', 'eg', 'a_b.yaml')); + }); + + test('modelPath throws without an entry group', async () => { + const l = await layout(undefined); + expect(() => l.modelPath('sales')).toThrow(/entry group/i); + }); + + test( + 'writeModelDocument creates the file, dirs, and indexes it', async () => { + const l = await layout('eg'); + expect(l.hasModel('sales')).toBe(false); + + l.writeModelDocument('sales', 'version: x\n'); + + const p = l.modelPath('sales'); + expect(fs.existsSync(p)).toBe(true); + expect(fs.readFileSync(p, 'utf8')).toBe('version: x\n'); + expect(l.hasModel('sales')).toBe(true); + // Indexed, so a subsequent read surfaces it as a model document. + expect(l.modelDocuments()).toEqual([ + {name: 'sales', text: 'version: x\n'} + ]); + }); + + test( + 'writeModelDocument overwrites an existing document (last-write-wins)', + async () => { + const l = await layout('eg'); + l.writeModelDocument('sales', 'first\n'); + l.writeModelDocument('sales', 'second\n'); + expect(fs.readFileSync(l.modelPath('sales'), 'utf8')).toBe('second\n'); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/serialize.test.ts b/toolbox/mdcode/tests/libts/semantic/serialize.test.ts new file mode 100644 index 00000000..81c4454c --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/serialize.test.ts @@ -0,0 +1,290 @@ +// Behavior specification for the semantic-model serializer +// (src/libts/semantic/serialize.ts). +// +// serialize.ts is the inverse of loader.ts: IR -> open-format YAML. The +// strongest guarantee is a round trip through the loader -- load a fixture to +// the IR, serialize it, load the serialized text again, and assert the two IRs +// are identical. That pins IR-level fidelity across every feature the loader +// produces (datasets, fields, datatypes, dimensions, labels, ai_context, +// custom_extensions, relationships, metrics, imported expressions) without +// hard-coding YAML text. Targeted structural tests cover the mapping details a +// round trip cannot isolate. + +import {describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as yaml from 'yaml'; + +import {Field, Metric, Relationship, SemanticModel} from '../../../src/libts/semantic/ir'; +import {loadModels} from '../../../src/libts/semantic/loader'; +import {modelDocument, serializeModel} from '../../../src/libts/semantic/serialize'; + +const FIXTURES = path.join(__dirname, 'fixtures'); + +function loadFixture(name: string): SemanticModel[] { + const text = fs.readFileSync(path.join(FIXTURES, name), 'utf8'); + return loadModels(text).models; +} + + +describe('loader <-> serialize round trip is IR-stable', () => { + // Each fixture exercises a different slice of the format: relationships + + // ai_context + synonyms + label + dimension; the full tpc-ds corpus with + // custom_extensions + unique_keys; explicit datatypes; and imported + // (vendor-dialect) expressions. + const fixtures = [ + 'star_orders_customer.yaml', + 'tpcds_retail.yaml', + 'sales_google_ext.yaml', + 'vendor_dialects.yaml', + 'lineitem_databricks_ext.yaml', + 'sales_bq_graph_target.yaml', + ]; + + for (const fixture of fixtures) { + test(`${fixture} survives IR -> YAML -> IR unchanged`, () => { + const original = loadFixture(fixture); + expect(original.length).toBeGreaterThan(0); + + for (const model of original) { + const {yaml: text} = serializeModel(model); + const reloaded = loadModels(text).models; + expect(reloaded).toHaveLength(1); + // IR-level equality: every field the loader keeps must match exactly. + expect(reloaded[0]).toEqual(model); + } + }); + } +}); + + +describe('serialized document structure', () => { + const model = loadFixture('star_orders_customer.yaml')[0]; + const doc = modelDocument(model) as any; + const sm = doc.semantic_model[0]; + + test('emits the supported version and a single model', () => { + expect(doc.version).toBe('0.2.0.dev0'); + expect(doc.semantic_model).toHaveLength(1); + expect(sm.name).toBe(model.name); + }); + + test('a dataset source is the opaque dataSource string, verbatim', () => { + const orders = sm.datasets.find((d: any) => d.name === 'orders'); + const entity = model.entities.find(e => e.name === 'orders')!; + expect(orders.source).toBe(entity.dataSource); + expect(typeof orders.source).toBe('string'); + }); + + test('primary_key mirrors the entity keys', () => { + const orders = sm.datasets.find((d: any) => d.name === 'orders'); + const entity = model.entities.find(e => e.name === 'orders')!; + expect(orders.primary_key).toEqual(entity.keys); + }); + + test('ai_context is emitted structurally (synonyms round-trip)', () => { + // The fixture annotates a field (o_orderdate) with synonyms; find it and + // assert the structured ai_context is emitted under that field. + const entity = model.entities.find( + e => e.fields.some(f => f.aiContext?.synonyms?.length))!; + const field = entity.fields.find(f => f.aiContext?.synonyms?.length)!; + const dsDoc = sm.datasets.find((d: any) => d.name === entity.name); + const fieldDoc = dsDoc.fields.find((f: any) => f.name === field.name); + expect(fieldDoc.ai_context.synonyms).toEqual(field.aiContext!.synonyms); + }); + + test('a relationship maps to from/to + positional columns', () => { + expect(sm.relationships.length).toBeGreaterThan(0); + const rel = model.relationships[0]; + const relDoc = sm.relationships[0]; + expect(relDoc.from).toBe(rel.source.entity); + expect(relDoc.to).toBe(rel.destination.entity); + expect(relDoc.from_columns).toEqual(rel.source.columns); + expect(relDoc.to_columns).toEqual(rel.destination.columns); + }); +}); + + +describe('expression + datatype + dimension mapping', () => { + test('an explicit datatype round-trips as `datatype`', () => { + const model = loadFixture('sales_google_ext.yaml')[0]; + const typed = model.entities.flatMap(e => e.fields).find(f => f.type); + expect(typed).toBeDefined(); + const {yaml: text} = serializeModel(model); + const reloaded = loadModels(text).models[0]; + const back = reloaded.entities.flatMap(e => e.fields) + .find(f => f.name === typed!.name)!; + expect(back.type).toBe(typed!.type); + }); + + test('a bare dimension marker survives as `dimension: {}`', () => { + const field: + Field = {name: 'ship_date', expression: 'e.ship_date', dimension: {}}; + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: ['k'], fields: [field]}], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const fieldDoc = doc.semantic_model[0].datasets[0].fields[0]; + expect(fieldDoc.dimension).toEqual({}); + // And it reloads back to a dimension field. + const reloaded = loadModels(serializeModel(model).yaml).models[0]; + expect(reloaded.entities[0].fields[0].dimension).toEqual({}); + }); + + test('an imported vendor expression is emitted under its own dialect', () => { + const field: Field = { + name: 'amt', + expression: 'e.amt', + importedExpression: 'e.amt::NUMBER', + importedDialect: 'SNOWFLAKE', + }; + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: ['k'], fields: [field]}], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const dialects = + doc.semantic_model[0].datasets[0].fields[0].expression.dialects; + const labels = dialects.map((d: any) => d.dialect); + expect(labels).toContain('SNOWFLAKE'); + // The canonical form is labeled BIGQUERY so the loader re-picks it exactly. + expect(labels).toContain('BIGQUERY'); + }); + + test('a metric does not emit its derived attach entity', () => { + const metric: Metric = { + name: 'total', + expression: 'SUM(orders.amt)', + entity: 'orders', + }; + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: ['k'], + fields: [{name: 'amt', expression: 'orders.amt'}], + }], + relationships: [], + metrics: [metric], + }; + const metricDoc = + (modelDocument(model) as any).semantic_model[0].metrics[0]; + expect(metricDoc).not.toHaveProperty('entity'); + // The loader re-derives it on reload. + const reloaded = loadModels(serializeModel(model).yaml).models[0]; + expect(reloaded.metrics[0].entity).toBe('orders'); + }); +}); + + +describe('lossy edges are flagged', () => { + test( + 'an association relationship warns and drops the junction detail', () => { + const rel: Relationship = { + name: 'enrollment', + source: {entity: 'student', columns: ['id']}, + destination: {entity: 'course', columns: ['id']}, + association: { + dataSource: 'p.d.enrollment', + keys: ['student_id', 'course_id'], + sourceColumns: ['student_id'], + destinationColumns: ['course_id'], + }, + }; + const model: SemanticModel = { + name: 'school', + entities: [ + { + name: 'student', + dataSource: 'p.d.student', + keys: ['id'], + fields: [] + }, + { + name: 'course', + dataSource: 'p.d.course', + keys: ['id'], + fields: [] + }, + ], + relationships: [rel], + metrics: [], + }; + const {yaml: text, warnings} = serializeModel(model); + expect(warnings.some(w => /association/i.test(w))).toBe(true); + // The direct-FK view is still emitted (from/to + columns), so it + // reloads. + const relDoc = yaml.parse(text).semantic_model[0].relationships[0]; + expect(relDoc.from).toBe('student'); + expect(relDoc.to).toBe('course'); + expect(relDoc.from_columns).toEqual(['id']); + }); +}); + + +describe('serialize flags loader-invalid reconstructions', () => { + test('a model with no entities warns that datasets is required', () => { + const model: SemanticModel = { + name: 'empty', + entities: [], + relationships: [], + metrics: [], + }; + const {warnings} = serializeModel(model); + expect(warnings.some(w => /no datasets/i.test(w))).toBe(true); + }); + + test('a field with no expression warns that one is required', () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'orphan'}], + }], + relationships: [], + metrics: [], + }; + const {warnings} = serializeModel(model); + expect(warnings.some(w => /field 'orphan'.*no expression/i.test(w))) + .toBe(true); + }); + + test( + 'an imported dialect colliding with the canonical label is relabeled', + () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: [], + fields: [{ + name: 'amt', + expression: 'orders.amt', + importedExpression: 'orders.AMT', + importedDialect: 'BIGQUERY', + }], + }], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const labels: string[] = + doc.semantic_model[0].datasets[0].fields[0].expression.dialects.map( + (d: any) => d.dialect); + // No duplicate dialect label: the imported form is relabeled off + // BIGQUERY so the loader does not pick between two BIGQUERY entries. + expect(new Set(labels).size).toBe(labels.length); + expect(labels).toContain('BIGQUERY'); + }); +}); From 2d7f9bf407609514e4c93d6d308d49f1dba1310e Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 06:22:44 +0000 Subject: [PATCH 24/34] mdcode: restructure pull into converter-scaffold files; add fixture goldens + docs Addresses PR review feedback on the KC pull leg: - Rename serialize.ts -> osi_converter.ts (the OSI <-> IR converter). The name now says what it converts between; a header banner notes it currently holds only the serialize direction and that the loader migrates in post-#278. - Extract the KC reader into kc_converter.ts and the network pull into pull_kc.ts, so the new capability lives in its own files rather than swelling knowledge_catalog.ts / deploy_knowledge_catalog.ts. Those two files return to their #278 state (emit-only / push-only). This is scaffolding for the eventual two-layer split (pure converters vs push/pull orchestration); the remaining halves move in once #278 merges, with no further file renames. - Reorganize the pull tests around committed golden artifacts: each corpus fixture now has an .osi.golden.yaml (IR -> OSI) and a .pull.golden.yaml (KC entries -> IR -> OSI). A reviewer sees the whole input and output as files and can diff the two to see exactly what a Knowledge Catalog round trip drops. Test files renamed to match their modules (osi_converter/kc_converter/pull_kc). - Document `kcmd pull` in docs/semantic-model.md: the --dry-run/--model flags, multiple models per entry group, last-write-wins overwrite policy, and the catalog-not-a-full-copy round-trip loss. --- toolbox/mdcode/docs/semantic-model.md | 65 ++++- .../semantic/deploy_knowledge_catalog.ts | 199 +------------ .../mdcode/src/libts/semantic/kc_converter.ts | 267 ++++++++++++++++++ .../src/libts/semantic/knowledge_catalog.ts | 263 +---------------- .../{serialize.ts => osi_converter.ts} | 19 +- toolbox/mdcode/src/libts/semantic/pull_kc.ts | 182 ++++++++++++ toolbox/mdcode/src/tool/commands.ts | 5 +- .../sales_bq_graph_target.osi.golden.yaml | 29 ++ .../sales_bq_graph_target.pull.golden.yaml | 25 ++ .../star_orders_customer.osi.golden.yaml | 86 ++++++ .../star_orders_customer.pull.golden.yaml | 61 ++++ .../fixtures/tpcds_date_edge.osi.golden.yaml | 218 ++++++++++++++ .../fixtures/tpcds_date_edge.pull.golden.yaml | 147 ++++++++++ ...alog.read.test.ts => kc_converter.test.ts} | 63 ++++- ...erialize.test.ts => osi_converter.test.ts} | 52 +++- ...e_catalog.pull.test.ts => pull_kc.test.ts} | 6 +- 16 files changed, 1214 insertions(+), 473 deletions(-) create mode 100644 toolbox/mdcode/src/libts/semantic/kc_converter.ts rename toolbox/mdcode/src/libts/semantic/{serialize.ts => osi_converter.ts} (93%) create mode 100644 toolbox/mdcode/src/libts/semantic/pull_kc.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml rename toolbox/mdcode/tests/libts/semantic/{knowledge_catalog.read.test.ts => kc_converter.test.ts} (73%) rename toolbox/mdcode/tests/libts/semantic/{serialize.test.ts => osi_converter.test.ts} (81%) rename toolbox/mdcode/tests/libts/semantic/{deploy_knowledge_catalog.pull.test.ts => pull_kc.test.ts} (98%) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 19d192ba..7d3c28a2 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -13,8 +13,9 @@ model to two destinations at once: Both are generated from the same source document — you never author them separately, and a single `push` keeps them in sync. -This guide covers authoring, deploying, and updating a model. For the Ossie -document format itself, see [ossie.apache.org](https://ossie.apache.org/). +This guide covers authoring, deploying, pulling back, and updating a model. +For the Ossie document format itself, see +[ossie.apache.org](https://ossie.apache.org/). ## Prerequisites @@ -220,3 +221,63 @@ Every push prints one line per destination summarizing what it did. For a Deployed 1 BigQuery Graph(s). Wrote 5 new and 2 updated Knowledge Catalog entries; removed 1 orphaned entry; linked 2 relationships; unlinked 1 orphaned link. ``` + +## Pull + +`kcmd pull` is the inverse of push's Knowledge Catalog leg: it reads the +`semantic-*` entries back from the catalog and reconstructs local model +documents at `catalog/EntryGroups//.yaml`. Use it to +recover a workspace from a catalog someone else deployed, or to see what the +catalog actually holds. + +```bash +kcmd pull +``` + +Pull reads only from Knowledge Catalog (never BigQuery). Its coordinates come +from the same scope you authored under (`..`). + +| Flag | Effect | +|------|--------| +| `--dry-run` | Reconstruct from the catalog and report what would be written, but write no files. | +| `--model ` | Pull a single model by name; other models in the entry group are left alone. | + +One entry group can hold **many models** — each `semantic-model` entry is a +separate anchor, and pull reconstructs one document per anchor. `--model` +narrows both the fetch and the write to a single anchor. + +Pull writes with the same last-write-wins policy as the core pull: a model that +already exists locally is overwritten in place, and a local-only document (one +with no matching catalog entry) is left untouched — pull never deletes. + +> **Note — pull recovers the catalog, not your authored document.** The catalog +> stores only what push wrote to it (see the note under [What gets created in +> Knowledge Catalog](#what-gets-created-in-knowledge-catalog)), so a pulled +> document comes back without the content the catalog never held: entity keys, +> `ai_context`, field labels, the original vendor SQL (`importedExpression`), +> and relationships (the graph edges live in the BigQuery property graph, not +> the catalog). A field's *role* survives as a bare `dimension: {}` marker, but +> its detail (`is_time`, and so on) does not. Keep your authored document as the +> source of truth; treat a pulled document as a faithful copy of the catalog +> metadata, not of the original model. + +## Permissions + +`push` needs access to whichever destinations you deploy to. + +**BigQuery** — for `--target bq` or `all`, and for the validation pre-flight: + +* `bigquery.jobs.create` in the deployment-target project — to run the deploy's + `CREATE OR REPLACE PROPERTY GRAPH` and the validation dry-run query +* read access on each entity's source table, so the dry-run can resolve it +* `bigquery.datasets.get` on the target dataset (region detection; optional — + push degrades gracefully without it) + +**Knowledge Catalog / Dataplex** — for `--target kc` or `all`: + +* `dataplex.entryGroups.useSemanticModelAspect` on the destination entry group +* `dataplex.entryGroups.useSchemaJoinEntryLink` and + `dataplex.entryGroups.useSchemaJoinAspect` when the model has relationships + +`kcmd pull` needs read access to the same entry group instead — to list its +entries and fetch each `semantic-*` entry with its aspects. diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 0bf69527..5676ec86 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -38,8 +38,7 @@ import {ApiResult} from '../gcp/api'; import * as context from '../gcp/context'; import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; -import {SemanticModel} from './ir'; -import {generateCatalogResources, idOf, KcResources, modelsFromCatalogResources} from './knowledge_catalog'; +import {generateCatalogResources, KcResources} from './knowledge_catalog'; import {LoadedModel} from './loader'; @@ -121,10 +120,6 @@ interface Counts { type EmittedModel = {model: string; resources: KcResources}; -// Upper bound on concurrent entry / entry-link writes within one model's wave -// (mirrors HYDRATE_CONCURRENCY on the pull side). -const WRITE_CONCURRENCY = 8; - // entries.create propagation retry: a just-created entry group can briefly 404. const ENTRY_CREATE_TRIES = 3; const ENTRY_CREATE_RETRY_MS = 3000; @@ -571,8 +566,7 @@ async function createEntries( const isMetric = (e: Entry) => (e.entryType ?? '').endsWith('/semantic-metric'); for (const wave of [children.filter(e => !isMetric(e)), children.filter(isMetric)]) { - const res = await mapConcurrent( - wave, WRITE_CONCURRENCY, e => writeEntry(cat, opts, e)); + const res = await Promise.all(wave.map(e => writeEntry(cat, opts, e))); const firstErr = res.find(r => r.error); if (firstErr) return {created, updated, error: firstErr.error}; for (const r of res) { @@ -599,8 +593,7 @@ async function createEntryLinks( cat: CatalogClient, opts: KcDeployOptions, links: EntryLink[]): Promise { if (!links.length) return {linked: 0}; - const res = await mapConcurrent( - links, WRITE_CONCURRENCY, l => writeEntryLink(cat, opts, l)); + const res = await Promise.all(links.map(l => writeEntryLink(cat, opts, l))); const firstErr = res.find(r => r.error); if (firstErr) return {linked: 0, error: firstErr.error}; return {linked: links.length}; @@ -693,6 +686,11 @@ function planSummary( } +// The id segment of a full entry/entryType resource name (after the last '/'). +function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + function isOk(res: {status: number}): boolean { return res.status === 200; } @@ -717,184 +715,3 @@ function isPropagating(res: {message?: string}): boolean { function errText(res: {status: number; message?: string}): string { return res.message?.trim() || `HTTP ${res.status}`; } - - -// --------------------------------------------------------------------------- -// Pull: Knowledge Catalog -> Semantic Model IR. -// -// The read counterpart of deployKnowledgeCatalog and the inverse of push. -// Unlike a write, a pull needs no server-side type provisioning -- only that -// the `semantic-*` entries exist. It enumerates the entry group, keeps the -// semantic entries, hydrates each one's aspect data (a BASIC list omits aspect -// data, so each entry is re-fetched with its aspect types -- an entity needs -// BOTH its `semantic-entity` aspect and the built-in `schema` aspect), and -// hands the hydrated entries to the pure reader (modelsFromCatalogResources). -// --------------------------------------------------------------------------- - -export interface KcPullOptions { - project: string; - location: string; - entryGroup: string; - model?: string; // limit to a single model by name (default: all) -} - -export interface KcPullResult { - models: SemanticModel[]; - warnings: string[]; -} - -// Upper bound on in-flight aspect-hydration fetches during a pull. -const HYDRATE_CONCURRENCY = 8; - -// Reads the semantic models back from a Knowledge Catalog entry group. Emits no -// console output; warnings (skipped entries, no match for --model, reader -// warnings) are returned for the caller to print. -export async function pullKnowledgeCatalog( - cat: CatalogClient, opts: KcPullOptions): Promise { - const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`; - const warnings: string[] = []; - - // Enumerate the group (paging is inherently sequential) and pick the semantic - // entries, then hydrate their aspects concurrently: a BASIC list omits aspect - // data, so each entry needs its own lookupEntry, and those fetches are - // independent. The pool preserves input order so warnings stay deterministic. - const targets: {entry: Entry; aspectTypes: string[]}[] = []; - for await (const entry of cat.listEntries( - opts.project, opts.location, opts.entryGroup)) { - const aspectTypes = semanticAspectTypes(entry.entryType); - if (aspectTypes) targets.push({entry, aspectTypes}); - // else: not part of a semantic model; ignore it. - } - - // When scoped to one model, hydrate only that model's entries -- its anchor - // (matched by name) plus the children pointing at it. A list already carries - // entrySource + parentEntry, so this avoids fetching every other model's - // aspects. No match short-circuits with just the not-found warning. - let scoped = targets; - if (opts.model) { - scoped = scopeToModel(targets, opts.model); - if (!scoped.length) { - return { - models: [], - warnings: [ - `no semantic model named '${opts.model}' found in ${destination}` - ], - }; - } - } - - const fetched = await mapConcurrent( - scoped, HYDRATE_CONCURRENCY, async ({entry, aspectTypes}) => { - const res = await cat.lookupEntry( - opts.project, opts.location, entry.name, aspectTypes); - if (res.status !== 200 || !res.result) { - return { - warning: `failed to fetch entry '${entry.name}' (status ${ - res.status}); skipped` - }; - } - return {entry: res.result}; - }); - - const hydrated: Entry[] = []; - for (const r of fetched) { - if (r.entry) - hydrated.push(r.entry); - else if (r.warning) - warnings.push(r.warning); - } - - const read = modelsFromCatalogResources(hydrated); - warnings.push(...read.warnings); - - // Defense in depth: keep only the requested model even if the reader surfaced - // another anchor (e.g. a child whose parentEntry pointed outside the scope). - let models = read.models; - if (opts.model) { - models = models.filter(m => m.name === opts.model); - if (!models.length) { - warnings.push( - `no semantic model named '${opts.model}' found in ${destination}`); - } - } - - return {models, warnings: [...new Set(warnings)]}; -} - - -// The aspect type resource names to hydrate for a semantic entry, derived from -// its entryType (the aspect types are the parallel resources in the same -// project/location). An entity carries two aspects: its `semantic-entity` -// aspect and the built-in `schema` aspect that holds its fields. Returns -// undefined for entries that are not part of a semantic model. -function semanticAspectTypes(entryType: string): string[]|undefined { - const marker = '/entryTypes/'; - const idx = entryType?.indexOf(marker) ?? -1; - if (idx < 0) return undefined; - const typeBase = entryType.slice(0, idx); - const t = entryType.slice(idx + marker.length); - const aspectType = (name: string) => `${typeBase}/aspectTypes/${name}`; - switch (t) { - case 'semantic-model': - return [aspectType('semantic-model')]; - case 'semantic-entity': - return [aspectType('semantic-entity'), aspectType('schema')]; - case 'semantic-metric': - return [aspectType('semantic-metric')]; - default: - return undefined; - } -} - - -// Restricts hydration targets to a single model: the semantic-model anchor -// whose name (entrySource.displayName, else the entry id) matches `model`, plus -// every child entry whose parentEntry is that anchor. Uses only list-level -// fields (no aspect data), so it runs before hydration and avoids fetching -// unrelated models' aspects. Returns [] when no anchor matches. -function scopeToModel( - targets: {entry: Entry; aspectTypes: string[]}[], - model: string): {entry: Entry; aspectTypes: string[]}[] { - const isAnchor = (t: {entry: Entry}) => - !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); - const allAnchorNames = new Set(targets.filter(isAnchor).map(t => t.entry.name)); - const matchedAnchorNames = new Set( - targets.filter(isAnchor) - .filter( - t => (t.entry.entrySource?.displayName ?? idOf(t.entry.name)) === - model) - .map(t => t.entry.name)); - if (!matchedAnchorNames.size) return []; - // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds - // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a - // project-id normalization mismatch) still belongs to it. Without this a - // scoped pull would drop children a full pull keeps. - const soleAnchor = - allAnchorNames.size === 1 ? [...allAnchorNames][0] : undefined; - const soleMatched = - soleAnchor !== undefined && matchedAnchorNames.has(soleAnchor); - return targets.filter( - t => matchedAnchorNames.has(t.entry.name) || - matchedAnchorNames.has(t.entry.parentEntry ?? '') || - (soleMatched && !isAnchor(t) && - !allAnchorNames.has(t.entry.parentEntry ?? ''))); -} - - -// Maps `items` through `fn` with at most `limit` calls in flight, returning -// results in input order (so downstream ordering stays deterministic). -async function mapConcurrent( - items: T[], limit: number, fn: (item: T) => Promise): Promise { - const results: R[] = new Array(items.length); - let next = 0; - async function worker(): Promise { - while (next < items.length) { - const i = next++; - results[i] = await fn(items[i]); - } - } - const workers = - Array.from({length: Math.min(limit, items.length)}, () => worker()); - await Promise.all(workers); - return results; -} diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts new file mode 100644 index 00000000..233f7b5e --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -0,0 +1,267 @@ +// Knowledge Catalog <-> Semantic Model IR converter. +// +// SCAFFOLD (naming for the end state): this file currently holds only the +// READ direction -- Knowledge Catalog entries -> IR (`modelsFromCatalogResources` +// and its helpers). The WRITE direction (IR -> KC entries, +// `generateCatalogResources`) still lives in `knowledge_catalog.ts` and moves +// here once PR4 (the KC push line) merges, at which point this becomes the full +// two-way converter and `knowledge_catalog.ts` goes away. Until then, "where is +// the KC emit code?" -> `knowledge_catalog.ts`. +// +// The reader is the inverse of `generateCatalogResources`: it reconstructs the +// IR from the entries a pull hydrated (see `pull_kc.pullKnowledgeCatalog`). +// `semantic-entity` / `semantic-metric` entries are grouped under their +// `semantic-model` anchor via `parentEntry`; entries of other types are ignored. +// Resources are matched by type-name SUFFIX, so a reader need not know which +// system-type project/location the emitter used. +// +// Fidelity is bounded by what the emitter persisted, so this read is the inverse +// of the WRITE, not of the authored document. It recovers names, descriptions, +// data sources, field datatypes (via the schema aspect) and DIMENSION roles, +// field/metric expressions, and each metric's attach entity (re-derived from its +// expression, as the loader does). It cannot recover what the emitter does not +// write: entity keys/unique keys, `ai_context`, field labels, `importedDialect`, +// `custom_extensions`, and relationships (the graph edges live in the BigQuery +// property graph, not the catalog). + +import type {Entry} from '../gcp/dataplex'; +import {DataType, Entity, Field, Metric, SemanticModel} from './ir'; +import {referencedEntityNames} from './sql_expr_utils'; + +export interface ReadResult { + models: SemanticModel[]; + warnings: string[]; +} + +/** + * Reconstructs the Semantic Model IR from Knowledge Catalog entries. + * + * Returns one model per `semantic-model` anchor plus any warnings (no anchor, + * an orphaned child, an entry missing its aspect data). Entries must already be + * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a + * BASIC list omits aspect data, so the puller re-fetches each entry first. + */ +export function modelsFromCatalogResources(entries: Entry[]): ReadResult { + const warnings: string[] = []; + + const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); + const entityEntries = + entries.filter(e => semanticType(e) === 'semantic-entity'); + const metricEntries = + entries.filter(e => semanticType(e) === 'semantic-metric'); + + if (!anchors.length) { + warnings.push('no semantic-model entry found; nothing to reconstruct'); + return {models: [], warnings: [...new Set(warnings)]}; + } + + // A child belongs to its anchor by parentEntry. When there is exactly one + // anchor, children whose parentEntry does not resolve (e.g. a project-id + // normalization mismatch) are still attached to it rather than dropped. + const anchorNames = new Set(anchors.map(a => a.name)); + const soleAnchor = anchors.length === 1 ? anchors[0].name : undefined; + const childrenOf = (anchorName: string, pool: Entry[]) => pool.filter( + e => e.parentEntry === anchorName || + (soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? ''))); + + const models = anchors.map(anchor => { + const name = anchor.entrySource?.displayName ?? idOf(anchor.name); + + const entities = childrenOf(anchor.name, entityEntries) + .map(e => readEntity(e, warnings)); + const entityNames = entities.map(e => e.name); + const metrics = childrenOf(anchor.name, metricEntries) + .map(e => readMetric(e, entityNames, warnings)); + + // Relationships are not published to the catalog (see the file header), so + // a reconstructed model always has an empty edge set. + const model: SemanticModel = {name, entities, relationships: [], metrics}; + const description = anchor.entrySource?.description; + if (description !== undefined) model.description = description; + return model; + }); + + // Flag children that resolved to no anchor at all (only possible with + // multiple anchors, where the sole-anchor fallback does not apply). + if (!soleAnchor) { + for (const child of [...entityEntries, ...metricEntries]) { + if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { + warnings.push(`entry '${ + child.name}' has no resolvable parent semantic-model; omitted`); + } + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// Reconstructs an entity from its `semantic-entity` aspect (the backing source) +// and the built-in `schema` aspect (its fields). Keys are not persisted by the +// emitter and so come back empty. +function readEntity(entry: Entry, warnings: string[]): Entity { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const semantic = aspectData(entry, 'semantic-entity'); + const schema = aspectData(entry, 'schema'); + if (!Object.keys(semantic).length) { + warnings.push(`entity '${ + name}': no semantic-entity aspect data (fetch with the aspect type)`); + } + + const dataSource = dataSourceFromResource(semantic?.source?.resources?.[0]); + if (!dataSource) { + warnings.push( + `entity '${name}': no backing data source in the semantic-entity ` + + `aspect; 'source' will be empty and the entity may not load`); + } + const entity: Entity = { + name, + dataSource, + keys: [], // not persisted by the emitter; unrecoverable on read + fields: asArray(schema.fields).map(fd => readField(fd, name, warnings)), + }; + const description = entry.entrySource?.description; + if (description !== undefined) entity.description = description; + return entity; +} + + +// Reconstructs a field from one `schema` aspect field record, inverting +// schemaAspectData: the datatype from dataType/metadataType, expressions from +// the nested `semantics` block, and the DIMENSION role back to a dimension +// marker. +function readField(fd: any, entityName: string, warnings: string[]): Field { + const name = fd?.name; + if (name === undefined || name === '') { + warnings.push( + `entity '${entityName}': a schema field is missing its name; the ` + + `field may not load`); + } + const field: Field = {name}; + const sem = fd?.semantics ?? {}; + if (sem.expression !== undefined) field.expression = sem.expression; + const type = irDataType(fd?.dataType, fd?.metadataType); + if (type !== undefined) field.type = type; + if (sem.role === 'DIMENSION') field.dimension = {}; + if (fd?.description !== undefined) field.description = fd.description; + return field; +} + + +// Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` +// is re-derived from the expression (as the loader does) rather than read from +// the aspect, so it stays consistent with the reconstructed entity set. +function readMetric( + entry: Entry, entityNames: string[], warnings: string[]): Metric { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const data = aspectData(entry, 'semantic-metric'); + if (data.expression === undefined) { + warnings.push(`metric '${name}': no expression in semantic-metric aspect`); + } + + const metric: Metric = {name}; + if (data.expression !== undefined) metric.expression = data.expression; + const exprForRefs = data.expression ?? ''; + const referenced = referencedEntityNames(exprForRefs, entityNames); + if (referenced.length === 1) { + metric.entity = referenced[0]; + } else if (exprForRefs && !referenced.length) { + // Parity with the loader's convertMetric: an expression that qualifies no + // known entity is flagged as potentially unplaceable downstream. + warnings.push( + `metric '${name}': expression references no known entity; it may not ` + + `be placeable downstream`); + } + // The emitter writes a required dataType, defaulting a typeless metric to + // NUMERIC (see metricAspectData); NUMERIC maps back to Decimal, so a metric + // authored without a datatype round-trips as an explicit Decimal rather than + // un-typed. (Dimensions differ: their STRING default reads back as un-typed.) + const type = irDataType(data.dataType, undefined); + if (type !== undefined) metric.type = type; + const description = entry.entrySource?.description; + if (description !== undefined) metric.description = description; + return metric; +} + + +// The inverse of columnDataType/columnMetadataType: maps the schema aspect's +// dataType (disambiguated by metadataType only for the STRING family) back to +// the IR's logical DataType. STRING + OTHER is Opaque; a plain STRING is read +// as un-typed (undefined) -- the loader's default -- since the emitter cannot +// distinguish an authored `String` from an un-typed field (both emit STRING). +function irDataType(dataType: string|undefined, metadataType: string|undefined): + DataType|undefined { + switch (dataType) { + case 'INT64': + return 'Integer'; + case 'NUMERIC': + return 'Decimal'; + case 'FLOAT64': + return 'Float'; + case 'BOOL': + return 'Boolean'; + case 'DATE': + return 'Date'; + case 'TIME': + return 'Time'; + case 'DATETIME': + return 'DateTime'; + case 'TIMESTAMP': + return 'DateTimeTz'; + case 'STRING': + return metadataType === 'OTHER' ? 'Opaque' : undefined; + default: + return undefined; + } +} + + +// The inverse of resourcePath: a BigQuery linked-resource URI becomes the +// canonical `project.dataset.table` string; anything else (a verbatim query or +// a passthrough reference) is returned unchanged. +function dataSourceFromResource(resource: string|undefined): string { + const value = (resource ?? '').trim(); + const m = value.match( + /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/); + return m ? `${m[1]}.${m[2]}.${m[3]}` : value; +} + + +// The bare `semantic-*` type of an entry, matched by entryType suffix so the +// system-type project/location need not be known. Returns undefined for entries +// that are not part of a semantic model. +function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| + 'semantic-metric'|undefined { + for (const t of ['semantic-model', 'semantic-entity', 'semantic-metric'] as + const) { + if (entry.entryType?.endsWith(`/entryTypes/${t}`)) return t; + } + return undefined; +} + + +// The `data` payload of an entry's aspect of the given bare type, matched by +// the aspect key's `.` suffix or the aspectType's `/aspectTypes/` +// suffix (robust to whichever system-type project/location the emitter used). +// Returns an empty object when the aspect is absent. +function aspectData(entry: Entry, type: string): Record { + const aspects = entry.aspects ?? {}; + for (const [key, aspect] of Object.entries(aspects)) { + if (key.endsWith(`.${type}`) || + aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { + return aspect.data ?? {}; + } + } + return {}; +} + + +// The id segment of a full entry resource name (after the last '/'). +export function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + + +function asArray(value: any): any[] { + return Array.isArray(value) ? value : []; +} diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index 5e5860c9..a3051b22 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -45,8 +45,7 @@ import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; import {bigQueryGraphTargets} from './deploy_bigquery'; -import {DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; -import {referencedEntityNames} from './sql_expr_utils'; +import {DataType, Entity, Metric, Relationship, SemanticModel} from './ir'; // Where the `semantic-*` and `schema` system types live: built-in types in // project `dataplex-types`, location `global`. Callers may override to reference @@ -540,263 +539,3 @@ function linkSlug(s: string): string { function unquote(part: string): string { return part.replace(/^[`"]/, '').replace(/[`"]$/, ''); } - - -// --------------------------------------------------------------------------- -// Reader: Knowledge Catalog entries -> Semantic Model IR. -// -// The inverse of generateCatalogResources: it reconstructs the IR from the -// entries a pull hydrated (see deploy_knowledge_catalog.pullKnowledgeCatalog). -// `semantic-entity` / `semantic-metric` entries are grouped under their -// `semantic-model` anchor via `parentEntry`; entries of other types are -// ignored. Resources are matched by type-name SUFFIX, so a reader need not know -// which system-type project/location the emitter used. -// -// Fidelity is bounded by what the emitter persisted, so this read is the -// inverse of the WRITE, not of the authored document. It recovers names, -// descriptions, data sources, field datatypes (via the schema aspect) and -// DIMENSION roles, field/metric expressions, and each -// metric's attach entity (re-derived from its expression, as the loader does). -// It cannot recover what the emitter does not write: entity keys/unique keys, -// `ai_context`, field labels, `importedDialect`, `custom_extensions`, and -// relationships (the graph edges live in the BigQuery property graph, not the -// catalog). -// --------------------------------------------------------------------------- - -export interface ReadResult { - models: SemanticModel[]; - warnings: string[]; -} - -/** - * Reconstructs the Semantic Model IR from Knowledge Catalog entries. - * - * Returns one model per `semantic-model` anchor plus any warnings (no anchor, - * an orphaned child, an entry missing its aspect data). Entries must already be - * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a - * BASIC list omits aspect data, so the puller re-fetches each entry first. - */ -export function modelsFromCatalogResources(entries: Entry[]): ReadResult { - const warnings: string[] = []; - - const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); - const entityEntries = - entries.filter(e => semanticType(e) === 'semantic-entity'); - const metricEntries = - entries.filter(e => semanticType(e) === 'semantic-metric'); - - if (!anchors.length) { - warnings.push('no semantic-model entry found; nothing to reconstruct'); - return {models: [], warnings: [...new Set(warnings)]}; - } - - // A child belongs to its anchor by parentEntry. When there is exactly one - // anchor, children whose parentEntry does not resolve (e.g. a project-id - // normalization mismatch) are still attached to it rather than dropped. - const anchorNames = new Set(anchors.map(a => a.name)); - const soleAnchor = anchors.length === 1 ? anchors[0].name : undefined; - const childrenOf = (anchorName: string, pool: Entry[]) => pool.filter( - e => e.parentEntry === anchorName || - (soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? ''))); - - const models = anchors.map(anchor => { - const name = anchor.entrySource?.displayName ?? idOf(anchor.name); - - const entities = childrenOf(anchor.name, entityEntries) - .map(e => readEntity(e, warnings)); - const entityNames = entities.map(e => e.name); - const metrics = childrenOf(anchor.name, metricEntries) - .map(e => readMetric(e, entityNames, warnings)); - - // Relationships are not published to the catalog (see the file header), so - // a reconstructed model always has an empty edge set. - const model: SemanticModel = {name, entities, relationships: [], metrics}; - const description = anchor.entrySource?.description; - if (description !== undefined) model.description = description; - return model; - }); - - // Flag children that resolved to no anchor at all (only possible with - // multiple anchors, where the sole-anchor fallback does not apply). - if (!soleAnchor) { - for (const child of [...entityEntries, ...metricEntries]) { - if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { - warnings.push(`entry '${ - child.name}' has no resolvable parent semantic-model; omitted`); - } - } - } - - return {models, warnings: [...new Set(warnings)]}; -} - - -// Reconstructs an entity from its `semantic-entity` aspect (the backing source) -// and the built-in `schema` aspect (its fields). Keys are not persisted by the -// emitter and so come back empty. -function readEntity(entry: Entry, warnings: string[]): Entity { - const name = entry.entrySource?.displayName ?? idOf(entry.name); - const semantic = aspectData(entry, 'semantic-entity'); - const schema = aspectData(entry, 'schema'); - if (!Object.keys(semantic).length) { - warnings.push(`entity '${ - name}': no semantic-entity aspect data (fetch with the aspect type)`); - } - - const dataSource = dataSourceFromResource(semantic?.source?.resources?.[0]); - if (!dataSource) { - warnings.push( - `entity '${name}': no backing data source in the semantic-entity ` + - `aspect; 'source' will be empty and the entity may not load`); - } - const entity: Entity = { - name, - dataSource, - keys: [], // not persisted by the emitter; unrecoverable on read - fields: asArray(schema.fields).map(fd => readField(fd, name, warnings)), - }; - const description = entry.entrySource?.description; - if (description !== undefined) entity.description = description; - return entity; -} - - -// Reconstructs a field from one `schema` aspect field record, inverting -// schemaAspectData: the datatype from dataType/metadataType, expressions from -// the nested `semantics` block, and the DIMENSION role back to a dimension -// marker. -function readField(fd: any, entityName: string, warnings: string[]): Field { - const name = fd?.name; - if (name === undefined || name === '') { - warnings.push( - `entity '${entityName}': a schema field is missing its name; the ` + - `field may not load`); - } - const field: Field = {name}; - const sem = fd?.semantics ?? {}; - if (sem.expression !== undefined) field.expression = sem.expression; - const type = irDataType(fd?.dataType, fd?.metadataType); - if (type !== undefined) field.type = type; - if (sem.role === 'DIMENSION') field.dimension = {}; - if (fd?.description !== undefined) field.description = fd.description; - return field; -} - - -// Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` -// is re-derived from the expression (as the loader does) rather than read from -// the aspect, so it stays consistent with the reconstructed entity set. -function readMetric( - entry: Entry, entityNames: string[], warnings: string[]): Metric { - const name = entry.entrySource?.displayName ?? idOf(entry.name); - const data = aspectData(entry, 'semantic-metric'); - if (data.expression === undefined) { - warnings.push(`metric '${name}': no expression in semantic-metric aspect`); - } - - const metric: Metric = {name}; - if (data.expression !== undefined) metric.expression = data.expression; - const exprForRefs = data.expression ?? ''; - const referenced = referencedEntityNames(exprForRefs, entityNames); - if (referenced.length === 1) { - metric.entity = referenced[0]; - } else if (exprForRefs && !referenced.length) { - // Parity with the loader's convertMetric: an expression that qualifies no - // known entity is flagged as potentially unplaceable downstream. - warnings.push( - `metric '${name}': expression references no known entity; it may not ` + - `be placeable downstream`); - } - // The emitter writes a required dataType, defaulting a typeless metric to - // NUMERIC (see metricAspectData); NUMERIC maps back to Decimal, so a metric - // authored without a datatype round-trips as an explicit Decimal rather than - // un-typed. (Dimensions differ: their STRING default reads back as un-typed.) - const type = irDataType(data.dataType, undefined); - if (type !== undefined) metric.type = type; - const description = entry.entrySource?.description; - if (description !== undefined) metric.description = description; - return metric; -} - - -// The inverse of columnDataType/columnMetadataType: maps the schema aspect's -// dataType (disambiguated by metadataType only for the STRING family) back to -// the IR's logical DataType. STRING + OTHER is Opaque; a plain STRING is read -// as un-typed (undefined) -- the loader's default -- since the emitter cannot -// distinguish an authored `String` from an un-typed field (both emit STRING). -function irDataType(dataType: string|undefined, metadataType: string|undefined): - DataType|undefined { - switch (dataType) { - case 'INT64': - return 'Integer'; - case 'NUMERIC': - return 'Decimal'; - case 'FLOAT64': - return 'Float'; - case 'BOOL': - return 'Boolean'; - case 'DATE': - return 'Date'; - case 'TIME': - return 'Time'; - case 'DATETIME': - return 'DateTime'; - case 'TIMESTAMP': - return 'DateTimeTz'; - case 'STRING': - return metadataType === 'OTHER' ? 'Opaque' : undefined; - default: - return undefined; - } -} - - -// The inverse of resourcePath: a BigQuery linked-resource URI becomes the -// canonical `project.dataset.table` string; anything else (a verbatim query or -// a passthrough reference) is returned unchanged. -function dataSourceFromResource(resource: string|undefined): string { - const value = (resource ?? '').trim(); - const m = value.match( - /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/); - return m ? `${m[1]}.${m[2]}.${m[3]}` : value; -} - - -// The bare `semantic-*` type of an entry, matched by entryType suffix so the -// system-type project/location need not be known. Returns undefined for entries -// that are not part of a semantic model. -function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| - 'semantic-metric'|undefined { - for (const t of ['semantic-model', 'semantic-entity', 'semantic-metric'] as - const) { - if (entry.entryType?.endsWith(`/entryTypes/${t}`)) return t; - } - return undefined; -} - - -// The `data` payload of an entry's aspect of the given bare type, matched by -// the aspect key's `.` suffix or the aspectType's `/aspectTypes/` -// suffix (robust to whichever system-type project/location the emitter used). -// Returns an empty object when the aspect is absent. -function aspectData(entry: Entry, type: string): Record { - const aspects = entry.aspects ?? {}; - for (const [key, aspect] of Object.entries(aspects)) { - if (key.endsWith(`.${type}`) || - aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { - return aspect.data ?? {}; - } - } - return {}; -} - - -// The id segment of a full entry resource name (after the last '/'). -export function idOf(name: string): string { - return name.split('/').pop() ?? name; -} - - -function asArray(value: any): any[] { - return Array.isArray(value) ? value : []; -} diff --git a/toolbox/mdcode/src/libts/semantic/serialize.ts b/toolbox/mdcode/src/libts/semantic/osi_converter.ts similarity index 93% rename from toolbox/mdcode/src/libts/semantic/serialize.ts rename to toolbox/mdcode/src/libts/semantic/osi_converter.ts index dd215100..61c4a99b 100644 --- a/toolbox/mdcode/src/libts/semantic/serialize.ts +++ b/toolbox/mdcode/src/libts/semantic/osi_converter.ts @@ -1,9 +1,16 @@ -// Serializes the Semantic Model IR (./ir) back to the open AI-first semantics -// format (YAML). This is the inverse of `loader.ts`: `loader` reads authored -// YAML into the IR; this module writes the IR out as a YAML document the loader -// can read back. It is the local-workspace sink for `pull` (Knowledge Catalog -// -> IR -> YAML), the counterpart to how `push` compiles YAML -> IR -> a -// destination. +// OSI (open, AI-first semantics) <-> Semantic Model IR converter. +// +// SCAFFOLD (naming for the end state): this file currently holds only the +// WRITE direction -- IR -> OSI YAML (`serializeModel`). The READ direction +// (OSI YAML -> IR) still lives in `loader.ts` and moves here once PR4 (the +// KC push line) merges, at which point this becomes the full two-way +// converter. Until then, "where is the OSI parser?" -> `loader.ts`. +// +// Serializing is the inverse of `loader.ts`: `loader` reads authored YAML +// into the IR; this module writes the IR out as a YAML document the loader +// can read back. It is the local-workspace sink for `pull` (Knowledge +// Catalog -> IR -> YAML), the counterpart to how `push` compiles YAML -> IR +// -> a destination. // // Fidelity is at the IR level, not byte-for-byte with a hand-authored file. The // loader normalizes several authoring conveniences into the IR at load time, so diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts new file mode 100644 index 00000000..f03edc3f --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -0,0 +1,182 @@ +// Pull: fetch a semantic model from a live Knowledge Catalog into the IR. +// +// The read-direction counterpart of `deployKnowledgeCatalog` (push, in +// `deploy_knowledge_catalog.ts`). Unlike a write, a pull needs no server-side +// type provisioning -- only that the `semantic-*` entries exist. It enumerates +// the entry group, keeps the semantic entries, hydrates each one's aspect data +// (a BASIC list omits aspect data, so each entry is re-fetched with its aspect +// types -- an entity needs BOTH its `semantic-entity` aspect and the built-in +// `schema` aspect), and hands the hydrated entries to the pure reader +// (`kc_converter.modelsFromCatalogResources`). + +import {CatalogClient, Entry} from '../gcp/dataplex'; +import {SemanticModel} from './ir'; +import {idOf, modelsFromCatalogResources} from './kc_converter'; + +export interface KcPullOptions { + project: string; + location: string; + entryGroup: string; + model?: string; // limit to a single model by name (default: all) +} + +export interface KcPullResult { + models: SemanticModel[]; + warnings: string[]; +} + +// Upper bound on in-flight aspect-hydration fetches during a pull. +const HYDRATE_CONCURRENCY = 8; + +// Reads the semantic models back from a Knowledge Catalog entry group. Emits no +// console output; warnings (skipped entries, no match for --model, reader +// warnings) are returned for the caller to print. +export async function pullKnowledgeCatalog( + cat: CatalogClient, opts: KcPullOptions): Promise { + const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`; + const warnings: string[] = []; + + // Enumerate the group (paging is inherently sequential) and pick the semantic + // entries, then hydrate their aspects concurrently: a BASIC list omits aspect + // data, so each entry needs its own lookupEntry, and those fetches are + // independent. The pool preserves input order so warnings stay deterministic. + const targets: {entry: Entry; aspectTypes: string[]}[] = []; + for await (const entry of cat.listEntries( + opts.project, opts.location, opts.entryGroup)) { + const aspectTypes = semanticAspectTypes(entry.entryType); + if (aspectTypes) targets.push({entry, aspectTypes}); + // else: not part of a semantic model; ignore it. + } + + // When scoped to one model, hydrate only that model's entries -- its anchor + // (matched by name) plus the children pointing at it. A list already carries + // entrySource + parentEntry, so this avoids fetching every other model's + // aspects. No match short-circuits with just the not-found warning. + let scoped = targets; + if (opts.model) { + scoped = scopeToModel(targets, opts.model); + if (!scoped.length) { + return { + models: [], + warnings: [ + `no semantic model named '${opts.model}' found in ${destination}` + ], + }; + } + } + + const fetched = await mapConcurrent( + scoped, HYDRATE_CONCURRENCY, async ({entry, aspectTypes}) => { + const res = await cat.lookupEntry( + opts.project, opts.location, entry.name, aspectTypes); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry '${entry.name}' (status ${ + res.status}); skipped` + }; + } + return {entry: res.result}; + }); + + const hydrated: Entry[] = []; + for (const r of fetched) { + if (r.entry) + hydrated.push(r.entry); + else if (r.warning) + warnings.push(r.warning); + } + + const read = modelsFromCatalogResources(hydrated); + warnings.push(...read.warnings); + + // Defense in depth: keep only the requested model even if the reader surfaced + // another anchor (e.g. a child whose parentEntry pointed outside the scope). + let models = read.models; + if (opts.model) { + models = models.filter(m => m.name === opts.model); + if (!models.length) { + warnings.push( + `no semantic model named '${opts.model}' found in ${destination}`); + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// The aspect type resource names to hydrate for a semantic entry, derived from +// its entryType (the aspect types are the parallel resources in the same +// project/location). An entity carries two aspects: its `semantic-entity` +// aspect and the built-in `schema` aspect that holds its fields. Returns +// undefined for entries that are not part of a semantic model. +function semanticAspectTypes(entryType: string): string[]|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + const typeBase = entryType.slice(0, idx); + const t = entryType.slice(idx + marker.length); + const aspectType = (name: string) => `${typeBase}/aspectTypes/${name}`; + switch (t) { + case 'semantic-model': + return [aspectType('semantic-model')]; + case 'semantic-entity': + return [aspectType('semantic-entity'), aspectType('schema')]; + case 'semantic-metric': + return [aspectType('semantic-metric')]; + default: + return undefined; + } +} + + +// Restricts hydration targets to a single model: the semantic-model anchor +// whose name (entrySource.displayName, else the entry id) matches `model`, plus +// every child entry whose parentEntry is that anchor. Uses only list-level +// fields (no aspect data), so it runs before hydration and avoids fetching +// unrelated models' aspects. Returns [] when no anchor matches. +function scopeToModel( + targets: {entry: Entry; aspectTypes: string[]}[], + model: string): {entry: Entry; aspectTypes: string[]}[] { + const isAnchor = (t: {entry: Entry}) => + !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); + const allAnchorNames = new Set(targets.filter(isAnchor).map(t => t.entry.name)); + const matchedAnchorNames = new Set( + targets.filter(isAnchor) + .filter( + t => (t.entry.entrySource?.displayName ?? idOf(t.entry.name)) === + model) + .map(t => t.entry.name)); + if (!matchedAnchorNames.size) return []; + // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds + // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a + // project-id normalization mismatch) still belongs to it. Without this a + // scoped pull would drop children a full pull keeps. + const soleAnchor = + allAnchorNames.size === 1 ? [...allAnchorNames][0] : undefined; + const soleMatched = + soleAnchor !== undefined && matchedAnchorNames.has(soleAnchor); + return targets.filter( + t => matchedAnchorNames.has(t.entry.name) || + matchedAnchorNames.has(t.entry.parentEntry ?? '') || + (soleMatched && !isAnchor(t) && + !allAnchorNames.has(t.entry.parentEntry ?? ''))); +} + + +// Maps `items` through `fn` with at most `limit` calls in flight, returning +// results in input order (so downstream ordering stays deterministic). +async function mapConcurrent( + items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results: R[] = new Array(items.length); + let next = 0; + async function worker(): Promise { + while (next < items.length) { + const i = next++; + results[i] = await fn(items[i]); + } + } + const workers = + Array.from({length: Math.min(limit, items.length)}, () => worker()); + await Promise.all(workers); + return results; +} diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index 59ce84e1..1c67d76f 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -14,7 +14,8 @@ import * as deploy from '../libts/semantic/deploy_bigquery'; import * as kc from '../libts/semantic/deploy_knowledge_catalog'; import {BigQueryClient} from '../libts/gcp/bigquery'; import {LoadedModel, loadSemanticModels} from '../libts/semantic/loader'; -import { serializeModel } from '../libts/semantic/serialize'; +import {pullKnowledgeCatalog} from '../libts/semantic/pull_kc'; +import {serializeModel} from '../libts/semantic/osi_converter'; import {validateBigQueryDataSources, validatePushRequirements} from '../libts/semantic/validate'; @@ -387,7 +388,7 @@ async function pullSemanticModel( : 'Pulling semantic model from Knowledge Catalog...'); const catalog = new dataplex.CatalogClient(ctx); - const result = await kc.pullKnowledgeCatalog(catalog, { + const result = await pullKnowledgeCatalog(catalog, { project: source.project, location: source.location, entryGroup: source.entryGroup, diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml new file mode 100644 index 00000000..ffdfa798 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml @@ -0,0 +1,29 @@ +version: 0.2.0.dev0 +semantic_model: + - name: sales + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": + ["//bigquery.googleapis.com/projects/demo/datasets/sales/propertyGraphs/sales_graph"]}' + datasets: + - name: orders + source: demo.sales.orders + primary_key: + - o_orderkey + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml new file mode 100644 index 00000000..f9e42580 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml @@ -0,0 +1,25 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: sales + datasets: + - name: orders + source: demo.sales.orders + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) + datatype: Decimal diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml new file mode 100644 index 00000000..40c22466 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml @@ -0,0 +1,86 @@ +version: 0.2.0.dev0 +semantic_model: + - name: sales + description: Sales orders with customer attributes + ai_context: + instructions: Use this model for order analysis. + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": + ["//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales"]}' + datasets: + - name: orders + source: samples.tpch.orders + primary_key: + - o_orderkey + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + description: Order identifier + - name: o_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_custkey + - name: o_orderdate + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderdate + label: Order Date + dimension: + is_time: true + ai_context: + synonyms: + - order date + - date + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + - name: customer + source: samples.tpch.customer + primary_key: + - c_custkey + fields: + - name: c_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: c_custkey + - name: c_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_name + description: Customer name + relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: + - o_custkey + to_columns: + - c_custkey + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) + description: Total order revenue + ai_context: + synonyms: + - revenue + - sales + - name: order_count + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(orders.o_orderkey) + description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml new file mode 100644 index 00000000..bf7cfc8d --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -0,0 +1,61 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: sales + description: Sales orders with customer attributes + datasets: + - name: orders + source: samples.tpch.orders + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + description: Order identifier + - name: o_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_custkey + - name: o_orderdate + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderdate + dimension: {} + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + - name: customer + source: samples.tpch.customer + fields: + - name: c_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: c_custkey + - name: c_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_name + description: Customer name + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) + datatype: Decimal + description: Total order revenue + - name: order_count + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(orders.o_orderkey) + datatype: Decimal + description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml new file mode 100644 index 00000000..27acf1e6 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml @@ -0,0 +1,218 @@ +version: 0.2.0.dev0 +semantic_model: + - name: tpcds_model + description: TPC-DS retail model + datasets: + - name: store_sales + source: tpcds.public.store_sales + primary_key: + - ss_item_sk + - ss_ticket_number + description: Fact table containing all store sales transactions + fields: + - name: ss_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_item_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_item_sk}" + dimension: + is_time: false + - name: ss_ticket_number + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ticket_number + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_ticket_number}" + dimension: + is_time: false + - name: ss_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_customer_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_customer_sk}" + dimension: + is_time: false + - name: ss_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_store_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_store_sk}" + dimension: + is_time: false + - name: ss_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: ss_quantity + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_quantity}" + description: Quantity of items sold + - name: ss_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_sales_price + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_sales_price}" + description: Sales price per unit + - name: ss_ext_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ext_sales_price + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_ext_sales_price}" + description: Extended sales price (quantity * price) + - name: ss_net_profit + expression: + dialects: + - dialect: BIGQUERY + expression: ss_net_profit + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_net_profit}" + description: Net profit from the sale + - name: customer + source: tpcds.public.customer + primary_key: + - c_customer_sk + description: Customer dimension with demographic information + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: c_customer_sk + dimension: + is_time: false + - name: c_first_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_first_name + dimension: + is_time: false + description: Customer first name + - name: c_last_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_last_name + dimension: + is_time: false + description: Customer last name + - name: item + source: tpcds.public.item + primary_key: + - i_item_sk + description: Item/Product dimension + fields: + - name: i_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: i_item_sk + dimension: + is_time: false + - name: i_brand + expression: + dialects: + - dialect: BIGQUERY + expression: i_brand + dimension: + is_time: false + - name: i_category + expression: + dialects: + - dialect: BIGQUERY + expression: i_category + dimension: + is_time: false + - name: i_current_price + expression: + dialects: + - dialect: BIGQUERY + expression: i_current_price + description: Current price of the item + - name: store + source: tpcds.public.store + primary_key: + - s_store_sk + description: Store dimension with location attributes + fields: + - name: s_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_sk + dimension: + is_time: false + - name: s_store_name + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_name + dimension: + is_time: false + - name: s_city + expression: + dialects: + - dialect: BIGQUERY + expression: s_city + dimension: + is_time: false + - name: s_state + expression: + dialects: + - dialect: BIGQUERY + expression: s_state + dimension: + is_time: false + - name: s_number_employees + expression: + dialects: + - dialect: BIGQUERY + expression: s_number_employees + description: Number of employees at the store + - name: date_dim + source: sqlgen-testing.demo.date_dim + description: Date dimension with calendar attributes + custom_extensions: + - vendor_name: GOODDATA + data: '{"date_dimension": true, "granularities": ["DAY", "WEEK", "MONTH", + "QUARTER", "YEAR"]}' + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - ss_sold_date_sk + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml new file mode 100644 index 00000000..7437f8ac --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -0,0 +1,147 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: tpcds_model + description: TPC-DS retail model + datasets: + - name: store_sales + source: tpcds.public.store_sales + description: Fact table containing all store sales transactions + fields: + - name: ss_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_item_sk + dimension: {} + - name: ss_ticket_number + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ticket_number + dimension: {} + - name: ss_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_customer_sk + dimension: {} + - name: ss_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_store_sk + dimension: {} + - name: ss_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: ss_quantity + description: Quantity of items sold + - name: ss_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_sales_price + description: Sales price per unit + - name: ss_ext_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ext_sales_price + description: Extended sales price (quantity * price) + - name: ss_net_profit + expression: + dialects: + - dialect: BIGQUERY + expression: ss_net_profit + description: Net profit from the sale + - name: customer + source: tpcds.public.customer + description: Customer dimension with demographic information + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: c_customer_sk + dimension: {} + - name: c_first_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_first_name + dimension: {} + description: Customer first name + - name: c_last_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_last_name + dimension: {} + description: Customer last name + - name: item + source: tpcds.public.item + description: Item/Product dimension + fields: + - name: i_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: i_item_sk + dimension: {} + - name: i_brand + expression: + dialects: + - dialect: BIGQUERY + expression: i_brand + dimension: {} + - name: i_category + expression: + dialects: + - dialect: BIGQUERY + expression: i_category + dimension: {} + - name: i_current_price + expression: + dialects: + - dialect: BIGQUERY + expression: i_current_price + description: Current price of the item + - name: store + source: tpcds.public.store + description: Store dimension with location attributes + fields: + - name: s_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_sk + dimension: {} + - name: s_store_name + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_name + dimension: {} + - name: s_city + expression: + dialects: + - dialect: BIGQUERY + expression: s_city + dimension: {} + - name: s_state + expression: + dialects: + - dialect: BIGQUERY + expression: s_state + dimension: {} + - name: s_number_employees + expression: + dialects: + - dialect: BIGQUERY + expression: s_number_employees + description: Number of employees at the store + - name: date_dim + source: sqlgen-testing.demo.date_dim + description: Date dimension with calendar attributes diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts similarity index 73% rename from toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts rename to toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index a8d5e447..d5ecb145 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -1,5 +1,5 @@ -// Behavior specification for the Knowledge Catalog reader -// (modelsFromCatalogResources in src/libts/semantic/knowledge_catalog.ts). +// Behavior specification for the KC converter's read direction +// (modelsFromCatalogResources in src/libts/semantic/kc_converter.ts). // // The reader is the inverse of the emitter (generateCatalogResources). The // central guarantee is an emitter -> reader round trip: emit a model's entries, @@ -12,9 +12,15 @@ // re-derivation, and parent/anchor grouping). import {describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; import {Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; -import {generateCatalogResources, modelsFromCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; +import {serializeModel} from '../../../src/libts/semantic/osi_converter'; + +const FIXTURES = path.join(__dirname, 'fixtures'); const OPTS = { project: 'dest', @@ -256,3 +262,54 @@ describe('metric expression referencing no known entity', () => { .toBe(true); }); }); + + +// -- Golden pull: the whole KC entries -> IR -> OSI YAML output. -- +// +// The round trip above proves the reader inverts the emitter in memory; this +// pins the reviewable artifact. For each corpus fixture it reads the committed +// emitter golden (`.knowledge_catalog.golden.json` -- the exact entries +// a push produced) back through the reader and serializes the reconstructed IR +// to `.pull.golden.yaml`. Open that next to the fixture's +// `.osi.golden.yaml` to see, as whole files, what a Knowledge Catalog round trip +// preserves and what it drops (keys, ai_context, labels, relationships). +// +// Regenerate after an intentional reader/serializer change: +// UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/kc_converter.test.ts +describe('golden pull: each corpus KC golden reconstructs to its exact YAML', + () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + const kcGoldenPath = (fixture: string) => path.join( + FIXTURES, + fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); + const pullGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.pull.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const kc = JSON.parse(fs.readFileSync(kcGoldenPath(fixture), 'utf8')); + const {models, warnings} = modelsFromCatalogResources(kc.entries); + // Reader warnings ride along as YAML comments so the golden shows + // the full outcome, not just the recovered document. + const header = warnings.length ? + warnings.map((w: string) => `# warning: ${w}`).join('\n') + '\n' : + '# (no warnings)\n'; + const actual = + header + models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = pullGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename(golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/serialize.test.ts b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts similarity index 81% rename from toolbox/mdcode/tests/libts/semantic/serialize.test.ts rename to toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts index 81c4454c..82943898 100644 --- a/toolbox/mdcode/tests/libts/semantic/serialize.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts @@ -1,7 +1,7 @@ -// Behavior specification for the semantic-model serializer -// (src/libts/semantic/serialize.ts). +// Behavior specification for the OSI converter's serialize direction +// (serializeModel in src/libts/semantic/osi_converter.ts). // -// serialize.ts is the inverse of loader.ts: IR -> open-format YAML. The +// The serializer is the inverse of loader.ts: IR -> open-format YAML. The // strongest guarantee is a round trip through the loader -- load a fixture to // the IR, serialize it, load the serialized text again, and assert the two IRs // are identical. That pins IR-level fidelity across every feature the loader @@ -17,7 +17,7 @@ import * as yaml from 'yaml'; import {Field, Metric, Relationship, SemanticModel} from '../../../src/libts/semantic/ir'; import {loadModels} from '../../../src/libts/semantic/loader'; -import {modelDocument, serializeModel} from '../../../src/libts/semantic/serialize'; +import {modelDocument, serializeModel} from '../../../src/libts/semantic/osi_converter'; const FIXTURES = path.join(__dirname, 'fixtures'); @@ -288,3 +288,47 @@ describe('serialize flags loader-invalid reconstructions', () => { expect(labels).toContain('BIGQUERY'); }); }); + + +// -- Golden corpus: the whole IR -> OSI YAML output, reviewable as a file. -- +// +// The round-trip tests above prove IR-level stability but never pin the exact +// YAML text. These goldens capture the full serialized document for a small +// corpus so a reviewer can open a `.yaml` next to its `.osi.golden.yaml` and see +// exactly what the serializer emits. The same corpus backs kc_converter's +// `.pull.golden.yaml`, so diffing the two shows what a Knowledge Catalog round +// trip loses relative to the authored source. +// +// Regenerate after an intentional serializer change: +// UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/osi_converter.test.ts +describe('golden OSI document: each corpus fixture serializes to its exact YAML', + () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + // Same load defaults as the KC e2e/pull goldens, so the OSI golden + // and the pull golden are directly comparable. + const LOAD = {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}; + const osiGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.osi.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + const models = loadModels(text, LOAD).models; + const actual = models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = osiGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename(golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts similarity index 98% rename from toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts rename to toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index e31a2f94..5a09cebd 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -1,5 +1,5 @@ // Tests for the semantic-model Knowledge Catalog pull leg -// (pullKnowledgeCatalog in src/libts/semantic/deploy_knowledge_catalog.ts). +// (pullKnowledgeCatalog in src/libts/semantic/pull_kc.ts). // // pullKnowledgeCatalog is the orchestration around the pure reader: enumerate // the entry group, hydrate each semantic entry's aspect data, and reconstruct @@ -7,7 +7,7 @@ // the fake serves are produced by the real emitter, so this exercises the true // list -> hydrate -> read path end to end (an entity is re-fetched with BOTH // its semantic-entity and schema aspects). The reader's own mapping is covered -// in knowledge_catalog.read.test.ts; the focus here is the fetch SEQUENCE: +// in kc_converter.test.ts; the focus here is the fetch SEQUENCE: // aspect hydration, the --model filter, skipped entries, and ignoring foreign // entries. @@ -15,7 +15,7 @@ import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; import {ApiResult} from '../../../src/libts/gcp/api'; import {CatalogClient, Entry} from '../../../src/libts/gcp/dataplex'; -import {pullKnowledgeCatalog} from '../../../src/libts/semantic/deploy_knowledge_catalog'; +import {pullKnowledgeCatalog} from '../../../src/libts/semantic/pull_kc'; import {SemanticModel} from '../../../src/libts/semantic/ir'; import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; From da163dc4d4e29cf048346f693622c777fc00c76a Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 06:29:59 +0000 Subject: [PATCH 25/34] mdcode: add TODO(#278) migration notes to the converter-scaffold files Each new converter/orchestration file now carries an actionable TODO spelling out how the scaffold collapses once #278 merges, so reviewers can see the plan: - osi_converter.ts: fold loader.ts (OSI read) in, delete loader.ts, repoint importers. - kc_converter.ts: fold generateCatalogResources (KC write) in, delete knowledge_catalog.ts, repoint importers, demote the shared idOf to a local. - pull_kc.ts: rename deploy_knowledge_catalog.ts -> push_kc.ts for push_kc/pull_kc symmetry (rename only, no logic moves). --- toolbox/mdcode/src/libts/semantic/kc_converter.ts | 8 ++++++++ toolbox/mdcode/src/libts/semantic/osi_converter.ts | 7 +++++++ toolbox/mdcode/src/libts/semantic/pull_kc.ts | 7 +++++++ 3 files changed, 22 insertions(+) diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 233f7b5e..25a67c0b 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -8,6 +8,14 @@ // two-way converter and `knowledge_catalog.ts` goes away. Until then, "where is // the KC emit code?" -> `knowledge_catalog.ts`. // +// TODO(#278): fold the KC WRITE direction in and drop the scaffold. Once +// #278 merges, move `generateCatalogResources` (and the `KcResources` type +// + emit helpers) out of `knowledge_catalog.ts` into this file, delete +// `knowledge_catalog.ts`, and repoint its importers (`deploy_knowledge_catalog.ts` +// and the emitter tests). `idOf` -- shared by both directions and re-exported +// from here only for `pull_kc` today -- becomes a plain local. Then this is +// the sole KC<->IR codec and the SCAFFOLD note above comes out. +// // The reader is the inverse of `generateCatalogResources`: it reconstructs the // IR from the entries a pull hydrated (see `pull_kc.pullKnowledgeCatalog`). // `semantic-entity` / `semantic-metric` entries are grouped under their diff --git a/toolbox/mdcode/src/libts/semantic/osi_converter.ts b/toolbox/mdcode/src/libts/semantic/osi_converter.ts index 61c4a99b..b8b260aa 100644 --- a/toolbox/mdcode/src/libts/semantic/osi_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/osi_converter.ts @@ -6,6 +6,13 @@ // KC push line) merges, at which point this becomes the full two-way // converter. Until then, "where is the OSI parser?" -> `loader.ts`. // +// TODO(#278): fold the OSI READ direction in and drop the scaffold. Once +// #278 merges, move `loadModels` / `fromDocument` / `loadSemanticModels` +// (and their Load* types) out of `loader.ts` into this file, delete +// `loader.ts`, and repoint its importers (`src/tool/commands.ts` and the +// load / validate / bigquery tests). Then this is the sole OSI<->IR codec +// and the SCAFFOLD note above comes out. +// // Serializing is the inverse of `loader.ts`: `loader` reads authored YAML // into the IR; this module writes the IR out as a YAML document the loader // can read back. It is the local-workspace sink for `pull` (Knowledge diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index f03edc3f..aebaa7b5 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -8,6 +8,13 @@ // types -- an entity needs BOTH its `semantic-entity` aspect and the built-in // `schema` aspect), and hands the hydrated entries to the pure reader // (`kc_converter.modelsFromCatalogResources`). +// +// TODO(#278): Layer 2 push/pull symmetry. This is the pull half over the +// pure `kc_converter` codec; the push half (`deployKnowledgeCatalog`) still +// lives in `deploy_knowledge_catalog.ts`. Once #278 merges, rename that +// file to `push_kc.ts` (repointing `src/tool/commands.ts` and its test) so +// the orchestration layer reads as `push_kc` / `pull_kc`. Rename only -- no +// logic moves. import {CatalogClient, Entry} from '../gcp/dataplex'; import {SemanticModel} from './ir'; From 5b41f6429c5afa2f168c716df81e045d288efd26 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 06:34:51 +0000 Subject: [PATCH 26/34] mdcode: state push/pull lossiness explicitly in the user guide Reviewers asked the docs to be clear about lossless vs lossy. Both directions are lossy; say so plainly and enumerate exactly what each drops: - Push to BigQuery is lossy: captures the queryable structure (node/edge tables, measures) but not descriptive metadata; non-reducible metrics are skipped. - Push to Knowledge Catalog is lossy: stores a metadata subset (keeps 1:1/1:N as schema-join links) and drops keys, ai_context, labels, vendor SQL, M:N. - Pull is lossy: recovers even less than the catalog holds (no relationships, no deploymentTargets). A push followed by a pull does not return the original file. --- toolbox/mdcode/docs/semantic-model.md | 49 ++++++++++++++++----------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 7d3c28a2..80b7db91 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -138,6 +138,12 @@ statement runs in the right region; without that permission it falls back to BigQuery's own location inference and warns. Nothing runs under `--validate-only`; add `--print` to see the DDL. +Push to BigQuery is **lossy**: the graph captures the queryable structure — +node tables, edge tables, and measures — but not descriptive metadata +(descriptions, `ai_context`, synonyms, labels), and a metric that does not +reduce to a single MEASURE is dropped with a warning. It is a query surface, +not a copy of your model. + ### What gets created in Knowledge Catalog Each element of your model maps to one catalog resource. Every resource type @@ -155,16 +161,19 @@ An entity entry carries its columns in the `schema` aspect (name, data type, and description per field); a `schema-join` link carries the relationship detail — the paired columns and foreign-key direction — in its aspect. -> **Note — the catalog is not a full copy of your model.** By default the SQL -> expressions are **not** written to Knowledge Catalog: the published system-type -> templates do not yet carry a per-field `semantics` block or a -> `semantic-metric.expression` field, so the default push omits them (pass -> `--emit-expressions` to write them once the templates gain the fields). The -> original vendor SQL (`importedExpression` — e.g. the MAQL or Snowflake form a -> metric was imported from) is never written either. All of it stays in your -> authored document and is still used when generating BigQuery SQL. Keep your -> model document as the source of truth: a model reconstructed only from the -> catalog would come back without its SQL. +> **Note — push to Knowledge Catalog is lossy.** The catalog holds metadata, +> not a full copy of your model. It **stores** names, descriptions, data +> sources, field datatypes and roles, and 1:1 / 1:N relationships (as +> `schema-join` links). By default it does **not** store the SQL expressions: +> the published system-type templates do not yet carry a per-field `semantics` +> block or a `semantic-metric.expression` field, so the default push omits them +> (pass `--emit-expressions` to write the canonical GoogleSQL/ANSI expression +> once the templates gain the fields). It never stores entity keys, `ai_context`, +> field labels, the original vendor SQL (`importedExpression` — e.g. the MAQL or +> Snowflake form a metric was imported from), or M:N relationships. Those stay in +> your authored document (and, for the edges, in the BigQuery property graph); the +> vendor SQL and expressions are still used when generating BigQuery SQL. Keep +> your model document as the source of truth. ## Validation @@ -250,16 +259,18 @@ Pull writes with the same last-write-wins policy as the core pull: a model that already exists locally is overwritten in place, and a local-only document (one with no matching catalog entry) is left untouched — pull never deletes. -> **Note — pull recovers the catalog, not your authored document.** The catalog -> stores only what push wrote to it (see the note under [What gets created in -> Knowledge Catalog](#what-gets-created-in-knowledge-catalog)), so a pulled -> document comes back without the content the catalog never held: entity keys, +> **Note — pull is lossy.** It reconstructs a model only from what push wrote +> to the catalog (see the note under [What gets created in Knowledge +> Catalog](#what-gets-created-in-knowledge-catalog)), and recovers even less +> than the catalog holds. A pulled document comes back without entity keys, > `ai_context`, field labels, the original vendor SQL (`importedExpression`), -> and relationships (the graph edges live in the BigQuery property graph, not -> the catalog). A field's *role* survives as a bare `dimension: {}` marker, but -> its detail (`is_time`, and so on) does not. Keep your authored document as the -> source of truth; treat a pulled document as a faithful copy of the catalog -> metadata, not of the original model. +> relationships (even the 1:1 / 1:N `schema-join` links push wrote — the edges +> live in the BigQuery property graph), and the `deploymentTargets` custom +> extension. A field's *role* survives as a bare `dimension: {}` marker, but its +> detail (`is_time`, and so on) does not. **A push followed by a pull does not +> return your original file** — treat a pulled document as a faithful copy of +> the catalog metadata, not of the authored model, and keep the authored +> document as the source of truth. ## Permissions From b0dcf579ef9a198a5c562c1171212a924eefc2c5 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 06:57:30 +0000 Subject: [PATCH 27/34] mdcode: recover relationships and deployment targets on pull Pull previously dropped two things push had already written to Knowledge Catalog: the model's deployment targets (stored in the semantic-model aspect) and its 1:1/1:N relationships (stored as schema-join entry links). The reader only opened per-entry aspects and pull only fetched entries, so both were silently lost even though the catalog held them. - kc_converter: read deploymentTargets back into the GOOGLE custom_extensions block, and invert schema-join links into Relationships -- endpoints resolved by data source, FK direction and columns from the join aspect. modelsFromCatalogResources grows an entryLinks argument. Relationship names come back normalized (lowercased/hyphenated): the emitter stores the name only in the link id. - pull_kc: add a second fetch pass over the entity entries via lookupEntryLinks, deduping the undirected links (each is returned from both endpoints). - Tests cover endpoint/direction recovery, name normalization, the M:N drop, deployment-target recovery, and the pull fetch+dedup path; the .pull.golden fixtures are regenerated and the docs pull note rewritten. M:N (association) relationships remain unrecovered -- push never emits them. Writer files are untouched. --- toolbox/mdcode/docs/semantic-model.md | 28 +- .../mdcode/src/libts/semantic/kc_converter.ts | 210 +++++++++++--- toolbox/mdcode/src/libts/semantic/pull_kc.ts | 76 +++++- .../sales_bq_graph_target.pull.golden.yaml | 3 + .../star_orders_customer.pull.golden.yaml | 11 + .../fixtures/tpcds_date_edge.pull.golden.yaml | 29 ++ .../tests/libts/semantic/kc_converter.test.ts | 257 +++++++++++++----- .../tests/libts/semantic/pull_kc.test.ts | 107 +++++++- 8 files changed, 583 insertions(+), 138 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 80b7db91..ec67acbd 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -259,18 +259,22 @@ Pull writes with the same last-write-wins policy as the core pull: a model that already exists locally is overwritten in place, and a local-only document (one with no matching catalog entry) is left untouched — pull never deletes. -> **Note — pull is lossy.** It reconstructs a model only from what push wrote -> to the catalog (see the note under [What gets created in Knowledge -> Catalog](#what-gets-created-in-knowledge-catalog)), and recovers even less -> than the catalog holds. A pulled document comes back without entity keys, -> `ai_context`, field labels, the original vendor SQL (`importedExpression`), -> relationships (even the 1:1 / 1:N `schema-join` links push wrote — the edges -> live in the BigQuery property graph), and the `deploymentTargets` custom -> extension. A field's *role* survives as a bare `dimension: {}` marker, but its -> detail (`is_time`, and so on) does not. **A push followed by a pull does not -> return your original file** — treat a pulled document as a faithful copy of -> the catalog metadata, not of the authored model, and keep the authored -> document as the source of truth. +> **Note — pull is lossy.** It reconstructs a model from what push wrote to the +> catalog (see the note under [What gets created in Knowledge +> Catalog](#what-gets-created-in-knowledge-catalog)), so it recovers the model +> structure — entities and fields, metrics, 1:1 / 1:N relationships (from the +> `schema-join` links), and the deployment targets. It does **not** recover the +> content the catalog never held: entity keys, `ai_context`, field labels, the +> original vendor SQL (`importedExpression`), and M:N relationships (whose edge +> lives only in the BigQuery property graph). Some recovered content also comes +> back normalized rather than verbatim: relationship names are +> lowercased/hyphenated (the catalog stores the name only in the link id), a +> field's *role* survives as a bare `dimension: {}` marker without its detail +> (`is_time`, and so on), and a metric authored without a datatype comes back as +> an explicit `Decimal`. **A push followed by a pull does not return your original +> file** — treat a pulled document as a faithful copy of the catalog metadata, +> not of the authored model, and keep the authored document as the source of +> truth. ## Permissions diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 25a67c0b..89dd0ad5 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -1,39 +1,47 @@ // Knowledge Catalog <-> Semantic Model IR converter. // // SCAFFOLD (naming for the end state): this file currently holds only the -// READ direction -- Knowledge Catalog entries -> IR (`modelsFromCatalogResources` -// and its helpers). The WRITE direction (IR -> KC entries, -// `generateCatalogResources`) still lives in `knowledge_catalog.ts` and moves -// here once PR4 (the KC push line) merges, at which point this becomes the full -// two-way converter and `knowledge_catalog.ts` goes away. Until then, "where is -// the KC emit code?" -> `knowledge_catalog.ts`. +// READ direction -- Knowledge Catalog entries -> IR +// (`modelsFromCatalogResources` and its helpers). The WRITE direction (IR -> KC +// entries, `generateCatalogResources`) still lives in `knowledge_catalog.ts` +// and moves here once PR4 (the KC push line) merges, at which point this +// becomes the full two-way converter and `knowledge_catalog.ts` goes away. +// Until then, "where is the KC emit code?" -> `knowledge_catalog.ts`. // // TODO(#278): fold the KC WRITE direction in and drop the scaffold. Once // #278 merges, move `generateCatalogResources` (and the `KcResources` type // + emit helpers) out of `knowledge_catalog.ts` into this file, delete -// `knowledge_catalog.ts`, and repoint its importers (`deploy_knowledge_catalog.ts` -// and the emitter tests). `idOf` -- shared by both directions and re-exported -// from here only for `pull_kc` today -- becomes a plain local. Then this is -// the sole KC<->IR codec and the SCAFFOLD note above comes out. +// `knowledge_catalog.ts`, and repoint its importers +// (`deploy_knowledge_catalog.ts` and the emitter tests). `idOf` -- shared by +// both directions and re-exported from here only for `pull_kc` today -- becomes +// a plain local. Then this is the sole KC<->IR codec and the SCAFFOLD note +// above comes out. // // The reader is the inverse of `generateCatalogResources`: it reconstructs the // IR from the entries a pull hydrated (see `pull_kc.pullKnowledgeCatalog`). // `semantic-entity` / `semantic-metric` entries are grouped under their -// `semantic-model` anchor via `parentEntry`; entries of other types are ignored. -// Resources are matched by type-name SUFFIX, so a reader need not know which -// system-type project/location the emitter used. +// `semantic-model` anchor via `parentEntry`; entries of other types are +// ignored. Resources are matched by type-name SUFFIX, so a reader need not know +// which system-type project/location the emitter used. // -// Fidelity is bounded by what the emitter persisted, so this read is the inverse -// of the WRITE, not of the authored document. It recovers names, descriptions, -// data sources, field datatypes (via the schema aspect) and DIMENSION roles, -// field/metric expressions, and each metric's attach entity (re-derived from its -// expression, as the loader does). It cannot recover what the emitter does not -// write: entity keys/unique keys, `ai_context`, field labels, `importedDialect`, -// `custom_extensions`, and relationships (the graph edges live in the BigQuery -// property graph, not the catalog). - -import type {Entry} from '../gcp/dataplex'; -import {DataType, Entity, Field, Metric, SemanticModel} from './ir'; +// Fidelity is bounded by what the emitter persisted, so this read is the +// inverse of the WRITE, not of the authored document. It recovers names, +// descriptions, data sources, field datatypes (via the schema aspect) and +// DIMENSION roles, field/metric expressions, each metric's attach entity +// (re-derived from its expression, as the loader does), the model's deployment +// targets (from the semantic-model aspect, back into the GOOGLE +// `custom_extensions` block), and 1:1 / 1:N relationships (from the +// `schema-join` entry links a pull fetched -- see +// `modelsFromCatalogResources`'s `entryLinks` argument). It cannot recover what +// the emitter does not write: entity keys/unique keys, `ai_context`, field +// labels, `importedDialect`, and many-to-many (association) relationships +// (whose edge lives only in the BigQuery property graph). Relationship NAMES +// come back normalized (lowercased/hyphenated), since the emitter encodes the +// name only in the link id (via `linkSlug`), not in the join aspect. + +import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; + +import {CustomExtension, DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; import {referencedEntityNames} from './sql_expr_utils'; export interface ReadResult { @@ -48,8 +56,14 @@ export interface ReadResult { * an orphaned child, an entry missing its aspect data). Entries must already be * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a * BASIC list omits aspect data, so the puller re-fetches each entry first. + * + * `entryLinks` are the `schema-join` links a pull fetched for the group (see + * `pull_kc`); each link between two of a model's entity entries is + * reconstructed as a 1:1 / 1:N relationship. Pass `[]` (the default) to + * reconstruct entities and metrics only. */ -export function modelsFromCatalogResources(entries: Entry[]): ReadResult { +export function modelsFromCatalogResources( + entries: Entry[], entryLinks: EntryLink[] = []): ReadResult { const warnings: string[] = []; const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); @@ -75,17 +89,26 @@ export function modelsFromCatalogResources(entries: Entry[]): ReadResult { const models = anchors.map(anchor => { const name = anchor.entrySource?.displayName ?? idOf(anchor.name); - const entities = childrenOf(anchor.name, entityEntries) - .map(e => readEntity(e, warnings)); + const entityEntriesForModel = childrenOf(anchor.name, entityEntries); + const entities = entityEntriesForModel.map(e => readEntity(e, warnings)); const entityNames = entities.map(e => e.name); const metrics = childrenOf(anchor.name, metricEntries) .map(e => readMetric(e, entityNames, warnings)); - // Relationships are not published to the catalog (see the file header), so - // a reconstructed model always has an empty edge set. - const model: SemanticModel = {name, entities, relationships: [], metrics}; + // Relationships come from the schema-join entry links whose two endpoints + // are both this model's entity entries (M:N edges were never published -- + // they live only in the BigQuery property graph -- so stay absent here). + const relationships = readRelationships( + entryLinks, name, new Set(entityEntriesForModel.map(e => e.name)), + dataSourceIndex(entities), warnings); + + const model: SemanticModel = {name, entities, relationships, metrics}; const description = anchor.entrySource?.description; if (description !== undefined) model.description = description; + // Deployment targets ride back in the same GOOGLE custom_extensions block + // the author wrote them in (the inverse of the emitter's modelAspectData). + const targets = readDeploymentTargets(anchor); + if (targets) model.customExtensions = [targets]; return model; }); @@ -248,13 +271,21 @@ function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| } -// The `data` payload of an entry's aspect of the given bare type, matched by -// the aspect key's `.` suffix or the aspectType's `/aspectTypes/` -// suffix (robust to whichever system-type project/location the emitter used). -// Returns an empty object when the aspect is absent. +// The `data` payload of an entry's aspect of the given bare type. See +// aspectDataOf; entries and entry links carry aspects in the same shape. function aspectData(entry: Entry, type: string): Record { - const aspects = entry.aspects ?? {}; - for (const [key, aspect] of Object.entries(aspects)) { + return aspectDataOf(entry.aspects, type); +} + + +// The `data` payload of an aspect of the given bare type from an aspect map, +// matched by the aspect key's `.` suffix or the aspectType's +// `/aspectTypes/` suffix (robust to whichever system-type +// project/location the emitter used). Returns an empty object when the aspect +// is absent. +function aspectDataOf(aspects: Record|undefined, type: string): + Record { + for (const [key, aspect] of Object.entries(aspects ?? {})) { if (key.endsWith(`.${type}`) || aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { return aspect.data ?? {}; @@ -264,6 +295,113 @@ function aspectData(entry: Entry, type: string): Record { } +// Recovers the model's deployment targets from its semantic-model aspect back +// into the GOOGLE custom_extension the author declared them in (the inverse of +// the emitter's modelAspectData). Returns undefined when the model has none. +function readDeploymentTargets(anchor: Entry): CustomExtension|undefined { + const targets = + asArray(aspectData(anchor, 'semantic-model').deploymentTargets) + .filter((t): t is string => typeof t === 'string' && t !== ''); + if (!targets.length) return undefined; + return { + vendorName: 'GOOGLE', + data: JSON.stringify({deploymentTargets: targets}) + }; +} + + +// Indexes reconstructed entities by their SQL data source, so a schema-join +// aspect (which names each side by its table, not the entity) can resolve its +// endpoints back to entity names. +function dataSourceIndex(entities: Entity[]): Map { + const index = new Map(); + for (const entity of entities) { + const dataSource = (entity.dataSource ?? '').trim(); + if (dataSource) index.set(dataSource, entity.name); + } + return index; +} + + +// Inverts schemaJoinAspectData (knowledge_catalog.ts): each schema-join link +// whose two endpoints are both this model's entity entries becomes one +// direct-FK relationship. The aspect encodes direction (`source` is the +// foreign-key side) and the paired join columns; the link id encodes the +// (normalized) name. Links are deduped by name -- schema-join is undirected, so +// a per-entry lookup can return the same link once from each endpoint. +function readRelationships( + links: EntryLink[], modelName: string, entityEntryNames: Set, + dataSourceToEntity: Map, + warnings: string[]): Relationship[] { + const prefix = linkNamePrefix(modelName); + const seen = new Set(); + const out: Relationship[] = []; + for (const link of links) { + if (!link.entryLinkType?.endsWith('/entryLinkTypes/schema-join')) continue; + const refs = link.entryReferences ?? []; + // Only a link whose BOTH endpoints are this model's entities belongs here. + if (refs.length !== 2 || !refs.every(r => entityEntryNames.has(r.name))) { + continue; + } + if (link.name) { + if (seen.has(link.name)) continue; + seen.add(link.name); + } + const join = asArray(aspectDataOf(link.aspects, 'schema-join').joins)[0]; + if (!join) { + warnings.push( + `entry link '${link.name}': no schema-join aspect data; the ` + + `relationship is skipped`); + continue; + } + const source = dataSourceToEntity.get((join.source?.name ?? '').trim()); + const destination = + dataSourceToEntity.get((join.target?.name ?? '').trim()); + if (!source || !destination) { + warnings.push( + `entry link '${link.name}': a join endpoint table does not match a ` + + `reconstructed entity; the relationship is skipped`); + continue; + } + const rel: Relationship = { + name: relationshipName(link.name, prefix), + source: {entity: source, columns: asArray(join.source?.fields)}, + destination: {entity: destination, columns: asArray(join.target?.fields)}, + }; + if (join.description !== undefined) rel.description = join.description; + out.push(rel); + } + return out; +} + + +// Best-effort recovery of the relationship name from the link id, which the +// emitter built as linkSlug(`-`) (lowercased/hyphenated -- see +// knowledge_catalog.ts). Strips the model prefix when present; the name comes +// back normalized, never verbatim. +function relationshipName( + linkName: string|undefined, modelPrefix: string): string { + const id = idOf(linkName ?? ''); + if (modelPrefix && id.startsWith(`${modelPrefix}-`)) { + const rest = id.slice(modelPrefix.length + 1); + if (rest) return rest; + } + return id; +} + + +// Mirrors the model-name portion of the emitter's linkSlug so the prefix a link +// id was built with can be stripped. Kept local rather than imported: this +// read-side module must not depend on the write-side emitter. +function linkNamePrefix(modelName: string): string { + return modelName.toLowerCase() + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-+/g, '-') + .replace(/^[^a-z]+/, '') + .replace(/^-+|-+$/g, ''); +} + + // The id segment of a full entry resource name (after the last '/'). export function idOf(name: string): string { return name.split('/').pop() ?? name; diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index aebaa7b5..24d338c3 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -16,7 +16,8 @@ // the orchestration layer reads as `push_kc` / `pull_kc`. Rename only -- no // logic moves. -import {CatalogClient, Entry} from '../gcp/dataplex'; +import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; + import {SemanticModel} from './ir'; import {idOf, modelsFromCatalogResources} from './kc_converter'; @@ -65,9 +66,8 @@ export async function pullKnowledgeCatalog( if (!scoped.length) { return { models: [], - warnings: [ - `no semantic model named '${opts.model}' found in ${destination}` - ], + warnings: + [`no semantic model named '${opts.model}' found in ${destination}`], }; } } @@ -93,7 +93,45 @@ export async function pullKnowledgeCatalog( warnings.push(r.warning); } - const read = modelsFromCatalogResources(hydrated); + // Second fetch pass: relationships are schema-join entry links, which the + // entry list/lookup does not return. The catalog exposes links only per + // referenced entry (:lookupEntryLinks), so fan out over the entity entries + // and dedup by link name -- schema-join is undirected, so each link comes + // back once from each of its two endpoints. + const entityEntries = hydrated.filter( + e => e.entryType?.endsWith('/entryTypes/semantic-entity')); + const linkResults = + await mapConcurrent(entityEntries, HYDRATE_CONCURRENCY, async entry => { + const linkType = schemaJoinLinkType(entry.entryType); + const res = await cat.lookupEntryLinks(opts.project, opts.location, { + entry: entry.name, + entryLinkTypes: linkType ? [linkType] : undefined, + }); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry links for '${entry.name}' (status ${ + res.status}); relationships may be incomplete`, + }; + } + return {links: res.result}; + }); + + const seenLinks = new Set(); + const entryLinks: EntryLink[] = []; + for (const r of linkResults) { + if (r.warning) { + warnings.push(r.warning); + continue; + } + for (const link of r.links ?? []) { + const key = link.name ?? ''; + if (key && seenLinks.has(key)) continue; + if (key) seenLinks.add(key); + entryLinks.push(link); + } + } + + const read = modelsFromCatalogResources(hydrated, entryLinks); warnings.push(...read.warnings); // Defense in depth: keep only the requested model even if the reader surfaced @@ -136,6 +174,19 @@ function semanticAspectTypes(entryType: string): string[]|undefined { } +// The schema-join entry link type resource, derived from an entity's entryType +// base (the link type is the parallel resource in the same project/location the +// emitter referenced). Used to filter :lookupEntryLinks to just the +// relationship links. Returns undefined for an entryType with no recognizable +// base. +function schemaJoinLinkType(entryType: string): string|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + return `${entryType.slice(0, idx)}/entryLinkTypes/schema-join`; +} + + // Restricts hydration targets to a single model: the semantic-model anchor // whose name (entrySource.displayName, else the entry id) matches `model`, plus // every child entry whose parentEntry is that anchor. Uses only list-level @@ -146,13 +197,14 @@ function scopeToModel( model: string): {entry: Entry; aspectTypes: string[]}[] { const isAnchor = (t: {entry: Entry}) => !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); - const allAnchorNames = new Set(targets.filter(isAnchor).map(t => t.entry.name)); - const matchedAnchorNames = new Set( - targets.filter(isAnchor) - .filter( - t => (t.entry.entrySource?.displayName ?? idOf(t.entry.name)) === - model) - .map(t => t.entry.name)); + const allAnchorNames = + new Set(targets.filter(isAnchor).map(t => t.entry.name)); + const matchedAnchorNames = + new Set(targets.filter(isAnchor) + .filter( + t => (t.entry.entrySource?.displayName ?? + idOf(t.entry.name)) === model) + .map(t => t.entry.name)); if (!matchedAnchorNames.size) return []; // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml index f9e42580..06dd8bdc 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml @@ -2,6 +2,9 @@ version: 0.2.0.dev0 semantic_model: - name: sales + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets":["//bigquery.googleapis.com/projects/demo/datasets/sales/propertyGraphs/sales_graph"]}' datasets: - name: orders source: demo.sales.orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml index bf7cfc8d..d7a9cbe1 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -3,6 +3,9 @@ version: 0.2.0.dev0 semantic_model: - name: sales description: Sales orders with customer attributes + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets":["//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales"]}' datasets: - name: orders source: samples.tpch.orders @@ -44,6 +47,14 @@ semantic_model: - dialect: BIGQUERY expression: c_name description: Customer name + relationships: + - name: orders-to-customer + from: orders + to: customer + from_columns: + - o_custkey + to_columns: + - c_custkey metrics: - name: total_revenue expression: diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml index 7437f8ac..1900fe12 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -145,3 +145,32 @@ semantic_model: - name: date_dim source: sqlgen-testing.demo.date_dim description: Date dimension with calendar attributes + relationships: + - name: store-sales-to-date-dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - ss_sold_date_sk + - name: store-sales-to-customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store-sales-to-item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store-sales-to-store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index d5ecb145..0e2f0eb6 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -2,22 +2,26 @@ // (modelsFromCatalogResources in src/libts/semantic/kc_converter.ts). // // The reader is the inverse of the emitter (generateCatalogResources). The -// central guarantee is an emitter -> reader round trip: emit a model's entries, -// read them back, and get an IR equal to the source WHERE the emitter is -// lossless. The write drops content by design (entity keys, ai_context, field -// labels, importedDialect, relationships -- see the emitter header), so the -// expected read-back is the source model with exactly those fields cleared. +// central guarantee is an emitter -> reader round trip: emit a model's entries +// AND entry links, read them back, and get an IR equal to the source WHERE the +// emitter is lossless. The write drops content by design (entity keys, +// ai_context, field labels, importedDialect, and many-to-many relationships -- +// see the emitter header), so the expected read-back is the source model with +// exactly those fields cleared. 1:1 / 1:N relationships and deployment targets +// DO round-trip (via schema-join links and the semantic-model aspect), except +// that relationship names come back normalized (lowercased/hyphenated). // Targeted tests pin the mapping details a round trip cannot isolate (the // dataType inverse, the DIMENSION role, resource-URI parsing, metric attach -// re-derivation, and parent/anchor grouping). +// re-derivation, relationship endpoint/direction recovery, and parent/anchor +// grouping). import {describe, expect, test} from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import {Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; -import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; import {modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; import {serializeModel} from '../../../src/libts/semantic/osi_converter'; const FIXTURES = path.join(__dirname, 'fixtures'); @@ -28,11 +32,11 @@ const OPTS = { entryGroup: 'eg' }; -// Emits a model to entries and reads it straight back. +// Emits a model to entries + entry links and reads it straight back. function roundTrip(model: SemanticModel): {models: SemanticModel[]; warnings: string[]} { - const {entries} = generateCatalogResources(model, OPTS); - return modelsFromCatalogResources(entries); + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + return modelsFromCatalogResources(entries, entryLinks); } @@ -75,6 +79,120 @@ describe('emitter -> reader round trip (lossless slice)', () => { }); +describe('relationship recovery (schema-join links -> IR)', () => { + // orders.o_custkey (the foreign-key side) references customer.c_custkey. + const twoEntities: Entity[] = [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'o_custkey', expression: 'orders.o_custkey'}], + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'c_custkey', expression: 'customer.c_custkey'}], + }, + ]; + + test( + 'a 1:N relationship recovers its endpoints, direction, and columns', + () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: + 'places', // already link-slug-safe, so it round-trips exactly + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const rels = roundTrip(model).models[0].relationships; + expect(rels).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + }); + + test( + 'a relationship name comes back normalized (lowercased/hyphenated)', + () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'Places_Order', // mixed case + underscore -> normalized on + // read + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships[0].name) + .toBe('places-order'); + }); + + test('a many-to-many (association) relationship is not recovered', () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'enrolls', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + association: { + dataSource: 'p.d.junction', + keys: ['id'], + sourceColumns: ['j_orderkey'], + destinationColumns: ['j_custkey'], + }, + }], + metrics: [], + }; + // The emitter never publishes M:N, so no schema-join link exists to read. + expect(roundTrip(model).models[0].relationships).toEqual([]); + }); +}); + + +describe( + 'deployment-target recovery (semantic-model aspect -> custom_extensions)', + () => { + test('the GOOGLE deployment targets ride back verbatim', () => { + const uri = + '//bigquery.googleapis.com/projects/p/datasets/d/propertyGraphs/g'; + const data = JSON.stringify({deploymentTargets: [uri]}); + const model: SemanticModel = { + name: 'sales', + customExtensions: [{vendorName: 'GOOGLE', data}], + entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + expect(roundTrip(model).models[0].customExtensions).toEqual([ + {vendorName: 'GOOGLE', data} + ]); + }); + + test( + 'a model with no deployment targets recovers no custom_extensions', + () => { + const model: SemanticModel = { + name: 'sales', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + expect(roundTrip(model).models[0].customExtensions).toBeUndefined(); + }); + }); + + describe('dataType inverse (schema aspect -> IR type)', () => { // Emit a one-field model of each IR type, read it back, and check the field's // reconstructed type. String and Opaque both emit dataType STRING; String @@ -246,21 +364,22 @@ describe('anchor / parent grouping', () => { describe('metric expression referencing no known entity', () => { - test('warns that the metric may be unplaceable and leaves it unattached', - () => { - const model: SemanticModel = { - name: 'm', - entities: - [{name: 'orders', dataSource: 'p.d.o', keys: [], fields: []}], - relationships: [], - // References `widgets`, which is not an entity of this model. - metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], - }; - const {models, warnings} = roundTrip(model); - expect(models[0].metrics[0].entity).toBeUndefined(); - expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) - .toBe(true); - }); + test( + 'warns that the metric may be unplaceable and leaves it unattached', + () => { + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'orders', dataSource: 'p.d.o', keys: [], fields: []}], + relationships: [], + // References `widgets`, which is not an entity of this model. + metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], + }; + const {models, warnings} = roundTrip(model); + expect(models[0].metrics[0].entity).toBeUndefined(); + expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) + .toBe(true); + }); }); @@ -268,48 +387,52 @@ describe('metric expression referencing no known entity', () => { // // The round trip above proves the reader inverts the emitter in memory; this // pins the reviewable artifact. For each corpus fixture it reads the committed -// emitter golden (`.knowledge_catalog.golden.json` -- the exact entries -// a push produced) back through the reader and serializes the reconstructed IR -// to `.pull.golden.yaml`. Open that next to the fixture's -// `.osi.golden.yaml` to see, as whole files, what a Knowledge Catalog round trip -// preserves and what it drops (keys, ai_context, labels, relationships). +// emitter golden (`.knowledge_catalog.golden.json` -- the exact +// entries and entry links a push produced) back through the reader and +// serializes the reconstructed IR to `.pull.golden.yaml`. Open that +// next to the fixture's `.osi.golden.yaml` to see, as whole files, what a +// Knowledge Catalog round trip preserves (including 1:1 / 1:N relationships and +// deployment targets) and what it drops (keys, ai_context, labels, vendor SQL, +// M:N relationships). // // Regenerate after an intentional reader/serializer change: // UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/kc_converter.test.ts -describe('golden pull: each corpus KC golden reconstructs to its exact YAML', - () => { - const CORPUS = [ - 'sales_bq_graph_target.yaml', - 'star_orders_customer.yaml', - 'tpcds_date_edge.yaml', - ]; - const kcGoldenPath = (fixture: string) => path.join( - FIXTURES, - fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); - const pullGoldenPath = (fixture: string) => - path.join(FIXTURES, fixture.replace(/\.yaml$/, '.pull.golden.yaml')); - - for (const fixture of CORPUS) { - test(fixture, () => { - const kc = JSON.parse(fs.readFileSync(kcGoldenPath(fixture), 'utf8')); - const {models, warnings} = modelsFromCatalogResources(kc.entries); - // Reader warnings ride along as YAML comments so the golden shows - // the full outcome, not just the recovered document. - const header = warnings.length ? - warnings.map((w: string) => `# warning: ${w}`).join('\n') + '\n' : - '# (no warnings)\n'; - const actual = - header + models.map(m => serializeModel(m).yaml).join('---\n'); - const golden = pullGoldenPath(fixture); - if (process.env.UPDATE_GOLDENS) { - fs.writeFileSync(golden, actual); - return; - } - if (!fs.existsSync(golden)) { - throw new Error(`missing golden ${ - path.basename(golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); - } - expect(actual).toBe(fs.readFileSync(golden, 'utf8')); - }); - } - }); +describe( + 'golden pull: each corpus KC golden reconstructs to its exact YAML', () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + const kcGoldenPath = (fixture: string) => path.join( + FIXTURES, + fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); + const pullGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.pull.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const kc = JSON.parse(fs.readFileSync(kcGoldenPath(fixture), 'utf8')); + const {models, warnings} = + modelsFromCatalogResources(kc.entries, kc.entryLinks ?? []); + // Reader warnings ride along as YAML comments so the golden shows + // the full outcome, not just the recovered document. + const header = warnings.length ? + warnings.map((w: string) => `# warning: ${w}`).join('\n') + '\n' : + '# (no warnings)\n'; + const actual = + header + models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = pullGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename( + golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index 5a09cebd..d5d22d42 100644 --- a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -6,18 +6,19 @@ // the IR. The catalog client is stubbed so no network call is made. The entries // the fake serves are produced by the real emitter, so this exercises the true // list -> hydrate -> read path end to end (an entity is re-fetched with BOTH -// its semantic-entity and schema aspects). The reader's own mapping is covered -// in kc_converter.test.ts; the focus here is the fetch SEQUENCE: -// aspect hydration, the --model filter, skipped entries, and ignoring foreign -// entries. +// its semantic-entity and schema aspects, and a second pass fetches each +// entity's schema-join links). The reader's own mapping is covered in +// kc_converter.test.ts; the focus here is the fetch SEQUENCE: aspect hydration, +// the relationship-link fetch (per-entry, deduped across endpoints), the +// --model filter, skipped entries, and ignoring foreign entries. import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; import {ApiResult} from '../../../src/libts/gcp/api'; -import {CatalogClient, Entry} from '../../../src/libts/gcp/dataplex'; -import {pullKnowledgeCatalog} from '../../../src/libts/semantic/pull_kc'; +import {CatalogClient, Entry, EntryLink} from '../../../src/libts/gcp/dataplex'; import {SemanticModel} from '../../../src/libts/semantic/ir'; import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {pullKnowledgeCatalog} from '../../../src/libts/semantic/pull_kc'; const OPTS = { project: 'dest', @@ -60,8 +61,13 @@ function err(status: number, message: string): ApiResult { // Stubs listEntries (yields `listed`) and lookupEntry (serves `served` by name, // or 404s an unknown name). `lookupFail` forces a failure for one entry name. -function stubClient(listed: Entry[], served: Entry[], lookupFail?: string): - {list: any; lookup: any} { +// `links` are served by lookupEntryLinks per referenced entry (so an undirected +// link comes back once from each endpoint, exercising the puller's dedup); +// `linkFail` forces a link-lookup failure for one entry name. +function stubClient( + listed: Entry[], served: Entry[], lookupFail?: string, + links: EntryLink[] = [], + linkFail?: string): {list: any; lookup: any; lookupLinks: any} { const byName = new Map(served.map(e => [e.name, e])); const list = spyOn(CatalogClient.prototype, 'listEntries') .mockImplementation(async function*() { @@ -73,7 +79,16 @@ function stubClient(listed: Entry[], served: Entry[], lookupFail?: string): const e = byName.get(name); return e ? ok(e) : err(404, 'not found'); }); - return {list, lookup}; + const lookupLinks = + spyOn(CatalogClient.prototype, 'lookupEntryLinks') + .mockImplementation(async (_p: any, _l: any, opts: any) => { + if (opts.entry === linkFail) return err(500, 'boom'); + const forEntry = links.filter( + l => + (l.entryReferences ?? []).some(r => r.name === opts.entry)); + return ok(forEntry); + }); + return {list, lookup, lookupLinks}; } afterEach(() => { @@ -121,6 +136,76 @@ describe('pullKnowledgeCatalog: happy path', () => { }); +describe('pullKnowledgeCatalog: relationship links', () => { + // A 1:N model: orders.o_custkey (the FK side) references customer.c_custkey. + const SALES_REL: SemanticModel = { + name: 'sales', + entities: [ + { + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], + fields: [{name: 'o_custkey', expression: 'orders.o_custkey'}], + }, + { + name: 'customer', + dataSource: 'demo.sales.customer', + keys: [], + fields: [{name: 'c_custkey', expression: 'customer.c_custkey'}], + }, + ], + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + + test( + 'a schema-join link reconstructs into a relationship exactly once', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + const {lookupLinks} = + stubClient(entries, entries, undefined, entryLinks); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models[0].relationships).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + expect(warnings).toHaveLength(0); + // Links are fetched per entity (2 entities), and the single undirected + // link -- returned from both endpoints -- is deduped to one edge. + expect(lookupLinks).toHaveBeenCalledTimes(2); + }); + + test( + 'a failed link lookup on one endpoint still recovers the edge from the ' + + 'other, and warns', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + const ordersEntry = + entries.find(e => (e.entrySource?.displayName) === 'orders')!; + // Fail the link lookup for the orders entity; the link still comes back + // from the customer endpoint. + stubClient(entries, entries, undefined, entryLinks, ordersEntry.name); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models[0].relationships.map(r => r.name)).toEqual(['places']); + expect(warnings.some( + w => /failed to fetch entry links/i.test(w) && + w.includes(ordersEntry.name))) + .toBe(true); + }); +}); + + describe('pullKnowledgeCatalog: filtering and robustness', () => { test('--model keeps only the named model', async () => { const other: SemanticModel = { @@ -215,8 +300,8 @@ describe('pullKnowledgeCatalog: filtering and robustness', () => { // drops a child a full pull returns). const metricEntry = entries.find(e => e.entryType.endsWith('/semantic-metric'))!; - metricEntry.parentEntry = - metricEntry.parentEntry!.replace('projects/dest/', 'projects/12345/'); + metricEntry.parentEntry = metricEntry.parentEntry!.replace( + 'projects/dest/', 'projects/12345/'); const {lookup} = stubClient(entries, entries); const cat = new CatalogClient({} as any); From e1982e6d79f224946320d4163bce05be2d481a18 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 07:10:56 +0000 Subject: [PATCH 28/34] mdcode: resolve pull relationship endpoints via entryReferences Address code-review findings on the gap-3 pull leg: - Resolve schema-join endpoints from the link's entryReferences, matched by entry id, instead of a dataSource->entity index. The id is unique per entity (fixes two entities sharing a table collapsing last-wins) and is stable across the project-number/id normalization lookupEntry applies to entries but lookupEntryLinks does not apply to link references (fixes relationships silently dropping on the live path). The schema-join aspect is now used only for FK direction + join columns; undecidable direction keeps the reference order and warns rather than dropping the edge. - Dedup entry links by a sorted endpoint-pair key when a link has no name (shared linkDedupKey, reused by pull_kc) so a nameless link returned from both endpoints is not counted twice. - Rewrite the pull 'lossy' note in the user guide as recovered-exactly / recovered-but-normalized / not-recovered bullets. Tests: shared-table endpoints, un-normalized project-number references, prefix-stripping across tricky model names, and nameless-link dedup. --- toolbox/mdcode/docs/semantic-model.md | 47 ++++--- .../mdcode/src/libts/semantic/kc_converter.ts | 111 ++++++++++------ toolbox/mdcode/src/libts/semantic/pull_kc.ts | 10 +- .../tests/libts/semantic/kc_converter.test.ts | 123 ++++++++++++++++++ .../tests/libts/semantic/pull_kc.test.ts | 19 +++ 5 files changed, 250 insertions(+), 60 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index ec67acbd..b3a1fcbb 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -259,22 +259,37 @@ Pull writes with the same last-write-wins policy as the core pull: a model that already exists locally is overwritten in place, and a local-only document (one with no matching catalog entry) is left untouched — pull never deletes. -> **Note — pull is lossy.** It reconstructs a model from what push wrote to the -> catalog (see the note under [What gets created in Knowledge -> Catalog](#what-gets-created-in-knowledge-catalog)), so it recovers the model -> structure — entities and fields, metrics, 1:1 / 1:N relationships (from the -> `schema-join` links), and the deployment targets. It does **not** recover the -> content the catalog never held: entity keys, `ai_context`, field labels, the -> original vendor SQL (`importedExpression`), and M:N relationships (whose edge -> lives only in the BigQuery property graph). Some recovered content also comes -> back normalized rather than verbatim: relationship names are -> lowercased/hyphenated (the catalog stores the name only in the link id), a -> field's *role* survives as a bare `dimension: {}` marker without its detail -> (`is_time`, and so on), and a metric authored without a datatype comes back as -> an explicit `Decimal`. **A push followed by a pull does not return your original -> file** — treat a pulled document as a faithful copy of the catalog metadata, -> not of the authored model, and keep the authored document as the source of -> truth. +> **Note — pull reconstructs what the catalog holds, not your original file.** +> Pull can only recover what push wrote (see the note under [What gets created in +> Knowledge Catalog](#what-gets-created-in-knowledge-catalog)). What that means in +> practice: +> +> **Recovered exactly** — these come back as authored: +> - Model structure: the model, its entities, and each entity's fields. +> - Field data source and data type; the field expression. +> - Metrics: name, expression, data type, and attach entity. +> - 1:1 / 1:N relationships: endpoints, foreign-key direction, and join columns +> (from the `schema-join` links). +> - Deployment targets. +> +> **Recovered, but normalized** — the content survives, the form changes: +> - Relationship *names* come back lowercased/hyphenated (the catalog stores the +> name only in the link id, e.g. `Places Order` → `places-order`). +> - A field marked as a dimension comes back as a bare `dimension: {}` marker, +> without its detail (`is_time`, and so on). +> - A metric authored with no data type comes back as an explicit `Decimal` +> (push must write a type, and defaults it to `NUMERIC`). +> +> **Not recovered** — push never wrote these, so pull cannot return them: +> - Entity keys / unique keys. +> - `ai_context`. +> - Field labels. +> - The original vendor SQL (`importedExpression`). +> - M:N relationships (the edge lives only in the BigQuery property graph). +> +> **So: a push followed by a pull does not return your original file.** Treat a +> pulled document as a faithful copy of the catalog metadata, not of the authored +> model, and keep the authored document as the source of truth. ## Permissions diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 89dd0ad5..3600d781 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -95,12 +95,21 @@ export function modelsFromCatalogResources( const metrics = childrenOf(anchor.name, metricEntries) .map(e => readMetric(e, entityNames, warnings)); + // Index this model's entities by their entry id, so a schema-join link can + // resolve its endpoints from the link's entryReferences. The id segment is + // stable across the project-number/id normalization that lookupEntry + // applies to entries but lookupEntryLinks does not apply to link + // references, so matching on it (not the full resource name) keeps + // live-fetched links resolvable. + const entityByEntryId = new Map(); + entityEntriesForModel.forEach( + (e, i) => entityByEntryId.set(idOf(e.name), entities[i])); + // Relationships come from the schema-join entry links whose two endpoints // are both this model's entity entries (M:N edges were never published -- // they live only in the BigQuery property graph -- so stay absent here). - const relationships = readRelationships( - entryLinks, name, new Set(entityEntriesForModel.map(e => e.name)), - dataSourceIndex(entities), warnings); + const relationships = + readRelationships(entryLinks, name, entityByEntryId, warnings); const model: SemanticModel = {name, entities, relationships, metrics}; const description = anchor.entrySource?.description; @@ -310,28 +319,22 @@ function readDeploymentTargets(anchor: Entry): CustomExtension|undefined { } -// Indexes reconstructed entities by their SQL data source, so a schema-join -// aspect (which names each side by its table, not the entity) can resolve its -// endpoints back to entity names. -function dataSourceIndex(entities: Entity[]): Map { - const index = new Map(); - for (const entity of entities) { - const dataSource = (entity.dataSource ?? '').trim(); - if (dataSource) index.set(dataSource, entity.name); - } - return index; -} - - // Inverts schemaJoinAspectData (knowledge_catalog.ts): each schema-join link // whose two endpoints are both this model's entity entries becomes one -// direct-FK relationship. The aspect encodes direction (`source` is the -// foreign-key side) and the paired join columns; the link id encodes the -// (normalized) name. Links are deduped by name -- schema-join is undirected, so -// a per-entry lookup can return the same link once from each endpoint. +// direct-FK relationship. +// +// Endpoint identity comes from the link's entryReferences (matched by entry id +// via `entityByEntryId`), NOT from the aspect's table names: the id is unique +// per entity (two entities can share a table) and survives the +// project-number/id normalization lookupEntry applies to entries but +// lookupEntryLinks does not apply to link references. The aspect then supplies +// the join columns and the direction -- its `source` side is the foreign-key +// side, named by its data source -- used only to orient the two endpoints. The +// link id encodes the (normalized) relationship name. Links are deduped -- +// schema-join is undirected, so a per-entry lookup can return the same link +// once from each endpoint. function readRelationships( - links: EntryLink[], modelName: string, entityEntryNames: Set, - dataSourceToEntity: Map, + links: EntryLink[], modelName: string, entityByEntryId: Map, warnings: string[]): Relationship[] { const prefix = linkNamePrefix(modelName); const seen = new Set(); @@ -339,14 +342,18 @@ function readRelationships( for (const link of links) { if (!link.entryLinkType?.endsWith('/entryLinkTypes/schema-join')) continue; const refs = link.entryReferences ?? []; - // Only a link whose BOTH endpoints are this model's entities belongs here. - if (refs.length !== 2 || !refs.every(r => entityEntryNames.has(r.name))) { - continue; - } - if (link.name) { - if (seen.has(link.name)) continue; - seen.add(link.name); - } + if (refs.length !== 2) continue; + // Resolve both endpoints from the link's references. A reference that is + // not one of this model's entities (another model, or a non-entity) means + // the link does not belong here -- skip it quietly, as with M:N edges. + const endA = entityByEntryId.get(idOf(refs[0].name)); + const endB = entityByEntryId.get(idOf(refs[1].name)); + if (!endA || !endB) continue; + + const key = linkDedupKey(link); + if (seen.has(key)) continue; + seen.add(key); + const join = asArray(aspectDataOf(link.aspects, 'schema-join').joins)[0]; if (!join) { warnings.push( @@ -354,19 +361,35 @@ function readRelationships( `relationship is skipped`); continue; } - const source = dataSourceToEntity.get((join.source?.name ?? '').trim()); - const destination = - dataSourceToEntity.get((join.target?.name ?? '').trim()); - if (!source || !destination) { + // Direction lives in the aspect: `source` is the foreign-key side, named by + // its data source. Orient the two endpoints by matching that name, keeping + // join.source.fields paired with the source side. When neither orientation + // matches -- or both do, because the endpoints share a data source -- the + // direction is undecidable, so keep the reference order and warn rather + // than drop the edge. + const srcName = (join.source?.name ?? '').trim(); + const tgtName = (join.target?.name ?? '').trim(); + const aSrc = (endA.dataSource ?? '').trim(); + const bSrc = (endB.dataSource ?? '').trim(); + const endAIsSource = aSrc === srcName && bSrc === tgtName; + const endBIsSource = bSrc === srcName && aSrc === tgtName; + let [source, destination] = [endA, endB]; + if (endBIsSource && !endAIsSource) { + [source, destination] = [endB, endA]; + } else if (!endAIsSource && !endBIsSource) { warnings.push( - `entry link '${link.name}': a join endpoint table does not match a ` + - `reconstructed entity; the relationship is skipped`); - continue; + `entry link '${link.name}': join direction does not match either ` + + `endpoint's data source; using the reference order`); + } else if (endAIsSource && endBIsSource) { + warnings.push( + `entry link '${link.name}': join direction is ambiguous (endpoints ` + + `share a data source); using the reference order`); } const rel: Relationship = { name: relationshipName(link.name, prefix), - source: {entity: source, columns: asArray(join.source?.fields)}, - destination: {entity: destination, columns: asArray(join.target?.fields)}, + source: {entity: source.name, columns: asArray(join.source?.fields)}, + destination: + {entity: destination.name, columns: asArray(join.target?.fields)}, }; if (join.description !== undefined) rel.description = join.description; out.push(rel); @@ -375,6 +398,16 @@ function readRelationships( } +// A stable dedup key for an entry link: its name when present, else its +// (order-independent) endpoint pair and type. Guards against a per-endpoint +// lookup returning an unnamed link once from each of its two endpoints. +export function linkDedupKey(link: EntryLink): string { + if (link.name) return link.name; + const refs = (link.entryReferences ?? []).map(r => r.name).sort(); + return `${refs.join('')}${link.entryLinkType ?? ''}`; +} + + // Best-effort recovery of the relationship name from the link id, which the // emitter built as linkSlug(`-`) (lowercased/hyphenated -- see // knowledge_catalog.ts). Strips the model prefix when present; the name comes diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index 24d338c3..a1ebae76 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -19,7 +19,7 @@ import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; import {SemanticModel} from './ir'; -import {idOf, modelsFromCatalogResources} from './kc_converter'; +import {idOf, linkDedupKey, modelsFromCatalogResources} from './kc_converter'; export interface KcPullOptions { project: string; @@ -96,7 +96,7 @@ export async function pullKnowledgeCatalog( // Second fetch pass: relationships are schema-join entry links, which the // entry list/lookup does not return. The catalog exposes links only per // referenced entry (:lookupEntryLinks), so fan out over the entity entries - // and dedup by link name -- schema-join is undirected, so each link comes + // and dedup (linkDedupKey) -- schema-join is undirected, so each link comes // back once from each of its two endpoints. const entityEntries = hydrated.filter( e => e.entryType?.endsWith('/entryTypes/semantic-entity')); @@ -124,9 +124,9 @@ export async function pullKnowledgeCatalog( continue; } for (const link of r.links ?? []) { - const key = link.name ?? ''; - if (key && seenLinks.has(key)) continue; - if (key) seenLinks.add(key); + const key = linkDedupKey(link); + if (seenLinks.has(key)) continue; + seenLinks.add(key); entryLinks.push(link); } } diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index 0e2f0eb6..88a58a22 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -156,6 +156,129 @@ describe('relationship recovery (schema-join links -> IR)', () => { // The emitter never publishes M:N, so no schema-join link exists to read. expect(roundTrip(model).models[0].relationships).toEqual([]); }); + + test( + 'endpoints resolve from the link references, not the shared table name', + () => { + // Two entities over the SAME table: a data-source index would collapse + // them (last write wins), so both endpoints must come from the link's + // entryReferences instead. Direction can't be told from the shared + // table, so the reader keeps the reference order and warns. + const sameTable: Entity[] = [ + { + name: 'parent_order', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'id', expression: 'parent_order.id'}], + }, + { + name: 'child_order', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'parent_id', expression: 'child_order.parent_id'}], + }, + ]; + const model: SemanticModel = { + name: 'sales', + entities: sameTable, + relationships: [{ + name: 'parents', + source: {entity: 'child_order', columns: ['parent_id']}, + destination: {entity: 'parent_order', columns: ['id']}, + }], + metrics: [], + }; + const {models, warnings} = roundTrip(model); + expect(models[0].relationships).toEqual([{ + name: 'parents', + source: {entity: 'child_order', columns: ['parent_id']}, + destination: {entity: 'parent_order', columns: ['id']}, + }]); + expect(warnings.some(w => /join direction is ambiguous/i.test(w))) + .toBe(true); + }); + + test( + 'endpoints resolve when link references carry an un-normalized project ' + + 'number', + () => { + // The live path is asymmetric: lookupEntry normalizes entry names to + // project IDs, but lookupEntryLinks returns references still carrying + // the numeric project. Matching on the (stable) entry id keeps the + // relationship resolvable rather than silently dropping it. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + const numeric = entryLinks.map( + l => ({ + ...l, + entryReferences: + (l.entryReferences ?? + []).map(r => ({ + ...r, + name: r.name.replace( + 'projects/dest/', 'projects/000000000000/'), + })), + })); + const {models} = modelsFromCatalogResources(entries, numeric); + expect(models[0].relationships).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + }); + + test( + 'the model prefix is stripped across tricky model names ' + + '(linkNamePrefix tracks the emitter slug)', + () => { + // Pins the read-side prefix reproduction to the emitter's linkSlug: if + // it drifts, the model prefix would survive and the recovered name + // would not equal the bare, normalized relationship name. + for (const modelName of ['Sales', 'Sales Analytics', '123 Sales!']) { + const model: SemanticModel = { + name: modelName, + entities: twoEntities, + relationships: [{ + name: 'rel_one', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships[0].name) + .toBe('rel-one'); + } + }); + + test('an unnamed link returned from both endpoints is deduped', () => { + // Defense in depth: if the catalog ever returns a nameless schema-join + // link, the per-endpoint fan-out yields it twice; dedup falls back to the + // endpoint pair so it becomes one relationship, not two. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + const nameless = entryLinks.map(l => ({...l, name: undefined})); + const {models} = modelsFromCatalogResources( + entries, [...nameless, ...nameless.map(l => ({...l}))]); + expect(models[0].relationships).toHaveLength(1); + }); }); diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index d5d22d42..2316003d 100644 --- a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -183,6 +183,25 @@ describe('pullKnowledgeCatalog: relationship links', () => { expect(lookupLinks).toHaveBeenCalledTimes(2); }); + test( + 'an unnamed link returned from both endpoints is deduped to one edge', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + // A nameless link can't dedup by name; the puller must fall back to the + // endpoint pair, or the per-endpoint fan-out would yield it twice. + const nameless = entryLinks.map(l => ({...l, name: undefined})); + const {lookupLinks} = stubClient(entries, entries, undefined, nameless); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, OPTS); + + // Deduped to a single edge (a nameless link has no id, so the name + // itself can't be recovered -- but it must not be counted twice). + expect(models[0].relationships).toHaveLength(1); + expect(models[0].relationships[0].source.entity).toBe('orders'); + expect(lookupLinks).toHaveBeenCalledTimes(2); + }); + test( 'a failed link lookup on one endpoint still recovers the edge from the ' + 'other, and warns', From e6d5e1f97652082ebf944f2dd3fc8190bccb0e86 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 07:29:14 +0000 Subject: [PATCH 29/34] mdcode: assert pull symmetry + complete the sales BQ golden Two fixture-coverage gaps from the round-trip review: 1. Symmetry assertion. The golden pull files let a human eyeball what a Knowledge Catalog round trip drops, but nothing asserted it. Add a symmetry test over the converter corpus: load the authored IR, run a full emit -> read round trip, and assert the result equals the authored IR reduced to the "KC floor" (stripToKcFloor) -- the documented losses and normalizations applied to both sides. An undocumented regression (a dropped column, a lost description, an un-stripped M:N edge) now fails here even though each individual loss is already pinned by a targeted test. Export linkNamePrefix so the normalizer reproduces the relationship slug rather than reimplementing it. 2. sales_bq_graph_target had OSI/KC/pull goldens but no BigQuery golden. Add it to the BigQuery corpus and generate the golden: a valid single-node property graph with a MEASURE, so the fixture now carries a complete four-arm round-trip suite. No production behavior change; reader/emitter untouched apart from the linkNamePrefix export. --- .../mdcode/src/libts/semantic/kc_converter.ts | 6 +- .../tests/libts/semantic/bigquery.e2e.test.ts | 1 + .../sales_bq_graph_target.bigquery.golden.sql | 13 ++ .../tests/libts/semantic/kc_converter.test.ts | 134 +++++++++++++++++- 4 files changed, 150 insertions(+), 4 deletions(-) create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 3600d781..7a04d7b2 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -425,8 +425,10 @@ function relationshipName( // Mirrors the model-name portion of the emitter's linkSlug so the prefix a link // id was built with can be stripped. Kept local rather than imported: this -// read-side module must not depend on the write-side emitter. -function linkNamePrefix(modelName: string): string { +// read-side module must not depend on the write-side emitter. Exported for the +// symmetry test, which reuses it to reproduce the normalized relationship name +// rather than reimplementing the slug rule. +export function linkNamePrefix(modelName: string): string { return modelName.toLowerCase() .replace(/[^a-z0-9-]+/g, '-') .replace(/-+/g, '-') diff --git a/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts b/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts index b16be403..17816f02 100644 --- a/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts @@ -41,6 +41,7 @@ const CORPUS = [ 'measure_lowering.yaml', 'metric_skips.yaml', 'keyless_dimension.yaml', + 'sales_bq_graph_target.yaml', ]; // Loads a fixture file to its IR. Split out from `build` so a test that only diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql new file mode 100644 index 00000000..51059fe9 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE PROPERTY GRAPH `sqlgen-testing.demo.sales` +NODE TABLES ( + `demo.sales.orders` AS orders + KEY(o_orderkey) + PROPERTIES( + o_orderkey, + o_totalprice, + MEASURE(SUM(o_totalprice)) AS total_revenue + ) +); + +-- warnings -- +-- note: no 'BIGQUERY' dialect for one or more expressions; using the portable 'ANSI_SQL' dialect verbatim ('BIGQUERY' accepts the ANSI core subset — supply 'BIGQUERY' variants only for BIGQUERY-specific SQL) diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index 88a58a22..91967af8 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -19,9 +19,10 @@ import {describe, expect, test} from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import {Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; -import {modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; +import {CustomExtension, Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; +import {linkNamePrefix, modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {loadModels} from '../../../src/libts/semantic/loader'; import {serializeModel} from '../../../src/libts/semantic/osi_converter'; const FIXTURES = path.join(__dirname, 'fixtures'); @@ -559,3 +560,132 @@ describe( }); } }); + + +// -- Symmetry: emit -> read drops ONLY the documented set. -- +// +// The golden pull above lets a human eyeball what a Knowledge Catalog round +// trip loses; this asserts it mechanically. For each corpus fixture it loads +// the authored IR, runs a full emit -> read round trip, and asserts the result +// equals the authored IR reduced to the "KC floor" -- the model with exactly +// the documented losses and normalizations applied (`stripToKcFloor`). Applying +// that same reduction to BOTH sides makes `toEqual` flag only UNDOCUMENTED +// divergence, so a new emitter/reader regression (a dropped column, a lost +// description, an un-stripped M:N edge) fails here even though every individual +// loss is already pinned by a targeted test above. +describe( + 'symmetry: an emit -> read round trip drops only documented fields', () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + // Same load defaults as the OSI / KC / pull goldens. + const LOAD = {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}; + + for (const fixture of CORPUS) { + test(fixture, () => { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + for (const authored of loadModels(text, LOAD).models) { + const {models, warnings} = roundTrip(authored); + expect(models).toHaveLength(1); + expect(stripToKcFloor(models[0])).toEqual(stripToKcFloor(authored)); + // A clean corpus model round-trips without reader warnings. + expect(warnings).toEqual([]); + } + }); + } + }); + + +// Reduces a model to the "KC floor": the most an emit -> read round trip can +// preserve, i.e. the authored model with exactly the documented Knowledge +// Catalog losses and normalizations applied. Idempotent (already-floored input +// is unchanged), so it can be applied to both sides of a round trip. Each +// transform mirrors one emitter/reader behavior pinned by a targeted test +// above. +function stripToKcFloor(model: SemanticModel): SemanticModel { + const m = structuredClone(model); + + // Model ai_context is not persisted; only a GOOGLE deployment-targets block + // rides back, re-serialized to the reader's canonical JSON form. + delete m.aiContext; + const ext = canonicalGoogleExt(m.customExtensions); + if (ext) { + m.customExtensions = ext; + } else { + delete m.customExtensions; + } + + for (const e of m.entities) { + e.keys = []; // keys / unique keys are never persisted + delete e.uniqueKeys; + delete e.aiContext; + delete e.customExtensions; + for (const f of e.fields) { + delete f.label; // display label is not persisted + delete f.aiContext; + delete f.importedExpression; // vendor SQL is not persisted + delete f.importedDialect; + delete f.customExtensions; + if (f.dimension) f.dimension = {}; // only the DIMENSION role survives + // String is indistinguishable from an un-typed field on read. + if (f.type === 'String') delete f.type; + } + } + + for (const metric of m.metrics) { + delete metric.aiContext; + delete metric.importedExpression; + delete metric.importedDialect; + delete metric.customExtensions; + // The emitter writes a required dataType, defaulting typeless -> NUMERIC, + // which reads back as Decimal; String collapses to un-typed like fields. + if (metric.type === undefined) { + metric.type = 'Decimal'; + } else if (metric.type === 'String') { + delete metric.type; + } + } + + // M:N (association) edges are never published; a 1:1 / 1:N name comes back + // normalized via the emitter's slug (its exact form is pinned above). + m.relationships = m.relationships.filter(r => !r.association).map(r => { + const rel = structuredClone(r); + rel.name = linkNamePrefix(rel.name); + delete rel.aiContext; + delete rel.customExtensions; + return rel; + }); + + return m; +} + + +// The inverse-canonical of the emitter's deployment-target persistence, +// matching the reader's readDeploymentTargets: keep only GOOGLE +// deploymentTargets and re-serialize them so an authored block and its +// round-tripped form (which the reader always JSON.stringifies afresh) compare +// equal. Returns undefined when the model declares none, exactly as the reader +// omits custom_extensions then. +function canonicalGoogleExt(exts: CustomExtension[]|undefined): + CustomExtension[]|undefined { + const targets: string[] = []; + for (const ext of exts ?? []) { + if (ext.vendorName !== 'GOOGLE') continue; + let parsed: any; + try { + parsed = JSON.parse(ext.data); + } catch { + continue; + } + for (const t of parsed?.deploymentTargets ?? []) { + if (typeof t === 'string' && t) targets.push(t); + } + } + return targets.length ? [{ + vendorName: 'GOOGLE', + data: JSON.stringify({deploymentTargets: targets}) + }] : + undefined; +} From efcc162eeb0c12aa0fb0fccc35785857954665c3 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 07:40:48 +0000 Subject: [PATCH 30/34] mdcode: drop the sales_bq_graph_target BigQuery golden The BigQuery corpus golden path names the graph from the test's build opts (sqlgen-testing.demo.sales), ignoring the fixture's deployment target -- so the golden neither reflected the fixture's purpose nor matched a real deploy (demo.sales.sales_graph). That target-driven name is already covered by deploy_bigquery.test.ts, making this golden redundant. Revert the corpus addition and remove the generated file; the fixture keeps its OSI/KC/pull goldens and the pull symmetry assertion. --- .../tests/libts/semantic/bigquery.e2e.test.ts | 1 - .../sales_bq_graph_target.bigquery.golden.sql | 13 ------------- 2 files changed, 14 deletions(-) delete mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql diff --git a/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts b/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts index 17816f02..b16be403 100644 --- a/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts @@ -41,7 +41,6 @@ const CORPUS = [ 'measure_lowering.yaml', 'metric_skips.yaml', 'keyless_dimension.yaml', - 'sales_bq_graph_target.yaml', ]; // Loads a fixture file to its IR. Split out from `build` so a test that only diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql deleted file mode 100644 index 51059fe9..00000000 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE OR REPLACE PROPERTY GRAPH `sqlgen-testing.demo.sales` -NODE TABLES ( - `demo.sales.orders` AS orders - KEY(o_orderkey) - PROPERTIES( - o_orderkey, - o_totalprice, - MEASURE(SUM(o_totalprice)) AS total_revenue - ) -); - --- warnings -- --- note: no 'BIGQUERY' dialect for one or more expressions; using the portable 'ANSI_SQL' dialect verbatim ('BIGQUERY' accepts the ANSI core subset — supply 'BIGQUERY' variants only for BIGQUERY-specific SQL) From ff04848e30c91b875d6dfcf0c138639f36162ce7 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 15:41:56 +0000 Subject: [PATCH 31/34] mdcode: fix pull reader review findings Address code-review findings on the KC->IR reader (kc_converter.ts): - readMetric: when the expression does not pin exactly one known entity (none, or several), fall back to the attach entity metricAspectData persisted instead of dropping it. A cross-entity metric now recovers its authored entity. - readField: skip a schema field with no name (warn) instead of emitting a Field with an undefined name into the entity. - linkNamePrefix: mirror linkSlug's 63-char cap and trailing-hyphen re-strip so the read-side prefix stays aligned with the emitter's link id. - Header comment: add importedExpression (vendor SQL) and String/Opaque- typed metrics to the documented round-trip loss list. Add regression tests for the metric-entity fallback and the nameless-field skip. --- .../mdcode/src/libts/semantic/kc_converter.ts | 48 +++++++++++---- .../tests/libts/semantic/kc_converter.test.ts | 60 +++++++++++++++++++ 2 files changed, 98 insertions(+), 10 deletions(-) diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 7a04d7b2..e732ad55 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -28,14 +28,18 @@ // inverse of the WRITE, not of the authored document. It recovers names, // descriptions, data sources, field datatypes (via the schema aspect) and // DIMENSION roles, field/metric expressions, each metric's attach entity -// (re-derived from its expression, as the loader does), the model's deployment -// targets (from the semantic-model aspect, back into the GOOGLE -// `custom_extensions` block), and 1:1 / 1:N relationships (from the -// `schema-join` entry links a pull fetched -- see +// (re-derived from its expression, as the loader does, else the value the +// emitter persisted), the model's deployment targets (from the semantic-model +// aspect, back into the GOOGLE `custom_extensions` block), and 1:1 / 1:N +// relationships (from the `schema-join` entry links a pull fetched -- see // `modelsFromCatalogResources`'s `entryLinks` argument). It cannot recover what // the emitter does not write: entity keys/unique keys, `ai_context`, field -// labels, `importedDialect`, and many-to-many (association) relationships -// (whose edge lives only in the BigQuery property graph). Relationship NAMES +// labels, `importedExpression`/`importedDialect` (the vendor-dialect SQL), and +// many-to-many (association) relationships (whose edge lives only in the +// BigQuery property graph). A `String`- or `Opaque`-typed METRIC also reads +// back un-typed: the metric aspect persists only `dataType` (both collapse to +// `STRING`, with no metadataType to disambiguate), so -- as with a plain STRING +// field -- the reader leaves it un-typed rather than guess. Relationship NAMES // come back normalized (lowercased/hyphenated), since the emitter encodes the // name only in the link id (via `linkSlug`), not in the join aspect. @@ -158,7 +162,9 @@ function readEntity(entry: Entry, warnings: string[]): Entity { name, dataSource, keys: [], // not persisted by the emitter; unrecoverable on read - fields: asArray(schema.fields).map(fd => readField(fd, name, warnings)), + fields: asArray(schema.fields) + .map(fd => readField(fd, name, warnings)) + .filter((f): f is Field => f !== undefined), }; const description = entry.entrySource?.description; if (description !== undefined) entity.description = description; @@ -170,12 +176,14 @@ function readEntity(entry: Entry, warnings: string[]): Entity { // schemaAspectData: the datatype from dataType/metadataType, expressions from // the nested `semantics` block, and the DIMENSION role back to a dimension // marker. -function readField(fd: any, entityName: string, warnings: string[]): Field { +function readField(fd: any, entityName: string, warnings: string[]): Field| + undefined { const name = fd?.name; if (name === undefined || name === '') { warnings.push( `entity '${entityName}': a schema field is missing its name; the ` + - `field may not load`); + `field is skipped`); + return undefined; } const field: Field = {name}; const sem = fd?.semantics ?? {}; @@ -203,8 +211,18 @@ function readMetric( if (data.expression !== undefined) metric.expression = data.expression; const exprForRefs = data.expression ?? ''; const referenced = referencedEntityNames(exprForRefs, entityNames); + const persistedEntity = + typeof data.entity === 'string' && data.entity !== '' ? data.entity : + undefined; if (referenced.length === 1) { + // The expression pins exactly one known entity; re-derive it (as the loader + // does) so the attach entity stays consistent with the reconstructed set. metric.entity = referenced[0]; + } else if (persistedEntity !== undefined) { + // The expression does not pin a single known entity (none, or several), so + // fall back to the attach entity the emitter persisted (metricAspectData) + // rather than dropping it. + metric.entity = persistedEntity; } else if (exprForRefs && !referenced.length) { // Parity with the loader's convertMetric: an expression that qualifies no // known entity is flagged as potentially unplaceable downstream. @@ -428,12 +446,22 @@ function relationshipName( // read-side module must not depend on the write-side emitter. Exported for the // symmetry test, which reuses it to reproduce the normalized relationship name // rather than reimplementing the slug rule. +// +// It reproduces linkSlug's normalization AND its 63-char cap + trailing-hyphen +// re-strip, so the prefix stays aligned with the id even when the combined +// `-` slug was truncated. (linkSlug's `|| 'link'` empty-slug +// fallback is intentionally omitted: an empty prefix strips nothing and +// relationshipName then returns the id verbatim, which is already correct. When +// the cap truncates into the model slug itself the relationship name is +// unrecoverable regardless -- relationshipName returns the truncated id.) export function linkNamePrefix(modelName: string): string { return modelName.toLowerCase() .replace(/[^a-z0-9-]+/g, '-') .replace(/-+/g, '-') .replace(/^[^a-z]+/, '') - .replace(/^-+|-+$/g, ''); + .replace(/^-+|-+$/g, '') + .slice(0, 63) + .replace(/-+$/, ''); } diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index 91967af8..e63d6bf0 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -392,6 +392,35 @@ describe('field mapping details', () => { expect(back.importedExpression).toBeUndefined(); expect(back.importedDialect).toBeUndefined(); }); + + test('a schema field missing its name is skipped with a warning', () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'f', expression: 'e.f'}], + }], + relationships: [], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + // Inject a nameless field record into the entity's schema aspect, as a + // malformed catalog could return; the reader must drop it, not emit a field + // with an undefined name. + for (const entry of entries) { + for (const aspect of Object.values(entry.aspects ?? {})) { + const fields = (aspect as {data?: {fields?: unknown[]}}).data?.fields; + if (Array.isArray(fields)) fields.push({dataType: 'INT64'}); + } + } + const {models, warnings} = modelsFromCatalogResources(entries, entryLinks); + expect(models[0].entities[0].fields.map(f => f.name)).toEqual(['f']); + expect( + warnings.some(w => /missing its name; the field is skipped/i.test(w))) + .toBe(true); + }); }); @@ -452,6 +481,37 @@ describe('metric attach entity is re-derived from the expression', () => { expect(byName.get('revenue')!.entity).toBe('orders'); expect(byName.get('mix')!.entity).toBeUndefined(); }); + + test( + 'a cross-entity metric falls back to the persisted attach entity', () => { + const model: SemanticModel = { + name: 'm', + entities: [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'amt', expression: 'orders.amt'}] + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'region', expression: 'customer.region'}] + }, + ], + relationships: [], + // Two entities in the expression, so it cannot be re-derived; the + // authored attach entity must ride back on the persisted aspect. + metrics: [{ + name: 'mix', + expression: 'SUM(orders.amt) / COUNT(customer.region)', + entity: 'orders', + }], + }; + const {models} = roundTrip(model); + expect(models[0].metrics[0].entity).toBe('orders'); + }); }); From 8768a40dda29ee339591ec628568bb4f1db112d3 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 01:33:02 +0000 Subject: [PATCH 32/34] mdcode: reconcile pull with the merged --emit-expressions gating PR #278 landed on main gating the KC SQL-expression fields off by default: the emitter now omits the per-field `semantics` block (field expression + role) and the metric expression unless `--emit-expressions` is set, and the committed emitter golden was regenerated without them. The pull leg was built against the old always-emit behavior, so after rebasing onto main it read back less than its tests and goldens assumed. Reconcile the reader to the new default: - readMetric no longer warns when a metric aspect has no expression -- that is now the expected default, not a malformed aspect. It still derives the attach entity from the expression when one is present, else from the persisted `entity`, and still warns on an expression that pins no known entity. - Reader header + user guide: field/metric expressions and the DIMENSION role are recovered only from an `--emit-expressions` push; a default push -> pull drops them. - Tests: split the round trip into roundTrip (default, drops the semantics block) and roundTripFull (`--emit-expressions`, keeps it); point the lossless-slice, DIMENSION, canonical-expression, and expression-derivation cases at the full round trip, and add a case pinning the default drop. The symmetry floor now strips field/metric expressions and the dimension role. - Regenerate the .pull.golden.yaml fixtures: they drop only expressions and bare dimension markers, matching the default emitter golden. --- toolbox/mdcode/docs/semantic-model.md | 15 ++- .../mdcode/src/libts/semantic/kc_converter.ts | 32 ++++--- .../sales_bq_graph_target.pull.golden.yaml | 12 --- .../star_orders_customer.pull.golden.yaml | 33 ------- .../fixtures/tpcds_date_edge.pull.golden.yaml | 94 ------------------- .../tests/libts/semantic/kc_converter.test.ts | 78 +++++++++++---- .../tests/libts/semantic/pull_kc.test.ts | 8 +- 7 files changed, 95 insertions(+), 177 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index b3a1fcbb..b4aaa106 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -266,17 +266,24 @@ with no matching catalog entry) is left untouched — pull never deletes. > > **Recovered exactly** — these come back as authored: > - Model structure: the model, its entities, and each entity's fields. -> - Field data source and data type; the field expression. -> - Metrics: name, expression, data type, and attach entity. +> - Field data source and data type. +> - Metrics: name, data type, and attach entity. > - 1:1 / 1:N relationships: endpoints, foreign-key direction, and join columns > (from the `schema-join` links). > - Deployment targets. > +> **Recovered only if pushed with `--emit-expressions`** — the per-field +> `semantics` block (expressions and the dimension role) and the metric +> expression are omitted from the catalog by default (see the note above), so +> pull returns them only when the push that wrote them used `--emit-expressions`: +> - Field expressions and metric expressions (the canonical GoogleSQL/ANSI form). +> - A field's dimension role, which comes back as a bare `dimension: {}` marker, +> without its detail (`is_time`, and so on). A default push drops the marker +> entirely. +> > **Recovered, but normalized** — the content survives, the form changes: > - Relationship *names* come back lowercased/hyphenated (the catalog stores the > name only in the link id, e.g. `Places Order` → `places-order`). -> - A field marked as a dimension comes back as a bare `dimension: {}` marker, -> without its detail (`is_time`, and so on). > - A metric authored with no data type comes back as an explicit `Decimal` > (push must write a type, and defaults it to `NUMERIC`). > diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index e732ad55..9c863290 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -26,15 +26,20 @@ // // Fidelity is bounded by what the emitter persisted, so this read is the // inverse of the WRITE, not of the authored document. It recovers names, -// descriptions, data sources, field datatypes (via the schema aspect) and -// DIMENSION roles, field/metric expressions, each metric's attach entity -// (re-derived from its expression, as the loader does, else the value the -// emitter persisted), the model's deployment targets (from the semantic-model -// aspect, back into the GOOGLE `custom_extensions` block), and 1:1 / 1:N -// relationships (from the `schema-join` entry links a pull fetched -- see -// `modelsFromCatalogResources`'s `entryLinks` argument). It cannot recover what -// the emitter does not write: entity keys/unique keys, `ai_context`, field -// labels, `importedExpression`/`importedDialect` (the vendor-dialect SQL), and +// descriptions, data sources, field datatypes (via the schema aspect), each +// metric's attach entity (re-derived from its expression, as the loader does, +// when the catalog holds one, else the value the emitter persisted), the +// model's deployment targets (from the semantic-model aspect, back into the +// GOOGLE `custom_extensions` block), and 1:1 / 1:N relationships (from the +// `schema-join` entry links a pull fetched -- see +// `modelsFromCatalogResources`'s `entryLinks` argument). The per-field +// `semantics` block -- field/metric expressions and the DIMENSION role -- is +// gated off the catalog by default: the emitter writes it only under +// `--emit-expressions` (see `KcGenerateOptions.emitExpressions`), so those +// three recover only when the push that wrote them enabled it, and a default +// push -> pull drops them. It cannot recover what the emitter never writes: +// entity keys/unique keys, `ai_context`, field labels, +// `importedExpression`/`importedDialect` (the vendor-dialect SQL), and // many-to-many (association) relationships (whose edge lives only in the // BigQuery property graph). A `String`- or `Opaque`-typed METRIC also reads // back un-typed: the metric aspect persists only `dataType` (both collapse to @@ -197,15 +202,14 @@ function readField(fd: any, entityName: string, warnings: string[]): Field| // Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` -// is re-derived from the expression (as the loader does) rather than read from -// the aspect, so it stays consistent with the reconstructed entity set. +// is re-derived from the expression (as the loader does) when the aspect holds +// one, else it falls back to the entity the emitter persisted. A default push +// omits the expression entirely (it is gated behind `--emit-expressions`), so +// an absent expression is expected, not an error, and does not warn. function readMetric( entry: Entry, entityNames: string[], warnings: string[]): Metric { const name = entry.entrySource?.displayName ?? idOf(entry.name); const data = aspectData(entry, 'semantic-metric'); - if (data.expression === undefined) { - warnings.push(`metric '${name}': no expression in semantic-metric aspect`); - } const metric: Metric = {name}; if (data.expression !== undefined) metric.expression = data.expression; diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml index 06dd8bdc..f966f619 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml @@ -10,19 +10,7 @@ semantic_model: source: demo.sales.orders fields: - name: o_orderkey - expression: - dialects: - - dialect: BIGQUERY - expression: o_orderkey - name: o_totalprice - expression: - dialects: - - dialect: BIGQUERY - expression: o_totalprice metrics: - name: total_revenue - expression: - dialects: - - dialect: BIGQUERY - expression: SUM(orders.o_totalprice) datatype: Decimal diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml index d7a9cbe1..ceb6bed1 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -12,40 +12,15 @@ semantic_model: description: One row per order fields: - name: o_orderkey - expression: - dialects: - - dialect: BIGQUERY - expression: o_orderkey description: Order identifier - name: o_custkey - expression: - dialects: - - dialect: BIGQUERY - expression: o_custkey - name: o_orderdate - expression: - dialects: - - dialect: BIGQUERY - expression: o_orderdate - dimension: {} - name: o_totalprice - expression: - dialects: - - dialect: BIGQUERY - expression: o_totalprice - name: customer source: samples.tpch.customer fields: - name: c_custkey - expression: - dialects: - - dialect: BIGQUERY - expression: c_custkey - name: c_name - expression: - dialects: - - dialect: BIGQUERY - expression: c_name description: Customer name relationships: - name: orders-to-customer @@ -57,16 +32,8 @@ semantic_model: - c_custkey metrics: - name: total_revenue - expression: - dialects: - - dialect: BIGQUERY - expression: SUM(orders.o_totalprice) datatype: Decimal description: Total order revenue - name: order_count - expression: - dialects: - - dialect: BIGQUERY - expression: COUNT(orders.o_orderkey) datatype: Decimal description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml index 1900fe12..ab45bace 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -9,138 +9,44 @@ semantic_model: description: Fact table containing all store sales transactions fields: - name: ss_item_sk - expression: - dialects: - - dialect: BIGQUERY - expression: ss_item_sk - dimension: {} - name: ss_ticket_number - expression: - dialects: - - dialect: BIGQUERY - expression: ss_ticket_number - dimension: {} - name: ss_customer_sk - expression: - dialects: - - dialect: BIGQUERY - expression: ss_customer_sk - dimension: {} - name: ss_store_sk - expression: - dialects: - - dialect: BIGQUERY - expression: ss_store_sk - dimension: {} - name: ss_quantity - expression: - dialects: - - dialect: BIGQUERY - expression: ss_quantity description: Quantity of items sold - name: ss_sales_price - expression: - dialects: - - dialect: BIGQUERY - expression: ss_sales_price description: Sales price per unit - name: ss_ext_sales_price - expression: - dialects: - - dialect: BIGQUERY - expression: ss_ext_sales_price description: Extended sales price (quantity * price) - name: ss_net_profit - expression: - dialects: - - dialect: BIGQUERY - expression: ss_net_profit description: Net profit from the sale - name: customer source: tpcds.public.customer description: Customer dimension with demographic information fields: - name: c_customer_sk - expression: - dialects: - - dialect: BIGQUERY - expression: c_customer_sk - dimension: {} - name: c_first_name - expression: - dialects: - - dialect: BIGQUERY - expression: c_first_name - dimension: {} description: Customer first name - name: c_last_name - expression: - dialects: - - dialect: BIGQUERY - expression: c_last_name - dimension: {} description: Customer last name - name: item source: tpcds.public.item description: Item/Product dimension fields: - name: i_item_sk - expression: - dialects: - - dialect: BIGQUERY - expression: i_item_sk - dimension: {} - name: i_brand - expression: - dialects: - - dialect: BIGQUERY - expression: i_brand - dimension: {} - name: i_category - expression: - dialects: - - dialect: BIGQUERY - expression: i_category - dimension: {} - name: i_current_price - expression: - dialects: - - dialect: BIGQUERY - expression: i_current_price description: Current price of the item - name: store source: tpcds.public.store description: Store dimension with location attributes fields: - name: s_store_sk - expression: - dialects: - - dialect: BIGQUERY - expression: s_store_sk - dimension: {} - name: s_store_name - expression: - dialects: - - dialect: BIGQUERY - expression: s_store_name - dimension: {} - name: s_city - expression: - dialects: - - dialect: BIGQUERY - expression: s_city - dimension: {} - name: s_state - expression: - dialects: - - dialect: BIGQUERY - expression: s_state - dimension: {} - name: s_number_employees - expression: - dialects: - - dialect: BIGQUERY - expression: s_number_employees description: Number of employees at the store - name: date_dim source: sqlgen-testing.demo.date_dim diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index e63d6bf0..4ccac618 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -7,13 +7,16 @@ // emitter is lossless. The write drops content by design (entity keys, // ai_context, field labels, importedDialect, and many-to-many relationships -- // see the emitter header), so the expected read-back is the source model with -// exactly those fields cleared. 1:1 / 1:N relationships and deployment targets -// DO round-trip (via schema-join links and the semantic-model aspect), except -// that relationship names come back normalized (lowercased/hyphenated). -// Targeted tests pin the mapping details a round trip cannot isolate (the -// dataType inverse, the DIMENSION role, resource-URI parsing, metric attach -// re-derivation, relationship endpoint/direction recovery, and parent/anchor -// grouping). +// exactly those fields cleared. It ALSO drops the per-field `semantics` block +// (field/metric expressions and the DIMENSION role) unless the push enabled +// `emitExpressions`, so a DEFAULT round trip (roundTrip) loses those too, while +// a full round trip (roundTripFull, emitExpressions: true) keeps them. 1:1 / +// 1:N relationships and deployment targets DO round-trip either way (via +// schema-join links and the semantic-model aspect), except that relationship +// names come back normalized (lowercased/hyphenated). Targeted tests pin the +// mapping details a round trip cannot isolate (the dataType inverse, the +// DIMENSION role, resource-URI parsing, metric attach re-derivation, +// relationship endpoint/direction recovery, and parent/anchor grouping). import {describe, expect, test} from 'bun:test'; import * as fs from 'node:fs'; @@ -33,18 +36,32 @@ const OPTS = { entryGroup: 'eg' }; -// Emits a model to entries + entry links and reads it straight back. +// Emits a model to entries + entry links and reads it straight back. The +// default push omits the per-field `semantics` block (expressions + role), so a +// default round trip drops those; use roundTripFull to exercise their recovery. function roundTrip(model: SemanticModel): {models: SemanticModel[]; warnings: string[]} { const {entries, entryLinks} = generateCatalogResources(model, OPTS); return modelsFromCatalogResources(entries, entryLinks); } +// A round trip through a `--emit-expressions` push: the catalog then holds the +// per-field `semantics` block and the metric expression, so field/metric +// expressions and DIMENSION roles round-trip too. +function roundTripFull(model: SemanticModel): + {models: SemanticModel[]; warnings: string[]} { + const {entries, entryLinks} = + generateCatalogResources(model, {...OPTS, emitExpressions: true}); + return modelsFromCatalogResources(entries, entryLinks); +} + describe('emitter -> reader round trip (lossless slice)', () => { // A model using only round-trippable content: no keys/ai_context/labels/ // relationships (all dropped by the write), and datatypes that invert - // cleanly. + // cleanly. Its fields and metric carry expressions and a DIMENSION role, so + // it round-trips losslessly only through a `--emit-expressions` push + // (roundTripFull); a default push omits the per-field `semantics` block. const source: SemanticModel = { name: 'sales', description: 'the sales model', @@ -73,7 +90,7 @@ describe('emitter -> reader round trip (lossless slice)', () => { }; test('reconstructs an IR equal to the source', () => { - const {models} = roundTrip(source); + const {models} = roundTripFull(source); expect(models).toHaveLength(1); expect(models[0]).toEqual(source); }); @@ -357,30 +374,46 @@ describe('dataType inverse (schema aspect -> IR type)', () => { describe('field mapping details', () => { - function readField(field: Entity['fields'][number]) { + function readWith( + rt: (m: SemanticModel) => {models: SemanticModel[]}, + field: Entity['fields'][number]) { const model: SemanticModel = { name: 'm', entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: [field]}], relationships: [], metrics: [], }; - return roundTrip(model).models[0].entities[0].fields[0]; + return rt(model).models[0].entities[0].fields[0]; } + // Default push (no per-field `semantics`) vs a `--emit-expressions` push. + const readField = (f: Entity['fields'][number]) => readWith(roundTrip, f); + const readFieldFull = (f: Entity['fields'][number]) => + readWith(roundTripFull, f); test('a DIMENSION role reads back as a dimension marker', () => { - const back = readField({name: 'd', expression: 'e.d', dimension: {}}); + // The role lives in the gated `semantics` block, so it survives only a + // `--emit-expressions` push. + const back = readFieldFull({name: 'd', expression: 'e.d', dimension: {}}); expect(back.dimension).toEqual({}); }); test('a non-dimension field has no dimension marker', () => { - const back = readField({name: 'f', expression: 'e.f'}); + const back = readFieldFull({name: 'f', expression: 'e.f'}); expect(back.dimension).toBeUndefined(); }); + test( + 'a default push drops the DIMENSION role with the semantics block', + () => { + const back = readField({name: 'd', expression: 'e.d', dimension: {}}); + expect(back.dimension).toBeUndefined(); + expect(back.expression).toBeUndefined(); + }); + test( 'an imported expression is not persisted to or recovered from the catalog', () => { - const back = readField({ + const back = readFieldFull({ name: 'amt', expression: 'e.amt', importedExpression: 'e.amt::NUMBER', @@ -476,7 +509,9 @@ describe('metric attach entity is re-derived from the expression', () => { }, ], }; - const {models} = roundTrip(model); + // Full push: the expression is in the catalog, so the entity is + // re-derived from it (single entity -> attach; two entities -> none). + const {models} = roundTripFull(model); const byName = new Map(models[0].metrics.map(m => [m.name, m])); expect(byName.get('revenue')!.entity).toBe('orders'); expect(byName.get('mix')!.entity).toBeUndefined(); @@ -559,7 +594,9 @@ describe('metric expression referencing no known entity', () => { // References `widgets`, which is not an entity of this model. metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], }; - const {models, warnings} = roundTrip(model); + // The unplaceable warning fires from the expression, so it needs a push + // that wrote one (a default push omits it, leaving nothing to check). + const {models, warnings} = roundTripFull(model); expect(models[0].metrics[0].entity).toBeUndefined(); expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) .toBe(true); @@ -688,7 +725,11 @@ function stripToKcFloor(model: SemanticModel): SemanticModel { delete f.importedExpression; // vendor SQL is not persisted delete f.importedDialect; delete f.customExtensions; - if (f.dimension) f.dimension = {}; // only the DIMENSION role survives + // A default push omits the per-field `semantics` block, so the field + // expression and the DIMENSION role are not persisted (they ride back + // only through a `--emit-expressions` push). + delete f.expression; + delete f.dimension; // String is indistinguishable from an un-typed field on read. if (f.type === 'String') delete f.type; } @@ -699,6 +740,7 @@ function stripToKcFloor(model: SemanticModel): SemanticModel { delete metric.importedExpression; delete metric.importedDialect; delete metric.customExtensions; + delete metric.expression; // omitted by a default push, like field ones // The emitter writes a required dataType, defaulting typeless -> NUMERIC, // which reads back as Decimal; String collapses to un-typed like fields. if (metric.type === undefined) { diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index 2316003d..83f5aee0 100644 --- a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -47,9 +47,13 @@ const SALES: SemanticModel = { }], }; -// The entries the emitter would have written for a model. +// The entries the emitter would have written for a model. Emitted with +// expressions on (a `--emit-expressions` push) so a model that carries field / +// metric expressions reconstructs exactly; the default push omits them (the +// converter tests in kc_converter.test.ts pin that gating). function entriesFor(model: SemanticModel): Entry[] { - return generateCatalogResources(model, OPTS).entries; + return generateCatalogResources(model, {...OPTS, emitExpressions: true}) + .entries; } function ok(result?: T): ApiResult { From b36b7585d15a597ce47526383786b44934ef7433 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 02:34:24 +0000 Subject: [PATCH 33/34] mdcode: document pull round-trip writer-side follow-ups Distinguish inherent pull losses from write-side limits: relationship names (not stored in the schema-join aspect) and non-canonical deployment targets (dropped on write) could round-trip faithfully with a writer change. Reader already recovers everything the catalog holds. --- toolbox/mdcode/docs/semantic-model.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index b4aaa106..21b3e5f4 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -298,6 +298,22 @@ with no matching catalog entry) is left untouched — pull never deletes. > pulled document as a faithful copy of the catalog metadata, not of the authored > model, and keep the authored document as the source of truth. +> **Note — writer-side follow-ups (not inherent to pull).** Two of the reductions +> above are limits of what push currently *writes*, not of what pull can recover. +> They are recorded here as write-side follow-ups; the reader (pull) already +> returns everything the catalog holds. +> +> - **Relationship names.** The `schema-join` aspect does not store the authored +> relationship name, so pull recovers it from the link id — which is lowercased +> and hyphenated. Persisting the name in the aspect on write would let pull +> return it verbatim. +> - **Non-canonical deployment targets.** Push persists a target only when it is a +> canonical BigQuery Graph URL +> (`//bigquery.googleapis.com/projects/.../datasets/.../propertyGraphs/...`). +> Other forms — a misspelled path, or the `projects/.../entryGroups/@bigquery/` +> entry form — are dropped on write, so pull has nothing to recover. Widening or +> normalizing the writer's accepted forms would let them round-trip. + ## Permissions `push` needs access to whichever destinations you deploy to. From abc370b1f29f25743be3c9b4663b1276ff7d96b6 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 03:48:54 +0000 Subject: [PATCH 34/34] mdcode: document malformed deployment-target rejection A non-canonical BigQuery Graph deployment target fails push at the validation gate, before any leg and for every --target, so nothing is written to BigQuery or Knowledge Catalog. Correct the earlier writer-side follow-up note, which wrongly implied such targets are silently dropped on write, and sharpen the relationship-name follow-up to name the server-side schema-join aspect-type template as the fix. --- toolbox/mdcode/docs/semantic-model.md | 41 +++++++++++++++++---------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 21b3e5f4..b111776f 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -180,7 +180,16 @@ paired columns and foreign-key direction — in its aspect. `push` and `--validate-only` run the same checks, **before either destination is touched**, so a model that cannot deploy fails fast instead of half-deploying: -* **Exactly one deployment target per model.** *(static)* +* **Exactly one deployment target per model, and it must be a valid BigQuery + Graph URI.** A model with no target — or with more than one — is rejected, and + so is a single target whose URI does not match + `//bigquery.googleapis.com/projects/

/datasets//propertyGraphs/` (for + example a `propertyGraph`/`propertyGraphs` typo, or a + `…/entryGroups/@bigquery/entries/…` entry form). The error names the offending + URI and the expected form. This gate runs before any destination leg and for + every `--target`, so a malformed target writes **nothing** — not to BigQuery + and **not to Knowledge Catalog**; the push aborts with a non-zero exit and no + entries are created. *(static)* * **Every metric on a BigQuery Graph model resolves to exactly one entity** — otherwise it would be dropped from the BigQuery Graph. Set the metric's attach entity, or scope its expression to a single entity. *(static)* @@ -298,21 +307,23 @@ with no matching catalog entry) is left untouched — pull never deletes. > pulled document as a faithful copy of the catalog metadata, not of the authored > model, and keep the authored document as the source of truth. -> **Note — writer-side follow-ups (not inherent to pull).** Two of the reductions -> above are limits of what push currently *writes*, not of what pull can recover. -> They are recorded here as write-side follow-ups; the reader (pull) already -> returns everything the catalog holds. +> **Note — writer-side follow-up (not inherent to pull).** One reduction above is +> a limit of what push currently *writes*, not of what pull can recover. It is +> recorded here as a write-side follow-up; the reader (pull) already returns +> everything the catalog holds. > -> - **Relationship names.** The `schema-join` aspect does not store the authored -> relationship name, so pull recovers it from the link id — which is lowercased -> and hyphenated. Persisting the name in the aspect on write would let pull -> return it verbatim. -> - **Non-canonical deployment targets.** Push persists a target only when it is a -> canonical BigQuery Graph URL -> (`//bigquery.googleapis.com/projects/.../datasets/.../propertyGraphs/...`). -> Other forms — a misspelled path, or the `projects/.../entryGroups/@bigquery/` -> entry form — are dropped on write, so pull has nothing to recover. Widening or -> normalizing the writer's accepted forms would let them round-trip. +> - **Relationship names.** The `schema-join` aspect type's `metadataTemplate` has +> no field for the relationship name, so push cannot store it and pull recovers +> it from the link id — which is lowercased and hyphenated (the entry-link id +> format forbids the original casing/underscores). Returning the name verbatim +> requires adding a name field to the built-in `schema-join` aspect type in +> Knowledge Catalog (server-side), after which the client write/read is trivial; +> it is the same class of gap as the `semantics` field that gates +> `--emit-expressions`. +> +> (A non-canonical deployment target is **not** a pull gap: push rejects it at the +> validation gate before any leg runs, so it is never written — see +> [Validation](#validation).) ## Permissions