diff --git a/toolbox/mdcode/README.md b/toolbox/mdcode/README.md index e69065aa..bc9718f4 100644 --- a/toolbox/mdcode/README.md +++ b/toolbox/mdcode/README.md @@ -183,6 +183,14 @@ declared project (the `` in the init scope), not the ambient `gcloud` project; write a fully-qualified `project.dataset.table` source when the tables live elsewhere. +`kcmd push` also deploys the model to Knowledge Catalog (entries for the model, +its entities and metrics, and `schema-join` links for its relationships). Use +`--target bq|kc|all` (default `all`) to choose destinations, `--print` to dump +each destination's generated artifact, and `--force-remove` to delete models +left in the entry group that the push no longer includes. See +[docs/semantic-model.md](docs/semantic-model.md) for the full guide, including +how re-push reconciles removed entities, metrics, relationships, and models. + NOTE: The CLI uses `gcloud` to obtain authentication tokens, so ensure you are authenticated via `gcloud auth application-default login`. ### MCP Server diff --git a/toolbox/mdcode/demo/README.md b/toolbox/mdcode/demo/README.md index b4da0fcb..4ffea73d 100644 --- a/toolbox/mdcode/demo/README.md +++ b/toolbox/mdcode/demo/README.md @@ -183,7 +183,10 @@ bun push.ts **Cleanup** -* Deletes the Dataplex EntryGroup and the custom `okf` aspect type. +* Deletes the Dataplex EntryGroup. The custom `okf` aspect type is left in + place: it is scoped to the project and location rather than to this demo, so + other OKF bundles in the same project carry their signal layer on it. The + command to remove it manually is printed at the end. ```bash bun cleanup.ts diff --git a/toolbox/mdcode/demo/okf/cleanup.ts b/toolbox/mdcode/demo/okf/cleanup.ts index 9fae7085..07f21190 100644 --- a/toolbox/mdcode/demo/okf/cleanup.ts +++ b/toolbox/mdcode/demo/okf/cleanup.ts @@ -14,10 +14,11 @@ function dataplex(cmd: string, data: string|null=null) { dataplex(`entry-groups delete ${entryGroup}`); console.log(`Deleted entry group ${entryGroup}`); -try { - dataplex(`aspect-types delete okf`); - console.log('Deleted custom aspect type okf'); -} -catch { - // Might not exist, or still referenced -} +// The okf aspect type is scoped to the project and location, not to this demo's +// entry group, so any other OKF bundle in the project attaches its signal layer +// to the same type. Deleting it here would strip that signal from bundles this +// demo does not own. +console.log( + `Left aspect type ${project}.${location}.okf in place (shared across OKF bundles). ` + + `To remove it: gcloud dataplex aspect-types delete okf --project ${project} --location ${location}` +); diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md new file mode 100644 index 00000000..b111776f --- /dev/null +++ b/toolbox/mdcode/docs/semantic-model.md @@ -0,0 +1,347 @@ +# Deploying a semantic model + +A *semantic model* describes your entities (tables), the metrics computed over +them, and the relationships between them, authored as a single +[Apache Ossie](https://ossie.apache.org/) document. `kcmd push` deploys one +model to two destinations at once: + +* **BigQuery** — a queryable `CREATE OR REPLACE PROPERTY GRAPH` over the model's + tables, so the model can be traversed and its metrics computed in SQL. +* **Knowledge Catalog** — catalog entries and links that make the + model discoverable as metadata. + +Both are generated from the same source document — you never author them +separately, and a single `push` keeps them in sync. + +This guide covers authoring, deploying, pulling back, and updating a model. +For the Ossie document format itself, see +[ossie.apache.org](https://ossie.apache.org/). + +## Prerequisites + +`kcmd` authenticates with your `gcloud` credentials. Log in once before pushing: + +```bash +gcloud auth application-default login +``` + +You also need read/write access to whichever destinations you deploy to. + +## 1. Author a model + +Create the local layout. The scope is the Knowledge Catalog entry group the +model will be published to, written as `..`: + +```bash +kcmd init --semantic-model my-project.us-central1.my_model +``` + +`init` provisions that entry group (idempotent — an existing group is fine) and +creates its local directory. Author the model at +`catalog/EntryGroups//.yaml`: + +```yaml +version: "0.2.0.dev0" + +semantic_model: + - name: sales # must match the .yaml filename + # Required: the deployment target, in a GOOGLE custom extension. `data` is a + # JSON string whose deploymentTargets holds the target graph URI (for now, + # exactly one). + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": ["//bigquery.googleapis.com/projects/my-project/datasets/sales/propertyGraphs/sales_graph"]}' + datasets: # each dataset becomes an entity + - name: orders + source: my-project.sales.orders # the backing BigQuery table + primary_key: [o_orderkey] + fields: + - name: o_orderkey + expression: {dialects: [{dialect: BIGQUERY, expression: o_orderkey}]} + - name: o_totalprice + expression: {dialects: [{dialect: BIGQUERY, expression: o_totalprice}]} + metrics: + - name: total_revenue + expression: {dialects: [{dialect: BIGQUERY, expression: SUM(orders.o_totalprice)}]} +``` + +### Deployment targets (required) + +Every model must declare exactly one **deployment target** — the BigQuery +property graph it deploys to — in a `GOOGLE` custom extension, as shown above. A +target is a URI of the form: + +``` +//bigquery.googleapis.com/projects//datasets//propertyGraphs/ +``` + +The target's project and dataset are where the property graph is created; the +same URIs are also recorded on the model's Knowledge Catalog entry. A model with +no deployment target — or with more than one — is rejected at push time (see +[Validation](#validation)). + +### Table sources + +Each entity's `source` is its backing BigQuery table. A `source` written as +`dataset.table` (two parts) is qualified with the scope's project — the +`` from `init`. Write the full `project.dataset.table` when a table +lives in another project. + +Sources are not limited to native BigQuery tables. A name with more than +three parts — for example a four-part `catalog.database.schema.table` +reference — points at a table in a **federated REST catalog**, such as an +Apache Iceberg table exposed through BigLake. Write it exactly as BigQuery +resolves the name, and validation resolves it the same way the deploy does (see +[Validation](#validation)). + +## 2. Push + +```bash +kcmd push +``` + +With no flags this deploys to **every** destination, BigQuery first. The flags +below select destinations, dry-run, and preview: + +```bash +kcmd push --target bq # BigQuery only +kcmd push --validate-only # run all checks, write nothing +kcmd push --print # also print the generated DDL / entry plan +``` + +| Flag | Effect | +|------|--------| +| `--target ` | Which destination(s) to deploy to; accepts a comma-separated list (`bq,kc`). Default `all`. | +| `--validate-only` | Run every validation check and report pass/fail, but write nothing. | +| `--print` | Print each destination's generated artifact (BigQuery DDL, Knowledge Catalog entry plan). Combine with `--validate-only` to preview without deploying. | +| `--force-remove` | Delete models in the entry group that this push no longer includes (see [Updating and removing models](#updating-and-removing-models)). | +| `--emit-expressions` | Also write the SQL-expression fields (per-field `schema.semantics` and `semantic-metric.expression`) to Knowledge Catalog. Off by default: the published system-type templates do not carry them yet. Knowledge Catalog push only. | + +Destinations always deploy BigQuery-first and fail fast, so a rejected model +never half-deploys. + +### What gets created in BigQuery + +`push` executes a single `CREATE OR REPLACE PROPERTY GRAPH` per deployment +target, in the project and dataset the target names. Each part of your model +becomes one part of that graph: + +| Model element | BigQuery construct | Notes | +|---|---|---| +| Model | `PROPERTY GRAPH` | named by the deployment-target URI | +| Entity | `NODE TABLE` | backed by the entity's `source` table, keyed by its primary key | +| Relationship | `EDGE TABLE` | connects the two entities' node tables | +| Metric | `MEASURE` on a node table | must reduce to one aggregate over one entity, or it is skipped with a warning | + +`push` reads the target dataset's location (`bigquery.datasets.get`) so the +statement runs in the right region; without that permission it falls back to +BigQuery's own location inference and warns. Nothing runs under +`--validate-only`; add `--print` to see the DDL. + +Push to BigQuery is **lossy**: the graph captures the queryable structure — +node tables, edge tables, and measures — but not descriptive metadata +(descriptions, `ai_context`, synonyms, labels), and a metric that does not +reduce to a single MEASURE is dropped with a warning. It is a query surface, +not a copy of your model. + +### What gets created in Knowledge Catalog + +Each element of your model maps to one catalog resource. Every resource type +below is a built-in system type under `dataplex-types/global` — push references +them, it never creates them. + +| Model element | Catalog resource | Kind | Id | +|---|---|---|---| +| Model | `semantic-model` | entry — anchor / parent of the rest | `` | +| Entity | `semantic-entity` (+ built-in `schema` aspect) | entry | `.entities.` | +| Metric | `semantic-metric` | entry | `.metrics.` | +| Relationship | `schema-join` | entry link between the two entity entries | derived from the relationship name | + +An entity entry carries its columns in the `schema` aspect (name, data type, and +description per field); a `schema-join` link carries the relationship detail — the +paired columns and foreign-key direction — in its aspect. + +> **Note — push to Knowledge Catalog is lossy.** The catalog holds metadata, +> not a full copy of your model. It **stores** names, descriptions, data +> sources, field datatypes and roles, and 1:1 / 1:N relationships (as +> `schema-join` links). By default it does **not** store the SQL expressions: +> the published system-type templates do not yet carry a per-field `semantics` +> block or a `semantic-metric.expression` field, so the default push omits them +> (pass `--emit-expressions` to write the canonical GoogleSQL/ANSI expression +> once the templates gain the fields). It never stores entity keys, `ai_context`, +> field labels, the original vendor SQL (`importedExpression` — e.g. the MAQL or +> Snowflake form a metric was imported from), or M:N relationships. Those stay in +> your authored document (and, for the edges, in the BigQuery property graph); the +> vendor SQL and expressions are still used when generating BigQuery SQL. Keep +> your model document as the source of truth. + +## Validation + +`push` and `--validate-only` run the same checks, **before either destination is +touched**, so a model that cannot deploy fails fast instead of half-deploying: + +* **Exactly one deployment target per model, and it must be a valid BigQuery + Graph URI.** A model with no target — or with more than one — is rejected, and + so is a single target whose URI does not match + `//bigquery.googleapis.com/projects/

/datasets//propertyGraphs/` (for + example a `propertyGraph`/`propertyGraphs` typo, or a + `…/entryGroups/@bigquery/entries/…` entry form). The error names the offending + URI and the expected form. This gate runs before any destination leg and for + every `--target`, so a malformed target writes **nothing** — not to BigQuery + and **not to Knowledge Catalog**; the push aborts with a non-zero exit and no + entries are created. *(static)* +* **Every metric on a BigQuery Graph model resolves to exactly one entity** — + otherwise it would be dropped from the BigQuery Graph. Set the metric's attach + entity, or scope its expression to a single entity. *(static)* +* **Every entity's source table is reachable.** Each `source` is probed with a + dry-run query, so BigQuery resolves it exactly as the deploy will — a + three-part `project.dataset.table`, a four-part federated REST-catalog name + (e.g. an Iceberg table via BigLake), or a quoted identifier all work. A table + that does not exist or that you cannot access fails the push, naming the table + and the entity; a `source` that is a query (not a table) is skipped. *(live — + needs BigQuery access)* + +The live table check runs for **every** `--target`, because the same tables back +both destinations — so even a Knowledge-Catalog-only push confirms the model +could deploy to BigQuery. + +## Updating and removing models + +Your model document is the source of truth. To change what is deployed, edit +the document and run `kcmd push` again — you never edit the catalog or the +BigQuery Graph by hand. Re-running is safe: each push makes the destinations +match the document as it stands now. + +**When you edit an entity, metric, or relationship** — push overwrites the +existing one in place; you don't get duplicates. + +**When you delete an entity or metric** — remove it from the document and +push. Push deletes the leftover catalog entry for you (the summary shows +`removed N orphaned entries`). + +**When you rename or delete a relationship** — push removes the old +`schema-join` link after writing the new ones, so a rename never leaves the +two entities disconnected (the summary shows `unlinked N orphaned links`). +Relationships owned by other models that share the entry group are left +untouched. + +**When you remove a whole model** — deleting its document does *not* remove it +from the catalog on the next push. Instead, push stops and names the model the +catalog still has that you no longer push, so you can't wipe out a model by +accident or by pushing from the wrong directory. When you do mean to remove it, +run push again with `--force-remove` and its entries and links are deleted +first. + +Every push prints one line per destination summarizing what it did. For a +`--target all` push: + +``` +Deployed 1 BigQuery Graph(s). +Wrote 5 new and 2 updated Knowledge Catalog entries; removed 1 orphaned entry; linked 2 relationships; unlinked 1 orphaned link. +``` + +## Pull + +`kcmd pull` is the inverse of push's Knowledge Catalog leg: it reads the +`semantic-*` entries back from the catalog and reconstructs local model +documents at `catalog/EntryGroups//.yaml`. Use it to +recover a workspace from a catalog someone else deployed, or to see what the +catalog actually holds. + +```bash +kcmd pull +``` + +Pull reads only from Knowledge Catalog (never BigQuery). Its coordinates come +from the same scope you authored under (`..`). + +| Flag | Effect | +|------|--------| +| `--dry-run` | Reconstruct from the catalog and report what would be written, but write no files. | +| `--model ` | Pull a single model by name; other models in the entry group are left alone. | + +One entry group can hold **many models** — each `semantic-model` entry is a +separate anchor, and pull reconstructs one document per anchor. `--model` +narrows both the fetch and the write to a single anchor. + +Pull writes with the same last-write-wins policy as the core pull: a model that +already exists locally is overwritten in place, and a local-only document (one +with no matching catalog entry) is left untouched — pull never deletes. + +> **Note — pull reconstructs what the catalog holds, not your original file.** +> Pull can only recover what push wrote (see the note under [What gets created in +> Knowledge Catalog](#what-gets-created-in-knowledge-catalog)). What that means in +> practice: +> +> **Recovered exactly** — these come back as authored: +> - Model structure: the model, its entities, and each entity's fields. +> - Field data source and data type. +> - Metrics: name, data type, and attach entity. +> - 1:1 / 1:N relationships: endpoints, foreign-key direction, and join columns +> (from the `schema-join` links). +> - Deployment targets. +> +> **Recovered only if pushed with `--emit-expressions`** — the per-field +> `semantics` block (expressions and the dimension role) and the metric +> expression are omitted from the catalog by default (see the note above), so +> pull returns them only when the push that wrote them used `--emit-expressions`: +> - Field expressions and metric expressions (the canonical GoogleSQL/ANSI form). +> - A field's dimension role, which comes back as a bare `dimension: {}` marker, +> without its detail (`is_time`, and so on). A default push drops the marker +> entirely. +> +> **Recovered, but normalized** — the content survives, the form changes: +> - Relationship *names* come back lowercased/hyphenated (the catalog stores the +> name only in the link id, e.g. `Places Order` → `places-order`). +> - A metric authored with no data type comes back as an explicit `Decimal` +> (push must write a type, and defaults it to `NUMERIC`). +> +> **Not recovered** — push never wrote these, so pull cannot return them: +> - Entity keys / unique keys. +> - `ai_context`. +> - Field labels. +> - The original vendor SQL (`importedExpression`). +> - M:N relationships (the edge lives only in the BigQuery property graph). +> +> **So: a push followed by a pull does not return your original file.** Treat a +> pulled document as a faithful copy of the catalog metadata, not of the authored +> model, and keep the authored document as the source of truth. + +> **Note — writer-side follow-up (not inherent to pull).** One reduction above is +> a limit of what push currently *writes*, not of what pull can recover. It is +> recorded here as a write-side follow-up; the reader (pull) already returns +> everything the catalog holds. +> +> - **Relationship names.** The `schema-join` aspect type's `metadataTemplate` has +> no field for the relationship name, so push cannot store it and pull recovers +> it from the link id — which is lowercased and hyphenated (the entry-link id +> format forbids the original casing/underscores). Returning the name verbatim +> requires adding a name field to the built-in `schema-join` aspect type in +> Knowledge Catalog (server-side), after which the client write/read is trivial; +> it is the same class of gap as the `semantics` field that gates +> `--emit-expressions`. +> +> (A non-canonical deployment target is **not** a pull gap: push rejects it at the +> validation gate before any leg runs, so it is never written — see +> [Validation](#validation).) + +## Permissions + +`push` needs access to whichever destinations you deploy to. + +**BigQuery** — for `--target bq` or `all`, and for the validation pre-flight: + +* `bigquery.jobs.create` in the deployment-target project — to run the deploy's + `CREATE OR REPLACE PROPERTY GRAPH` and the validation dry-run query +* read access on each entity's source table, so the dry-run can resolve it +* `bigquery.datasets.get` on the target dataset (region detection; optional — + push degrades gracefully without it) + +**Knowledge Catalog / Dataplex** — for `--target kc` or `all`: + +* `dataplex.entryGroups.useSemanticModelAspect` on the destination entry group +* `dataplex.entryGroups.useSchemaJoinEntryLink` and + `dataplex.entryGroups.useSchemaJoinAspect` when the model has relationships + +`kcmd pull` needs read access to the same entry group instead — to list its +entries and fetch each `semantic-*` entry with its aspects. diff --git a/toolbox/mdcode/src/libts/gcp/api.ts b/toolbox/mdcode/src/libts/gcp/api.ts index 596bca6e..ffb2f43f 100644 --- a/toolbox/mdcode/src/libts/gcp/api.ts +++ b/toolbox/mdcode/src/libts/gcp/api.ts @@ -46,6 +46,12 @@ export class ApiClient { return this._requestRetry('PATCH', url, queryParams, body); } + async _delete(resourceName: string, + queryParams?: Record): Promise> { + const url = `${this._endpoint}/${this._pathPrefix}/${resourceName}`; + return this._requestRetry('DELETE', url, queryParams); + } + private async _requestRetry(method: string, url: string, queryParams?: Record, diff --git a/toolbox/mdcode/src/libts/gcp/bigquery.ts b/toolbox/mdcode/src/libts/gcp/bigquery.ts index e684a922..e8670d0e 100644 --- a/toolbox/mdcode/src/libts/gcp/bigquery.ts +++ b/toolbox/mdcode/src/libts/gcp/bigquery.ts @@ -60,6 +60,17 @@ export class BigQueryClient extends api.ApiClient { return await this._get(name, params); } + // Fetches a single table's metadata. Used as a cheap existence/access probe + // before deploy: a 200 means the table is reachable, a 404 that it does not + // exist, a 403 that the caller cannot see it. `selectedFields` trims the + // response to the reference alone, since only the status is consulted. + async getTable(project: string, dataset: string, table: string): Promise> { + const name = `projects/${project}/datasets/${dataset}/tables/${table}`; + const params: Record = { selectedFields: 'tableReference' }; + + return await this._get(name, params); + } + async *listTables(project: string, dataset: string): AsyncGenerator
{ const name = `projects/${project}/datasets/${dataset}/tables`; @@ -87,13 +98,18 @@ export class BigQueryClient extends api.ApiClient { // Executes a SQL statement (including DDL) synchronously via jobs.query. // When `location` is set it pins the job's processing location so it agrees // with getQueryResults/getJob; otherwise BigQuery infers it from the - // referenced tables. - async query(project: string, sql: string, location?: string): Promise> { + // referenced tables. With `dryRun` the statement is validated -- name + // resolution, table existence and access -- but not executed, so it serves as + // a cheap pre-flight probe over any reference form BigQuery can resolve. + async query(project: string, sql: string, location?: string, dryRun?: boolean): Promise> { const name = `projects/${project}/queries`; const body: Record = { query: sql, useLegacySql: false }; if (location) { body.location = location; } + if (dryRun) { + body.dryRun = true; + } return await this._post(name, body); } diff --git a/toolbox/mdcode/src/libts/gcp/dataplex.ts b/toolbox/mdcode/src/libts/gcp/dataplex.ts index b86a048c..8aa92daa 100644 --- a/toolbox/mdcode/src/libts/gcp/dataplex.ts +++ b/toolbox/mdcode/src/libts/gcp/dataplex.ts @@ -49,11 +49,38 @@ export interface Entry { aspects?: Record; } +export interface EntryReference { + // Full resource name of the referenced entry. + name: string; + // Path within the entry that is referenced; empty means the entry itself. + path?: string; + // SOURCE / TARGET for a directed link; UNSPECIFIED for an undirected one. + type: 'UNSPECIFIED' | 'SOURCE' | 'TARGET'; +} + +export interface EntryLink { + // Server-assigned (output only); set by the emitter to the destination name so + // the publisher can address it for an in-place update. + name?: string; + entryLinkType: string; + // Exactly two references. + entryReferences: EntryReference[]; + // At most one Dataplex-owned aspect (e.g. schema-join), keyed as + // `project.location.aspectType`. + aspects?: Record; +} + interface EntryList { entries: Entry[]; nextPageToken?: string; } +interface EntryLinkList { + // Absent when the referenced entry has no links of the requested type(s). + entryLinks?: EntryLink[]; + nextPageToken?: string; +} + export class CatalogClient extends api.ApiClient { @@ -193,6 +220,12 @@ export class CatalogClient extends api.ApiClient { return res; } + async deleteEntry(project: string, location: string, entryGroup: string, + entry: string): Promise> { + const name = `${catalogContainer(project, location, entryGroup)}/entries/${entry}`; + return await this._delete(name); + } + async createEntryGroup(project: string, location: string, entryGroupId: string, entryGroup?: EntryGroup): Promise> { const parent = catalogContainer(project, location); @@ -205,6 +238,74 @@ export class CatalogClient extends api.ApiClient { return res; } + async createEntryLink(project: string, location: string, entryGroup: string, + entryLinkId: string, + entryLink: EntryLink): Promise> { + const parent = catalogContainer(project, location, entryGroup); + const resourceName = `${parent}/entryLinks`; + + const params: Record = { entryLinkId }; + + return await this._post(resourceName, entryLink, params); + } + + // Patches an existing entry link. UpdateEntryLink has no update mask: the + // aspects present in the request body are the ones written, narrowed to + // `aspectKeys` (each `project.location.aspectType`) when given. The entry + // references and link type are immutable server-side. + async updateEntryLink(entryLink: EntryLink, + aspectKeys?: string[]): Promise> { + const params: Record = {}; + if (aspectKeys && aspectKeys.length) { + params.aspectKeys = aspectKeys; + } + return await this._patch(entryLink.name!, entryLink, params); + } + + async deleteEntryLink(project: string, location: string, entryGroup: string, + entryLinkId: string): Promise> { + const name = `${ + catalogContainer(project, location, entryGroup)}/entryLinks/${ + entryLinkId}`; + return await this._delete(name); + } + + // Returns every entry link that references `entry` (its full resource name), + // draining all pages. There is no list-entry-links collection API; the server + // only exposes links per referenced entry, via the location-scoped + // :lookupEntryLinks custom verb (mirroring :lookupEntry). `entryLinkTypes` + // optionally filters to specific link types (server caps it at 10); `entryMode` + // filters by the entry's role in the link (SOURCE/TARGET). The server caps a + // page at 10 links, so this follows nextPageToken until exhausted and returns + // the flat list. A non-200 on any page aborts and is returned as-is. + async lookupEntryLinks( + project: string, location: string, + opts: {entry: string; entryLinkTypes?: string[]; + entryMode?: 'UNSPECIFIED' | 'SOURCE' | 'TARGET'}): + Promise> { + const container = `${catalogContainer(project, location)}:lookupEntryLinks`; + const links: EntryLink[] = []; + let pageToken: string | undefined = undefined; + do { + const params: Record = { + entry: opts.entry, + entryLinkTypes: opts.entryLinkTypes, + entryMode: opts.entryMode, + pageSize: 10, + pageToken, + }; + const res = await this._get(container, params); + if (res.status != 200) { + return { status: res.status, message: res.message }; + } + for (const link of res.result?.entryLinks ?? []) { + links.push(link); + } + pageToken = res.result?.nextPageToken; + } while (pageToken); + return { status: 200, result: links }; + } + } diff --git a/toolbox/mdcode/src/libts/layouts/semantic-model.ts b/toolbox/mdcode/src/libts/layouts/semantic-model.ts index 214fe930..42730af7 100644 --- a/toolbox/mdcode/src/libts/layouts/semantic-model.ts +++ b/toolbox/mdcode/src/libts/layouts/semantic-model.ts @@ -91,6 +91,36 @@ export class SemanticModelLayout implements CatalogLayout { return docs; } + // True when a model document with this handle already exists on disk. `pull` + // uses it to report which files it would overwrite vs. create. + hasModel(name: string): boolean { + return fs.existsSync(this.modelPath(name)); + } + + // The absolute path a model document with this handle maps to: + // `/EntryGroups//.yaml`. Path separators in the + // model name are replaced so a name still yields a single flat file. Requires + // the layout to be scoped to an entry group (the semantic-model source always + // is). + modelPath(name: string): string { + if (!this._entryGroup) { + throw new Error( + 'SemanticModel layout has no entry group; cannot resolve a model path.'); + } + const file = `${name.replace(/[/\\]/g, '_')}.yaml`; + return path.join(this._catalogPath, 'EntryGroups', this._entryGroup, file); + } + + // Writes a model's serialized document to its path, creating the EntryGroup + // directory if needed, and indexes it so a later modelDocuments() sees it. + // This is the sink `pull` writes reconstructed models to. + writeModelDocument(name: string, text: string): void { + const localPath = this.modelPath(name); + fs.mkdirSync(path.dirname(localPath), {recursive: true}); + fs.writeFileSync(localPath, text); + this._index.set(name, localPath); + } + // The Knowledge Catalog entry-level members are not applicable to this // push-only layout; the model is authored as a single Ossie document, not as // per-entry Knowledge Catalog files. These are wired when KC-resource emit diff --git a/toolbox/mdcode/src/libts/semantic/deploy.ts b/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts similarity index 59% rename from toolbox/mdcode/src/libts/semantic/deploy.ts rename to toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts index 5af49505..a5d35679 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_bigquery.ts @@ -1,12 +1,11 @@ // Deploys a semantic model's BigQuery Graph. // // This is the BigQuery leg of `kcmd push` for the semantic-model scope: it -// parses each authored Ossie document into the semantic IR (loader), lowers it -// to `CREATE OR REPLACE PROPERTY GRAPH` DDL (generator), and executes that DDL -// against the BigQuery project named by the model's GOOGLE deployment target. -// -// The Knowledge Catalog resource emit (entries + entryLinks) for the semantic -// model is a follow-on; `push` currently deploys only the BigQuery Graph. +// consumes models already parsed into the semantic IR (see loadSemanticModels, +// shared with the Knowledge Catalog leg so a `--target all` push parses each +// document once), lowers each to `CREATE OR REPLACE PROPERTY GRAPH` DDL +// (generator), and executes that DDL against the BigQuery project named by the +// model's GOOGLE deployment target. // // This is a library module: it emits no console output. The generated DDL and // any loader/generator warnings are returned in `DeployResult` for the CLI @@ -19,7 +18,7 @@ import * as context from '../gcp/context'; import {generatePropertyGraph} from './bigquery'; import {SemanticModel} from './ir'; -import {loadModels} from './loader'; +import {LoadedModel} from './loader'; // Our own vendor tag in the Ossie `custom_extensions` list. @@ -30,22 +29,12 @@ const GOOGLE_VENDOR = 'GOOGLE'; // The capture groups are restricted to valid BigQuery identifier characters: // the components are interpolated into DDL unescaped (see qualifyGraph), so a // permissive `[^/]+` would let backticks or semicolons through. A URI that -// carries the BigQuery Graph prefix but fails this full match is collected as -// `malformed` (see bigQueryGraphTargets) and named in the deploy error, rather -// than being silently skipped and later misreported as "no target declared". +// fails this full match is collected as `malformed` (see +// googleDeploymentTargets) and named in the deploy/validation error, rather than +// being silently skipped and later misreported as "no target declared". const BQ_GRAPH_TARGET = /^\/\/bigquery\.googleapis\.com\/projects\/([A-Za-z0-9_-]+)\/datasets\/([A-Za-z0-9_-]+)\/propertyGraphs\/([A-Za-z0-9_-]+)$/; -// A deployment target that "looks like" a BigQuery Graph URI but fails the -// strict match above: it carries the bigquery.googleapis host (even under a -// typo'd scheme such as `https://`, or a truncated host missing `.com`) or the -// graph-specific `/propertyGraph(s)/` path segment. Such a URI is reported as -// malformed (see bigQueryGraphTargets) rather than silently dropped, so a -// host/scheme/segment/identifier typo surfaces instead of being misreported as -// "no target declared". An unrelated destination (e.g. a Dataplex URI) matches -// neither this hint nor the strict form and is left alone. -const BQ_GRAPH_TARGET_HINT = /bigquery\.googleapis|\/propertyGraphs?\//; - export interface BigQueryGraphTarget { project: string; @@ -77,14 +66,20 @@ export interface DeployResult { } -// Extracts the BigQuery Graph deployment targets from a model's GOOGLE -// custom extension. The extension `data` is an opaque, vendor-serialized JSON -// string (the loader keeps it verbatim); we own its `deploymentTargets` shape. -export function bigQueryGraphTargets(model: SemanticModel): - {targets: BigQueryGraphTarget[]; malformed: string[]} { +// Reads a model's GOOGLE custom_extension(s) in a single pass and returns the +// deployment-target facts every caller derives from them: every declared +// deploymentTarget URI (`uris`), the subset that parse as BigQuery Graph targets +// (`targets`), and the URIs that did not parse as a BigQuery Graph target +// (`malformed`, kept so a caller can name a typo instead of silently skipping). +// The extension `data` is an opaque, vendor-serialized JSON string (the loader +// keeps it verbatim); we own its `deploymentTargets` shape. Throws on malformed +// extension JSON. bigQueryGraphTargets and deploymentTargetUris are thin views +// over this, and validation calls it directly so a push parses each extension +// once rather than once per reader. +export function googleDeploymentTargets(model: SemanticModel): + {uris: string[]; targets: BigQueryGraphTarget[]; malformed: string[]} { + const uris: string[] = []; const targets: BigQueryGraphTarget[] = []; - // BigQuery-prefixed URIs that failed the strict match (typo, bad identifier - // char). Kept so the caller can name them instead of silently skipping. const malformed: string[] = []; for (const ext of model.customExtensions ?? []) { @@ -100,31 +95,52 @@ export function bigQueryGraphTargets(model: SemanticModel): model.name}': GOOGLE custom_extension 'data' is not valid JSON.`); } - const uris = data?.deploymentTargets; - if (!Array.isArray(uris)) { + const list = data?.deploymentTargets; + if (!Array.isArray(list)) { continue; } - for (const uri of uris) { + for (const uri of list) { if (typeof uri !== 'string') { continue; } + uris.push(uri); const m = uri.match(BQ_GRAPH_TARGET); if (m) { targets.push({project: m[1], dataset: m[2], graphName: m[3], uri}); - } else if (BQ_GRAPH_TARGET_HINT.test(uri)) { - // Looks like a BigQuery Graph target but doesn't parse (host, scheme, - // path-segment, or identifier-char typo); a plain Dataplex/other URI is - // not our concern and is left alone. + } else { + // We only support BigQuery Graph deployment targets, so any URI that + // does not parse as one -- a host/scheme/segment/identifier typo, or an + // unrelated destination -- is collected as malformed and rejected + // (validate.ts, and the deploy leg) rather than silently ignored. malformed.push(uri); } } } + return {uris, targets, malformed}; +} + + +// The BigQuery Graph deployment targets a model declares in its GOOGLE custom +// extension, plus any BigQuery-prefixed URIs that failed to parse. A view over +// googleDeploymentTargets. +export function bigQueryGraphTargets(model: SemanticModel): + {targets: BigQueryGraphTarget[]; malformed: string[]} { + const {targets, malformed} = googleDeploymentTargets(model); return {targets, malformed}; } +// Every deploymentTarget URI a model declares, regardless of whether each parses +// as a BigQuery Graph target. Validation uses this to require that a model +// declares at least one deployment target; bigQueryGraphTargets narrows to the +// BigQuery graphs. Throws on malformed extension JSON. +export function deploymentTargetUris(model: SemanticModel): string[] { + return googleDeploymentTargets(model).uris; +} + + // Upper bound on getQueryResults polls before we give up on a job that never // reports completion (each poll long-polls the server for up to 10s). const MAX_QUERY_POLLS = 30; @@ -242,20 +258,14 @@ async function datasetLocation( } -// Deploys the BigQuery Graph for each authored model document. Emits no console -// output; the generated DDL and any warnings are returned for the caller to -// print. -// -// `defaultProject` qualifies a dataset `source` that omits its project. The -// caller should pass the scope's declared project (a deterministic, -// user-authored value) rather than relying on the ambient gcloud project, which -// can silently drift away from where the model's tables live. It cannot default -// to each deployment target's own project: a document is parsed once, before -// its targets are known, and may declare several graphs across projects. +// Deploys the BigQuery Graph for each pre-loaded model. Emits no console output; +// the generated DDL and any warnings are returned for the caller to print. The +// models are parsed once by the caller (loadSemanticModels) and shared with the +// Knowledge Catalog leg; each carries its originating document name for +// diagnostics. export async function deployBigQuery( - docs: {name: string; text: string}[], ctx: context.ApiContext, - options: DeployOptions = {}, - defaultProject?: string): Promise { + models: LoadedModel[], ctx: context.ApiContext, + options: DeployOptions = {}): Promise { const bigQuery = new BigQueryClient(ctx); const ddl: string[] = []; const warnings: string[] = []; @@ -282,93 +292,76 @@ export async function deployBigQuery( return loc; }; - for (const doc of docs) { - // A document that fails to parse (or violates the model schema) is an - // authoring error. Report it against the specific document rather than - // letting the loader's exception propagate as an uncaught stack trace, and - // name the document so a bad file in a multi-document push is identifiable. - let loaded; + for (const {document, model} of models) { + modelsSeen++; + let targets: BigQueryGraphTarget[]; + let malformed: string[]; try { - loaded = - loadModels(doc.text, {defaultProject: defaultProject ?? ctx.project}); + ({targets, malformed} = bigQueryGraphTargets(model)); } catch (err: any) { - return fail(`Model document '${doc.name}': ${err.message || err}`); + return fail( + `Model '${model.name}' (${document}): ${err.message || err}`); + } + if (!targets.length) { + // A malformed-but-present target must not be reported as "none + // declared" -- that sends the author hunting for an extension they + // already wrote. Name the URIs that failed to parse. + if (malformed.length) { + return fail( + `Model '${model.name}' (${document}) declares BigQuery Graph ` + + `deploymentTarget(s) that could not be parsed: ` + + `${malformed.join(', ')}. Expected //bigquery.googleapis.com/` + + `projects/

/datasets//propertyGraphs/.`); + } + return fail( + `Model '${model.name}' (${document}) declares no BigQuery Graph ` + + `deploymentTarget in a GOOGLE custom_extension; nothing to deploy.`); } - for (const w of loaded.warnings) { - warnings.push(`[${doc.name}] ${w}`); + // Malformed targets alongside valid ones: surface them so a typo'd URI is + // not silently dropped when the deploy otherwise succeeds. + for (const bad of malformed) { + warnings.push(`[${ + model.name}] ignoring unparseable BigQuery Graph target: ${bad}`); } - for (const model of loaded.models) { - modelsSeen++; - let targets: BigQueryGraphTarget[]; - let malformed: string[]; + for (const target of targets) { + const gen = generatePropertyGraph(model, { + project: target.project, + dataset: target.dataset, + graphName: target.graphName, + }); + for (const w of gen.warnings) { + warnings.push(`[${model.name} -> ${target.graphName}] ${w}`); + } + ddl.push(`-- ${target.uri}\n${gen.ddl}`); + + if (options.validateOnly) { + continue; + } + + let location: string|undefined; try { - ({targets, malformed} = bigQueryGraphTargets(model)); + location = await locationFor(target); } catch (err: any) { return fail( - `Model '${model.name}' (${doc.name}): ${err.message || err}`); + `Failed to deploy '${target.graphName}': ${err.message || err}`); } - if (!targets.length) { - // A malformed-but-present target must not be reported as "none - // declared" -- that sends the author hunting for an extension they - // already wrote. Name the URIs that failed to parse. - if (malformed.length) { - return fail( - `Model '${model.name}' (${doc.name}) declares BigQuery Graph ` + - `deploymentTarget(s) that could not be parsed: ` + - `${malformed.join(', ')}. Expected //bigquery.googleapis.com/` + - `projects/

/datasets//propertyGraphs/.`); - } - return fail( - `Model '${model.name}' (${doc.name}) declares no BigQuery Graph ` + - `deploymentTarget in a GOOGLE custom_extension; nothing to deploy.`); - } - // Malformed targets alongside valid ones: surface them so a typo'd URI is - // not silently dropped when the deploy otherwise succeeds. - for (const bad of malformed) { - warnings.push(`[${ - model.name}] ignoring unparseable BigQuery Graph target: ${bad}`); + const started = await bigQuery.query(target.project, gen.ddl, location); + const outcome = await awaitQueryDone( + bigQuery, target.project, started, maxPolls, backoffMs); + if (!outcome.ok) { + // These are CREATE OR REPLACE statements executed one at a time, so + // any graphs already deployed in this run have mutated production and + // are not rolled back. Surface that alongside the failing target. + const partial = deployed ? + ` (${deployed} graph(s) already deployed in this run; ` + + `CREATE OR REPLACE changes are not rolled back)` : + ''; + return fail(`Failed to deploy '${target.graphName}': ${ + outcome.error}${partial}`); } - for (const target of targets) { - const gen = generatePropertyGraph(model, { - project: target.project, - dataset: target.dataset, - graphName: target.graphName, - }); - for (const w of gen.warnings) { - warnings.push(`[${model.name} -> ${target.graphName}] ${w}`); - } - ddl.push(`-- ${target.uri}\n${gen.ddl}`); - - if (options.validateOnly) { - continue; - } - - let location: string|undefined; - try { - location = await locationFor(target); - } catch (err: any) { - return fail( - `Failed to deploy '${target.graphName}': ${err.message || err}`); - } - const started = await bigQuery.query(target.project, gen.ddl, location); - const outcome = await awaitQueryDone( - bigQuery, target.project, started, maxPolls, backoffMs); - if (!outcome.ok) { - // These are CREATE OR REPLACE statements executed one at a time, so - // any graphs already deployed in this run have mutated production and - // are not rolled back. Surface that alongside the failing target. - const partial = deployed ? - ` (${deployed} graph(s) already deployed in this run; ` + - `CREATE OR REPLACE changes are not rolled back)` : - ''; - return fail(`Failed to deploy '${target.graphName}': ${ - outcome.error}${partial}`); - } - - deployed++; - } + deployed++; } } diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts new file mode 100644 index 00000000..5676ec86 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -0,0 +1,717 @@ +// Deploys a semantic model's Knowledge Catalog resources. +// +// This is the Knowledge Catalog leg of `kcmd push` for the semantic-model +// scope, the counterpart to `deploy_bigquery.ts`. It consumes models already +// parsed into the semantic IR (see loadSemanticModels, shared with the BigQuery +// leg so a `--target all` push parses each document once), maps each to catalog +// Entries + Aspects (the pure emitter in knowledge_catalog.ts), and writes them +// through the Knowledge Catalog client. +// +// Types: the `semantic-model`/`semantic-entity`/`semantic-metric` entry and +// aspect types — and the built-in `schema` aspect — are built-in system types +// in `dataplex-types/global`. Push does NOT provision any type, nor the entry +// group (that is created at `init`); it only writes entries. The caller needs +// `dataplex.entryGroups.useSemanticModelAspect` on the destination entry group. +// +// Publish sequence (mirrors the BigQuery leg's structure): +// * Create each model's entries in array order: the semantic-model anchor +// first (it is the parentEntry of every entity/metric entry), then the +// children concurrently. No entry-group or type creation -- the entry +// group is provisioned at `init` and the system types are built-in. +// * A re-push upserts: an entry that already exists is updated in place. +// * Reconcile deletions: an entity or metric removed from a still-present +// model leaves an orphaned entry under its anchor; after writing, delete +// any entry this push owns (by entry-id prefix) that was not re-emitted. +// * Relationship edges are published as schema-join entry links between the +// two entity entries, written after that model's entries (both endpoints +// must exist first). A re-push upserts the link's aspect; a many-to-many +// (association) edge is not published yet (the emitter warns and skips it). +// The caller additionally needs `dataplex.entryGroups.useSchemaJoinEntryLink` +// and `useSchemaJoinAspect` on the destination entry group. +// +// This is a library module: it emits no console output. Warnings and the +// dry-run plan are returned in `KcDeployResult` for the CLI (commands.ts) to +// print. +// + +import {ApiResult} from '../gcp/api'; +import * as context from '../gcp/context'; +import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; + +import {generateCatalogResources, KcResources} from './knowledge_catalog'; +import {LoadedModel} from './loader'; + + +export interface KcDeployOptions { + // Project that owns the destination entry group. Flag overrides are applied + // over the catalog.yaml scope defaults before this is set. + project: string; + // Location (region) of the destination entry group, e.g. `global` or `us`. + location: string; + // Id of the destination entry group (provisioned at `init`, not by push). + entryGroup: string; + // Project the built-in `semantic-*` / `schema` system types are referenced + // from. Defaults to `dataplex-types`, where these types live; overridable + // only so hermetic tests can point at a fixture types project. The emitted + // entries are otherwise unchanged. + systemTypeProject?: string; + // Location the built-in system types are referenced from. Default `global`. + systemTypeLocation?: string; + // Emit the SQL-expression fields not yet in the published system-type + // templates (per-field `schema.semantics` and `semantic-metric.expression`). + // Off by default so the push matches the live types; see + // KcGenerateOptions.emitExpressions. + emitExpressions?: boolean; + // Compile and report only; never writes to the catalog (a dry run). + validateOnly?: boolean; + // Delete models already in the entry group that this push does not re-emit -- + // a removed or renamed model's entries and links. Without it, an unrecognized + // model in the group is a hard error rather than a silent orphan. + forceRemove?: boolean; + // How many times to try entries.create before giving up: a just-created entry + // group can briefly 404, and the create path retries that window. Overridable + // so tests exercise the retry without burning wall-clock. Default + // ENTRY_CREATE_TRIES. + entryCreateTries?: number; + // Delay between entries.create retries, in ms. Default ENTRY_CREATE_RETRY_MS. + entryCreateRetryMs?: number; +} + +export interface KcDeployResult { + // Whether the push (or --validate-only run) completed without error. + success: boolean; + // On failure, a human-readable reason; unset on success. + details?: string; + // Loader and emitter warnings collected across all documents (e.g. a skipped + // many-to-many relationship, or a metric that could not be lowered). + warnings: string[]; + // New entries created in the entry group (0 for validateOnly). + created: number; + // Existing entries updated in place by an idempotent re-push (0 for + // validateOnly). + updated: number; + // Entries deleted: those orphaned by removed entities/metrics, plus every + // entry of a --force-remove'd model (0 for validateOnly). + deleted: number; + // Relationship (schema-join) entry links written -- created or upserted (0 for + // validateOnly). + linked: number; + // Orphaned schema-join links deleted -- from relationships dropped or renamed + // on a still-present model, and from force-removed models (0 for validateOnly). + unlinked: number; + // A human-readable plan of what would be written: the sole output of a + // validateOnly run, and also returned (for --print) on a real push. + plan: string[]; +} + + +// Running tallies threaded through the write phase so a partial failure still +// reports what had been done. Field names match KcDeployResult so this spreads +// straight into the result. +interface Counts { + created: number; + updated: number; + deleted: number; + linked: number; + unlinked: number; +} + +// One authored model paired with the catalog resources it emitted. +type EmittedModel = {model: string; resources: KcResources}; + + +// entries.create propagation retry: a just-created entry group can briefly 404. +const ENTRY_CREATE_TRIES = 3; +const ENTRY_CREATE_RETRY_MS = 3000; + +function sleep(ms: number): Promise { + return new Promise(r => setTimeout(r, ms)); +} + + +// Deploys every authored model's Knowledge Catalog resources. The body is the +// sequence of phases, each a helper below: +// emitModels -- turn the model into catalog resources (pure) +// buildPlan -- the dry-run plan (and stop here for --validate-only) +// listEntryGroup -- snapshot the group once, before any write +// guardForeignModels -- refuse (or --force-remove) models no longer pushed +// writeModels -- create/upsert each model's entries and links +// reconcileDeletions -- delete entries orphaned by removed entities/metrics +// Emits no console output; warnings and the plan are returned in KcDeployResult +// for the caller (commands.ts) to print. +export async function deployKnowledgeCatalog( + models: LoadedModel[], ctx: context.ApiContext, + opts: KcDeployOptions): Promise { + const warnings: string[] = []; + const counts: Counts = + {created: 0, updated: 0, deleted: 0, linked: 0, unlinked: 0}; + let plan: string[] = []; + // Builds the return value from the running state; details is set only on a + // failure, and counts/plan reflect whatever had been done when called. + const result = (success: boolean, details?: string): KcDeployResult => + ({success, warnings, ...counts, plan, ...(details ? {details} : {})}); + + // Emit every model to catalog resources up front (pure -- no network). + const emit = emitModels(models, opts); + warnings.push(...emit.warnings); + if (emit.error) return result(false, emit.error); + const emitted = emit.emitted; + + // Exactly one model per entry group is supported for now. An empty workspace + // is a clean no-op under --validate-only and a configuration error on a real + // push; more than one model in a single push is always rejected (which also + // means two models can never race for the same entry id). + if (emitted.length !== 1) { + if (!emitted.length) { + if (opts.validateOnly) { + warnings.push( + 'No semantic model documents found; nothing to validate.'); + return result(true); + } + return result( + false, 'No semantic model documents found; nothing to deploy.'); + } + return result( + false, + `entry group '${opts.entryGroup}' would receive ${ + emitted.length} models, but only one model per entry group is ` + + `supported; split them into separate entry groups.`); + } + + // The plan is built for every push (printed with --print) and is the only + // output of a --validate-only run, which writes nothing. + plan = buildPlan(emitted, opts); + if (opts.validateOnly) return result(true); + + // From here on we write. The entry group is provisioned at `init`, not here; + // a missing group surfaces as a clear entry-creation error, and createEntries + // rides out the brief post-init propagation window. + const cat = new CatalogClient(ctx); + + // Snapshot the entry group once, before any write: the same listing feeds the + // foreign-model guard, link reconciliation, and deletion reconciliation. A + // re-emitted entry is never a deletion candidate, so a pre-write snapshot is + // correct for all three. + const listing = await listEntryGroup(cat, opts); + if (listing.error) return result(false, listing.error); + const existing = listing.entries; + + // Whole-model lifecycle: refuse (or, with --force-remove, delete) any model + // the group still holds that this push no longer includes. + const guard = await guardForeignModels(cat, opts, emitted, existing, counts); + if (guard.error) return result(false, guard.error); + + // Write each model's entries and relationship links. + const written = await writeModels(cat, opts, emitted, existing, counts); + if (written.error) return result(false, written.error); + + // Finally, delete entries orphaned by entities/metrics removed from a + // still-present model since its last push. + const recon = await reconcileDeletions(cat, opts, emitted, existing); + if (recon.error) return result(false, recon.error); + counts.deleted += recon.deleted; + + return result(true); +} + + +// Turns every authored model into its catalog resources. Pure -- no network I/O +// -- so the dry-run plan and any generation warnings are produced even when a +// later write fails. A malformed GOOGLE custom_extension (reached via the +// semantic-model aspect) throws; report it against its document, as the BigQuery +// leg does, rather than letting it escape as an uncaught stack trace. +function emitModels(models: LoadedModel[], opts: KcDeployOptions): + {emitted: EmittedModel[]; warnings: string[]; error?: string} { + const emitted: EmittedModel[] = []; + const warnings: string[] = []; + for (const {document, model} of models) { + let resources: KcResources; + try { + resources = generateCatalogResources(model, { + project: opts.project, + location: opts.location, + entryGroup: opts.entryGroup, + systemTypeProject: opts.systemTypeProject, + systemTypeLocation: opts.systemTypeLocation, + emitExpressions: opts.emitExpressions, + }); + } catch (err: any) { + return { + emitted, warnings, + error: `Model '${model.name}' (${document}): ${err.message || err}`, + }; + } + for (const w of resources.warnings) warnings.push(`[${model.name}] ${w}`); + emitted.push({model: model.name, resources}); + } + return {emitted, warnings}; +} + + +// The full dry-run plan across all models (one block per model; see planSummary). +function buildPlan(emitted: EmittedModel[], opts: KcDeployOptions): string[] { + const plan: string[] = []; + for (const {model, resources} of emitted) { + plan.push(...planSummary(model, resources, opts)); + } + return plan; +} + + +// Whole-model lifecycle guard. A `semantic-model` anchor already in the group +// whose id this push does not re-emit belongs to a model whose document is gone +// (removed or renamed). Refuse the push and name them -- unless --force-remove, +// which deletes each such model's links and entries first (tallied into counts). +async function guardForeignModels( + cat: CatalogClient, opts: KcDeployOptions, emitted: EmittedModel[], + existing: Entry[], counts: Counts): Promise<{error?: string}> { + const pushedAnchors = + new Set(emitted.map(e => idOf(e.resources.entries[0].name))); + const foreignAnchors = existing + .filter(e => (e.entryType ?? '').endsWith('/semantic-model')) + .map(e => idOf(e.name)) + .filter(id => !pushedAnchors.has(id)); + if (!foreignAnchors.length) return {}; + if (!opts.forceRemove) { + return { + error: + `entry group '${opts.entryGroup}' already contains model(s) this ` + + `push does not include: ${foreignAnchors.join(', ')}. Re-run with ` + + `--force-remove to delete them, or add their documents to this push.`, + }; + } + const removed = await removeForeignModels(cat, opts, existing, foreignAnchors); + if (removed.error) return {error: removed.error}; + counts.deleted += removed.deleted; + counts.unlinked += removed.unlinked; + return {}; +} + + +// Writes every model's entries and relationship links, in model order. For each +// model: create/upsert its entries (anchor first), write its schema-join links +// (both endpoints must exist first), then drop any link it owns but no longer +// emits (a dropped or renamed relationship). Progress accumulates into counts, so +// a mid-way failure still reports what had been written. +async function writeModels( + cat: CatalogClient, opts: KcDeployOptions, emitted: EmittedModel[], + existing: Entry[], counts: Counts): Promise<{error?: string}> { + for (const {model, resources} of emitted) { + const entries = await createEntries(cat, opts, resources.entries); + if (entries.error) return {error: `Model '${model}': ${entries.error}`}; + counts.created += entries.created; + counts.updated += entries.updated; + + // Links reference this model's entity entries, so they follow the entries + // above (both endpoints must exist first). + const links = await createEntryLinks(cat, opts, resources.entryLinks); + if (links.error) return {error: `Model '${model}': ${links.error}`}; + counts.linked += links.linked; + + // Then drop any schema-join link this model owns but no longer emits (a + // relationship dropped or renamed), after its current links are written so a + // rename never leaves the pair with no link between them. + const relLinks = await reconcileLinks(cat, opts, resources, existing); + if (relLinks.error) return {error: `Model '${model}': ${relLinks.error}`}; + counts.unlinked += relLinks.unlinked; + } + return {}; +} + + +interface ReconcileOutcome { + deleted: number; + error?: string; +} + +// Removes the catalog entries left behind when you delete an entity or metric +// from a model and push again -- the entries the model no longer emits. Only +// entries this push OWNS are ever touched: an entry whose id is a pushed model's +// anchor, or that lives under that anchor's `.entities.` / +// `.metrics.` namespace. Entries belonging to other models that share the +// entry group are left alone, and an anchor (always re-emitted) is never deleted +// here. +// +// Deleting a whole model is handled by the --force-remove guard, and orphaned +// relationship links by reconcileLinks. This step reads the pre-write snapshot +// `existing`, so it issues no list call of its own (a re-emitted entry is never a +// deletion candidate, so the snapshot stays correct). +function reconcileDeletions( + cat: CatalogClient, opts: KcDeployOptions, emitted: EmittedModel[], + existing: Entry[]): Promise { + const emittedIds = new Set(); + const anchorIds = new Set(); + const childPrefixes: string[] = []; + for (const {resources} of emitted) { + for (const e of resources.entries) emittedIds.add(idOf(e.name)); + // entries[0] is the model anchor (the emitter writes it first). + const anchorId = idOf(resources.entries[0].name); + anchorIds.add(anchorId); + childPrefixes.push(`${anchorId}.entities.`, `${anchorId}.metrics.`); + } + const owned = (id: string) => + anchorIds.has(id) || childPrefixes.some(p => id.startsWith(p)); + + const orphans = existing.map(e => idOf(e.name)) + .filter(id => owned(id) && !emittedIds.has(id)); + + return deleteOrphanEntries(cat, opts, orphans); +} + +async function deleteOrphanEntries( + cat: CatalogClient, opts: KcDeployOptions, + orphans: string[]): Promise { + let deleted = 0; + for (const id of orphans) { + const res = + await cat.deleteEntry(opts.project, opts.location, opts.entryGroup, id); + // A 404 means it is already gone -- reconciliation's goal is met either way. + if (isOk(res) || res.status === 404) { + deleted++; + continue; + } + return {deleted, error: `deleting orphaned entry '${id}': ${errText(res)}`}; + } + return {deleted}; +} + + +// The schema-join entry-link type name, matching what the emitter stamps on each +// link (Namer.typeName('entryLink', 'schema-join')). Used to filter +// lookupEntryLinks to the links this leg owns. +function schemaJoinLinkType(opts: KcDeployOptions): string { + const proj = opts.systemTypeProject ?? 'dataplex-types'; + const loc = opts.systemTypeLocation ?? 'global'; + return `projects/${proj}/locations/${loc}/entryLinkTypes/schema-join`; +} + + +// Snapshots the destination entry group's entries once, before any write. A +// brand-new entry group can briefly fail to list its entries collection (the +// same propagation window createEntryWithRetry rides out); treat only that +// not-yet-visible error as empty -- there is nothing to guard against or +// reconcile, and the create path retries the window. The match mirrors +// isPropagating (entry-group-scoped) so any OTHER listing failure -- a real +// backend error, a permission problem -- is surfaced rather than masked. +async function listEntryGroup( + cat: CatalogClient, + opts: KcDeployOptions): Promise<{entries: Entry[]; error?: string}> { + const entries: Entry[] = []; + try { + for await (const entry of cat.listEntries( + opts.project, opts.location, opts.entryGroup)) { + entries.push(entry); + } + } catch (err: any) { + const msg = err.message || String(err); + if (/may not exist/i.test(msg) || + /entry group .*(not found|does not exist)/i.test(msg)) { + return {entries: []}; + } + return {entries: [], error: `listing entries in entry group '${ + opts.entryGroup}': ${msg}`}; + } + return {entries}; +} + + +interface LinkReconcileOutcome { + unlinked: number; + error?: string; +} + +// Deletes the schema-join links referencing any of `entityNames` (full entry +// resource names) for which `shouldDelete` returns true. Links are looked up per +// referenced entry -- the only server-side access path -- so a link between two +// of the entities is returned twice; a `seen` set dedups it. A 404 on delete +// counts as success (already gone). +async function deleteOwnedLinks( + cat: CatalogClient, opts: KcDeployOptions, entityNames: string[], + shouldDelete: (link: EntryLink) => boolean): + Promise { + const linkType = schemaJoinLinkType(opts); + const seen = new Set(); + let unlinked = 0; + for (const entry of entityNames) { + const res = await cat.lookupEntryLinks( + opts.project, opts.location, {entry, entryLinkTypes: [linkType]}); + if (!isOk(res)) { + return {unlinked, error: `looking up entry links for '${idOf(entry)}': ${ + errText(res)}`}; + } + for (const link of res.result ?? []) { + const id = idOf(link.name ?? ''); + if (seen.has(id)) continue; + seen.add(id); + if (!shouldDelete(link)) continue; + const del = await cat.deleteEntryLink( + opts.project, opts.location, opts.entryGroup, id); + if (isOk(del) || del.status === 404) { + unlinked++; + continue; + } + return {unlinked, error: `deleting entry link '${id}': ${errText(del)}`}; + } + } + return {unlinked}; +} + + +// Reconciles a still-present model's schema-join links: deletes any link this +// model OWNS (both endpoints under its `.entities.` namespace) that the +// model no longer emits -- a relationship dropped or renamed since the last +// push. A link touching an entry outside this model is never treated as owned, +// so a shared entry group is safe. +function reconcileLinks( + cat: CatalogClient, opts: KcDeployOptions, resources: KcResources, + existing: Entry[]): Promise { + const anchorId = idOf(resources.entries[0].name); + const entityPrefix = `${anchorId}.entities.`; + // Look up links via the model's entity entries KNOWN TO THE SERVER (the + // pre-write snapshot), not the ones this push emits. Only a server-side entry + // can already carry a link, and enumerating the snapshot also reaches a link + // both of whose endpoints were removed in this push -- neither is re-emitted, + // but both entries are still present in `existing` until reconcileDeletions + // deletes them at the end. A brand-new model has no such entries, so it issues + // no lookups at all. + const entityNames = existing.map(e => e.name) + .filter(name => idOf(name).startsWith(entityPrefix)); + if (!entityNames.length) return Promise.resolve({unlinked: 0}); + + const emittedLinkIds = + new Set(resources.entryLinks.map(l => idOf(l.name ?? ''))); + const ownedByModel = (link: EntryLink) => + link.entryReferences.length === 2 && + link.entryReferences.every(r => idOf(r.name).startsWith(entityPrefix)); + + return deleteOwnedLinks( + cat, opts, entityNames, + link => + ownedByModel(link) && !emittedLinkIds.has(idOf(link.name ?? ''))); +} + + +// Deletes models already in the entry group that this push does not re-emit +// (--force-remove). For each foreign anchor: remove its schema-join links first +// (they reference entries about to be deleted), then its entries -- the anchor +// and its `.entities.` / `.metrics.` children present in the +// pre-write listing `existing`. +async function removeForeignModels( + cat: CatalogClient, opts: KcDeployOptions, existing: Entry[], + foreignAnchors: string[]): + Promise<{deleted: number; unlinked: number; error?: string}> { + let deleted = 0; + let unlinked = 0; + for (const anchor of foreignAnchors) { + const entityPrefix = `${anchor}.entities.`; + const metricPrefix = `${anchor}.metrics.`; + const owned = existing.filter(e => { + const id = idOf(e.name); + return id === anchor || id.startsWith(entityPrefix) || + id.startsWith(metricPrefix); + }); + const entityNames = owned + .filter(e => (e.entryType ?? '').endsWith('/semantic-entity')) + .map(e => e.name); + + // The whole model is going away, so every schema-join link referencing one + // of its entities is orphaned -- delete them all. + const links = await deleteOwnedLinks(cat, opts, entityNames, () => true); + if (links.error) return {deleted, unlinked, error: links.error}; + unlinked += links.unlinked; + + for (const e of owned) { + const id = idOf(e.name); + const res = await cat.deleteEntry( + opts.project, opts.location, opts.entryGroup, id); + if (isOk(res) || res.status === 404) { + deleted++; + continue; + } + return {deleted, unlinked, + error: `deleting entry '${id}' of removed model '${anchor}': ${ + errText(res)}`}; + } + } + return {deleted, unlinked}; +} + + +interface EntriesOutcome { + created: number; + updated: number; + error?: string; +} + +// Creates a model's entries. The anchor (entries[0]) is the parent of every +// child and is written first. Entity entries are then written before metric +// entries -- a metric's aspect references its entity by name, so the entity must +// exist first -- and within each of those two waves the entries are independent +// and written concurrently. An entry that already exists is updated in place +// (idempotent re-push). +async function createEntries( + cat: CatalogClient, opts: KcDeployOptions, + entries: Entry[]): Promise { + if (!entries.length) return {created: 0, updated: 0}; + const [anchor, ...children] = entries; + + const anchorRes = await writeEntry(cat, opts, anchor); + if (anchorRes.error) return {created: 0, updated: 0, error: anchorRes.error}; + + let created = anchorRes.updated ? 0 : 1; + let updated = anchorRes.updated ? 1 : 0; + + // Entities first, then metrics (a metric references its entity); each wave is + // written concurrently. + const isMetric = (e: Entry) => (e.entryType ?? '').endsWith('/semantic-metric'); + for (const wave of [children.filter(e => !isMetric(e)), + children.filter(isMetric)]) { + const res = await Promise.all(wave.map(e => writeEntry(cat, opts, e))); + const firstErr = res.find(r => r.error); + if (firstErr) return {created, updated, error: firstErr.error}; + for (const r of res) { + if (r.updated) + updated++; + else + created++; + } + } + return {created, updated}; +} + + +interface LinksOutcome { + linked: number; + error?: string; +} + +// Writes a model's schema-join entry links. Both endpoint entries already exist +// (createEntries ran first for this model), and links are independent of each +// other, so they are written concurrently. A link that already exists is upserted +// (its aspect refreshed). +async function createEntryLinks( + cat: CatalogClient, opts: KcDeployOptions, + links: EntryLink[]): Promise { + if (!links.length) return {linked: 0}; + const res = await Promise.all(links.map(l => writeEntryLink(cat, opts, l))); + const firstErr = res.find(r => r.error); + if (firstErr) return {linked: 0, error: firstErr.error}; + return {linked: links.length}; +} + +// Writes one entry link: create, then fall back to an in-place aspect update if +// it already exists. A link's entry references and type are immutable, so a +// re-push only refreshes the aspect (the join detail). A relationship whose id +// changed writes a new link and leaves the old one; reconcileLinks deletes such +// orphaned links after this model's links are written. +async function writeEntryLink( + cat: CatalogClient, opts: KcDeployOptions, + link: EntryLink): Promise<{error?: string}> { + const linkId = idOf(link.name ?? ''); + const res = await cat.createEntryLink( + opts.project, opts.location, opts.entryGroup, linkId, link); + if (isExists(res)) { + const upd = await cat.updateEntryLink( + {name: link.name, aspects: link.aspects} as EntryLink, + Object.keys(link.aspects ?? {})); + if (!isOk(upd)) return {error: `entry link '${linkId}': ${errText(upd)}`}; + return {}; + } + if (!isOk(res)) return {error: `entry link '${linkId}': ${errText(res)}`}; + return {}; +} + + +interface WriteOutcome { + updated?: boolean; // true when the entry already existed and was updated + error?: string; +} + +// Writes one entry: create (retrying the group-propagation window), then fall +// back to update-in-place if it already exists. +async function writeEntry( + cat: CatalogClient, opts: KcDeployOptions, + entry: Entry): Promise { + const entryId = idOf(entry.name); + let res = await createEntryWithRetry(cat, opts, entryId, entry); + if (isExists(res)) { + // Idempotent re-push: refresh the existing entry's source + aspects. + const upd = await cat.updateEntry( + entry, ['entry_source', 'aspects'], Object.keys(entry.aspects ?? {})); + if (!isOk(upd)) return {error: `entry '${entryId}': ${errText(upd)}`}; + return {updated: true}; + } + if (!isOk(res)) return {error: `entry '${entryId}': ${errText(res)}`}; + return {}; +} + +// entries.create can briefly 404 on a just-created entry group; retry that +// window. +async function createEntryWithRetry( + cat: CatalogClient, opts: KcDeployOptions, entryId: string, + entry: Entry): Promise> { + const tries = opts.entryCreateTries ?? ENTRY_CREATE_TRIES; + const retryMs = opts.entryCreateRetryMs ?? ENTRY_CREATE_RETRY_MS; + let res = await cat.createEntry( + opts.project, opts.location, opts.entryGroup, entryId, entry); + for (let attempt = 1; attempt < tries; attempt++) { + if (isOk(res) || isExists(res) || !isPropagating(res)) break; + await sleep(retryMs); + res = await cat.createEntry( + opts.project, opts.location, opts.entryGroup, entryId, entry); + } + return res; +} + + +// A human-readable summary of what a (dry-run) push would write for one model. +function planSummary( + model: string, resources: KcResources, opts: KcDeployOptions): string[] { + const dest = `${opts.project}.${opts.location}.${opts.entryGroup}`; + const lines = [ + `Knowledge Catalog plan for '${model}' (destination ${dest}):`, + ` ${resources.entries.length} entr${ + resources.entries.length === 1 ? 'y' : 'ies'}:`, + ...resources.entries.map( + e => ` - ${idOf(e.name)} (${idOf(e.entryType)})`), + ]; + if (resources.entryLinks.length) { + lines.push( + ` ${resources.entryLinks.length} schema-join link${ + resources.entryLinks.length === 1 ? '' : 's'}:`, + ...resources.entryLinks.map( + l => ` - ${idOf(l.name ?? '')}`)); + } + return lines; +} + + +// The id segment of a full entry/entryType resource name (after the last '/'). +function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + +function isOk(res: {status: number}): boolean { + return res.status === 200; +} + +// A create that failed because the resource already exists — treated as success +// for idempotent provisioning and re-push. The API layer preserves the HTTP +// status (gcp/api.ts), so the 409 ALREADY_EXISTS status is authoritative; no +// need to match the error text. +function isExists(res: {status: number}): boolean { + return res.status === 409; +} + +// A transient "not visible yet" error worth retrying, matching the propagation +// phrasing specifically rather than a bare "not found" (which also covers a +// genuinely missing aspect/entry type — a real, non-transient failure). +function isPropagating(res: {message?: string}): boolean { + const msg = res.message ?? ''; + return /may not exist/i.test(msg) || + /entry group .*(not found|does not exist)/i.test(msg); +} + +function errText(res: {status: number; message?: string}): string { + return res.message?.trim() || `HTTP ${res.status}`; +} diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts new file mode 100644 index 00000000..9c863290 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -0,0 +1,480 @@ +// Knowledge Catalog <-> Semantic Model IR converter. +// +// SCAFFOLD (naming for the end state): this file currently holds only the +// READ direction -- Knowledge Catalog entries -> IR +// (`modelsFromCatalogResources` and its helpers). The WRITE direction (IR -> KC +// entries, `generateCatalogResources`) still lives in `knowledge_catalog.ts` +// and moves here once PR4 (the KC push line) merges, at which point this +// becomes the full two-way converter and `knowledge_catalog.ts` goes away. +// Until then, "where is the KC emit code?" -> `knowledge_catalog.ts`. +// +// TODO(#278): fold the KC WRITE direction in and drop the scaffold. Once +// #278 merges, move `generateCatalogResources` (and the `KcResources` type +// + emit helpers) out of `knowledge_catalog.ts` into this file, delete +// `knowledge_catalog.ts`, and repoint its importers +// (`deploy_knowledge_catalog.ts` and the emitter tests). `idOf` -- shared by +// both directions and re-exported from here only for `pull_kc` today -- becomes +// a plain local. Then this is the sole KC<->IR codec and the SCAFFOLD note +// above comes out. +// +// The reader is the inverse of `generateCatalogResources`: it reconstructs the +// IR from the entries a pull hydrated (see `pull_kc.pullKnowledgeCatalog`). +// `semantic-entity` / `semantic-metric` entries are grouped under their +// `semantic-model` anchor via `parentEntry`; entries of other types are +// ignored. Resources are matched by type-name SUFFIX, so a reader need not know +// which system-type project/location the emitter used. +// +// Fidelity is bounded by what the emitter persisted, so this read is the +// inverse of the WRITE, not of the authored document. It recovers names, +// descriptions, data sources, field datatypes (via the schema aspect), each +// metric's attach entity (re-derived from its expression, as the loader does, +// when the catalog holds one, else the value the emitter persisted), the +// model's deployment targets (from the semantic-model aspect, back into the +// GOOGLE `custom_extensions` block), and 1:1 / 1:N relationships (from the +// `schema-join` entry links a pull fetched -- see +// `modelsFromCatalogResources`'s `entryLinks` argument). The per-field +// `semantics` block -- field/metric expressions and the DIMENSION role -- is +// gated off the catalog by default: the emitter writes it only under +// `--emit-expressions` (see `KcGenerateOptions.emitExpressions`), so those +// three recover only when the push that wrote them enabled it, and a default +// push -> pull drops them. It cannot recover what the emitter never writes: +// entity keys/unique keys, `ai_context`, field labels, +// `importedExpression`/`importedDialect` (the vendor-dialect SQL), and +// many-to-many (association) relationships (whose edge lives only in the +// BigQuery property graph). A `String`- or `Opaque`-typed METRIC also reads +// back un-typed: the metric aspect persists only `dataType` (both collapse to +// `STRING`, with no metadataType to disambiguate), so -- as with a plain STRING +// field -- the reader leaves it un-typed rather than guess. Relationship NAMES +// come back normalized (lowercased/hyphenated), since the emitter encodes the +// name only in the link id (via `linkSlug`), not in the join aspect. + +import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; + +import {CustomExtension, DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; +import {referencedEntityNames} from './sql_expr_utils'; + +export interface ReadResult { + models: SemanticModel[]; + warnings: string[]; +} + +/** + * Reconstructs the Semantic Model IR from Knowledge Catalog entries. + * + * Returns one model per `semantic-model` anchor plus any warnings (no anchor, + * an orphaned child, an entry missing its aspect data). Entries must already be + * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a + * BASIC list omits aspect data, so the puller re-fetches each entry first. + * + * `entryLinks` are the `schema-join` links a pull fetched for the group (see + * `pull_kc`); each link between two of a model's entity entries is + * reconstructed as a 1:1 / 1:N relationship. Pass `[]` (the default) to + * reconstruct entities and metrics only. + */ +export function modelsFromCatalogResources( + entries: Entry[], entryLinks: EntryLink[] = []): ReadResult { + const warnings: string[] = []; + + const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); + const entityEntries = + entries.filter(e => semanticType(e) === 'semantic-entity'); + const metricEntries = + entries.filter(e => semanticType(e) === 'semantic-metric'); + + if (!anchors.length) { + warnings.push('no semantic-model entry found; nothing to reconstruct'); + return {models: [], warnings: [...new Set(warnings)]}; + } + + // A child belongs to its anchor by parentEntry. When there is exactly one + // anchor, children whose parentEntry does not resolve (e.g. a project-id + // normalization mismatch) are still attached to it rather than dropped. + const anchorNames = new Set(anchors.map(a => a.name)); + const soleAnchor = anchors.length === 1 ? anchors[0].name : undefined; + const childrenOf = (anchorName: string, pool: Entry[]) => pool.filter( + e => e.parentEntry === anchorName || + (soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? ''))); + + const models = anchors.map(anchor => { + const name = anchor.entrySource?.displayName ?? idOf(anchor.name); + + const entityEntriesForModel = childrenOf(anchor.name, entityEntries); + const entities = entityEntriesForModel.map(e => readEntity(e, warnings)); + const entityNames = entities.map(e => e.name); + const metrics = childrenOf(anchor.name, metricEntries) + .map(e => readMetric(e, entityNames, warnings)); + + // Index this model's entities by their entry id, so a schema-join link can + // resolve its endpoints from the link's entryReferences. The id segment is + // stable across the project-number/id normalization that lookupEntry + // applies to entries but lookupEntryLinks does not apply to link + // references, so matching on it (not the full resource name) keeps + // live-fetched links resolvable. + const entityByEntryId = new Map(); + entityEntriesForModel.forEach( + (e, i) => entityByEntryId.set(idOf(e.name), entities[i])); + + // Relationships come from the schema-join entry links whose two endpoints + // are both this model's entity entries (M:N edges were never published -- + // they live only in the BigQuery property graph -- so stay absent here). + const relationships = + readRelationships(entryLinks, name, entityByEntryId, warnings); + + const model: SemanticModel = {name, entities, relationships, metrics}; + const description = anchor.entrySource?.description; + if (description !== undefined) model.description = description; + // Deployment targets ride back in the same GOOGLE custom_extensions block + // the author wrote them in (the inverse of the emitter's modelAspectData). + const targets = readDeploymentTargets(anchor); + if (targets) model.customExtensions = [targets]; + return model; + }); + + // Flag children that resolved to no anchor at all (only possible with + // multiple anchors, where the sole-anchor fallback does not apply). + if (!soleAnchor) { + for (const child of [...entityEntries, ...metricEntries]) { + if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { + warnings.push(`entry '${ + child.name}' has no resolvable parent semantic-model; omitted`); + } + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// Reconstructs an entity from its `semantic-entity` aspect (the backing source) +// and the built-in `schema` aspect (its fields). Keys are not persisted by the +// emitter and so come back empty. +function readEntity(entry: Entry, warnings: string[]): Entity { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const semantic = aspectData(entry, 'semantic-entity'); + const schema = aspectData(entry, 'schema'); + if (!Object.keys(semantic).length) { + warnings.push(`entity '${ + name}': no semantic-entity aspect data (fetch with the aspect type)`); + } + + const dataSource = dataSourceFromResource(semantic?.source?.resources?.[0]); + if (!dataSource) { + warnings.push( + `entity '${name}': no backing data source in the semantic-entity ` + + `aspect; 'source' will be empty and the entity may not load`); + } + const entity: Entity = { + name, + dataSource, + keys: [], // not persisted by the emitter; unrecoverable on read + fields: asArray(schema.fields) + .map(fd => readField(fd, name, warnings)) + .filter((f): f is Field => f !== undefined), + }; + const description = entry.entrySource?.description; + if (description !== undefined) entity.description = description; + return entity; +} + + +// Reconstructs a field from one `schema` aspect field record, inverting +// schemaAspectData: the datatype from dataType/metadataType, expressions from +// the nested `semantics` block, and the DIMENSION role back to a dimension +// marker. +function readField(fd: any, entityName: string, warnings: string[]): Field| + undefined { + const name = fd?.name; + if (name === undefined || name === '') { + warnings.push( + `entity '${entityName}': a schema field is missing its name; the ` + + `field is skipped`); + return undefined; + } + const field: Field = {name}; + const sem = fd?.semantics ?? {}; + if (sem.expression !== undefined) field.expression = sem.expression; + const type = irDataType(fd?.dataType, fd?.metadataType); + if (type !== undefined) field.type = type; + if (sem.role === 'DIMENSION') field.dimension = {}; + if (fd?.description !== undefined) field.description = fd.description; + return field; +} + + +// Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` +// is re-derived from the expression (as the loader does) when the aspect holds +// one, else it falls back to the entity the emitter persisted. A default push +// omits the expression entirely (it is gated behind `--emit-expressions`), so +// an absent expression is expected, not an error, and does not warn. +function readMetric( + entry: Entry, entityNames: string[], warnings: string[]): Metric { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const data = aspectData(entry, 'semantic-metric'); + + const metric: Metric = {name}; + if (data.expression !== undefined) metric.expression = data.expression; + const exprForRefs = data.expression ?? ''; + const referenced = referencedEntityNames(exprForRefs, entityNames); + const persistedEntity = + typeof data.entity === 'string' && data.entity !== '' ? data.entity : + undefined; + if (referenced.length === 1) { + // The expression pins exactly one known entity; re-derive it (as the loader + // does) so the attach entity stays consistent with the reconstructed set. + metric.entity = referenced[0]; + } else if (persistedEntity !== undefined) { + // The expression does not pin a single known entity (none, or several), so + // fall back to the attach entity the emitter persisted (metricAspectData) + // rather than dropping it. + metric.entity = persistedEntity; + } else if (exprForRefs && !referenced.length) { + // Parity with the loader's convertMetric: an expression that qualifies no + // known entity is flagged as potentially unplaceable downstream. + warnings.push( + `metric '${name}': expression references no known entity; it may not ` + + `be placeable downstream`); + } + // The emitter writes a required dataType, defaulting a typeless metric to + // NUMERIC (see metricAspectData); NUMERIC maps back to Decimal, so a metric + // authored without a datatype round-trips as an explicit Decimal rather than + // un-typed. (Dimensions differ: their STRING default reads back as un-typed.) + const type = irDataType(data.dataType, undefined); + if (type !== undefined) metric.type = type; + const description = entry.entrySource?.description; + if (description !== undefined) metric.description = description; + return metric; +} + + +// The inverse of columnDataType/columnMetadataType: maps the schema aspect's +// dataType (disambiguated by metadataType only for the STRING family) back to +// the IR's logical DataType. STRING + OTHER is Opaque; a plain STRING is read +// as un-typed (undefined) -- the loader's default -- since the emitter cannot +// distinguish an authored `String` from an un-typed field (both emit STRING). +function irDataType(dataType: string|undefined, metadataType: string|undefined): + DataType|undefined { + switch (dataType) { + case 'INT64': + return 'Integer'; + case 'NUMERIC': + return 'Decimal'; + case 'FLOAT64': + return 'Float'; + case 'BOOL': + return 'Boolean'; + case 'DATE': + return 'Date'; + case 'TIME': + return 'Time'; + case 'DATETIME': + return 'DateTime'; + case 'TIMESTAMP': + return 'DateTimeTz'; + case 'STRING': + return metadataType === 'OTHER' ? 'Opaque' : undefined; + default: + return undefined; + } +} + + +// The inverse of resourcePath: a BigQuery linked-resource URI becomes the +// canonical `project.dataset.table` string; anything else (a verbatim query or +// a passthrough reference) is returned unchanged. +function dataSourceFromResource(resource: string|undefined): string { + const value = (resource ?? '').trim(); + const m = value.match( + /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/); + return m ? `${m[1]}.${m[2]}.${m[3]}` : value; +} + + +// The bare `semantic-*` type of an entry, matched by entryType suffix so the +// system-type project/location need not be known. Returns undefined for entries +// that are not part of a semantic model. +function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| + 'semantic-metric'|undefined { + for (const t of ['semantic-model', 'semantic-entity', 'semantic-metric'] as + const) { + if (entry.entryType?.endsWith(`/entryTypes/${t}`)) return t; + } + return undefined; +} + + +// The `data` payload of an entry's aspect of the given bare type. See +// aspectDataOf; entries and entry links carry aspects in the same shape. +function aspectData(entry: Entry, type: string): Record { + return aspectDataOf(entry.aspects, type); +} + + +// The `data` payload of an aspect of the given bare type from an aspect map, +// matched by the aspect key's `.` suffix or the aspectType's +// `/aspectTypes/` suffix (robust to whichever system-type +// project/location the emitter used). Returns an empty object when the aspect +// is absent. +function aspectDataOf(aspects: Record|undefined, type: string): + Record { + for (const [key, aspect] of Object.entries(aspects ?? {})) { + if (key.endsWith(`.${type}`) || + aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { + return aspect.data ?? {}; + } + } + return {}; +} + + +// Recovers the model's deployment targets from its semantic-model aspect back +// into the GOOGLE custom_extension the author declared them in (the inverse of +// the emitter's modelAspectData). Returns undefined when the model has none. +function readDeploymentTargets(anchor: Entry): CustomExtension|undefined { + const targets = + asArray(aspectData(anchor, 'semantic-model').deploymentTargets) + .filter((t): t is string => typeof t === 'string' && t !== ''); + if (!targets.length) return undefined; + return { + vendorName: 'GOOGLE', + data: JSON.stringify({deploymentTargets: targets}) + }; +} + + +// Inverts schemaJoinAspectData (knowledge_catalog.ts): each schema-join link +// whose two endpoints are both this model's entity entries becomes one +// direct-FK relationship. +// +// Endpoint identity comes from the link's entryReferences (matched by entry id +// via `entityByEntryId`), NOT from the aspect's table names: the id is unique +// per entity (two entities can share a table) and survives the +// project-number/id normalization lookupEntry applies to entries but +// lookupEntryLinks does not apply to link references. The aspect then supplies +// the join columns and the direction -- its `source` side is the foreign-key +// side, named by its data source -- used only to orient the two endpoints. The +// link id encodes the (normalized) relationship name. Links are deduped -- +// schema-join is undirected, so a per-entry lookup can return the same link +// once from each endpoint. +function readRelationships( + links: EntryLink[], modelName: string, entityByEntryId: Map, + warnings: string[]): Relationship[] { + const prefix = linkNamePrefix(modelName); + const seen = new Set(); + const out: Relationship[] = []; + for (const link of links) { + if (!link.entryLinkType?.endsWith('/entryLinkTypes/schema-join')) continue; + const refs = link.entryReferences ?? []; + if (refs.length !== 2) continue; + // Resolve both endpoints from the link's references. A reference that is + // not one of this model's entities (another model, or a non-entity) means + // the link does not belong here -- skip it quietly, as with M:N edges. + const endA = entityByEntryId.get(idOf(refs[0].name)); + const endB = entityByEntryId.get(idOf(refs[1].name)); + if (!endA || !endB) continue; + + const key = linkDedupKey(link); + if (seen.has(key)) continue; + seen.add(key); + + const join = asArray(aspectDataOf(link.aspects, 'schema-join').joins)[0]; + if (!join) { + warnings.push( + `entry link '${link.name}': no schema-join aspect data; the ` + + `relationship is skipped`); + continue; + } + // Direction lives in the aspect: `source` is the foreign-key side, named by + // its data source. Orient the two endpoints by matching that name, keeping + // join.source.fields paired with the source side. When neither orientation + // matches -- or both do, because the endpoints share a data source -- the + // direction is undecidable, so keep the reference order and warn rather + // than drop the edge. + const srcName = (join.source?.name ?? '').trim(); + const tgtName = (join.target?.name ?? '').trim(); + const aSrc = (endA.dataSource ?? '').trim(); + const bSrc = (endB.dataSource ?? '').trim(); + const endAIsSource = aSrc === srcName && bSrc === tgtName; + const endBIsSource = bSrc === srcName && aSrc === tgtName; + let [source, destination] = [endA, endB]; + if (endBIsSource && !endAIsSource) { + [source, destination] = [endB, endA]; + } else if (!endAIsSource && !endBIsSource) { + warnings.push( + `entry link '${link.name}': join direction does not match either ` + + `endpoint's data source; using the reference order`); + } else if (endAIsSource && endBIsSource) { + warnings.push( + `entry link '${link.name}': join direction is ambiguous (endpoints ` + + `share a data source); using the reference order`); + } + const rel: Relationship = { + name: relationshipName(link.name, prefix), + source: {entity: source.name, columns: asArray(join.source?.fields)}, + destination: + {entity: destination.name, columns: asArray(join.target?.fields)}, + }; + if (join.description !== undefined) rel.description = join.description; + out.push(rel); + } + return out; +} + + +// A stable dedup key for an entry link: its name when present, else its +// (order-independent) endpoint pair and type. Guards against a per-endpoint +// lookup returning an unnamed link once from each of its two endpoints. +export function linkDedupKey(link: EntryLink): string { + if (link.name) return link.name; + const refs = (link.entryReferences ?? []).map(r => r.name).sort(); + return `${refs.join('')}${link.entryLinkType ?? ''}`; +} + + +// Best-effort recovery of the relationship name from the link id, which the +// emitter built as linkSlug(`-`) (lowercased/hyphenated -- see +// knowledge_catalog.ts). Strips the model prefix when present; the name comes +// back normalized, never verbatim. +function relationshipName( + linkName: string|undefined, modelPrefix: string): string { + const id = idOf(linkName ?? ''); + if (modelPrefix && id.startsWith(`${modelPrefix}-`)) { + const rest = id.slice(modelPrefix.length + 1); + if (rest) return rest; + } + return id; +} + + +// Mirrors the model-name portion of the emitter's linkSlug so the prefix a link +// id was built with can be stripped. Kept local rather than imported: this +// read-side module must not depend on the write-side emitter. Exported for the +// symmetry test, which reuses it to reproduce the normalized relationship name +// rather than reimplementing the slug rule. +// +// It reproduces linkSlug's normalization AND its 63-char cap + trailing-hyphen +// re-strip, so the prefix stays aligned with the id even when the combined +// `-` slug was truncated. (linkSlug's `|| 'link'` empty-slug +// fallback is intentionally omitted: an empty prefix strips nothing and +// relationshipName then returns the id verbatim, which is already correct. When +// the cap truncates into the model slug itself the relationship name is +// unrecoverable regardless -- relationshipName returns the truncated id.) +export function linkNamePrefix(modelName: string): string { + return modelName.toLowerCase() + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-+/g, '-') + .replace(/^[^a-z]+/, '') + .replace(/^-+|-+$/g, '') + .slice(0, 63) + .replace(/-+$/, ''); +} + + +// The id segment of a full entry resource name (after the last '/'). +export function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + + +function asArray(value: any): any[] { + return Array.isArray(value) ? value : []; +} diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts new file mode 100644 index 00000000..a3051b22 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -0,0 +1,541 @@ +// Generates Knowledge Catalog resources from the Semantic Model IR. +// +// The IR (./ir) is pure semantics. This module is its Knowledge Catalog +// emitter, the counterpart to `bigquery.ts`: it maps the model to catalog +// Entries, each carrying the `semantic-*` Aspect(s) that describe it. Like +// `bigquery.ts` it is a PURE function of the IR: no GCP calls, no I/O. The +// orchestration layer (`deploy_knowledge_catalog.ts`, the counterpart to +// `deploy_bigquery.ts`) drives this emitter and writes the resulting resources +// via the Knowledge Catalog client. +// +// Target schema: the `semantic-model`, `semantic-entity`, and `semantic-metric` +// entry/aspect types are built-in system types under `dataplex-types/global`, +// alongside the built-in `schema` aspect. This emitter references them; it never +// provisions them. Each entry type declares `required_aspects`, so the emitted aspect set +// per entry is a hard contract: +// * semantic-model entry -> { semantic-model } +// * semantic-entity entry -> { semantic-entity, schema } +// * semantic-metric entry -> { semantic-metric } +// +// Aspect data shapes mirror the aspect types' CLOSED metadataTemplates exactly +// (a server aspect type rejects an undeclared data field): +// * semantic-model = { deploymentTargets: string[] } +// * semantic-entity = { source: { resources: string[], importedSystem?, +// importedResource? } } +// * semantic-metric = { entity?, dataType (required) } +// * schema = { fields: [{ name, dataType, metadataType, +// description? }] } +// +// The SQL-expression fields -- `semantic-metric.expression` and the per-field +// `schema.semantics` { expression, role } block -- are NOT in the published +// system-type templates yet, so they are gated behind +// KcGenerateOptions.emitExpressions (off by default) and omitted above. +// +// Relationships become `schema-join` entry links between the two entity entries. +// schema-join is a built-in, undirected entry link type in `dataplex-types/global` +// whose required `schema-join` aspect carries the join detail (the paired join +// columns, JOIN vs FOREIGN_KEY, USER inference). The join's direction -- which +// side holds the foreign key -- is preserved inside that aspect, not by the link. +// Many-to-many (association / junction-table) edges are not emitted yet: a +// junction is two joins through a third table, which schema-join's single +// source/target pair does not model; the emitter warns and skips them (the edge +// still lives in the BigQuery property graph, see bigquery.ts). +// + +import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; + +import {bigQueryGraphTargets} from './deploy_bigquery'; +import {DataType, Entity, Metric, Relationship, SemanticModel} from './ir'; + +// Where the `semantic-*` and `schema` system types live: built-in types in +// project `dataplex-types`, location `global`. Callers may override to reference +// them from a staging project. +const DEFAULT_TYPE_PROJECT = 'dataplex-types'; +const DEFAULT_TYPE_LOCATION = 'global'; + +export interface KcGenerateOptions { + project: string; // project the entries are created in (destination) + location: string; // destination location + entryGroup: string; // destination entry group + systemTypeProject?: string; // default 'dataplex-types' + systemTypeLocation?: string; // default 'global' + // Emit the SQL-expression fields the published Dataplex system-type templates + // do not carry yet: the per-field `schema.semantics` block (expression + + // role) and `semantic-metric.expression`. Off by default so a push matches + // the live types; flip on once the templates gain these fields. Enabling it + // against today's types fails the push with an ASPECT_*_PARSING_FAILURE on + // the unknown property. + emitExpressions?: boolean; +} + +export interface KcResources { + entries: Entry[]; + entryLinks: EntryLink[]; + warnings: string[]; +} + +/** + * Generates the Knowledge Catalog resources for a semantic model. + * + * Returns the entries (model anchor first, then entities and metrics), the + * schema-join entry links for the model's relationships, and any warnings + * collected while mapping the IR (missing keys, un-typed metrics, skipped M:N + * relationships). The resources reference the built-in system types; they do not + * create them. + */ +export function generateCatalogResources( + model: SemanticModel, opts: KcGenerateOptions): KcResources { + const warnings: string[] = []; + const entities = model.entities ?? []; + const metrics = model.metrics ?? []; + const relationships = model.relationships ?? []; + + if (!entities.length) { + warnings.push( + 'model has no entities; only the semantic-model entry will be generated'); + } + + const names = new Namer(opts); + const emitExpr = opts.emitExpressions ?? false; + + // Entry ids must be unique within the entry group. Two source names that + // normalize to the same id (see slug), or exact duplicates the loader did not + // reject, would otherwise emit two entries with the same name and have the + // later write silently overwrite the earlier. Track ids and skip a collision. + const seen = new Set(); + + // The model entry is the anchor and the parent of every entity/metric entry, + // so it is created first: it is entries[0] and the publisher writes in array + // order. Reserve its id up front so nothing else can claim it. + const modelId = names.modelId(model); + const modelEntryName = names.entry(modelId); + claim(seen, modelId, 'entry', `model '${model.name}'`, warnings); + + const entries: Entry[] = [{ + name: modelEntryName, + entryType: names.typeName('entry', 'semantic-model'), + entrySource: source(model.name, model.description), + aspects: aspectMap(names, { + 'semantic-model': modelAspectData(model), + }), + }]; + + // Maps an entity's model name to its published entry name, so a relationship + // can resolve its endpoints to the entries the link references. Only entities + // actually emitted (not skipped for a duplicate id) are recorded. + const entityEntryName = new Map(); + for (const entity of entities) { + const entityId = names.entityId(model, entity); + if (!claim(seen, entityId, 'entry', `entity '${entity.name}'`, warnings)) + continue; + entityEntryName.set(entity.name, names.entry(entityId)); + if (!entity.keys || !entity.keys.length) { + warnings.push( + `entity '${entity.name}': no keys declared in the source model`); + } + entries.push({ + name: names.entry(entityId), + entryType: names.typeName('entry', 'semantic-entity'), + parentEntry: modelEntryName, + entrySource: source(entity.name, entity.description), + // required_aspects: semantic-entity AND the built-in schema. + aspects: aspectMap(names, { + 'semantic-entity': entityAspectData(entity), + 'schema': schemaAspectData(entity, emitExpr), + }), + }); + } + + for (const metric of metrics) { + const metricId = names.metricId(model, metric); + if (!claim(seen, metricId, 'entry', `metric '${metric.name}'`, warnings)) + continue; + entries.push({ + name: names.entry(metricId), + entryType: names.typeName('entry', 'semantic-metric'), + parentEntry: modelEntryName, + entrySource: source(metric.name, metric.description), + aspects: aspectMap(names, { + 'semantic-metric': metricAspectData(metric, warnings, emitExpr), + }), + }); + } + + // Relationships map to schema-join entry links between their endpoint entries. + const entryLinks: EntryLink[] = []; + const seenLinks = new Set(); + for (const rel of relationships) { + const link = relationshipLink( + names, model, rel, entityEntryName, seenLinks, warnings); + if (link) entryLinks.push(link); + } + + return {entries, entryLinks, warnings: [...new Set(warnings)]}; +} + + +// Builds the schema-join entry link for one relationship, or undefined when it +// cannot be published. A many-to-many (association) edge is skipped -- a junction +// is two joins, which the single source/target schema-join does not model -- and +// so is an edge whose endpoint entity was not emitted (e.g. skipped for a +// duplicate id). Both cases warn; the BigQuery property graph still carries the +// edge. +function relationshipLink( + names: Namer, model: SemanticModel, rel: Relationship, + entityEntryName: Map, seenLinks: Set, + warnings: string[]): EntryLink|undefined { + if (rel.association) { + warnings.push( + `relationship '${rel.name}': many-to-many (association) edges are not ` + + `published to Knowledge Catalog yet; the edge lives in the BigQuery ` + + `property graph.`); + return undefined; + } + const src = entityEntryName.get(rel.source.entity); + const dst = entityEntryName.get(rel.destination.entity); + if (!src || !dst) { + const missing = !src ? rel.source.entity : rel.destination.entity; + warnings.push( + `relationship '${rel.name}': endpoint entity '${missing}' is not a ` + + `published entity; the relationship link is skipped.`); + return undefined; + } + const linkId = names.linkId(model, rel); + if (!claim( + seenLinks, linkId, 'entry link', `relationship '${rel.name}'`, + warnings)) + return undefined; + + return { + name: names.entryLink(linkId), + entryLinkType: names.typeName('entryLink', 'schema-join'), + // schema-join is undirected: both endpoints are UNSPECIFIED references. The + // join direction lives in the aspect (source is the foreign-key side). + entryReferences: [ + {name: src, type: 'UNSPECIFIED'}, + {name: dst, type: 'UNSPECIFIED'}, + ], + aspects: aspectMap(names, { + 'schema-join': schemaJoinAspectData(model, rel), + }), + }; +} + +// The schema-join aspect for a direct foreign key: a single join pairing the +// source (FK) columns to the destination (key) columns positionally. Each side's +// `name` is the entity's SQL table representation (its dataSource). `type` is +// FOREIGN_KEY (only direct FKs are emitted today); `inferenceSource` is USER +// because the join is author-declared, and `userManaged` stops Dataplex from +// overwriting it. Field names/nesting mirror the schema-join aspect type's +// metadataTemplate. +function schemaJoinAspectData( + model: SemanticModel, rel: Relationship): Record { + const srcEntity = entityByName(model, rel.source.entity); + const dstEntity = entityByName(model, rel.destination.entity); + return { + joins: [compact({ + source: { + name: srcEntity ? srcEntity.dataSource : rel.source.entity, + fields: rel.source.columns, + }, + target: { + name: dstEntity ? dstEntity.dataSource : rel.destination.entity, + fields: rel.destination.columns, + }, + description: rel.description, + type: 'FOREIGN_KEY', + inferenceSource: 'USER', + })], + userManaged: true, + }; +} + +function entityByName(model: SemanticModel, name: string): Entity|undefined { + return (model.entities ?? []).find(e => e.name === name); +} + + +// --------------------------------------------------------------------------- +// Aspect-data mappers. Field names/nesting mirror the server aspect types' +// CLOSED metadataTemplates exactly (see file header); the golden tests pin +// these shapes so any schema-driven change is a visible, reviewable diff. +// --------------------------------------------------------------------------- + +// semantic-model: the BigQuery Graph deployment target URIs this model deploys +// to (the same targets the BigQuery leg executes against). Empty data is valid; +// the aspect is still attached to satisfy the entry type's required_aspects. +function modelAspectData(model: SemanticModel): Record { + const {targets} = bigQueryGraphTargets(model); + return compact({ + deploymentTargets: targets.length ? targets.map(t => t.uri) : undefined, + }); +} + +// semantic-entity: the base table(s) backing the entity. `source` and its +// `resources` array are required by the aspect type. importedSystem/ +// importedResource have no IR source today and are left unset. +function entityAspectData(entity: Entity): Record { + return { + source: compact({ + resources: [resourcePath(entity.dataSource)], + }), + }; +} + +// The built-in schema aspect, carrying each field's column type plus the new +// per-field `semantics` block (expression / role). name, +// dataType, and metadataType are required per column. +function schemaAspectData( + entity: Entity, emitExpressions: boolean): Record { + return { + fields: (entity.fields ?? + []).map(f => compact({ + name: f.name, + dataType: columnDataType(f.type), + metadataType: columnMetadataType(f.type), + description: f.description, + // The per-field `semantics` block (expression + role) is + // not in the published `schema` aspect template yet, so + // it is gated off by default (see + // KcGenerateOptions.emitExpressions); compact() then + // drops the undefined key. + semantics: emitExpressions ? compact({ + expression: f.expression, + // A field with any dimension metadata is a dimension; + // otherwise DEFAULT. + role: f.dimension ? 'DIMENSION' : 'DEFAULT', + }) : undefined, + })), + }; +} + +// semantic-metric: the model-level aggregate. `dataType` is required by the +// aspect type; when the model does not declare one, fall back to NUMERIC +// (decimal) and warn (metrics are aggregates, so an exact numeric is the +// sensible default; dimensions, in schemaAspectData, default to STRING) rather +// than emit an invalid aspect. +function metricAspectData( + metric: Metric, warnings: string[], + emitExpressions: boolean): Record { + let dataType = metric.type ? columnDataType(metric.type) : undefined; + if (!dataType) { + warnings.push( + `metric '${ + metric.name}': no datatype in the source model; defaulting the ` + + `required semantic-metric.dataType to 'NUMERIC'`); + dataType = 'NUMERIC'; + } + return compact({ + entity: metric.entity, + dataType, + // `expression` is not in the published semantic-metric aspect template yet; + // gated off by default (see KcGenerateOptions.emitExpressions). + expression: emitExpressions ? metric.expression : undefined, + }); +} + + +// Maps the IR's logical DataType to a column data-type string for the schema +// aspect's required `dataType`. A conventional GoogleSQL type name; the field +// is a free string server-side. Unknown/undefined falls back to STRING. +function columnDataType(type: DataType|undefined): string { + switch (type) { + case 'Integer': + return 'INT64'; + case 'Decimal': + return 'NUMERIC'; + case 'Float': + return 'FLOAT64'; + case 'Boolean': + return 'BOOL'; + case 'Date': + return 'DATE'; + case 'Time': + return 'TIME'; + case 'DateTime': + return 'DATETIME'; + case 'DateTimeTz': + return 'TIMESTAMP'; + case 'String': + case 'Opaque': + default: + return 'STRING'; + } +} + +// Maps the IR's logical DataType to the schema aspect's required `metadataType` +// enum (BOOLEAN/NUMBER/STRING/BYTES/DATETIME/TIMESTAMP/GEOSPATIAL/STRUCT/RANGE/ +// OTHER). Numeric types collapse to NUMBER; unknown/undefined falls back to +// STRING (the loader leaves most fields un-typed). +function columnMetadataType(type: DataType|undefined): string { + switch (type) { + case 'Integer': + case 'Decimal': + case 'Float': + return 'NUMBER'; + case 'Boolean': + return 'BOOLEAN'; + case 'Date': + case 'Time': + case 'DateTime': + return 'DATETIME'; + case 'DateTimeTz': + return 'TIMESTAMP'; + case 'Opaque': + return 'OTHER'; + case 'String': + default: + return 'STRING'; + } +} + +// Maps the IR's opaque, canonical `dataSource` to a catalog resource string. A +// clean three-part `project.dataset.table` becomes the BigQuery linked-resource +// URI; anything else (a query, an under/over-qualified ref) is passed through +// verbatim so nothing is lost. +// TODO: settle the linked-resource form for Iceberg / BigLake tables, which may +// need a different URI shape than the BigQuery managed-table one below. +function resourcePath(dataSource: string): string { + const trimmed = (dataSource ?? '').trim(); + if (!trimmed || /\s/.test(trimmed)) return trimmed; + const parts = trimmed.split('.').map(unquote); + if (parts.length === 3 && parts.every(p => p.length)) { + return `//bigquery.googleapis.com/projects/${parts[0]}/datasets/${ + parts[1]}/tables/${parts[2]}`; + } + return trimmed; +} + + +// --------------------------------------------------------------------------- +// Naming and small helpers. +// --------------------------------------------------------------------------- + +// Builds the fully-qualified resource names for a destination. Kept in one +// place so entry/type name construction is consistent and the emitter body +// reads as pure mapping. +class Namer { + private readonly typeProj: string; + private readonly typeLoc: string; + constructor(private readonly opts: KcGenerateOptions) { + this.typeProj = opts.systemTypeProject ?? DEFAULT_TYPE_PROJECT; + this.typeLoc = opts.systemTypeLocation ?? DEFAULT_TYPE_LOCATION; + } + + // Full resource name of a system type. `kind` selects the collection. + typeName(kind: 'entry'|'aspect'|'entryLink', name: string): string { + return `projects/${this.typeProj}/locations/${this.typeLoc}/${kind}Types/${ + name}`; + } + + // Aspect-map key: the `project.location.type` reference form the client keys + // an entry's aspects by (see dataplex._nameToTypeRef / _fixEntry). + aspectRef(name: string): string { + return `${this.typeProj}.${this.typeLoc}.${name}`; + } + + entry(entryId: string): string { + return `${this.container()}/entries/${entryId}`; + } + + entryLink(linkId: string): string { + return `${this.container()}/entryLinks/${linkId}`; + } + + private container(): string { + return `projects/${this.opts.project}/locations/${ + this.opts.location}/entryGroups/${this.opts.entryGroup}`; + } + + modelId(model: SemanticModel): string { + return slug(model.name); + } + entityId(model: SemanticModel, entity: Entity): string { + return `${slug(model.name)}.entities.${slug(entity.name)}`; + } + metricId(model: SemanticModel, metric: Metric): string { + return `${slug(model.name)}.metrics.${slug(metric.name)}`; + } + // Entry link ids are more restricted than entry ids: lowercase letters, + // numbers and hyphens only, starting with a letter (see linkSlug). + linkId(model: SemanticModel, rel: Relationship): string { + return linkSlug(`${model.name}-${rel.name}`); + } +} + + +// Wraps aspect data (keyed by bare type id) as the client's aspect map: each +// key is the `project.location.type` reference form, each value the +// fully-qualified aspectType plus its data. +function aspectMap(names: Namer, byType: Record>): + Record { + const out: Record = {}; + for (const [type, data] of Object.entries(byType)) { + out[names.aspectRef(type)] = { + aspectType: names.typeName('aspect', type), + data + }; + } + return out; +} + +// The native catalog entry source (display name + description), separate from +// the semantic-* aspect copy that carries full model fidelity. +function source( + displayName: string, description?: string): Entry['entrySource'] { + return compact({displayName, description}) as Entry['entrySource']; +} + +// Drops undefined-valued keys so the emitted JSON (and its golden) only shows +// fields the model actually set — matching how the BigQuery emitter omits empty +// OPTIONS rather than rendering blanks. +function compact>(obj: T): T { + const out: Record = {}; + for (const [k, v] of Object.entries(obj)) { + if (v !== undefined) out[k] = v; + } + return out as T; +} + +// Records a generated entry id in `seen`, returning true when it is new. On a +// repeat it warns and returns false so the caller skips that resource: two +// entries sharing one name would otherwise have the later write silently +// overwrite the earlier one on publish. +function claim( + seen: Set, id: string, kind: string, label: string, + warnings: string[]): boolean { + if (seen.has(id)) { + warnings.push( + `${label}: generated ${kind} id '${id}' duplicates an earlier one; ` + + `skipped (rename to avoid overwriting it on publish)`); + return false; + } + seen.add(id); + return true; +} + +// Entry IDs allow letters, numbers, underscores, hyphens, and periods; map +// anything else to an underscore so a model/entity name with spaces or other +// characters still yields a valid, stable ID. +function slug(s: string): string { + return s.replace(/[^A-Za-z0-9_.-]/g, '_'); +} + +// Entry link ids allow only lowercase letters, numbers and hyphens, must start +// with a letter, must end with a letter or number, and are capped at 63 chars. +// Lowercase, map any other character to a hyphen, collapse runs, then trim +// leading non-letters and edge hyphens so the id satisfies the API contract. +function linkSlug(s: string): string { + let out = s.toLowerCase() + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-+/g, '-') + .replace(/^[^a-z]+/, '') + .replace(/^-+|-+$/g, '') + .slice(0, 63) + .replace(/-+$/, ''); + return out || 'link'; +} + +function unquote(part: string): string { + return part.replace(/^[`"]/, '').replace(/[`"]$/, ''); +} diff --git a/toolbox/mdcode/src/libts/semantic/loader.ts b/toolbox/mdcode/src/libts/semantic/loader.ts index 86ba27eb..5b61607b 100644 --- a/toolbox/mdcode/src/libts/semantic/loader.ts +++ b/toolbox/mdcode/src/libts/semantic/loader.ts @@ -478,3 +478,57 @@ function parseSource(source: string, opts: LoadOptions, function unquote(part: string): string { return part.replace(/^[`"]/, '').replace(/[`"]$/, ''); } + + +// A single model parsed from one authored model file, tagged with that file's +// name so a consumer (a deploy leg) can attribute warnings and errors back to +// the file the author wrote. +export interface LoadedModel { + // The model file this was parsed from: the `.yaml` basename the layout + // discovered (e.g. `sales` for `sales.yaml`), not a filesystem path. Used + // only to prefix this model's warnings/errors so they point at the author's + // file; not part of the deployed IR. + document: string; + model: SemanticModel; +} + +export interface LoadedModels { + models: LoadedModel[]; + // Loader warnings across all documents, each prefixed with its document name. + warnings: string[]; + // Set when a document failed to parse or violated the schema, naming the + // document. `models` then holds whatever parsed before the failure; callers + // should treat a set `error` as fatal and not deploy. + error?: string; +} + +/** + * Loads every authored document into the IR once, so a multi-destination push + * parses and validates each model a single time and fans the result out to each + * deploy leg (BigQuery, Knowledge Catalog) rather than re-parsing per leg. + * + * A parse/schema error is returned as `error` (naming the document) rather than + * thrown, mirroring how the deploy legs previously reported it; loader warnings + * are prefixed with their document name. + */ +export function loadSemanticModels( + docs: {name: string; text: string}[], + opts: LoadOptions = {}): LoadedModels { + const models: LoadedModel[] = []; + const warnings: string[] = []; + for (const doc of docs) { + let loaded: LoadResult; + try { + loaded = loadModels(doc.text, opts); + } catch (err: any) { + return { + models, + warnings, + error: `Model document '${doc.name}': ${err.message || err}`, + }; + } + for (const w of loaded.warnings) warnings.push(`[${doc.name}] ${w}`); + for (const model of loaded.models) models.push({document: doc.name, model}); + } + return {models, warnings}; +} diff --git a/toolbox/mdcode/src/libts/semantic/osi_converter.ts b/toolbox/mdcode/src/libts/semantic/osi_converter.ts new file mode 100644 index 00000000..b8b260aa --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/osi_converter.ts @@ -0,0 +1,270 @@ +// OSI (open, AI-first semantics) <-> Semantic Model IR converter. +// +// SCAFFOLD (naming for the end state): this file currently holds only the +// WRITE direction -- IR -> OSI YAML (`serializeModel`). The READ direction +// (OSI YAML -> IR) still lives in `loader.ts` and moves here once PR4 (the +// KC push line) merges, at which point this becomes the full two-way +// converter. Until then, "where is the OSI parser?" -> `loader.ts`. +// +// TODO(#278): fold the OSI READ direction in and drop the scaffold. Once +// #278 merges, move `loadModels` / `fromDocument` / `loadSemanticModels` +// (and their Load* types) out of `loader.ts` into this file, delete +// `loader.ts`, and repoint its importers (`src/tool/commands.ts` and the +// load / validate / bigquery tests). Then this is the sole OSI<->IR codec +// and the SCAFFOLD note above comes out. +// +// Serializing is the inverse of `loader.ts`: `loader` reads authored YAML +// into the IR; this module writes the IR out as a YAML document the loader +// can read back. It is the local-workspace sink for `pull` (Knowledge +// Catalog -> IR -> YAML), the counterpart to how `push` compiles YAML -> IR +// -> a destination. +// +// Fidelity is at the IR level, not byte-for-byte with a hand-authored file. The +// loader normalizes several authoring conveniences into the IR at load time, so +// they are already gone before serialization and cannot be reproduced here: +// * per-dialect `expression.dialects[]` variants collapse to at most two +// forms +// (a target/canonical `expression` + an `importedExpression`); only those +// are re-emitted, each under a single dialect label. +// * comments and key ordering are not preserved. +// What the loader DOES keep on the IR round-trips here: names, descriptions, +// `ai_context` (instructions / synonyms / examples), `custom_extensions` +// (verbatim), keys and unique keys, data sources, field datatypes / labels / +// dimension flags, expressions, and relationship join columns. See +// serialize.test.ts. +// +// An `association` (junction-table) relationship has no open-format syntax (the +// loader cannot produce one), so only its direct foreign-key view (from/to + +// columns) is serialized; the junction detail is dropped with a note. + +import * as yaml from 'yaml'; + +import {AiContext, CustomExtension, Entity, Field, Metric, Relationship, SemanticModel,} from './ir'; + +// The schema version the loader was written against; re-emitted verbatim so a +// serialized document loads without a version-mismatch warning. Mirrors +// loader.SUPPORTED_VERSION. +const SERIALIZED_VERSION = '0.2.0.dev0'; + +// The dialect label for the IR's target/canonical `expression`. The IR does not +// record which authored dialect that string came from (the loader picked it +// from the target dialect or the ANSI_SQL fallback and discarded the label), +// and its contract is "GoogleSQL-valid". BIGQUERY is the loader's default +// target dialect, so labeling the canonical form BIGQUERY makes the loader +// re-pick it exactly on reload -- a clean round trip with no dialect-fallback +// note. +const CANONICAL_DIALECT = 'BIGQUERY'; + +// The dialect label used for an `importedExpression` whose `importedDialect` +// was lost (e.g. read back from a Knowledge Catalog aspect that does not +// persist the dialect). A non-target, non-canonical label so the loader +// re-reads it as the imported vendor form rather than the canonical expression. +const UNKNOWN_IMPORTED_DIALECT = 'IMPORTED'; + +export interface SerializeResult { + yaml: string; + warnings: string[]; +} + +/** + * Serializes a single semantic model to a YAML document string in the open + * AI-first semantics format. The document contains exactly this one model; + * `pull` writes one file per model + * (catalog/EntryGroups//.yaml). + * + * Warnings flag IR content that has no loadable representation (an association + * relationship's junction detail), so the caller can surface the lossy edge. + */ +export function serializeModel(model: SemanticModel): SerializeResult { + const warnings: string[] = []; + const text = yaml.stringify(modelDocument(model, warnings)); + return {yaml: text, warnings: [...new Set(warnings)]}; +} + +// Builds the plain document object (version + one model) that yaml.stringify +// renders. Kept separate so tests can assert the structure without parsing +// YAML. +export function modelDocument( + model: SemanticModel, warnings: string[] = []): Record { + return { + version: SERIALIZED_VERSION, + semantic_model: [modelDoc(model, warnings)], + }; +} + +function modelDoc( + model: SemanticModel, warnings: string[]): Record { + // `datasets` is required (min 1) by the loader. A reconstructed model with no + // entities (e.g. every entity fetch failed during a pull) would serialize to + // a document the loader rejects; emit the (empty) array but flag it so the + // lossy edge is visible rather than surfacing later as an opaque load error. + const datasets = (model.entities ?? []).map(e => datasetDoc(e, warnings)); + if (!datasets.length) { + warnings.push( + `model '${model.name}': no datasets (entities); the document requires ` + + `at least one and will not load until an entity is present.`); + } + return compact({ + name: model.name, + description: model.description, + ai_context: aiContextDoc(model.aiContext), + custom_extensions: customExtensionsDoc(model.customExtensions), + datasets, + relationships: nonEmpty( + (model.relationships ?? []).map(r => relationshipDoc(r, warnings))), + metrics: nonEmpty((model.metrics ?? []).map(m => metricDoc(m, warnings))), + }); +} + +function datasetDoc( + entity: Entity, warnings: string[]): Record { + return compact({ + name: entity.name, + source: entity.dataSource, + primary_key: nonEmpty(entity.keys), + unique_keys: nonEmpty(entity.uniqueKeys), + description: entity.description, + ai_context: aiContextDoc(entity.aiContext), + fields: nonEmpty((entity.fields ?? []).map(f => fieldDoc(f, warnings))), + custom_extensions: customExtensionsDoc(entity.customExtensions), + }); +} + +function fieldDoc(field: Field, warnings: string[]): Record { + const expression = expressionDoc( + field.expression, field.importedExpression, field.importedDialect); + if (!expression) { + warnings.push( + `field '${field.name}': no expression; the loader requires one per ` + + `field and the document will not load until it is set.`); + } + return compact({ + name: field.name, + expression, + datatype: field.type, + label: field.label, + dimension: dimensionDoc(field), + description: field.description, + ai_context: aiContextDoc(field.aiContext), + custom_extensions: customExtensionsDoc(field.customExtensions), + }); +} + +function metricDoc(metric: Metric, warnings: string[]): Record { + // `entity` is derived by the loader from the expression's entity qualifiers, + // so it is intentionally not emitted: the loader recomputes it on reload. + const expression = expressionDoc( + metric.expression, metric.importedExpression, metric.importedDialect); + if (!expression) { + warnings.push( + `metric '${metric.name}': no expression; the loader requires one per ` + + `metric and the document will not load until it is set.`); + } + return compact({ + name: metric.name, + expression, + datatype: metric.type, + description: metric.description, + ai_context: aiContextDoc(metric.aiContext), + custom_extensions: customExtensionsDoc(metric.customExtensions), + }); +} + +// Inverts loader.convertRelationship: `from`/`to` are the endpoint entities and +// `from_columns`/`to_columns` are their positional join columns. An association +// (junction-table) edge has no open-format syntax, so only this direct-FK view +// is emitted and the junction detail is flagged. +function relationshipDoc( + rel: Relationship, warnings: string[]): Record { + if (rel.association) { + warnings.push( + `relationship '${ + rel.name}': association (junction-table) detail has no ` + + `open-format representation and is not serialized; only its foreign-key ` + + `endpoints are written.`); + } + return compact({ + name: rel.name, + from: rel.source.entity, + to: rel.destination.entity, + from_columns: nonEmpty(rel.source.columns), + to_columns: nonEmpty(rel.destination.columns), + description: rel.description, + ai_context: aiContextDoc(rel.aiContext), + custom_extensions: customExtensionsDoc(rel.customExtensions), + }); +} + +// Renders the `expression` object matching the loader's schema (a `dialects` +// array of {dialect, expression}). Emits the target/canonical form under +// CANONICAL_DIALECT and the imported vendor form under its own dialect, so the +// loader re-picks each into the same IR field. Returns undefined when neither +// form is present (the loader requires an expression, but a pathological field +// with none is dropped rather than fabricated). +function expressionDoc( + expression: string|undefined, importedExpression: string|undefined, + importedDialect: string|undefined): Record|undefined { + const dialects: {dialect: string; expression: string}[] = []; + if (expression !== undefined) { + dialects.push({dialect: CANONICAL_DIALECT, expression}); + } + if (importedExpression !== undefined) { + let label = importedDialect ?? UNKNOWN_IMPORTED_DIALECT; + // Never emit two dialect entries under the same label: the loader would + // pick between them non-deterministically. If the imported form's dialect + // collides with the canonical label already pushed, fall back to the + // imported placeholder. + if (expression !== undefined && label === CANONICAL_DIALECT) { + label = UNKNOWN_IMPORTED_DIALECT; + } + dialects.push({dialect: label, expression: importedExpression}); + } + return dialects.length ? {dialects} : undefined; +} + +// Emits the field's `dimension` block when present, inverting +// loader.convertField (which sets `dimension = {}` for a bare marker and copies +// `is_time`). The key is emitted whenever the IR carries dimension metadata, +// even for an empty marker, so a dimension field reloads as a dimension. +function dimensionDoc(field: Field): Record|undefined { + if (!field.dimension) return undefined; + return compact({is_time: field.dimension.isTime}); +} + +// Emits the structured `ai_context` (the only authoring path the loader reads +// synonyms/instructions/examples from), inverting loader.normalizeAiContext. +// Returns undefined when the context carries nothing, so `compact` drops the +// key. +function aiContextDoc(ai: AiContext|undefined): Record|undefined { + if (!ai) return undefined; + const doc = compact({ + instructions: ai.instructions, + synonyms: nonEmpty(ai.synonyms), + examples: nonEmpty(ai.examples), + }); + return Object.keys(doc).length ? doc : undefined; +} + +// Emits vendor `custom_extensions` verbatim (`vendorName` -> `vendor_name`), +// inverting loader.toCustomExtensions. Returns undefined when there are none. +function customExtensionsDoc(exts: CustomExtension[]|undefined): + Record[]|undefined { + if (!exts || !exts.length) return undefined; + return exts.map(e => ({vendor_name: e.vendorName, data: e.data})); +} + +// Drops undefined-valued keys so the emitted YAML only shows fields the model +// actually set (matching the emitters' `compact` convention). +function compact>(obj: T): T { + const out: Record = {}; + for (const [k, v] of Object.entries(obj)) { + if (v !== undefined) out[k] = v; + } + return out as T; +} + +// Returns the array, or undefined when empty/absent, so an empty list is +// omitted rather than rendered as `[]`. +function nonEmpty(items: T[]|undefined): T[]|undefined { + return items && items.length ? items : undefined; +} diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts new file mode 100644 index 00000000..a1ebae76 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -0,0 +1,241 @@ +// Pull: fetch a semantic model from a live Knowledge Catalog into the IR. +// +// The read-direction counterpart of `deployKnowledgeCatalog` (push, in +// `deploy_knowledge_catalog.ts`). Unlike a write, a pull needs no server-side +// type provisioning -- only that the `semantic-*` entries exist. It enumerates +// the entry group, keeps the semantic entries, hydrates each one's aspect data +// (a BASIC list omits aspect data, so each entry is re-fetched with its aspect +// types -- an entity needs BOTH its `semantic-entity` aspect and the built-in +// `schema` aspect), and hands the hydrated entries to the pure reader +// (`kc_converter.modelsFromCatalogResources`). +// +// TODO(#278): Layer 2 push/pull symmetry. This is the pull half over the +// pure `kc_converter` codec; the push half (`deployKnowledgeCatalog`) still +// lives in `deploy_knowledge_catalog.ts`. Once #278 merges, rename that +// file to `push_kc.ts` (repointing `src/tool/commands.ts` and its test) so +// the orchestration layer reads as `push_kc` / `pull_kc`. Rename only -- no +// logic moves. + +import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; + +import {SemanticModel} from './ir'; +import {idOf, linkDedupKey, modelsFromCatalogResources} from './kc_converter'; + +export interface KcPullOptions { + project: string; + location: string; + entryGroup: string; + model?: string; // limit to a single model by name (default: all) +} + +export interface KcPullResult { + models: SemanticModel[]; + warnings: string[]; +} + +// Upper bound on in-flight aspect-hydration fetches during a pull. +const HYDRATE_CONCURRENCY = 8; + +// Reads the semantic models back from a Knowledge Catalog entry group. Emits no +// console output; warnings (skipped entries, no match for --model, reader +// warnings) are returned for the caller to print. +export async function pullKnowledgeCatalog( + cat: CatalogClient, opts: KcPullOptions): Promise { + const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`; + const warnings: string[] = []; + + // Enumerate the group (paging is inherently sequential) and pick the semantic + // entries, then hydrate their aspects concurrently: a BASIC list omits aspect + // data, so each entry needs its own lookupEntry, and those fetches are + // independent. The pool preserves input order so warnings stay deterministic. + const targets: {entry: Entry; aspectTypes: string[]}[] = []; + for await (const entry of cat.listEntries( + opts.project, opts.location, opts.entryGroup)) { + const aspectTypes = semanticAspectTypes(entry.entryType); + if (aspectTypes) targets.push({entry, aspectTypes}); + // else: not part of a semantic model; ignore it. + } + + // When scoped to one model, hydrate only that model's entries -- its anchor + // (matched by name) plus the children pointing at it. A list already carries + // entrySource + parentEntry, so this avoids fetching every other model's + // aspects. No match short-circuits with just the not-found warning. + let scoped = targets; + if (opts.model) { + scoped = scopeToModel(targets, opts.model); + if (!scoped.length) { + return { + models: [], + warnings: + [`no semantic model named '${opts.model}' found in ${destination}`], + }; + } + } + + const fetched = await mapConcurrent( + scoped, HYDRATE_CONCURRENCY, async ({entry, aspectTypes}) => { + const res = await cat.lookupEntry( + opts.project, opts.location, entry.name, aspectTypes); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry '${entry.name}' (status ${ + res.status}); skipped` + }; + } + return {entry: res.result}; + }); + + const hydrated: Entry[] = []; + for (const r of fetched) { + if (r.entry) + hydrated.push(r.entry); + else if (r.warning) + warnings.push(r.warning); + } + + // Second fetch pass: relationships are schema-join entry links, which the + // entry list/lookup does not return. The catalog exposes links only per + // referenced entry (:lookupEntryLinks), so fan out over the entity entries + // and dedup (linkDedupKey) -- schema-join is undirected, so each link comes + // back once from each of its two endpoints. + const entityEntries = hydrated.filter( + e => e.entryType?.endsWith('/entryTypes/semantic-entity')); + const linkResults = + await mapConcurrent(entityEntries, HYDRATE_CONCURRENCY, async entry => { + const linkType = schemaJoinLinkType(entry.entryType); + const res = await cat.lookupEntryLinks(opts.project, opts.location, { + entry: entry.name, + entryLinkTypes: linkType ? [linkType] : undefined, + }); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry links for '${entry.name}' (status ${ + res.status}); relationships may be incomplete`, + }; + } + return {links: res.result}; + }); + + const seenLinks = new Set(); + const entryLinks: EntryLink[] = []; + for (const r of linkResults) { + if (r.warning) { + warnings.push(r.warning); + continue; + } + for (const link of r.links ?? []) { + const key = linkDedupKey(link); + if (seenLinks.has(key)) continue; + seenLinks.add(key); + entryLinks.push(link); + } + } + + const read = modelsFromCatalogResources(hydrated, entryLinks); + warnings.push(...read.warnings); + + // Defense in depth: keep only the requested model even if the reader surfaced + // another anchor (e.g. a child whose parentEntry pointed outside the scope). + let models = read.models; + if (opts.model) { + models = models.filter(m => m.name === opts.model); + if (!models.length) { + warnings.push( + `no semantic model named '${opts.model}' found in ${destination}`); + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// The aspect type resource names to hydrate for a semantic entry, derived from +// its entryType (the aspect types are the parallel resources in the same +// project/location). An entity carries two aspects: its `semantic-entity` +// aspect and the built-in `schema` aspect that holds its fields. Returns +// undefined for entries that are not part of a semantic model. +function semanticAspectTypes(entryType: string): string[]|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + const typeBase = entryType.slice(0, idx); + const t = entryType.slice(idx + marker.length); + const aspectType = (name: string) => `${typeBase}/aspectTypes/${name}`; + switch (t) { + case 'semantic-model': + return [aspectType('semantic-model')]; + case 'semantic-entity': + return [aspectType('semantic-entity'), aspectType('schema')]; + case 'semantic-metric': + return [aspectType('semantic-metric')]; + default: + return undefined; + } +} + + +// The schema-join entry link type resource, derived from an entity's entryType +// base (the link type is the parallel resource in the same project/location the +// emitter referenced). Used to filter :lookupEntryLinks to just the +// relationship links. Returns undefined for an entryType with no recognizable +// base. +function schemaJoinLinkType(entryType: string): string|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + return `${entryType.slice(0, idx)}/entryLinkTypes/schema-join`; +} + + +// Restricts hydration targets to a single model: the semantic-model anchor +// whose name (entrySource.displayName, else the entry id) matches `model`, plus +// every child entry whose parentEntry is that anchor. Uses only list-level +// fields (no aspect data), so it runs before hydration and avoids fetching +// unrelated models' aspects. Returns [] when no anchor matches. +function scopeToModel( + targets: {entry: Entry; aspectTypes: string[]}[], + model: string): {entry: Entry; aspectTypes: string[]}[] { + const isAnchor = (t: {entry: Entry}) => + !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); + const allAnchorNames = + new Set(targets.filter(isAnchor).map(t => t.entry.name)); + const matchedAnchorNames = + new Set(targets.filter(isAnchor) + .filter( + t => (t.entry.entrySource?.displayName ?? + idOf(t.entry.name)) === model) + .map(t => t.entry.name)); + if (!matchedAnchorNames.size) return []; + // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds + // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a + // project-id normalization mismatch) still belongs to it. Without this a + // scoped pull would drop children a full pull keeps. + const soleAnchor = + allAnchorNames.size === 1 ? [...allAnchorNames][0] : undefined; + const soleMatched = + soleAnchor !== undefined && matchedAnchorNames.has(soleAnchor); + return targets.filter( + t => matchedAnchorNames.has(t.entry.name) || + matchedAnchorNames.has(t.entry.parentEntry ?? '') || + (soleMatched && !isAnchor(t) && + !allAnchorNames.has(t.entry.parentEntry ?? ''))); +} + + +// Maps `items` through `fn` with at most `limit` calls in flight, returning +// results in input order (so downstream ordering stays deterministic). +async function mapConcurrent( + items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results: R[] = new Array(items.length); + let next = 0; + async function worker(): Promise { + while (next < items.length) { + const i = next++; + results[i] = await fn(items[i]); + } + } + const workers = + Array.from({length: Math.min(limit, items.length)}, () => worker()); + await Promise.all(workers); + return results; +} diff --git a/toolbox/mdcode/src/libts/semantic/validate.ts b/toolbox/mdcode/src/libts/semantic/validate.ts new file mode 100644 index 00000000..8c9248a7 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/validate.ts @@ -0,0 +1,161 @@ +// Push-time validation gate for a semantic model. +// +// Runs once over the shared, already-parsed models (see loadSemanticModels) +// before any destination leg, so a real `kcmd push` AND a `--validate-only` dry +// run enforce the same requirements. Returns one message per violation (an +// empty array means valid); the caller (commands.ts) prints them and aborts the +// push. Kept separate from the loader -- which validates a document against the +// schema -- because these are deployment requirements, not schema rules, and +// they read the GOOGLE deployment-target extension the BigQuery leg owns. + +import {BigQueryClient} from '../gcp/bigquery'; +import {googleDeploymentTargets} from './deploy_bigquery'; +import {SemanticModel} from './ir'; +import {LoadedModel} from './loader'; + +// Checks every model against the push requirements and returns the collected +// error messages (empty when all models pass), each tagged with the model's +// source document so the author can find it. +export function validatePushRequirements(models: LoadedModel[]): string[] { + const errors: string[] = []; + for (const {document, model} of models) { + let deployInfo: ReturnType; + try { + // One pass over the model's GOOGLE extension(s): both checks below read + // the same parse rather than re-parsing the JSON per reader. + deployInfo = googleDeploymentTargets(model); + } catch (err: any) { + // Malformed GOOGLE extension JSON: surface it as a validation error here + // rather than letting it throw out of a later leg as an uncaught stack. + errors.push(`${err.message || err} (${document})`); + continue; + } + + // Every model must declare exactly one deployment target -- a single + // BigQuery Graph URI (we do not support zero or several graphs per model). + if (deployInfo.uris.length !== 1) { + errors.push( + `model '${model.name}' (${document}) declares ${ + deployInfo.uris.length} deploymentTargets; exactly one BigQuery ` + + `Graph target is required under its GOOGLE custom_extension.`); + } else if (deployInfo.malformed.length) { + // The single target is present but is not a valid BigQuery Graph URI. + errors.push( + `model '${model.name}' (${document}) deploymentTarget '${ + deployInfo.malformed[0]}' is not a valid BigQuery Graph URI; ` + + `expected //bigquery.googleapis.com/projects/

/datasets//` + + `propertyGraphs/.`); + } + + // A model that targets a BigQuery graph must have every metric resolve to a + // single entity, or the metric cannot lower to a MEASURE and would be + // silently dropped from the graph. The loader sets metric.entity only when + // the expression resolves to exactly one entity, so an unset entity is the + // "references zero or multiple entities" case. + if (deployInfo.targets.length > 0) { + for (const metric of model.metrics ?? []) { + if (!metric.entity) { + errors.push( + `metric '${metric.name}' in model '${model.name}' (${ + document}) targets a BigQuery graph but does not resolve to a ` + + `single entity; set its attach entity or scope its expression to ` + + `one entity.`); + } + } + } + } + return errors; +} + + +// Live pre-flight over the same models: confirms every entity's BigQuery source +// table is reachable BEFORE any destination leg runs, so a push -- to BigQuery +// or to Knowledge Catalog -- fails fast when the model could not deploy, rather +// than surfacing a missing table only once the BigQuery leg executes its DDL. +// The entity sources are BigQuery tables regardless of --target, so this runs +// for every destination and for --validate-only. +// +// Each distinct source is probed with a dry-run query (`SELECT 1 FROM `, +// suffixed `WHERE FALSE` so it scans no data), +// so BigQuery resolves the reference exactly as the generated DDL will. That +// covers every reference form the generator emits -- a three-part +// `project.dataset.table`, a four-part federated REST-catalog / Lakehouse name +// (e.g. an Apache Iceberg table via BigLake), and quoted identifiers -- rather +// than only a three-part name. A source the loader kept verbatim because it is a +// query (contains whitespace) is not a table and is skipped. The dry-run is +// billed to the model's BigQuery deployment-target project (the same project the +// deploy runs against), falling back to `defaultProject`. Each distinct (billing +// project, reference) pair is probed once. Returns one message per unreachable +// table (empty when all pass). +export async function validateBigQueryDataSources( + models: LoadedModel[], bq: BigQueryClient, + defaultProject: string): Promise { + // Dedup by billing project + reference so a table shared across + // entities/models is probed once; keep the first reference for a locatable + // error message. + const refs = new Map(); + for (const {document, model} of models) { + const project = billingProject(model, defaultProject); + for (const entity of model.entities ?? []) { + const ref = probeableRef(entity.dataSource); + if (!ref) continue; + const key = `${project}\u0000${ref}`; + if (!refs.has(key)) { + refs.set(key, { + project, ref, document, model: model.name, entity: entity.name, + }); + } + } + } + + const errors: string[] = []; + for (const {project, ref, document, model, entity} of refs.values()) { + const res = + await bq.query( + project, `SELECT 1 FROM \`${ref}\` WHERE FALSE`, undefined, true); + if (res.status === 200) continue; + const msg = res.message?.trim() || `HTTP ${res.status}`; + const why = /not found/i.test(msg) ? + 'does not exist' : + /access denied|permission denied|not authorized|does not have permission/i + .test(msg) ? + 'is not accessible (permission denied)' : + `could not be verified (${msg})`; + errors.push( + `entity '${entity}' in model '${model}' (${document}) references ` + + `BigQuery table '${ref}', which ${why}; the model cannot be deployed. ` + + `Create the table or grant access to it, or fix the entity's source.`); + } + return errors; +} + + +// The BigQuery project a model's deploy -- and thus its dry-run pre-flight -- +// bills to: the project of the model's first BigQuery Graph deployment target +// (where the CREATE PROPERTY GRAPH runs), falling back to the scope's default +// project when the model declares no parseable BigQuery Graph target. +// googleDeploymentTargets is safe here: validatePushRequirements ran first and +// already rejected a malformed GOOGLE extension. +function billingProject(model: SemanticModel, defaultProject: string): string { + try { + return googleDeploymentTargets(model).targets[0]?.project ?? defaultProject; + } catch { + return defaultProject; + } +} + + +// A source that can be probed as a BigQuery table: the canonical `dataSource`, +// trimmed, or null when it is not a table reference -- empty, or a query the +// loader kept verbatim (contains whitespace). Unlike a tables.get probe this +// imposes no part-count limit, so a three-part `project.dataset.table` and a +// four-part REST-catalog / Lakehouse name are both returned for the dry-run to +// resolve. +function probeableRef(dataSource: string|undefined): string|null { + const trimmed = (dataSource ?? '').trim(); + if (!trimmed || /\s/.test(trimmed)) return null; + return trimmed; +} diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index 23ef2ef2..1c67d76f 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -10,7 +10,13 @@ import * as context from '../libts/gcp/context'; import { Sources } from '../libts/source'; import { SemanticModelLayout } from '../libts/layouts/semantic-model'; import { SemanticModelSource } from '../libts/sources/semantic-model'; -import * as deploy from '../libts/semantic/deploy'; +import * as deploy from '../libts/semantic/deploy_bigquery'; +import * as kc from '../libts/semantic/deploy_knowledge_catalog'; +import {BigQueryClient} from '../libts/gcp/bigquery'; +import {LoadedModel, loadSemanticModels} from '../libts/semantic/loader'; +import {pullKnowledgeCatalog} from '../libts/semantic/pull_kc'; +import {serializeModel} from '../libts/semantic/osi_converter'; +import {validateBigQueryDataSources, validatePushRequirements} from '../libts/semantic/validate'; export interface InitOptions { @@ -23,8 +29,78 @@ export interface InitOptions { export interface PushOptions { + // Generic push flag for non-semantic-model (CatalogSync) scopes; + // forwarded to CatalogSync.push. The semantic-model legs ignore it. + // The catch-all "force the push" toggle -- distinct from + // `forceRemove` below, which specifically authorizes deleting models + // this push no longer includes. force?: boolean; + // Run every validation check and report pass/fail, but write nothing + // to any destination (a dry run). Applies to both push paths. validateOnly?: boolean; + // Delete Knowledge Catalog models already in the entry group that this push + // does not include (a removed or renamed model). Without it, an unrecognized + // model in the group fails the push. Semantic-model KC push only. + // Unlike `force` above, this authorizes a destructive delete rather + // than overriding a conflict. + forceRemove?: boolean; + // Semantic-model push destination(s): 'bq', 'kc', 'all' (default), or a + // comma-separated list (e.g. 'bq,kc'). Ignored for non-semantic-model scopes. + target?: string; + // Print each pushed destination's generated artifact in that destination's + // native format (BigQuery Graph -> SQL DDL, Knowledge Catalog -> the entry + // plan), each block labeled by destination. Scope which destinations run with + // --target. Works with or without --validate-only. Semantic-model push only. + print?: boolean; + // Emit the SQL-expression fields not yet supported by the published Knowledge + // Catalog system-type templates (per-field schema semantics and the metric + // expression). Off by default so a push matches the live types; enable once + // the templates gain these fields. Semantic-model KC push only. + emitExpressions?: boolean; +} + + +export type PushTarget = 'bigquery' | 'kc'; + +// All known semantic-model push destinations, in canonical run order. `all` +// expands to this list, and resolveTargets always emits in this order so the +// run is deterministic and BigQuery-first fail-fast holds regardless of how the +// user ordered the flag. Append new destinations here as they land. +const DESTINATIONS: PushTarget[] = ['bigquery', 'kc']; + +// The default when --target is omitted: push to every destination. +const DEFAULT_TARGET = 'all'; + +// User-typeable aliases for a single destination. +const TARGET_ALIASES: Record = { + bq: 'bigquery', + bigquery: 'bigquery', + kc: 'kc', +}; + +// Resolves a --target flag value to its ordered, de-duplicated destinations, or +// undefined if any token is unrecognized (the caller reports the error). +// Accepts a comma-separated list ('bq,kc'), the keyword 'all' (every +// destination), and defaults to 'bq'. The result is always in canonical +// DESTINATIONS order. +export function resolveTargets(target?: string): PushTarget[] | undefined { + const tokens = (target ?? DEFAULT_TARGET) + .toLowerCase() + .split(',') + .map(t => t.trim()) + .filter(t => t.length); + if (!tokens.length) return undefined; + const selected = new Set(); + for (const tok of tokens) { + if (tok === 'all') { + DESTINATIONS.forEach(d => selected.add(d)); + continue; + } + const dest = TARGET_ALIASES[tok]; + if (!dest) return undefined; + selected.add(dest); + } + return DESTINATIONS.filter(d => selected.has(d)); } @@ -50,8 +126,23 @@ export async function init(options: InitOptions): Promise { } else if (options.semanticModel) { manifest = await kcmd.CatalogManifest.initWithSemanticModel(options.semanticModel, ctx); - const entryGroup = manifest.source.entryGroup!; - fs.mkdirSync(path.join('catalog', 'EntryGroups', entryGroup), { recursive: true }); + const source = manifest.source as SemanticModelSource; + // Provision the destination entry group now, at init, so push writes only + // entries -- matching how the standard layout operates (its push creates + // entries, never the entry group). Idempotent: an already-existing group + // (409) is success. + const catalog = new dataplex.CatalogClient(ctx); + const res = await catalog.createEntryGroup( + source.project, source.location, source.entryGroup); + if (res.status !== 200 && res.status !== 409) { + console.error( + `Error: failed to create entry group '${source.name}': ` + + `${res.message || res.status}`); + return 1; + } + fs.mkdirSync( + path.join('catalog', 'EntryGroups', source.entryGroup), + { recursive: true }); } else { console.error('Error: Must provide --entry-group, --bigquery-dataset, --kb, or --semantic-model'); @@ -69,15 +160,20 @@ export async function init(options: InitOptions): Promise { } -export async function pull(): Promise { +export interface PullOptions { + // Reconstruct + report only; never writes a file. Mirrors push --validate-only. + dryRun?: boolean; + // Limit the pull to a single model by name (default: all in the entry group). + model?: string; +} + + +export async function pull(options: PullOptions = {}): Promise { const ctx = context.ApiContext.default(); const snapshot = await kcmd.CatalogSnapshot.fromPath('.', ctx); if (snapshot.manifest.source.type === Sources.SEMANTIC_MODEL) { - console.log( - 'Semantic-model scope: nothing to pull. Knowledge Catalog resource ' + - 'pull for the semantic model is not yet implemented.'); - return 0; + return await pullSemanticModel(ctx, snapshot, options); } const catalog = new dataplex.CatalogClient(ctx); @@ -106,32 +202,68 @@ export async function push(options: PushOptions): Promise { // (see createLayout), so this cast is safe. const layout = snapshot.layout as SemanticModelLayout; const source = snapshot.manifest.source as SemanticModelSource; - console.log('Pushing semantic model (BigQuery Graph)...'); - const result = await deploy.deployBigQuery( - layout.modelDocuments(), ctx, options, source.project); - for (const w of result.warnings) { + const targets = resolveTargets(options.target); + if (!targets) { + console.error( + `Error: invalid --target '${options.target}'; expected bq, kc, all, ` + + `or a comma-separated list (e.g. bq,kc).`); + return 1; + } + + // Load + validate every model ONCE, then fan the parsed models out to each + // destination leg. Both legs consume the same IR, so a `--target all` push + // parses each document a single time instead of once per leg. A parse error + // fails the whole push before any destination runs. defaultProject is the + // scope's declared project (deterministic) rather than the ambient gcloud + // project, which can drift from where the model's tables live. + const docs = layout.modelDocuments(); + const loaded = loadSemanticModels( + docs, {defaultProject: source.project ?? ctx.project}); + if (loaded.error) { + console.error('Error:', loaded.error); + return 1; + } + for (const w of loaded.warnings) { console.warn(`Warning: ${w}`); } - if (options.validateOnly) { - for (const block of result.ddl) { - console.log(`${block}\n`); + + // Enforce push-time requirements once over the shared models, before any + // destination runs: every model must declare a deployment target, and a + // BigQuery-graph-targeting model's metrics must each resolve to one entity. + // This is also the --validate-only path, so a dry run reports the same + // failures. + const validationErrors = validatePushRequirements(loaded.models); + if (validationErrors.length) { + for (const e of validationErrors) { + console.error(`Error: ${e}`); } + return 1; } - if (result.success) { - if (options.validateOnly) { - console.log('Validation complete; no changes applied.'); - } - else { - console.log( - `Deployed ${result.deployed} BigQuery Graph(s). ` + - `Knowledge Catalog resource emit for the semantic model is not yet implemented.`); + // Live pre-flight over the same models, before any destination runs: every + // entity's BigQuery source table must be reachable, so a push to either + // BigQuery or Knowledge Catalog fails fast when the model could not deploy. + // Runs for every --target and for --validate-only. + const accessErrors = await validateBigQueryDataSources( + loaded.models, new BigQueryClient(ctx), source.project ?? ctx.project); + if (accessErrors.length) { + for (const e of accessErrors) { + console.error(`Error: ${e}`); } - return 0; + return 1; } - console.error('Error pushing semantic model:', result.details); - return 1; + + // Run the resolved destinations in canonical order (BigQuery first); the + // early return below fails fast, skipping later legs when an earlier one + // fails. + for (const target of targets) { + const code = target === 'bigquery' + ? await pushBigQuery(loaded.models, ctx, options) + : await pushKnowledgeCatalog(loaded.models, ctx, options, source); + if (code !== 0) return code; + } + return 0; } const catalog = new dataplex.CatalogClient(ctx); @@ -149,3 +281,163 @@ export async function push(options: PushOptions): Promise { return 1; } } + + +// Deploys the semantic model's BigQuery Graph leg (over the pre-loaded models) +// and prints the result. Returns a process exit code (0 on success). +async function pushBigQuery( + models: LoadedModel[], ctx: context.ApiContext, + options: PushOptions): Promise { + console.log(options.validateOnly + ? 'Validating semantic model for BigQuery Graph...' + : 'Pushing semantic model (BigQuery Graph)...'); + const result = await deploy.deployBigQuery(models, ctx, options); + + for (const w of result.warnings) { + console.warn(`Warning: ${w}`); + } + if (options.print) { + console.log('-- BigQuery Graph --'); + for (const block of result.ddl) { + console.log(`${block}\n`); + } + } + + if (!result.success) { + console.error('Error pushing semantic model to BigQuery:', result.details); + return 1; + } + console.log(options.validateOnly + ? 'Validation complete; no changes applied.' + : `Deployed ${result.deployed} BigQuery Graph(s).`); + return 0; +} + + +// Deploys the semantic model's Knowledge Catalog leg (over the pre-loaded +// models) and prints the result. The destination coordinates come from the +// scope (project.location.entryGroup). Returns a process exit code (0 on +// success). +async function pushKnowledgeCatalog( + models: LoadedModel[], ctx: context.ApiContext, + options: PushOptions, source: SemanticModelSource): Promise { + console.log(options.validateOnly + ? 'Validating semantic model for Knowledge Catalog...' + : 'Pushing semantic model (Knowledge Catalog)...'); + const result = await kc.deployKnowledgeCatalog(models, ctx, { + project: source.project, + location: source.location, + entryGroup: source.entryGroup, + validateOnly: options.validateOnly, + forceRemove: options.forceRemove, + emitExpressions: options.emitExpressions, + }); + + for (const w of result.warnings) { + console.warn(`Warning: ${w}`); + } + if (options.print) { + console.log('-- Knowledge Catalog --'); + for (const line of result.plan) { + console.log(line); + } + } + + if (!result.success) { + console.error( + 'Error pushing semantic model to Knowledge Catalog:', result.details); + return 1; + } + const n = result.created + result.updated; + const removed = result.deleted + ? `; removed ${result.deleted} orphaned entr${ + result.deleted === 1 ? 'y' : 'ies'}` + : ''; + const linked = result.linked + ? `; linked ${result.linked} relationship${result.linked === 1 ? '' : 's'}` + : ''; + const unlinked = result.unlinked + ? `; unlinked ${result.unlinked} orphaned link${ + result.unlinked === 1 ? '' : 's'}` + : ''; + console.log(options.validateOnly + ? 'Validation complete; no changes applied.' + : `Wrote ${result.created} new and ${result.updated} updated ` + + `Knowledge Catalog entr${n === 1 ? 'y' : 'ies'}${removed}${linked}${ + unlinked}.`); + return 0; +} + + +// Pulls the semantic model's Knowledge Catalog entries back into local model +// documents (catalog/EntryGroups//.yaml) and prints the +// result. The destination coordinates come from the scope +// (project.location.entryGroup). Overwrite policy matches the core pull: +// last-write-wins, local-only documents are left untouched (never deleted). +// Returns a process exit code (0 on success). +async function pullSemanticModel( + ctx: context.ApiContext, snapshot: kcmd.CatalogSnapshot, + options: PullOptions): Promise { + // The semantic-model source always resolves to the SemanticModel layout + // (see createLayout), so these casts are safe. + const layout = snapshot.layout as SemanticModelLayout; + const source = snapshot.manifest.source as SemanticModelSource; + + console.log(options.dryRun + ? 'Reconstructing semantic model from Knowledge Catalog (dry run)...' + : 'Pulling semantic model from Knowledge Catalog...'); + + const catalog = new dataplex.CatalogClient(ctx); + const result = await pullKnowledgeCatalog(catalog, { + project: source.project, + location: source.location, + entryGroup: source.entryGroup, + model: options.model, + }); + + for (const w of result.warnings) { + console.warn(`Warning: ${w}`); + } + + if (!result.models.length) { + console.log('No semantic models found; nothing to pull.'); + return 0; + } + + let created = 0; + let updated = 0; + // Guard against two reconstructed models whose names map to the same file + // (path-separator sanitizing, or two anchors sharing a display name): the + // later write would silently clobber the earlier. Track written paths so the + // collision is reported and the dry-run/real counts agree on the repeat. + const writtenBy = new Map(); + for (const model of result.models) { + const serialized = serializeModel(model); + for (const w of serialized.warnings) { + console.warn(`Warning: [${model.name}] ${w}`); + } + const target = layout.modelPath(model.name); + const prior = writtenBy.get(target); + if (prior !== undefined && prior !== model.name) { + console.warn( + `Warning: models '${prior}' and '${model.name}' both map to ` + + `${target}; the later overwrites the earlier -- rename one model.`); + } + const existed = writtenBy.has(target) || layout.hasModel(model.name); + writtenBy.set(target, model.name); + if (options.dryRun) { + console.log(` would ${existed ? 'update' : 'create'} ${target}`); + } + else { + layout.writeModelDocument(model.name, serialized.yaml); + console.log(` ${existed ? 'updated' : 'created'} ${target}`); + } + if (existed) updated++; + else created++; + } + + console.log(options.dryRun + ? `Dry run: would write ${created} new and ${updated} updated model document(s).` + : `Wrote ${created} new and ${updated} updated model document(s).`); + return 0; +} diff --git a/toolbox/mdcode/src/tool/main.ts b/toolbox/mdcode/src/tool/main.ts index 75aa710e..84cceff9 100644 --- a/toolbox/mdcode/src/tool/main.ts +++ b/toolbox/mdcode/src/tool/main.ts @@ -25,10 +25,12 @@ cli.command('init', 'Initialize a new catalog snapshot') cli.command('pull', 'Pull catalog entries') - .action(async () => { + .option('--dry-run', 'Reconstruct and report only; do not write files (semantic-model scope)') + .option('--model ', 'Limit the pull to a single model by name (semantic-model scope)') + .action(async (options) => { let exitCode = 1; try { - exitCode = await commands.pull(); + exitCode = await commands.pull(options); } catch (err: any) { console.error('Error:', err.message || err); @@ -40,7 +42,11 @@ cli.command('pull', 'Pull catalog entries') cli.command('push', 'Push catalog entries') .option('--force', 'Force push changes') + .option('--force-remove', 'Delete Knowledge Catalog models in the entry group that this push does not include (removed/renamed models); semantic-model push only') + .option('--emit-expressions', 'Emit SQL-expression fields not yet in the published Knowledge Catalog system-type templates (per-field schema semantics, metric expression); off by default, enable once the templates support them; semantic-model push only') .option('--validate-only', 'Only validate changes without applying') + .option('--target ', 'Semantic-model push destination(s): bq, kc, all (default), or a comma-separated list (e.g. bq,kc)') + .option('--print', 'Print each pushed destination\'s generated artifact in its native format (BigQuery Graph SQL DDL, Knowledge Catalog entry plan); scope with --target (semantic-model push only)') .action(async (options) => { let exitCode = 1; try { diff --git a/toolbox/mdcode/tests/libts/gcp/dataplex.test.ts b/toolbox/mdcode/tests/libts/gcp/dataplex.test.ts new file mode 100644 index 00000000..6aa54e80 --- /dev/null +++ b/toolbox/mdcode/tests/libts/gcp/dataplex.test.ts @@ -0,0 +1,116 @@ +// Behavior spec for the entry-link client methods added for KC push +// reconciliation: CatalogClient.lookupEntryLinks (paginated, location-scoped +// custom verb) and CatalogClient.deleteEntryLink. Spies on the low-level +// _get/_delete so it pins the exact URL + query-param shape without a live +// Dataplex. + +import {describe, expect, spyOn, test} from 'bun:test'; + +import {ApiContext} from '../../../src/libts/gcp/context'; +import {CatalogClient, EntryLink} from '../../../src/libts/gcp/dataplex'; + +const CTX = new ApiContext('test-project', 'us', 'test-token'); +const SCHEMA_JOIN = + 'projects/dataplex-types/locations/global/entryLinkTypes/schema-join'; +const ENTRY = 'projects/proj/locations/us/entryGroups/g/entries/e1'; + +function link(id: string): EntryLink { + return { + name: `projects/proj/locations/us/entryGroups/g/entryLinks/${id}`, + entryLinkType: SCHEMA_JOIN, + entryReferences: [], + }; +} + +function idOf(l: EntryLink): string { + return l.name!.split('/').pop()!; +} + +describe('CatalogClient.lookupEntryLinks', () => { + test('issues a location-scoped :lookupEntryLinks GET with entry + filters', + async () => { + const client = new CatalogClient(CTX); + const get = spyOn(client as any, '_get') + .mockImplementation( + async () => + ({status: 200, result: {entryLinks: [link('a')]}})); + + const res = await client.lookupEntryLinks('proj', 'us', { + entry: ENTRY, + entryLinkTypes: [SCHEMA_JOIN], + entryMode: 'SOURCE', + }); + + expect(res.status).toBe(200); + expect(res.result?.map(idOf)).toEqual(['a']); + expect(get).toHaveBeenCalledTimes(1); + const [url, params] = get.mock.calls[0] as [string, any]; + expect(url).toBe('projects/proj/locations/us:lookupEntryLinks'); + expect(params.entry).toBe(ENTRY); + expect(params.entryLinkTypes).toEqual([SCHEMA_JOIN]); + expect(params.entryMode).toBe('SOURCE'); + expect(params.pageSize).toBe(10); + }); + + test('drains every page and flattens the links', async () => { + const client = new CatalogClient(CTX); + let call = 0; + const get = spyOn(client as any, '_get').mockImplementation(async () => { + call++; + if (call === 1) { + return { + status: 200, + result: {entryLinks: [link('a'), link('b')], nextPageToken: 'T'}, + }; + } + return {status: 200, result: {entryLinks: [link('c')]}}; + }); + + const res = await client.lookupEntryLinks('proj', 'us', {entry: ENTRY}); + + expect(get).toHaveBeenCalledTimes(2); + // The second page must carry the token the first page returned. + expect((get.mock.calls[1][1] as any).pageToken).toBe('T'); + expect(res.result?.map(idOf)).toEqual(['a', 'b', 'c']); + }); + + test('a non-200 page aborts and is returned as-is', async () => { + const client = new CatalogClient(CTX); + const get = + spyOn(client as any, '_get') + .mockImplementation(async () => ({status: 403, message: 'denied'})); + + const res = await client.lookupEntryLinks('proj', 'us', {entry: ENTRY}); + + expect(res.status).toBe(403); + expect(res.message).toBe('denied'); + expect(res.result).toBeUndefined(); + expect(get).toHaveBeenCalledTimes(1); + }); + + test('an entry with no links returns an empty list, not undefined', + async () => { + const client = new CatalogClient(CTX); + spyOn(client as any, '_get') + .mockImplementation(async () => ({status: 200, result: {}})); + + const res = await client.lookupEntryLinks('proj', 'us', {entry: ENTRY}); + + expect(res.status).toBe(200); + expect(res.result).toEqual([]); + }); +}); + +describe('CatalogClient.deleteEntryLink', () => { + test('DELETEs the fully-qualified entry-link resource', async () => { + const client = new CatalogClient(CTX); + const del = spyOn(client as any, '_delete') + .mockImplementation(async () => ({status: 200})); + + await client.deleteEntryLink('proj', 'us', 'grp', 'my-link'); + + expect(del).toHaveBeenCalledTimes(1); + expect(del.mock.calls[0][0]) + .toBe('projects/proj/locations/us/entryGroups/grp/entryLinks/my-link'); + }); +}); diff --git a/toolbox/mdcode/tests/libts/mocks.ts b/toolbox/mdcode/tests/libts/mocks.ts index 3c3cc20a..6518e450 100644 --- a/toolbox/mdcode/tests/libts/mocks.ts +++ b/toolbox/mdcode/tests/libts/mocks.ts @@ -169,6 +169,15 @@ export class BigQueryClientMock extends bigquery.BigQueryClient { return { status: 404, message: 'Not found' }; } + async getTable(project: string, dataset: string, table: string): Promise> { + const name = `projects/${project}/datasets/${dataset}/tables/${table}`; + const resource = this.mockTables.get(name); + if (resource) { + return { status: 200, result: resource }; + } + return { status: 404, message: 'Not found' }; + } + async *listTables(project: string, dataset: string): AsyncGenerator { for (const table of this.mockTables.values()) { if (table.tableReference.projectId === project && table.tableReference.datasetId === dataset) { @@ -176,4 +185,38 @@ export class BigQueryClientMock extends bigquery.BigQueryClient { } } } + + // Reachable sources for the dry-run probe (validateBigQueryDataSources). Use + // for reference forms tables.get cannot address, e.g. a four-part REST-catalog + // name; a three-part table added via addMockTable is reachable too. + public mockSources: Set = new Set(); + + addMockSource(ref: string) { + this.mockSources.add(ref); + } + + // Dry-run table probe: a query of the form `SELECT 1 FROM \`\`` resolves + // against the reachable set (mockSources, plus any three-part table added via + // addMockTable). An unknown reference returns a BigQuery-style "Not found" so + // the validator reports it; a non-probe query is a benign success. + async query(project: string, sql: string, location?: string, dryRun?: boolean): + Promise> { + const m = /FROM `([^`]+)`/.exec(sql); + if (dryRun && m) { + return this.isReachableSource(m[1]) ? + {status: 200, result: {}} : + {status: 400, message: `Not found: Table ${m[1]} was not found`}; + } + return {status: 200, result: {}}; + } + + private isReachableSource(ref: string): boolean { + if (this.mockSources.has(ref)) return true; + const parts = ref.split('.'); + if (parts.length === 3) { + return this.mockTables.has( + `projects/${parts[0]}/datasets/${parts[1]}/tables/${parts[2]}`); + } + return false; + } } diff --git a/toolbox/mdcode/tests/libts/semantic/deploy.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_bigquery.test.ts similarity index 87% rename from toolbox/mdcode/tests/libts/semantic/deploy.test.ts rename to toolbox/mdcode/tests/libts/semantic/deploy_bigquery.test.ts index 7d930e17..b24595ec 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/deploy_bigquery.test.ts @@ -1,5 +1,5 @@ // Tests for the semantic-model BigQuery deploy leg -// (src/libts/semantic/deploy.ts). +// (src/libts/semantic/deploy_bigquery.ts). // // `bigQueryGraphTargets` is exercised as a pure function; `deployBigQuery` is // exercised end to end over an Ossie fixture (loader -> IR -> generator -> @@ -11,11 +11,22 @@ import * as path from 'node:path'; import * as bq from '../../../src/libts/gcp/bigquery'; import {ApiContext} from '../../../src/libts/gcp/context'; -import {bigQueryGraphTargets, deployBigQuery} from '../../../src/libts/semantic/deploy'; +import {bigQueryGraphTargets, deployBigQuery} from '../../../src/libts/semantic/deploy_bigquery'; import {SemanticModel} from '../../../src/libts/semantic/ir'; +import {loadSemanticModels} from '../../../src/libts/semantic/loader'; const CTX = new ApiContext('test-project', 'us', 'test-token'); +// The deploy leg now consumes models already parsed by loadSemanticModels +// (shared with the Knowledge Catalog leg). These tests still author documents, +// so this helper parses them the way commands.ts does. defaultProject mirrors +// the scope's declared project. +function models(docs: {name: string; text: string}[], defaultProject = 'test-project') { + const r = loadSemanticModels(docs, {defaultProject}); + if (r.error) throw new Error(r.error); + return r.models; +} + const FIXTURES = path.join(__dirname, 'fixtures'); const BQ_URI = @@ -28,7 +39,7 @@ const OSSIE = const OSSIE_NO_TARGET = fs.readFileSync(path.join(FIXTURES, 'sales_no_target.yaml'), 'utf8'); // An existing fixture whose only GOOGLE deployment target is a (non-BigQuery) -// Dataplex URI; exercises the deploy leg's "no BigQuery Graph target" path. +// Dataplex URI; exercises the deploy leg's unparseable-target path. const OSSIE_DATAPLEX_ONLY = fs.readFileSync(path.join(FIXTURES, 'sales_google_ext.yaml'), 'utf8'); // A model whose only GOOGLE target carries the BigQuery prefix but fails the @@ -73,14 +84,15 @@ describe('bigQueryGraphTargets', () => { expect(targets).toEqual([]); }); - test('ignores deployment targets that are not BigQuery Graphs', () => { - // A non-BigQuery URI is not our concern: neither a target nor malformed. + test('reports a non-BigQuery deployment target as malformed', () => { + // We only support BigQuery Graph targets, so any other URI does not parse + // as one and is collected as malformed (rejected), not silently ignored. const dataplexUri = 'projects/demo/locations/us/entryGroups/@bigquery/entries/sales_graph'; const {targets, malformed} = bigQueryGraphTargets( modelWithExtension(JSON.stringify({deploymentTargets: [dataplexUri]}))); expect(targets).toEqual([]); - expect(malformed).toEqual([]); + expect(malformed).toEqual([dataplexUri]); }); test('returns [] for a model with no custom extensions', () => { @@ -117,9 +129,8 @@ describe('bigQueryGraphTargets', () => { test('reports a host/scheme-typo target as malformed, not silently dropped', () => { // A truncated host (`.co` not `.com`) and an `https://` scheme both - // fail the strict match, yet clearly intend a BigQuery Graph target. - // They must be reported (via the /propertyGraphs?/ + host hint) rather - // than dropped and later misreported as "no target declared". + // fail the strict match; both are reported as malformed rather than + // dropped and later misreported as "no target declared". const hostTypo = '//bigquery.googleapis.co/projects/demo/datasets/sales/propertyGraphs/g'; const scheme = @@ -146,7 +157,7 @@ describe('deployBigQuery', () => { const querySpy = spyOn(bq.BigQueryClient.prototype, 'query'); try { const res = await deployBigQuery( - [{name: 'sales', text: OSSIE}], CTX, {validateOnly: true}); + models([{name: 'sales', text: OSSIE}]), CTX, {validateOnly: true}); expect(res.success).toBe(true); expect(res.deployed).toBe(0); expect(querySpy).not.toHaveBeenCalled(); @@ -169,7 +180,7 @@ describe('deployBigQuery', () => { .mockImplementation( async () => ({status: 200, result: {jobComplete: true}})); try { - const res = await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + const res = await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(true); expect(res.deployed).toBe(1); expect(querySpy).toHaveBeenCalledTimes(1); @@ -201,7 +212,7 @@ describe('deployBigQuery', () => { .mockImplementation( async () => ({status: 200, result: {jobComplete: true}})); try { - const res = await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + const res = await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(true); expect(pollSpy).toHaveBeenCalledTimes(1); // getQueryResults(project, jobId, location) -- jobId is the second arg. @@ -225,7 +236,7 @@ describe('deployBigQuery', () => { async () => ({status: 200, result: {jobComplete: false}})); try { const res = - await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.details).toContain('no job reference to poll'); } finally { @@ -256,7 +267,7 @@ describe('deployBigQuery', () => { async () => ({status: 200, result: {status: {}}})); try { const res = - await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(true); expect(jobSpy).toHaveBeenCalledTimes(1); } finally { @@ -287,7 +298,7 @@ describe('deployBigQuery', () => { }, })); try { - const res = await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + const res = await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.details).toContain('Not found: table'); } finally { @@ -313,7 +324,7 @@ describe('deployBigQuery', () => { })); try { const res = - await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.details).toContain('graph already exists'); } finally { @@ -328,7 +339,7 @@ describe('deployBigQuery', () => { .mockImplementation( async () => ({status: 400, message: 'bad DDL'})); try { - const res = await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + const res = await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.details).toContain('bad DDL'); } finally { @@ -354,7 +365,7 @@ describe('deployBigQuery', () => { .mockImplementation(async () => stuck); try { const res = await deployBigQuery( - [{name: 'sales', text: OSSIE}], CTX, + models([{name: 'sales', text: OSSIE}]), CTX, {maxQueryPolls: 3, pollBackoffMs: 0}); expect(res.success).toBe(false); expect(res.details).toContain('did not complete after 3 polls'); @@ -373,7 +384,7 @@ describe('deployBigQuery', () => { .mockImplementation(async () => ({status: 404} as any)); const querySpy = spyOn(bq.BigQueryClient.prototype, 'query'); try { - const res = await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + const res = await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.details).toContain('demo.sales not found'); expect(querySpy).not.toHaveBeenCalled(); @@ -398,7 +409,7 @@ describe('deployBigQuery', () => { async () => ({status: 200, result: {jobComplete: true}})); try { const res = - await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(true); expect(querySpy.mock.calls[0][2]).toBeUndefined(); expect(res.warnings.some( @@ -422,7 +433,7 @@ describe('deployBigQuery', () => { async () => ({status: 200, result: {jobComplete: true}})); try { const res = await deployBigQuery( - [{name: 'a', text: OSSIE}, {name: 'b', text: OSSIE}], CTX, {}); + models([{name: 'a', text: OSSIE}, {name: 'b', text: OSSIE}]), CTX, {}); expect(res.success).toBe(true); expect(res.deployed).toBe(2); expect(dsSpy).toHaveBeenCalledTimes(1); @@ -438,7 +449,7 @@ describe('deployBigQuery', () => { // A typo'd URI that carries the BigQuery prefix must be reported as // unparseable, not as "declares no deploymentTarget". const res = await deployBigQuery( - [{name: 'sales', text: OSSIE_BAD_TARGET}], CTX, + models([{name: 'sales', text: OSSIE_BAD_TARGET}]), CTX, {validateOnly: true}); expect(res.success).toBe(false); expect(res.details).toContain('could not be parsed'); @@ -462,7 +473,7 @@ describe('deployBigQuery', () => { }); try { const res = await deployBigQuery( - [{name: 'a', text: OSSIE}, {name: 'b', text: OSSIE}], CTX, {}); + models([{name: 'a', text: OSSIE}, {name: 'b', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.deployed).toBe(1); expect(res.details).toContain('boom'); @@ -473,37 +484,25 @@ describe('deployBigQuery', () => { } }); - test( - 'fails with the document name when a document cannot be parsed', - async () => { - // A malformed doc must produce a clean, document-scoped failure rather - // than an uncaught loader exception. - const res = await deployBigQuery( - [ - {name: 'sales', text: OSSIE}, {name: 'broken', text: 'other: 1\n'} - ], - CTX, {validateOnly: true}); - expect(res.success).toBe(false); - expect(res.details).toContain('Model document \'broken\''); - }); - test( 'fails when a model declares no BigQuery deployment target', async () => { const res = await deployBigQuery( - [{name: 'sales', text: OSSIE_NO_TARGET}], CTX, + models([{name: 'sales', text: OSSIE_NO_TARGET}]), CTX, {validateOnly: true}); expect(res.success).toBe(false); expect(res.details).toContain('no BigQuery Graph'); }); test( - 'fails when the only deployment target is a non-BigQuery destination', + 'fails when the only deployment target is not a BigQuery Graph URI', async () => { + // A non-BigQuery destination does not parse as a BigQuery Graph target, + // so it is reported as unparseable rather than "none declared". const res = await deployBigQuery( - [{name: 'sales', text: OSSIE_DATAPLEX_ONLY}], CTX, + models([{name: 'sales', text: OSSIE_DATAPLEX_ONLY}]), CTX, {validateOnly: true}); expect(res.success).toBe(false); - expect(res.details).toContain('no BigQuery Graph'); + expect(res.details).toContain('could not be parsed'); }); test( @@ -513,8 +512,8 @@ describe('deployBigQuery', () => { // ('scope-proj'), not the ambient ctx.project ('test-project'): the // scope's declared project is deterministic where gcloud's is not. const res = await deployBigQuery( - [{name: 'sales', text: OSSIE_UNQUALIFIED_SOURCE}], CTX, - {validateOnly: true}, 'scope-proj'); + models([{name: 'sales', text: OSSIE_UNQUALIFIED_SOURCE}], 'scope-proj'), + CTX, {validateOnly: true}); expect(res.success).toBe(true); const ddl = res.ddl.join('\n'); expect(ddl).toContain('`scope-proj.sales.orders`'); @@ -547,7 +546,7 @@ describe('deployBigQuery', () => { spyOn(bq.BigQueryClient.prototype, 'query') .mockImplementation(async () => ({status: 200} as any)); try { - const res = await deployBigQuery([{name: 'sales', text: OSSIE}], CTX, {}); + const res = await deployBigQuery(models([{name: 'sales', text: OSSIE}]), CTX, {}); expect(res.success).toBe(false); expect(res.details).toContain('no response body'); } finally { diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts new file mode 100644 index 00000000..1d637505 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts @@ -0,0 +1,637 @@ +// Tests for the semantic-model Knowledge Catalog deploy leg +// (src/libts/semantic/deploy_knowledge_catalog.ts). +// +// `deployKnowledgeCatalog` is exercised end to end over an Ossie fixture +// (loader -> IR -> emitter -> writes), with the catalog client stubbed so no +// network call is made. The focus is the publish SEQUENCE the emitter +// goldens cannot show: anchor-first entry writes (the entry group is +// provisioned at `init`, not here), idempotent upsert on re-push, delete +// reconciliation, and the dry-run plan. + +import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import {ApiResult} from '../../../src/libts/gcp/api'; +import {ApiContext} from '../../../src/libts/gcp/context'; +import {CatalogClient} from '../../../src/libts/gcp/dataplex'; +import {deployKnowledgeCatalog} from '../../../src/libts/semantic/deploy_knowledge_catalog'; +import {loadSemanticModels} from '../../../src/libts/semantic/loader'; + +const CTX = new ApiContext('test-project', 'us', 'test-token'); +const FIXTURES = path.join(__dirname, 'fixtures'); + +// A loader-valid model: one entity (orders) + one metric (total_revenue), so a +// push writes exactly three entries (model anchor + entity + metric). +const OSSIE = + fs.readFileSync(path.join(FIXTURES, 'sales_bq_graph_target.yaml'), 'utf8'); +const DOCS = [{name: 'sales.yaml', text: OSSIE}]; + +// A model with a direct-FK relationship: 5 entries (anchor + 2 entities + 2 +// metrics) plus 1 schema-join entry link (orders -> customer). Used to exercise +// the link write path the OSSIE fixture (no relationship) cannot. +const STAR = + fs.readFileSync(path.join(FIXTURES, 'star_orders_customer.yaml'), 'utf8'); +const STAR_DOCS = [{name: 'star.yaml', text: STAR}]; + +// The deploy leg now consumes models already parsed by loadSemanticModels +// (shared with the BigQuery leg). These tests author documents, so this helper +// parses them the way commands.ts does. +function models(docs: {name: string; text: string}[]) { + const r = loadSemanticModels(docs, {defaultProject: 'test-project'}); + if (r.error) throw new Error(r.error); + return r.models; +} + +// A destination entry name for a given entry id, as the catalog returns it from +// listEntries; reconciliation keys off the id (the last path segment). +function entryName(id: string): string { + return `projects/dest/locations/us/entryGroups/eg/entries/${id}`; +} + +// The same loader-valid model, but its GOOGLE custom_extension carries invalid +// JSON: the doc parses, yet the emitter (via bigQueryGraphTargets) throws while +// building the semantic-model aspect. Surgically swap only the `data:` value so +// the rest of the document stays valid. +const MALFORMED_EXTENSION = + OSSIE.replace(/data: '[^\n]*'/, 'data: \'not valid json\''); + +// entryCreateTries: 1 keeps the propagation-retry loop from sleeping in tests. +const OPTS = { + project: 'dest', + location: 'us', + entryGroup: 'eg', + entryCreateTries: 1 +}; + +// bun's spyOn accumulates calls across tests; restore originals after each so +// per-test call counts and ordering assertions are isolated. +afterEach(() => { + mock.restore(); +}); + +function ok(result?: T): ApiResult { + return {status: 200, result}; +} +function err(status: number, message: string): ApiResult { + return {status, message}; +} + +// Stubs createEntryGroup + createEntry (+ optionally updateEntry) and the entry- +// link writes, returning the spies so a test can assert call counts and +// ordering. +function stubClient(opts: { + group?: ApiResult, + create?: (entryId: string) => ApiResult, + update?: ApiResult, + // Entries the destination entry group already holds (yielded by listEntries). + // A bare id defaults to a generic entryType; pass {id, type} to stage a + // specific one (e.g. a foreign semantic-model anchor). Default: none. + existing?: (string | {id: string; type: string})[], + del?: (entryId: string) => ApiResult, + // Result of createEntryLink, keyed by the entry-link id. Default: 200. + createLink?: (linkId: string) => ApiResult, + updateLink?: ApiResult, + // Entry links the destination returns from lookupEntryLinks, keyed by the + // referenced entry's full resource name. Default: none (ok, empty list). + links?: (entry: string) => ApiResult, + // Result of deleteEntryLink, keyed by the entry-link id. Default: 200. + delLink?: (linkId: string) => ApiResult, +} = {}) { + const group = spyOn(CatalogClient.prototype, 'createEntryGroup') + .mockImplementation(async () => opts.group ?? ok({})); + const create = spyOn(CatalogClient.prototype, 'createEntry') + .mockImplementation( + async (_p, _l, _eg, entryId) => + (opts.create ?? (() => ok({})))(entryId)); + const update = spyOn(CatalogClient.prototype, 'updateEntry') + .mockImplementation(async () => opts.update ?? ok({})); + const list = spyOn(CatalogClient.prototype, 'listEntries') + .mockImplementation(async function*() { + for (const e of opts.existing ?? []) { + const id = typeof e === 'string' ? e : e.id; + const entryType = + typeof e === 'string' ? 'semantic' : e.type; + yield {name: entryName(id), entryType} as any; + } + }); + const del = spyOn(CatalogClient.prototype, 'deleteEntry') + .mockImplementation( + async (_p, _l, _eg, entryId) => + (opts.del ?? (() => ok({})))(entryId)); + const createLink = spyOn(CatalogClient.prototype, 'createEntryLink') + .mockImplementation( + async (_p, _l, _eg, linkId) => + (opts.createLink ?? (() => ok({})))(linkId)); + const updateLink = spyOn(CatalogClient.prototype, 'updateEntryLink') + .mockImplementation( + async () => opts.updateLink ?? ok({})); + const lookupLinks = + spyOn(CatalogClient.prototype, 'lookupEntryLinks') + .mockImplementation( + async (_p, _l, o: any) => (opts.links ?? (() => ok([])))(o.entry)); + const delLink = spyOn(CatalogClient.prototype, 'deleteEntryLink') + .mockImplementation( + async (_p, _l, _eg, linkId) => + (opts.delLink ?? (() => ok({})))(linkId)); + return {group, create, update, list, + del, createLink, updateLink, + lookupLinks, delLink}; +} + + +describe('deployKnowledgeCatalog: happy path', () => { + test('writes entries anchor-first, provisioning nothing', async () => { + const {group, create, update} = stubClient(); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.created).toBe(3); + expect(result.updated).toBe(0); + + // Push provisions neither the entry group (created at `init`) nor any type + // (the semantic types are built-in): it only writes the three entries. + expect(group).not.toHaveBeenCalled(); + expect(create).toHaveBeenCalledTimes(3); + expect(update).not.toHaveBeenCalled(); + + // The model anchor is written before its children. + const firstEntryId = create.mock.calls[0][3]; + expect(firstEntryId).toBe('sales'); + }); +}); + + +describe('deployKnowledgeCatalog: re-push upserts', () => { + test('an entry that already exists is updated in place', async () => { + const {create, update} = stubClient({ + create: () => err(409, 'entry already exists'), + update: ok({}), + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.created).toBe(0); + expect(result.updated).toBe(3); + expect(create).toHaveBeenCalledTimes(3); + expect(update).toHaveBeenCalledTimes(3); + }); +}); + + +describe('deployKnowledgeCatalog: relationship entry links', () => { + test('a direct-FK relationship is written as one schema-join link', async () => { + const {create, createLink, updateLink} = stubClient(); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.created).toBe(5); // anchor + 2 entities + 2 metrics + expect(result.linked).toBe(1); + expect(create).toHaveBeenCalledTimes(5); + expect(createLink).toHaveBeenCalledTimes(1); + expect(updateLink).not.toHaveBeenCalled(); + // Links are written to the same destination the entries are. + const [project, location, entryGroup, linkId] = createLink.mock.calls[0]; + expect(project).toBe('dest'); + expect(location).toBe('us'); + expect(entryGroup).toBe('eg'); + expect(linkId).toBe('sales-orders-to-customer'); + }); + + test('a link that already exists is upserted via updateEntryLink', async () => { + const {createLink, updateLink} = stubClient({ + createLink: () => err(409, 'entry link already exists'), + updateLink: ok({}), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.linked).toBe(1); + expect(createLink).toHaveBeenCalledTimes(1); + expect(updateLink).toHaveBeenCalledTimes(1); + // The upsert narrows the patch to the link's aspect keys (schema-join); + // UpdateEntryLink has no update mask, so a stale ['aspects'] mask would 400. + const aspectKeys = updateLink.mock.calls[0][1] ?? []; + expect(aspectKeys.some((k: string) => k.endsWith('schema-join'))).toBe(true); + }); + + test('a failed link write fails the push, naming the link', async () => { + stubClient({ + createLink: () => err(500, 'boom'), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(false); + expect(result.details).toContain('sales-orders-to-customer'); + }); + + test('a model with no relationships writes no links', async () => { + const {createLink} = stubClient(); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.linked).toBe(0); + expect(createLink).not.toHaveBeenCalled(); + }); +}); + + +describe('deployKnowledgeCatalog: validateOnly', () => { + test('writes nothing and returns a plan', async () => { + const {group, create} = stubClient(); + + const result = + await deployKnowledgeCatalog(models(DOCS), CTX, {...OPTS, validateOnly: true}); + + expect(result.success).toBe(true); + expect(result.created).toBe(0); + expect(group).not.toHaveBeenCalled(); + expect(create).not.toHaveBeenCalled(); + expect(result.plan.join('\n')).toContain('Knowledge Catalog plan'); + expect(result.plan.join('\n')).toContain('sales'); + }); +}); + + +describe('deployKnowledgeCatalog: failures', () => { + test('a failed anchor write stops before its children', async () => { + const {create} = stubClient({ + create: (id) => id === 'sales' ? err(500, 'boom') : ok({}), + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(false); + expect(create).toHaveBeenCalledTimes(1); // anchor only; children skipped + }); + + test( + 'a model that throws during emit fails with the model + document named', + async () => { + // A parseable doc whose GOOGLE extension is invalid JSON makes the + // emitter throw; the publisher must report it, not crash the push. + const {create} = stubClient(); + const docs = [{name: 'broken.yaml', text: MALFORMED_EXTENSION}]; + + const result = await deployKnowledgeCatalog(models(docs), CTX, OPTS); + + expect(result.success).toBe(false); + expect(result.details).toContain('broken.yaml'); + expect(result.details).toContain('sales'); // the model name + expect(create).not.toHaveBeenCalled(); + }); + + test( + 'more than one model in a single push is rejected before any write', + async () => { + // Only one model per entry group is supported, so a push carrying two + // documents is rejected up front -- which also means two models can + // never race for the same entry id. + const {group, create} = stubClient(); + const docs = [ + {name: 'a.yaml', text: OSSIE}, + {name: 'b.yaml', text: OSSIE}, + ]; + + const result = await deployKnowledgeCatalog(models(docs), CTX, OPTS); + + expect(result.success).toBe(false); + expect(result.details).toContain('one model per entry group'); + expect(group).not.toHaveBeenCalled(); + expect(create).not.toHaveBeenCalled(); + }); + + test('no documents is a hard error for a real push', async () => { + stubClient(); + const result = await deployKnowledgeCatalog([], CTX, OPTS); + expect(result.success).toBe(false); + expect(result.details).toContain('No semantic model documents'); + }); + + test('no documents is a clean no-op for validateOnly', async () => { + stubClient(); + const result = + await deployKnowledgeCatalog([], CTX, {...OPTS, validateOnly: true}); + expect(result.success).toBe(true); + }); +}); + + +describe('deployKnowledgeCatalog: delete reconciliation', () => { + // The fixture model 'sales' emits exactly three ids: the anchor 'sales', the + // entity 'sales.entities.orders', and the metric 'sales.metrics.total_revenue'. + const EMITTED = ['sales', 'sales.entities.orders', 'sales.metrics.total_revenue']; + + test('deletes entities/metrics removed from the model, scoped by owner', async () => { + // The group also holds two orphans owned by the 'sales' anchor (an entity + // and a metric no longer in the model) plus two entries owned by a + // different model. Only the two orphans under 'sales' must be deleted. + const {del, list} = stubClient({ + existing: [ + ...EMITTED, + 'sales.entities.removed', + 'sales.metrics.removed', + 'other', + 'other.entities.x', + ], + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.deleted).toBe(2); + expect(list).toHaveBeenCalledTimes(1); + const deletedIds = del.mock.calls.map(c => c[3]).sort(); + expect(deletedIds).toEqual(['sales.entities.removed', 'sales.metrics.removed']); + }); + + test('deletes nothing when the model still emits every entry', async () => { + const {del} = stubClient({existing: EMITTED}); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.deleted).toBe(0); + expect(del).not.toHaveBeenCalled(); + }); + + test('a 404 on an orphan is tolerated (already gone)', async () => { + const {del} = stubClient({ + existing: [...EMITTED, 'sales.entities.removed'], + del: () => err(404, 'not found'), + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.deleted).toBe(1); + expect(del).toHaveBeenCalledTimes(1); + }); + + test('a failed delete fails the push, naming the entry', async () => { + const {del} = stubClient({ + existing: [...EMITTED, 'sales.metrics.removed'], + del: () => err(500, 'boom'), + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(false); + expect(result.details).toContain('sales.metrics.removed'); + expect(del).toHaveBeenCalledTimes(1); + }); + + test('validateOnly never lists or deletes (offline)', async () => { + const {list, del} = stubClient({existing: ['sales.entities.removed']}); + + const result = + await deployKnowledgeCatalog(models(DOCS), CTX, {...OPTS, validateOnly: true}); + + expect(result.success).toBe(true); + expect(result.deleted).toBe(0); + expect(list).not.toHaveBeenCalled(); + expect(del).not.toHaveBeenCalled(); + }); +}); + + +// Built-in system entry types, as the catalog reports them on existing entries. +const MODEL_TYPE = + 'projects/dataplex-types/locations/global/entryTypes/semantic-model'; +const ENTITY_TYPE = + 'projects/dataplex-types/locations/global/entryTypes/semantic-entity'; +const METRIC_TYPE = + 'projects/dataplex-types/locations/global/entryTypes/semantic-metric'; + +// A schema-join entry link as lookupEntryLinks returns it: `refIds` are the two +// endpoint entry ids (undirected, both UNSPECIFIED). +function linkEntry(id: string, refIds: string[]): any { + return { + name: `projects/dest/locations/us/entryGroups/eg/entryLinks/${id}`, + entryLinkType: + 'projects/dataplex-types/locations/global/entryLinkTypes/schema-join', + entryReferences: refIds.map(r => ({name: entryName(r), type: 'UNSPECIFIED'})), + }; +} + + +describe('deployKnowledgeCatalog: link reconciliation', () => { + // The STAR model 'sales' has entities orders + customer and emits exactly one + // schema-join link, 'sales-orders-to-customer'. + const KEPT = linkEntry( + 'sales-orders-to-customer', + ['sales.entities.orders', 'sales.entities.customer']); + + // The model's entity entries as the destination already holds them (a + // re-push). reconcileLinks looks up links via these server-known entities, so + // they must be in the pre-write listing for a link to be discoverable. + const STAR_ENTITIES = [ + {id: 'sales.entities.orders', type: ENTITY_TYPE}, + {id: 'sales.entities.customer', type: ENTITY_TYPE}, + ]; + + test('deletes an owned schema-join link the model no longer emits', + async () => { + // The server also still holds a link to a dropped relationship (both + // endpoints under this model). It is returned from both endpoints it + // touches (orders + customer), so the dedup path is exercised too. + const orphan = linkEntry( + 'sales-orders-to-supplier', + ['sales.entities.orders', 'sales.entities.supplier']); + const {delLink} = stubClient({ + existing: STAR_ENTITIES, + links: (entry: string) => entry.endsWith('sales.entities.orders') + ? ok([orphan, KEPT]) + : ok([KEPT]), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.unlinked).toBe(1); + expect(delLink).toHaveBeenCalledTimes(1); + expect(delLink.mock.calls[0][3]).toBe('sales-orders-to-supplier'); + }); + + test('keeps a link the model still emits', async () => { + const {delLink} = stubClient({ + existing: STAR_ENTITIES, + links: (entry: string) => + entry.endsWith('sales.entities.orders') ? ok([KEPT]) : ok([KEPT]), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.unlinked).toBe(0); + expect(delLink).not.toHaveBeenCalled(); + }); + + test('never deletes a link that touches an entry outside the model', + async () => { + // One endpoint is another model's entity: not owned, so not this model's + // to reconcile even though it references one of our entities. + const foreign = linkEntry( + 'sales-orders-to-external', + ['sales.entities.orders', 'other.entities.x']); + const {delLink} = stubClient({ + existing: STAR_ENTITIES, + links: (entry: string) => + entry.endsWith('sales.entities.orders') ? ok([foreign]) : ok([]), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.unlinked).toBe(0); + expect(delLink).not.toHaveBeenCalled(); + }); + + test('a failed link delete fails the push, naming the link', async () => { + const orphan = linkEntry( + 'sales-orders-to-supplier', + ['sales.entities.orders', 'sales.entities.supplier']); + const {delLink} = stubClient({ + existing: STAR_ENTITIES, + links: (entry: string) => + entry.endsWith('sales.entities.orders') ? ok([orphan]) : ok([]), + delLink: () => err(500, 'boom'), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(false); + expect(result.details).toContain('sales-orders-to-supplier'); + expect(delLink).toHaveBeenCalledTimes(1); + }); + + test('validateOnly never looks up or deletes links (offline)', async () => { + const {lookupLinks, delLink} = stubClient({ + links: () => ok([linkEntry( + 'sales-orders-to-supplier', + ['sales.entities.orders', 'sales.entities.supplier'])]), + }); + + const result = await deployKnowledgeCatalog( + models(STAR_DOCS), CTX, {...OPTS, validateOnly: true}); + + expect(result.success).toBe(true); + expect(lookupLinks).not.toHaveBeenCalled(); + expect(delLink).not.toHaveBeenCalled(); + }); + + test('deletes a link whose BOTH endpoints were removed in this push', + async () => { + // The dropped relationship's two endpoints are both entities removed from + // the model, so the orphan link is reachable ONLY via those removed + // entries in the pre-write snapshot -- never via a still-emitted entity. + // Enumerating the snapshot (not the emitted set) is what finds it. + const orphan = linkEntry( + 'sales-supplier-to-warehouse', + ['sales.entities.supplier', 'sales.entities.warehouse']); + const {delLink} = stubClient({ + existing: [ + ...STAR_ENTITIES, + {id: 'sales.entities.supplier', type: ENTITY_TYPE}, + {id: 'sales.entities.warehouse', type: ENTITY_TYPE}, + ], + links: (entry: string) => + entry.endsWith('sales.entities.supplier') || + entry.endsWith('sales.entities.warehouse') + ? ok([orphan]) + : ok([]), + }); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.unlinked).toBe(1); + expect(delLink.mock.calls.map(c => c[3])) + .toEqual(['sales-supplier-to-warehouse']); + }); + + test('a first push (empty group) issues no link lookups', async () => { + // No entity of the model exists on the server yet, so there is nothing to + // reconcile and reconcileLinks makes no lookupEntryLinks call. + const {lookupLinks, delLink} = stubClient(); + + const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(lookupLinks).not.toHaveBeenCalled(); + expect(delLink).not.toHaveBeenCalled(); + }); +}); + + +describe('deployKnowledgeCatalog: whole-model removal (--force-remove)', () => { + test('a model already in the group that this push omits fails without the flag', + async () => { + const {create, del} = stubClient({ + existing: [{id: 'old', type: MODEL_TYPE}], + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(false); + expect(result.details).toContain('old'); + expect(result.details).toContain('--force-remove'); + // Nothing is written or deleted -- the guard runs before any mutation. + expect(create).not.toHaveBeenCalled(); + expect(del).not.toHaveBeenCalled(); + }); + + test('--force-remove drops the omitted model (entries + links), then writes', + async () => { + const orphanLink = linkEntry( + 'old-a-to-b', ['old.entities.a', 'old.entities.b']); + const {create, del, delLink} = stubClient({ + existing: [ + {id: 'old', type: MODEL_TYPE}, + {id: 'old.entities.a', type: ENTITY_TYPE}, + {id: 'old.entities.b', type: ENTITY_TYPE}, + {id: 'old.metrics.m', type: METRIC_TYPE}, + ], + links: (entry: string) => + entry.endsWith('old.entities.a') ? ok([orphanLink]) : ok([]), + }); + + const result = await deployKnowledgeCatalog( + models(DOCS), CTX, {...OPTS, forceRemove: true}); + + expect(result.success).toBe(true); + // The foreign model's 4 entries and its 1 link are removed... + expect(result.deleted).toBe(4); + expect(result.unlinked).toBe(1); + expect(delLink.mock.calls.map(c => c[3])).toEqual(['old-a-to-b']); + expect(del.mock.calls.map(c => c[3]).sort()).toEqual( + ['old', 'old.entities.a', 'old.entities.b', 'old.metrics.m']); + // ...and the current model is still written (3 entries). + expect(create).toHaveBeenCalledTimes(3); + }); + + test('an existing anchor this push re-emits is not treated as foreign', + async () => { + // The group already holds the same model being pushed: a normal re-push, + // no --force-remove required, nothing removed. + const {del} = stubClient({ + existing: [ + {id: 'sales', type: MODEL_TYPE}, + {id: 'sales.entities.orders', type: ENTITY_TYPE}, + {id: 'sales.metrics.total_revenue', type: METRIC_TYPE}, + ], + }); + + const result = await deployKnowledgeCatalog(models(DOCS), CTX, OPTS); + + expect(result.success).toBe(true); + expect(result.deleted).toBe(0); + expect(del).not.toHaveBeenCalled(); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.knowledge_catalog.golden.json new file mode 100644 index 00000000..7c56b706 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.knowledge_catalog.golden.json @@ -0,0 +1,79 @@ +{ + "entries": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-model", + "entrySource": { + "displayName": "sales" + }, + "aspects": { + "dataplex-types.global.semantic-model": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-model", + "data": { + "deploymentTargets": [ + "//bigquery.googleapis.com/projects/demo/datasets/sales/propertyGraphs/sales_graph" + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.entities.orders", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "orders" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/demo/datasets/sales/tables/orders" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "o_orderkey", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "o_totalprice", + "dataType": "STRING", + "metadataType": "STRING" + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.metrics.total_revenue", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-metric", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "total_revenue" + }, + "aspects": { + "dataplex-types.global.semantic-metric": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-metric", + "data": { + "entity": "orders", + "dataType": "NUMERIC" + } + } + } + } + ], + "entryLinks": [], + "warnings": [ + "metric 'total_revenue': no datatype in the source model; defaulting the required semantic-metric.dataType to 'NUMERIC'" + ] +} diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml new file mode 100644 index 00000000..ffdfa798 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml @@ -0,0 +1,29 @@ +version: 0.2.0.dev0 +semantic_model: + - name: sales + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": + ["//bigquery.googleapis.com/projects/demo/datasets/sales/propertyGraphs/sales_graph"]}' + datasets: + - name: orders + source: demo.sales.orders + primary_key: + - o_orderkey + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml new file mode 100644 index 00000000..f966f619 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml @@ -0,0 +1,16 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: sales + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets":["//bigquery.googleapis.com/projects/demo/datasets/sales/propertyGraphs/sales_graph"]}' + datasets: + - name: orders + source: demo.sales.orders + fields: + - name: o_orderkey + - name: o_totalprice + metrics: + - name: total_revenue + datatype: Decimal diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.bigquery.golden.sql b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.bigquery.golden.sql index 8120bcb1..ff4bbb89 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.bigquery.golden.sql +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.bigquery.golden.sql @@ -8,7 +8,8 @@ NODE TABLES ( o_custkey, o_orderdate OPTIONS(description="Order Date\n\nTime dimension.\n\nSynonyms: order date, date"), o_totalprice, - MEASURE(SUM(o_totalprice)) AS total_revenue OPTIONS(description="Total order revenue\n\nSynonyms: revenue, sales") + MEASURE(SUM(o_totalprice)) AS total_revenue OPTIONS(description="Total order revenue\n\nSynonyms: revenue, sales"), + MEASURE(COUNT(o_orderkey)) AS order_count OPTIONS(description="Number of orders") ), `samples.tpch.customer` AS customer KEY(c_custkey) @@ -27,5 +28,3 @@ OPTIONS(description="Sales orders with customer attributes\n\nUse this model for -- warnings -- -- note: no 'BIGQUERY' dialect for one or more expressions; using the portable 'ANSI_SQL' dialect verbatim ('BIGQUERY' accepts the ANSI core subset — supply 'BIGQUERY' variants only for BIGQUERY-specific SQL) --- metric 'order_count': expression references no known entity; it may not be placeable downstream --- metric 'order_count' references no known entity; skipped (cannot be a single MEASURE) diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json new file mode 100644 index 00000000..91354101 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json @@ -0,0 +1,191 @@ +{ + "entries": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-model", + "entrySource": { + "displayName": "sales", + "description": "Sales orders with customer attributes" + }, + "aspects": { + "dataplex-types.global.semantic-model": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-model", + "data": { + "deploymentTargets": [ + "//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales" + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.entities.orders", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "orders", + "description": "One row per order" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/samples/datasets/tpch/tables/orders" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "o_orderkey", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Order identifier" + }, + { + "name": "o_custkey", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "o_orderdate", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "o_totalprice", + "dataType": "STRING", + "metadataType": "STRING" + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.entities.customer", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "customer" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/samples/datasets/tpch/tables/customer" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "c_custkey", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "c_name", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Customer name" + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.metrics.total_revenue", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-metric", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "total_revenue", + "description": "Total order revenue" + }, + "aspects": { + "dataplex-types.global.semantic-metric": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-metric", + "data": { + "entity": "orders", + "dataType": "NUMERIC" + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.metrics.order_count", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-metric", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "order_count", + "description": "Number of orders" + }, + "aspects": { + "dataplex-types.global.semantic-metric": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-metric", + "data": { + "entity": "orders", + "dataType": "NUMERIC" + } + } + } + } + ], + "entryLinks": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entryLinks/sales-orders-to-customer", + "entryLinkType": "projects/dataplex-types/locations/global/entryLinkTypes/schema-join", + "entryReferences": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.entities.orders", + "type": "UNSPECIFIED" + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.entities.customer", + "type": "UNSPECIFIED" + } + ], + "aspects": { + "dataplex-types.global.schema-join": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema-join", + "data": { + "joins": [ + { + "source": { + "name": "samples.tpch.orders", + "fields": [ + "o_custkey" + ] + }, + "target": { + "name": "samples.tpch.customer", + "fields": [ + "c_custkey" + ] + }, + "type": "FOREIGN_KEY", + "inferenceSource": "USER" + } + ], + "userManaged": true + } + } + } + } + ], + "warnings": [ + "metric 'total_revenue': no datatype in the source model; defaulting the required semantic-metric.dataType to 'NUMERIC'", + "metric 'order_count': no datatype in the source model; defaulting the required semantic-metric.dataType to 'NUMERIC'" + ] +} diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml new file mode 100644 index 00000000..40c22466 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml @@ -0,0 +1,86 @@ +version: 0.2.0.dev0 +semantic_model: + - name: sales + description: Sales orders with customer attributes + ai_context: + instructions: Use this model for order analysis. + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": + ["//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales"]}' + datasets: + - name: orders + source: samples.tpch.orders + primary_key: + - o_orderkey + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + description: Order identifier + - name: o_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_custkey + - name: o_orderdate + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderdate + label: Order Date + dimension: + is_time: true + ai_context: + synonyms: + - order date + - date + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + - name: customer + source: samples.tpch.customer + primary_key: + - c_custkey + fields: + - name: c_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: c_custkey + - name: c_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_name + description: Customer name + relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: + - o_custkey + to_columns: + - c_custkey + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) + description: Total order revenue + ai_context: + synonyms: + - revenue + - sales + - name: order_count + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(orders.o_orderkey) + description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml new file mode 100644 index 00000000..ceb6bed1 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -0,0 +1,39 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: sales + description: Sales orders with customer attributes + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets":["//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales"]}' + datasets: + - name: orders + source: samples.tpch.orders + description: One row per order + fields: + - name: o_orderkey + description: Order identifier + - name: o_custkey + - name: o_orderdate + - name: o_totalprice + - name: customer + source: samples.tpch.customer + fields: + - name: c_custkey + - name: c_name + description: Customer name + relationships: + - name: orders-to-customer + from: orders + to: customer + from_columns: + - o_custkey + to_columns: + - c_custkey + metrics: + - name: total_revenue + datatype: Decimal + description: Total order revenue + - name: order_count + datatype: Decimal + description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.yaml index f9bf2958..15d4a5cf 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.yaml @@ -1,14 +1,18 @@ # Test fixture -- AI-first semantics format (v0.2.0.dev0). # -# Minimal star schema: orders (fact) -> customer (dimension). Exercises +# Minimal star schema: orders (fact) -> customer (dimension), with a GOOGLE +# BigQuery Graph deployment target so the model is executable. Exercises # ai_context at model / field / metric level, a time dimension (is_time), a -# field label, and a COUNT(*) metric that carries no entity qualifier. +# field label, and an entity-scoped COUNT metric. version: "0.2.0.dev0" semantic_model: - name: sales description: Sales orders with customer attributes + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": ["//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales"]}' ai_context: instructions: "Use this model for order analysis." datasets: @@ -77,5 +81,5 @@ semantic_model: expression: dialects: - dialect: ANSI_SQL - expression: COUNT(*) + expression: COUNT(orders.o_orderkey) description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json new file mode 100644 index 00000000..cc872725 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json @@ -0,0 +1,430 @@ +{ + "entries": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-model", + "entrySource": { + "displayName": "tpcds_model", + "description": "TPC-DS retail model" + }, + "aspects": { + "dataplex-types.global.semantic-model": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-model", + "data": {} + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store_sales", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "store_sales", + "description": "Fact table containing all store sales transactions" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/tpcds/datasets/public/tables/store_sales" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "ss_item_sk", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "ss_ticket_number", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "ss_customer_sk", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "ss_store_sk", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "ss_quantity", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Quantity of items sold" + }, + { + "name": "ss_sales_price", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Sales price per unit" + }, + { + "name": "ss_ext_sales_price", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Extended sales price (quantity * price)" + }, + { + "name": "ss_net_profit", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Net profit from the sale" + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.customer", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "customer", + "description": "Customer dimension with demographic information" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/tpcds/datasets/public/tables/customer" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "c_customer_sk", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "c_first_name", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Customer first name" + }, + { + "name": "c_last_name", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Customer last name" + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.item", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "item", + "description": "Item/Product dimension" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/tpcds/datasets/public/tables/item" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "i_item_sk", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "i_brand", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "i_category", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "i_current_price", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Current price of the item" + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "store", + "description": "Store dimension with location attributes" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/tpcds/datasets/public/tables/store" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "s_store_sk", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "s_store_name", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "s_city", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "s_state", + "dataType": "STRING", + "metadataType": "STRING" + }, + { + "name": "s_number_employees", + "dataType": "STRING", + "metadataType": "STRING", + "description": "Number of employees at the store" + } + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.date_dim", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "date_dim", + "description": "Date dimension with calendar attributes" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/tables/date_dim" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [] + } + } + } + } + ], + "entryLinks": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entryLinks/tpcds-model-store-sales-to-date-dim", + "entryLinkType": "projects/dataplex-types/locations/global/entryLinkTypes/schema-join", + "entryReferences": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store_sales", + "type": "UNSPECIFIED" + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.date_dim", + "type": "UNSPECIFIED" + } + ], + "aspects": { + "dataplex-types.global.schema-join": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema-join", + "data": { + "joins": [ + { + "source": { + "name": "tpcds.public.store_sales", + "fields": [ + "ss_sold_date_sk" + ] + }, + "target": { + "name": "sqlgen-testing.demo.date_dim", + "fields": [ + "ss_sold_date_sk" + ] + }, + "type": "FOREIGN_KEY", + "inferenceSource": "USER" + } + ], + "userManaged": true + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entryLinks/tpcds-model-store-sales-to-customer", + "entryLinkType": "projects/dataplex-types/locations/global/entryLinkTypes/schema-join", + "entryReferences": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store_sales", + "type": "UNSPECIFIED" + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.customer", + "type": "UNSPECIFIED" + } + ], + "aspects": { + "dataplex-types.global.schema-join": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema-join", + "data": { + "joins": [ + { + "source": { + "name": "tpcds.public.store_sales", + "fields": [ + "ss_customer_sk" + ] + }, + "target": { + "name": "tpcds.public.customer", + "fields": [ + "c_customer_sk" + ] + }, + "type": "FOREIGN_KEY", + "inferenceSource": "USER" + } + ], + "userManaged": true + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entryLinks/tpcds-model-store-sales-to-item", + "entryLinkType": "projects/dataplex-types/locations/global/entryLinkTypes/schema-join", + "entryReferences": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store_sales", + "type": "UNSPECIFIED" + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.item", + "type": "UNSPECIFIED" + } + ], + "aspects": { + "dataplex-types.global.schema-join": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema-join", + "data": { + "joins": [ + { + "source": { + "name": "tpcds.public.store_sales", + "fields": [ + "ss_item_sk" + ] + }, + "target": { + "name": "tpcds.public.item", + "fields": [ + "i_item_sk" + ] + }, + "type": "FOREIGN_KEY", + "inferenceSource": "USER" + } + ], + "userManaged": true + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entryLinks/tpcds-model-store-sales-to-store", + "entryLinkType": "projects/dataplex-types/locations/global/entryLinkTypes/schema-join", + "entryReferences": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store_sales", + "type": "UNSPECIFIED" + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.entities.store", + "type": "UNSPECIFIED" + } + ], + "aspects": { + "dataplex-types.global.schema-join": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema-join", + "data": { + "joins": [ + { + "source": { + "name": "tpcds.public.store_sales", + "fields": [ + "ss_store_sk" + ] + }, + "target": { + "name": "tpcds.public.store", + "fields": [ + "s_store_sk" + ] + }, + "type": "FOREIGN_KEY", + "inferenceSource": "USER" + } + ], + "userManaged": true + } + } + } + } + ], + "warnings": [ + "entity 'date_dim': no keys declared in the source model" + ] +} diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml new file mode 100644 index 00000000..27acf1e6 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml @@ -0,0 +1,218 @@ +version: 0.2.0.dev0 +semantic_model: + - name: tpcds_model + description: TPC-DS retail model + datasets: + - name: store_sales + source: tpcds.public.store_sales + primary_key: + - ss_item_sk + - ss_ticket_number + description: Fact table containing all store sales transactions + fields: + - name: ss_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_item_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_item_sk}" + dimension: + is_time: false + - name: ss_ticket_number + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ticket_number + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_ticket_number}" + dimension: + is_time: false + - name: ss_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_customer_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_customer_sk}" + dimension: + is_time: false + - name: ss_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_store_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_store_sk}" + dimension: + is_time: false + - name: ss_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: ss_quantity + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_quantity}" + description: Quantity of items sold + - name: ss_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_sales_price + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_sales_price}" + description: Sales price per unit + - name: ss_ext_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ext_sales_price + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_ext_sales_price}" + description: Extended sales price (quantity * price) + - name: ss_net_profit + expression: + dialects: + - dialect: BIGQUERY + expression: ss_net_profit + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_net_profit}" + description: Net profit from the sale + - name: customer + source: tpcds.public.customer + primary_key: + - c_customer_sk + description: Customer dimension with demographic information + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: c_customer_sk + dimension: + is_time: false + - name: c_first_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_first_name + dimension: + is_time: false + description: Customer first name + - name: c_last_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_last_name + dimension: + is_time: false + description: Customer last name + - name: item + source: tpcds.public.item + primary_key: + - i_item_sk + description: Item/Product dimension + fields: + - name: i_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: i_item_sk + dimension: + is_time: false + - name: i_brand + expression: + dialects: + - dialect: BIGQUERY + expression: i_brand + dimension: + is_time: false + - name: i_category + expression: + dialects: + - dialect: BIGQUERY + expression: i_category + dimension: + is_time: false + - name: i_current_price + expression: + dialects: + - dialect: BIGQUERY + expression: i_current_price + description: Current price of the item + - name: store + source: tpcds.public.store + primary_key: + - s_store_sk + description: Store dimension with location attributes + fields: + - name: s_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_sk + dimension: + is_time: false + - name: s_store_name + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_name + dimension: + is_time: false + - name: s_city + expression: + dialects: + - dialect: BIGQUERY + expression: s_city + dimension: + is_time: false + - name: s_state + expression: + dialects: + - dialect: BIGQUERY + expression: s_state + dimension: + is_time: false + - name: s_number_employees + expression: + dialects: + - dialect: BIGQUERY + expression: s_number_employees + description: Number of employees at the store + - name: date_dim + source: sqlgen-testing.demo.date_dim + description: Date dimension with calendar attributes + custom_extensions: + - vendor_name: GOODDATA + data: '{"date_dimension": true, "granularities": ["DAY", "WEEK", "MONTH", + "QUARTER", "YEAR"]}' + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - ss_sold_date_sk + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml new file mode 100644 index 00000000..ab45bace --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -0,0 +1,82 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: tpcds_model + description: TPC-DS retail model + datasets: + - name: store_sales + source: tpcds.public.store_sales + description: Fact table containing all store sales transactions + fields: + - name: ss_item_sk + - name: ss_ticket_number + - name: ss_customer_sk + - name: ss_store_sk + - name: ss_quantity + description: Quantity of items sold + - name: ss_sales_price + description: Sales price per unit + - name: ss_ext_sales_price + description: Extended sales price (quantity * price) + - name: ss_net_profit + description: Net profit from the sale + - name: customer + source: tpcds.public.customer + description: Customer dimension with demographic information + fields: + - name: c_customer_sk + - name: c_first_name + description: Customer first name + - name: c_last_name + description: Customer last name + - name: item + source: tpcds.public.item + description: Item/Product dimension + fields: + - name: i_item_sk + - name: i_brand + - name: i_category + - name: i_current_price + description: Current price of the item + - name: store + source: tpcds.public.store + description: Store dimension with location attributes + fields: + - name: s_store_sk + - name: s_store_name + - name: s_city + - name: s_state + - name: s_number_employees + description: Number of employees at the store + - name: date_dim + source: sqlgen-testing.demo.date_dim + description: Date dimension with calendar attributes + relationships: + - name: store-sales-to-date-dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - ss_sold_date_sk + - name: store-sales-to-customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store-sales-to-item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store-sales-to-store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts new file mode 100644 index 00000000..4ccac618 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -0,0 +1,793 @@ +// Behavior specification for the KC converter's read direction +// (modelsFromCatalogResources in src/libts/semantic/kc_converter.ts). +// +// The reader is the inverse of the emitter (generateCatalogResources). The +// central guarantee is an emitter -> reader round trip: emit a model's entries +// AND entry links, read them back, and get an IR equal to the source WHERE the +// emitter is lossless. The write drops content by design (entity keys, +// ai_context, field labels, importedDialect, and many-to-many relationships -- +// see the emitter header), so the expected read-back is the source model with +// exactly those fields cleared. It ALSO drops the per-field `semantics` block +// (field/metric expressions and the DIMENSION role) unless the push enabled +// `emitExpressions`, so a DEFAULT round trip (roundTrip) loses those too, while +// a full round trip (roundTripFull, emitExpressions: true) keeps them. 1:1 / +// 1:N relationships and deployment targets DO round-trip either way (via +// schema-join links and the semantic-model aspect), except that relationship +// names come back normalized (lowercased/hyphenated). Targeted tests pin the +// mapping details a round trip cannot isolate (the dataType inverse, the +// DIMENSION role, resource-URI parsing, metric attach re-derivation, +// relationship endpoint/direction recovery, and parent/anchor grouping). + +import {describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import {CustomExtension, Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; +import {linkNamePrefix, modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {loadModels} from '../../../src/libts/semantic/loader'; +import {serializeModel} from '../../../src/libts/semantic/osi_converter'; + +const FIXTURES = path.join(__dirname, 'fixtures'); + +const OPTS = { + project: 'dest', + location: 'us', + entryGroup: 'eg' +}; + +// Emits a model to entries + entry links and reads it straight back. The +// default push omits the per-field `semantics` block (expressions + role), so a +// default round trip drops those; use roundTripFull to exercise their recovery. +function roundTrip(model: SemanticModel): + {models: SemanticModel[]; warnings: string[]} { + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + return modelsFromCatalogResources(entries, entryLinks); +} + +// A round trip through a `--emit-expressions` push: the catalog then holds the +// per-field `semantics` block and the metric expression, so field/metric +// expressions and DIMENSION roles round-trip too. +function roundTripFull(model: SemanticModel): + {models: SemanticModel[]; warnings: string[]} { + const {entries, entryLinks} = + generateCatalogResources(model, {...OPTS, emitExpressions: true}); + return modelsFromCatalogResources(entries, entryLinks); +} + + +describe('emitter -> reader round trip (lossless slice)', () => { + // A model using only round-trippable content: no keys/ai_context/labels/ + // relationships (all dropped by the write), and datatypes that invert + // cleanly. Its fields and metric carry expressions and a DIMENSION role, so + // it round-trips losslessly only through a `--emit-expressions` push + // (roundTripFull); a default push omits the per-field `semantics` block. + const source: SemanticModel = { + name: 'sales', + description: 'the sales model', + entities: [{ + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], // keys are not persisted; keep empty so the round trip matches + fields: [ + {name: 'o_orderkey', expression: 'orders.o_orderkey', type: 'Integer'}, + { + name: 'o_orderdate', + expression: 'orders.o_orderdate', + type: 'Date', + dimension: {}, + description: 'order date', + }, + ], + }], + relationships: [], + metrics: [{ + name: 'total_revenue', + expression: 'SUM(orders.o_totalprice)', + entity: 'orders', + type: 'Decimal', + }], + }; + + test('reconstructs an IR equal to the source', () => { + const {models} = roundTripFull(source); + expect(models).toHaveLength(1); + expect(models[0]).toEqual(source); + }); +}); + + +describe('relationship recovery (schema-join links -> IR)', () => { + // orders.o_custkey (the foreign-key side) references customer.c_custkey. + const twoEntities: Entity[] = [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'o_custkey', expression: 'orders.o_custkey'}], + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'c_custkey', expression: 'customer.c_custkey'}], + }, + ]; + + test( + 'a 1:N relationship recovers its endpoints, direction, and columns', + () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: + 'places', // already link-slug-safe, so it round-trips exactly + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const rels = roundTrip(model).models[0].relationships; + expect(rels).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + }); + + test( + 'a relationship name comes back normalized (lowercased/hyphenated)', + () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'Places_Order', // mixed case + underscore -> normalized on + // read + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships[0].name) + .toBe('places-order'); + }); + + test('a many-to-many (association) relationship is not recovered', () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'enrolls', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + association: { + dataSource: 'p.d.junction', + keys: ['id'], + sourceColumns: ['j_orderkey'], + destinationColumns: ['j_custkey'], + }, + }], + metrics: [], + }; + // The emitter never publishes M:N, so no schema-join link exists to read. + expect(roundTrip(model).models[0].relationships).toEqual([]); + }); + + test( + 'endpoints resolve from the link references, not the shared table name', + () => { + // Two entities over the SAME table: a data-source index would collapse + // them (last write wins), so both endpoints must come from the link's + // entryReferences instead. Direction can't be told from the shared + // table, so the reader keeps the reference order and warns. + const sameTable: Entity[] = [ + { + name: 'parent_order', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'id', expression: 'parent_order.id'}], + }, + { + name: 'child_order', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'parent_id', expression: 'child_order.parent_id'}], + }, + ]; + const model: SemanticModel = { + name: 'sales', + entities: sameTable, + relationships: [{ + name: 'parents', + source: {entity: 'child_order', columns: ['parent_id']}, + destination: {entity: 'parent_order', columns: ['id']}, + }], + metrics: [], + }; + const {models, warnings} = roundTrip(model); + expect(models[0].relationships).toEqual([{ + name: 'parents', + source: {entity: 'child_order', columns: ['parent_id']}, + destination: {entity: 'parent_order', columns: ['id']}, + }]); + expect(warnings.some(w => /join direction is ambiguous/i.test(w))) + .toBe(true); + }); + + test( + 'endpoints resolve when link references carry an un-normalized project ' + + 'number', + () => { + // The live path is asymmetric: lookupEntry normalizes entry names to + // project IDs, but lookupEntryLinks returns references still carrying + // the numeric project. Matching on the (stable) entry id keeps the + // relationship resolvable rather than silently dropping it. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + const numeric = entryLinks.map( + l => ({ + ...l, + entryReferences: + (l.entryReferences ?? + []).map(r => ({ + ...r, + name: r.name.replace( + 'projects/dest/', 'projects/000000000000/'), + })), + })); + const {models} = modelsFromCatalogResources(entries, numeric); + expect(models[0].relationships).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + }); + + test( + 'the model prefix is stripped across tricky model names ' + + '(linkNamePrefix tracks the emitter slug)', + () => { + // Pins the read-side prefix reproduction to the emitter's linkSlug: if + // it drifts, the model prefix would survive and the recovered name + // would not equal the bare, normalized relationship name. + for (const modelName of ['Sales', 'Sales Analytics', '123 Sales!']) { + const model: SemanticModel = { + name: modelName, + entities: twoEntities, + relationships: [{ + name: 'rel_one', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships[0].name) + .toBe('rel-one'); + } + }); + + test('an unnamed link returned from both endpoints is deduped', () => { + // Defense in depth: if the catalog ever returns a nameless schema-join + // link, the per-endpoint fan-out yields it twice; dedup falls back to the + // endpoint pair so it becomes one relationship, not two. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + const nameless = entryLinks.map(l => ({...l, name: undefined})); + const {models} = modelsFromCatalogResources( + entries, [...nameless, ...nameless.map(l => ({...l}))]); + expect(models[0].relationships).toHaveLength(1); + }); +}); + + +describe( + 'deployment-target recovery (semantic-model aspect -> custom_extensions)', + () => { + test('the GOOGLE deployment targets ride back verbatim', () => { + const uri = + '//bigquery.googleapis.com/projects/p/datasets/d/propertyGraphs/g'; + const data = JSON.stringify({deploymentTargets: [uri]}); + const model: SemanticModel = { + name: 'sales', + customExtensions: [{vendorName: 'GOOGLE', data}], + entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + expect(roundTrip(model).models[0].customExtensions).toEqual([ + {vendorName: 'GOOGLE', data} + ]); + }); + + test( + 'a model with no deployment targets recovers no custom_extensions', + () => { + const model: SemanticModel = { + name: 'sales', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + expect(roundTrip(model).models[0].customExtensions).toBeUndefined(); + }); + }); + + +describe('dataType inverse (schema aspect -> IR type)', () => { + // Emit a one-field model of each IR type, read it back, and check the field's + // reconstructed type. String and Opaque both emit dataType STRING; String + // (indistinguishable from an un-typed field) reads back as undefined, while + // Opaque is disambiguated by metadataType OTHER. + const cases: [Metric['type']|undefined, Metric['type']|undefined][] = [ + ['Integer', 'Integer'], + ['Decimal', 'Decimal'], + ['Float', 'Float'], + ['Boolean', 'Boolean'], + ['Date', 'Date'], + ['Time', 'Time'], + ['DateTime', 'DateTime'], + ['DateTimeTz', 'DateTimeTz'], + ['Opaque', 'Opaque'], + ['String', undefined], // collapses to un-typed + [undefined, undefined], // un-typed stays un-typed + ]; + + for (const [type, expected] of cases) { + test(`${type ?? 'un-typed'} -> ${expected ?? 'un-typed'}`, () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'f', expression: 'e.f', ...(type ? {type} : {})}], + }], + relationships: [], + metrics: [], + }; + const back = roundTrip(model).models[0].entities[0].fields[0]; + expect(back.type).toBe(expected as any); + }); + } +}); + + +describe('field mapping details', () => { + function readWith( + rt: (m: SemanticModel) => {models: SemanticModel[]}, + field: Entity['fields'][number]) { + const model: SemanticModel = { + name: 'm', + entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: [field]}], + relationships: [], + metrics: [], + }; + return rt(model).models[0].entities[0].fields[0]; + } + // Default push (no per-field `semantics`) vs a `--emit-expressions` push. + const readField = (f: Entity['fields'][number]) => readWith(roundTrip, f); + const readFieldFull = (f: Entity['fields'][number]) => + readWith(roundTripFull, f); + + test('a DIMENSION role reads back as a dimension marker', () => { + // The role lives in the gated `semantics` block, so it survives only a + // `--emit-expressions` push. + const back = readFieldFull({name: 'd', expression: 'e.d', dimension: {}}); + expect(back.dimension).toEqual({}); + }); + + test('a non-dimension field has no dimension marker', () => { + const back = readFieldFull({name: 'f', expression: 'e.f'}); + expect(back.dimension).toBeUndefined(); + }); + + test( + 'a default push drops the DIMENSION role with the semantics block', + () => { + const back = readField({name: 'd', expression: 'e.d', dimension: {}}); + expect(back.dimension).toBeUndefined(); + expect(back.expression).toBeUndefined(); + }); + + test( + 'an imported expression is not persisted to or recovered from the catalog', + () => { + const back = readFieldFull({ + name: 'amt', + expression: 'e.amt', + importedExpression: 'e.amt::NUMBER', + importedDialect: 'SNOWFLAKE', + }); + expect(back.expression).toBe('e.amt'); + // The emitter no longer writes importedExpression/importedDialect, so + // neither survives the round trip. + expect(back.importedExpression).toBeUndefined(); + expect(back.importedDialect).toBeUndefined(); + }); + + test('a schema field missing its name is skipped with a warning', () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'f', expression: 'e.f'}], + }], + relationships: [], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + // Inject a nameless field record into the entity's schema aspect, as a + // malformed catalog could return; the reader must drop it, not emit a field + // with an undefined name. + for (const entry of entries) { + for (const aspect of Object.values(entry.aspects ?? {})) { + const fields = (aspect as {data?: {fields?: unknown[]}}).data?.fields; + if (Array.isArray(fields)) fields.push({dataType: 'INT64'}); + } + } + const {models, warnings} = modelsFromCatalogResources(entries, entryLinks); + expect(models[0].entities[0].fields.map(f => f.name)).toEqual(['f']); + expect( + warnings.some(w => /missing its name; the field is skipped/i.test(w))) + .toBe(true); + }); +}); + + +describe('data source resource-path parsing', () => { + function readDataSource(dataSource: string): string { + const model: SemanticModel = { + name: 'm', + entities: [{name: 'e', dataSource, keys: [], fields: []}], + relationships: [], + metrics: [], + }; + return roundTrip(model).models[0].entities[0].dataSource; + } + + test( + 'a three-part BigQuery reference round-trips through the resource URI', + () => { + expect(readDataSource('proj.ds.tbl')).toBe('proj.ds.tbl'); + }); + + test('a verbatim query source is preserved unchanged', () => { + const query = 'SELECT * FROM t'; + expect(readDataSource(query)).toBe(query); + }); +}); + + +describe('metric attach entity is re-derived from the expression', () => { + test( + 'a single-entity metric attaches; a cross-entity metric does not', () => { + const model: SemanticModel = { + name: 'm', + entities: [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'amt', expression: 'orders.amt'}] + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'region', expression: 'customer.region'}] + }, + ], + relationships: [], + metrics: [ + {name: 'revenue', expression: 'SUM(orders.amt)', entity: 'orders'}, + { + name: 'mix', + expression: 'SUM(orders.amt) / COUNT(customer.region)' + }, + ], + }; + // Full push: the expression is in the catalog, so the entity is + // re-derived from it (single entity -> attach; two entities -> none). + const {models} = roundTripFull(model); + const byName = new Map(models[0].metrics.map(m => [m.name, m])); + expect(byName.get('revenue')!.entity).toBe('orders'); + expect(byName.get('mix')!.entity).toBeUndefined(); + }); + + test( + 'a cross-entity metric falls back to the persisted attach entity', () => { + const model: SemanticModel = { + name: 'm', + entities: [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'amt', expression: 'orders.amt'}] + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'region', expression: 'customer.region'}] + }, + ], + relationships: [], + // Two entities in the expression, so it cannot be re-derived; the + // authored attach entity must ride back on the persisted aspect. + metrics: [{ + name: 'mix', + expression: 'SUM(orders.amt) / COUNT(customer.region)', + entity: 'orders', + }], + }; + const {models} = roundTrip(model); + expect(models[0].metrics[0].entity).toBe('orders'); + }); +}); + + +describe('anchor / parent grouping', () => { + test('no semantic-model entry yields no models and a warning', () => { + const {models, warnings} = modelsFromCatalogResources([]); + expect(models).toHaveLength(0); + expect(warnings.some(w => /no semantic-model entry/i.test(w))).toBe(true); + }); + + test('two models keep their own children by parentEntry', () => { + const a = generateCatalogResources( + { + name: 'a', + entities: [{name: 'ea', dataSource: 'p.d.a', keys: [], fields: []}], + relationships: [], + metrics: [], + }, + OPTS); + const b = generateCatalogResources( + { + name: 'b', + entities: [{name: 'eb', dataSource: 'p.d.b', keys: [], fields: []}], + relationships: [], + metrics: [], + }, + OPTS); + const {models} = modelsFromCatalogResources([...a.entries, ...b.entries]); + const byName = new Map(models.map(m => [m.name, m])); + expect(byName.get('a')!.entities.map(e => e.name)).toEqual(['ea']); + expect(byName.get('b')!.entities.map(e => e.name)).toEqual(['eb']); + }); +}); + + +describe('metric expression referencing no known entity', () => { + test( + 'warns that the metric may be unplaceable and leaves it unattached', + () => { + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'orders', dataSource: 'p.d.o', keys: [], fields: []}], + relationships: [], + // References `widgets`, which is not an entity of this model. + metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], + }; + // The unplaceable warning fires from the expression, so it needs a push + // that wrote one (a default push omits it, leaving nothing to check). + const {models, warnings} = roundTripFull(model); + expect(models[0].metrics[0].entity).toBeUndefined(); + expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) + .toBe(true); + }); +}); + + +// -- Golden pull: the whole KC entries -> IR -> OSI YAML output. -- +// +// The round trip above proves the reader inverts the emitter in memory; this +// pins the reviewable artifact. For each corpus fixture it reads the committed +// emitter golden (`.knowledge_catalog.golden.json` -- the exact +// entries and entry links a push produced) back through the reader and +// serializes the reconstructed IR to `.pull.golden.yaml`. Open that +// next to the fixture's `.osi.golden.yaml` to see, as whole files, what a +// Knowledge Catalog round trip preserves (including 1:1 / 1:N relationships and +// deployment targets) and what it drops (keys, ai_context, labels, vendor SQL, +// M:N relationships). +// +// Regenerate after an intentional reader/serializer change: +// UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/kc_converter.test.ts +describe( + 'golden pull: each corpus KC golden reconstructs to its exact YAML', () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + const kcGoldenPath = (fixture: string) => path.join( + FIXTURES, + fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); + const pullGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.pull.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const kc = JSON.parse(fs.readFileSync(kcGoldenPath(fixture), 'utf8')); + const {models, warnings} = + modelsFromCatalogResources(kc.entries, kc.entryLinks ?? []); + // Reader warnings ride along as YAML comments so the golden shows + // the full outcome, not just the recovered document. + const header = warnings.length ? + warnings.map((w: string) => `# warning: ${w}`).join('\n') + '\n' : + '# (no warnings)\n'; + const actual = + header + models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = pullGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename( + golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); + + +// -- Symmetry: emit -> read drops ONLY the documented set. -- +// +// The golden pull above lets a human eyeball what a Knowledge Catalog round +// trip loses; this asserts it mechanically. For each corpus fixture it loads +// the authored IR, runs a full emit -> read round trip, and asserts the result +// equals the authored IR reduced to the "KC floor" -- the model with exactly +// the documented losses and normalizations applied (`stripToKcFloor`). Applying +// that same reduction to BOTH sides makes `toEqual` flag only UNDOCUMENTED +// divergence, so a new emitter/reader regression (a dropped column, a lost +// description, an un-stripped M:N edge) fails here even though every individual +// loss is already pinned by a targeted test above. +describe( + 'symmetry: an emit -> read round trip drops only documented fields', () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + // Same load defaults as the OSI / KC / pull goldens. + const LOAD = {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}; + + for (const fixture of CORPUS) { + test(fixture, () => { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + for (const authored of loadModels(text, LOAD).models) { + const {models, warnings} = roundTrip(authored); + expect(models).toHaveLength(1); + expect(stripToKcFloor(models[0])).toEqual(stripToKcFloor(authored)); + // A clean corpus model round-trips without reader warnings. + expect(warnings).toEqual([]); + } + }); + } + }); + + +// Reduces a model to the "KC floor": the most an emit -> read round trip can +// preserve, i.e. the authored model with exactly the documented Knowledge +// Catalog losses and normalizations applied. Idempotent (already-floored input +// is unchanged), so it can be applied to both sides of a round trip. Each +// transform mirrors one emitter/reader behavior pinned by a targeted test +// above. +function stripToKcFloor(model: SemanticModel): SemanticModel { + const m = structuredClone(model); + + // Model ai_context is not persisted; only a GOOGLE deployment-targets block + // rides back, re-serialized to the reader's canonical JSON form. + delete m.aiContext; + const ext = canonicalGoogleExt(m.customExtensions); + if (ext) { + m.customExtensions = ext; + } else { + delete m.customExtensions; + } + + for (const e of m.entities) { + e.keys = []; // keys / unique keys are never persisted + delete e.uniqueKeys; + delete e.aiContext; + delete e.customExtensions; + for (const f of e.fields) { + delete f.label; // display label is not persisted + delete f.aiContext; + delete f.importedExpression; // vendor SQL is not persisted + delete f.importedDialect; + delete f.customExtensions; + // A default push omits the per-field `semantics` block, so the field + // expression and the DIMENSION role are not persisted (they ride back + // only through a `--emit-expressions` push). + delete f.expression; + delete f.dimension; + // String is indistinguishable from an un-typed field on read. + if (f.type === 'String') delete f.type; + } + } + + for (const metric of m.metrics) { + delete metric.aiContext; + delete metric.importedExpression; + delete metric.importedDialect; + delete metric.customExtensions; + delete metric.expression; // omitted by a default push, like field ones + // The emitter writes a required dataType, defaulting typeless -> NUMERIC, + // which reads back as Decimal; String collapses to un-typed like fields. + if (metric.type === undefined) { + metric.type = 'Decimal'; + } else if (metric.type === 'String') { + delete metric.type; + } + } + + // M:N (association) edges are never published; a 1:1 / 1:N name comes back + // normalized via the emitter's slug (its exact form is pinned above). + m.relationships = m.relationships.filter(r => !r.association).map(r => { + const rel = structuredClone(r); + rel.name = linkNamePrefix(rel.name); + delete rel.aiContext; + delete rel.customExtensions; + return rel; + }); + + return m; +} + + +// The inverse-canonical of the emitter's deployment-target persistence, +// matching the reader's readDeploymentTargets: keep only GOOGLE +// deploymentTargets and re-serialize them so an authored block and its +// round-tripped form (which the reader always JSON.stringifies afresh) compare +// equal. Returns undefined when the model declares none, exactly as the reader +// omits custom_extensions then. +function canonicalGoogleExt(exts: CustomExtension[]|undefined): + CustomExtension[]|undefined { + const targets: string[] = []; + for (const ext of exts ?? []) { + if (ext.vendorName !== 'GOOGLE') continue; + let parsed: any; + try { + parsed = JSON.parse(ext.data); + } catch { + continue; + } + for (const t of parsed?.deploymentTargets ?? []) { + if (typeof t === 'string' && t) targets.push(t); + } + } + return targets.length ? [{ + vendorName: 'GOOGLE', + data: JSON.stringify({deploymentTargets: targets}) + }] : + undefined; +} diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts new file mode 100644 index 00000000..5452ec55 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts @@ -0,0 +1,88 @@ +// End-to-end tests for the Knowledge Catalog destination: real-shaped fixtures +// run the full file -> IR -> catalog-resources path. +// +// The primary check is a GOLDEN test: for every fixture in the corpus, the +// generated Entries (with their `semantic-*` / `schema` aspects) plus the +// emitter warnings are compared byte-for-byte against a committed +// `.knowledge_catalog.golden.json`. The golden is the reviewable "big +// picture" — open a `.yaml` next to its `.knowledge_catalog.golden.json` to see +// the full input and the exact catalog resources it maps to; a changed aspect +// shape, dropped field, or reordered entry shows up as a diff. +// +// Regenerate goldens after an intentional emitter change: +// UPDATE_GOLDENS=1 npx bun test \ +// ./tests/libts/semantic/knowledge_catalog.e2e.test.ts +// then read the diff before committing. +// +// Focused behavior (warnings, schema-join link emission, metric dataType +// fallback) is asserted in knowledge_catalog.test.ts; the publisher's write +// sequence in deploy_knowledge_catalog.test.ts. + +import {describe, expect, test} from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; + +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {loadModels, LoadOptions} from '../../../src/libts/semantic/loader'; + +const FIXTURES = path.join(__dirname, 'fixtures'); + +// Fixtures that get a KC golden. Chosen to exercise the distinct mappings: +// sales_bq_graph_target -> model aspect deploymentTargets + un-typed metric +// (dataType fallback); star_orders_customer -> a direct-FK relationship +// (schema-join link) + multiple entities/metrics; tpcds_date_edge -> +// temporal field types. +const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', +]; + +// A fixed destination + default (dataplex-types/global) system types, so the +// golden pins the exact resource names the emitter produces. +const GEN_OPTS = { + project: 'sqlgen-testing', + location: 'us', + entryGroup: 'semantic' +}; + +function loadFixture(fixture: string, load: LoadOptions = {}) { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + return loadModels( + text, + {defaultProject: 'sqlgen-testing', defaultDataset: 'demo', ...load}); +} + +// The exact artifact a golden captures: the generated entries, then the +// relationship entry links, then the emitter warnings, as pretty JSON so a +// reviewer sees the full resource shapes. +function render(fixture: string): string { + const {models} = loadFixture(fixture); + const {entries, entryLinks, warnings} = + generateCatalogResources(models[0], GEN_OPTS); + return JSON.stringify({entries, entryLinks, warnings}, null, 2) + '\n'; +} + +const goldenPath = (fixture: string) => path.join( + FIXTURES, fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); + + +describe( + 'golden KC resources: each corpus fixture maps to its exact catalog entries', + () => { + for (const fixture of CORPUS) { + test(fixture, () => { + const actual = render(fixture); + const golden = goldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename(golden)} — run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts new file mode 100644 index 00000000..621f402a --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts @@ -0,0 +1,376 @@ +// Behavior specification for the Knowledge Catalog emitter +// (src/libts/semantic/knowledge_catalog.ts). +// +// The readable "big picture" goldens live in `knowledge_catalog.e2e.test.ts` (a +// corpus of `.yaml` inputs, each with a committed +// `.knowledge_catalog.golden.json`). This file holds what a loader +// fixture +// CANNOT express, because the open AI-first format the loader reads is a subset +// of the IR: +// - the logical DataType -> schema `dataType`/`metadataType` mapping and the +// DIMENSION role (the corpus fixtures declare no field `datatype`, so every +// field falls back to STRING/DEFAULT); +// - the `dataSource` -> resource-path forms (BigQuery URI vs. verbatim); +// - IR-contract cases the loader never produces (a metric with an explicit +// type, a cross-entity metric with no attach entity, duplicate ids). + +import {describe, expect, test} from 'bun:test'; + +import {DataType, Entity, SemanticModel} from '../../../src/libts/semantic/ir'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; + +const OPTS = { + project: 'dest-proj', + location: 'us', + entryGroup: 'eg' +}; +// The expression fields (per-field schema semantics, metric expression) are +// gated off by default; this turns them on to assert their content. +const OPTS_EXPR = {...OPTS, emitExpressions: true}; + +// A one-entity model whose single field carries the given IR type + dimension, +// so a test can read back the emitted schema aspect for that field. +function modelWithField( + type: DataType|undefined, dimension = false): SemanticModel { + const entity: Entity = { + name: 'e', + dataSource: 'p.d.t', + keys: ['k'], + fields: [{ + name: 'f', + expression: 'e.f', + ...(type ? {type} : {}), + ...(dimension ? {dimension: {}} : {}), + }], + }; + return {name: 'm', entities: [entity], relationships: [], metrics: []}; +} + +// The schema-aspect field record for the sole field of modelWithField. +function schemaField( + model: SemanticModel, opts = OPTS): Record { + const {entries} = generateCatalogResources(model, opts); + const entity = entries.find(e => e.entryType.endsWith('/semantic-entity'))!; + const schema = entity.aspects!['dataplex-types.global.schema'].data!; + return schema.fields[0]; +} + + +describe( + 'logical DataType maps to the schema aspect dataType + metadataType', + () => { + const cases: [DataType, string, string][] = [ + ['String', 'STRING', 'STRING'], + ['Integer', 'INT64', 'NUMBER'], + ['Decimal', 'NUMERIC', 'NUMBER'], + ['Float', 'FLOAT64', 'NUMBER'], + ['Boolean', 'BOOL', 'BOOLEAN'], + ['Date', 'DATE', 'DATETIME'], + ['Time', 'TIME', 'DATETIME'], + ['DateTime', 'DATETIME', 'DATETIME'], + ['DateTimeTz', 'TIMESTAMP', 'TIMESTAMP'], + ['Opaque', 'STRING', 'OTHER'], + ]; + for (const [type, dataType, metadataType] of cases) { + test(`${type} -> ${dataType} / ${metadataType}`, () => { + const f = schemaField(modelWithField(type)); + expect(f.dataType).toBe(dataType); + expect(f.metadataType).toBe(metadataType); + }); + } + + test('an un-typed field falls back to STRING / STRING', () => { + const f = schemaField(modelWithField(undefined)); + expect(f.dataType).toBe('STRING'); + expect(f.metadataType).toBe('STRING'); + }); + }); + + +describe( + 'a field with dimension metadata gets role DIMENSION, else DEFAULT', () => { + test('dimension -> DIMENSION', () => { + expect(schemaField(modelWithField('String', true), OPTS_EXPR).semantics.role) + .toBe('DIMENSION'); + }); + test('no dimension -> DEFAULT', () => { + expect(schemaField(modelWithField('String', false), OPTS_EXPR).semantics.role) + .toBe('DEFAULT'); + }); + }); + + +describe('the schema aspect carries the target expression in semantics', () => { + test('the target expression is kept; imported vendor SQL is not emitted', () => { + const model: SemanticModel = { + name: 'm', + relationships: [], + metrics: [], + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: ['k'], + fields: [{ + name: 'f', + expression: 'CAST(e.f AS INT64)', + importedExpression: 'e.f::int', + importedDialect: 'SNOWFLAKE', + }], + }], + }; + const f = schemaField(model, OPTS_EXPR); + expect(f.semantics.expression).toBe('CAST(e.f AS INT64)'); + // importedExpression is the vendor/MAQL form; KC has no consumer for it. + expect(f.semantics.importedExpression).toBeUndefined(); + }); +}); + + +describe('the schema semantics block is gated behind emitExpressions', () => { + test('omitted by default; the non-gated columns are still emitted', () => { + const f = schemaField(modelWithField('String', true)); + expect(f.semantics).toBeUndefined(); + expect(f.dataType).toBe('STRING'); + expect(f.metadataType).toBe('STRING'); + }); + test('emitExpressions re-adds the semantics block (expression + role)', () => { + const f = schemaField(modelWithField('String', true), OPTS_EXPR); + expect(f.semantics).toEqual({expression: 'e.f', role: 'DIMENSION'}); + }); +}); + + +describe('entity dataSource maps to a resource path', () => { + function resources(dataSource: string): string[] { + const model: SemanticModel = { + name: 'm', + relationships: [], + metrics: [], + entities: [{name: 'e', dataSource, keys: ['k'], fields: []}], + }; + const {entries} = generateCatalogResources(model, OPTS); + const entity = entries.find(e => e.entryType.endsWith('/semantic-entity'))!; + return entity.aspects!['dataplex-types.global.semantic-entity'] + .data!.source.resources; + } + + test( + 'a clean project.dataset.table becomes a BigQuery linked-resource URI', + () => { + expect(resources('proj.ds.tbl')).toEqual([ + '//bigquery.googleapis.com/projects/proj/datasets/ds/tables/tbl', + ]); + }); + test('a query-like source is kept verbatim', () => { + expect(resources('SELECT 1')).toEqual(['SELECT 1']); + }); + test('an under-qualified reference is kept verbatim', () => { + expect(resources('ds.tbl')).toEqual(['ds.tbl']); + }); +}); + + +describe('semantic-metric aspect', () => { + function metricData(model: SemanticModel, opts = OPTS) { + const {entries, warnings} = generateCatalogResources(model, opts); + const metric = entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + return { + data: metric.aspects!['dataplex-types.global.semantic-metric'].data!, + warnings + }; + } + + test( + 'a typed metric carries its mapped dataType and attach entity, no warning', + () => { + const model: SemanticModel = { + name: 'm', + entities: [], + relationships: [], + metrics: [ + {name: 'rev', expression: 'SUM(o.p)', entity: 'o', type: 'Decimal'} + ], + }; + const {data, warnings} = metricData(model, OPTS_EXPR); + expect(data).toEqual( + {entity: 'o', dataType: 'NUMERIC', expression: 'SUM(o.p)'}); + expect(warnings.some(w => w.includes('dataType'))).toBe(false); + }); + + test('an un-typed metric falls back to NUMERIC dataType and warns', () => { + const model: SemanticModel = { + name: 'm', + entities: [], + relationships: [], + metrics: [{name: 'rev', expression: 'COUNT(*)'}], + }; + const {data, warnings} = metricData(model); + expect(data.dataType).toBe('NUMERIC'); + expect(data.entity).toBeUndefined(); // cross-entity / unattached + expect(warnings.some( + w => w.includes('metric \'rev\'') && w.includes('NUMERIC'))) + .toBe(true); + }); + + test('the expression is gated: omitted by default, kept with emitExpressions', + () => { + const model: SemanticModel = { + name: 'm', + entities: [], + relationships: [], + metrics: [{ + name: 'rev', expression: 'SUM(o.p)', entity: 'o', type: 'Decimal' + }], + }; + expect(metricData(model).data).toEqual({entity: 'o', dataType: 'NUMERIC'}); + expect(metricData(model, OPTS_EXPR).data.expression).toBe('SUM(o.p)'); + }); +}); + + +describe('model-level structure', () => { + test( + 'a model with no BigQuery targets emits an empty semantic-model aspect', + () => { + const model: SemanticModel = + {name: 'm', entities: [], relationships: [], metrics: []}; + const {entries, warnings} = generateCatalogResources(model, OPTS); + const anchor = entries[0]; + expect(anchor.entryType.endsWith('/semantic-model')).toBe(true); + expect(anchor.aspects!['dataplex-types.global.semantic-model'].data) + .toEqual({}); + expect(warnings).toContain( + 'model has no entities; only the semantic-model entry will be generated'); + }); + + test('the anchor is emitted first and is the parent of every child', () => { + const model: SemanticModel = { + name: 'm', + relationships: [], + entities: [{name: 'e', dataSource: 'p.d.t', keys: ['k'], fields: []}], + metrics: + [{name: 'rev', expression: 'SUM(e.x)', entity: 'e', type: 'Integer'}], + }; + const {entries} = generateCatalogResources(model, OPTS); + expect(entries[0].entryType.endsWith('/semantic-model')).toBe(true); + const anchorName = entries[0].name; + for (const child of entries.slice(1)) { + expect(child.parentEntry).toBe(anchorName); + } + }); + + test( + 'two entities that normalize to the same id: the duplicate is skipped + warned', + () => { + // 'a b' and 'a-b' both slug to 'a_b' vs 'a-b'? '-' is allowed, space -> + // '_'. Use names that both map to 'a_b': 'a b' and 'a.b' would differ + // ('.' kept). 'a b' and 'a/b' both become 'a_b'. + const model: SemanticModel = { + name: 'm', + relationships: [], + metrics: [], + entities: [ + {name: 'a b', dataSource: 'p.d.t1', keys: ['k'], fields: []}, + {name: 'a/b', dataSource: 'p.d.t2', keys: ['k'], fields: []}, + ], + }; + const {entries, warnings} = generateCatalogResources(model, OPTS); + const entityEntries = + entries.filter(e => e.entryType.endsWith('/semantic-entity')); + expect(entityEntries.length).toBe(1); + expect(warnings.some(w => w.includes('duplicates an earlier one'))) + .toBe(true); + }); +}); + + +describe('relationships map to schema-join entry links', () => { + // A two-entity model joined by a direct foreign key (orders.custkey -> + // customer.custkey), the common case the loader produces from + // `relationships`. + function directFkModel(): SemanticModel { + return { + name: 'm', + metrics: [], + entities: [ + {name: 'orders', dataSource: 'p.d.orders', keys: ['o_key'], fields: []}, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: ['c_key'], + fields: [] + }, + ], + relationships: [{ + name: 'orders_to_customer', + source: {entity: 'orders', columns: ['custkey']}, + destination: {entity: 'customer', columns: ['c_key']}, + description: 'each order belongs to a customer', + }], + }; + } + + test('a direct FK becomes one FOREIGN_KEY schema-join link', () => { + const {entryLinks, warnings} = + generateCatalogResources(directFkModel(), OPTS); + expect(entryLinks.length).toBe(1); + const link = entryLinks[0]; + + // Link id is slugged and undirected: two UNSPECIFIED references naming the + // endpoint entities' entries, typed schema-join. + expect(link.name!.endsWith('/entryLinks/m-orders-to-customer')).toBe(true); + expect(link.entryLinkType.endsWith('/entryLinkTypes/schema-join')) + .toBe(true); + expect(link.entryReferences.map(r => r.type)).toEqual([ + 'UNSPECIFIED', 'UNSPECIFIED' + ]); + expect(link.entryReferences[0].name.endsWith('/entries/m.entities.orders')) + .toBe(true); + expect( + link.entryReferences[1].name.endsWith('/entries/m.entities.customer')) + .toBe(true); + + // The join direction + column pairing live in the aspect: source is the FK + // side, each side's `name` is the entity's dataSource, type is FOREIGN_KEY. + const aspect = link.aspects!['dataplex-types.global.schema-join'].data!; + expect(aspect.userManaged).toBe(true); + expect(aspect.joins.length).toBe(1); + const join = aspect.joins[0]; + expect(join.type).toBe('FOREIGN_KEY'); + expect(join.inferenceSource).toBe('USER'); + expect(join.source).toEqual({name: 'p.d.orders', fields: ['custkey']}); + expect(join.target).toEqual({name: 'p.d.customer', fields: ['c_key']}); + expect(join.description).toBe('each order belongs to a customer'); + expect(warnings.length).toBe(0); + }); + + test( + 'a many-to-many (association) edge is skipped and warned, no link', + () => { + const model = directFkModel(); + model.relationships[0].association = { + dataSource: 'p.d.order_customer', + keys: ['id'], + sourceColumns: ['o_key'], + destinationColumns: ['c_key'], + }; + const {entryLinks, warnings} = generateCatalogResources(model, OPTS); + expect(entryLinks.length).toBe(0); + expect(warnings.some( + w => w.includes('orders_to_customer') && + w.includes('many-to-many'))) + .toBe(true); + }); + + test('an edge to an unpublished entity is skipped and warned', () => { + const model = directFkModel(); + // Point the destination at an entity the model does not declare, so it is + // never emitted and the link has no endpoint entry to reference. + model.relationships[0].destination.entity = 'ghost'; + const {entryLinks, warnings} = generateCatalogResources(model, OPTS); + expect(entryLinks.length).toBe(0); + expect(warnings.some( + w => w.includes('orders_to_customer') && w.includes('ghost'))) + .toBe(true); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/load_semantic_models.test.ts b/toolbox/mdcode/tests/libts/semantic/load_semantic_models.test.ts new file mode 100644 index 00000000..02b6c6e3 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/load_semantic_models.test.ts @@ -0,0 +1,86 @@ +// Tests for the shared document loader (loadSemanticModels in +// src/libts/semantic/loader.ts). +// +// loadSemanticModels parses every authored document into the IR ONCE so a +// multi-destination push (BigQuery + Knowledge Catalog) validates each model a +// single time and fans the result out to both legs. It tags each model with its +// document, prefixes loader warnings with the document name, and returns a +// parse/schema error (naming the document) rather than throwing. + +import {describe, expect, test} from 'bun:test'; + +import {loadSemanticModels} from '../../../src/libts/semantic/loader'; + +// A minimal loader-valid document: one model, one dataset (with a primary key +// so it does not warn), no metrics. +function doc(modelName: string, source = 'proj.ds.tbl'): string { + return `semantic_model: + - name: ${modelName} + datasets: + - name: orders + source: ${source} + primary_key: [id] + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id +`; +} + +describe('loadSemanticModels', () => { + test( + 'flattens models across documents, tagging each with its document', + () => { + const res = loadSemanticModels([ + {name: 'a.yaml', text: doc('sales')}, + {name: 'b.yaml', text: doc('ops')}, + ]); + expect(res.error).toBeUndefined(); + expect(res.models.map(m => m.document)).toEqual(['a.yaml', 'b.yaml']); + expect(res.models.map(m => m.model.name)).toEqual(['sales', 'ops']); + }); + + test('prefixes loader warnings with the originating document name', () => { + // A dataset with no primary_key makes the loader warn; the warning must + // carry the document name so a multi-document push stays diagnosable. + const noKey = `semantic_model: + - name: sales + datasets: + - name: orders + source: proj.ds.tbl +`; + const res = loadSemanticModels([{name: 'sales.yaml', text: noKey}]); + expect(res.error).toBeUndefined(); + expect(res.warnings.some(w => w.startsWith('[sales.yaml] '))).toBe(true); + }); + + test('returns a document-named error on a parse/schema failure', () => { + // The second document is not a valid model; the error must name it and the + // models parsed before it are still returned. + const res = loadSemanticModels([ + {name: 'good.yaml', text: doc('sales')}, + {name: 'broken.yaml', text: 'semantic_model: [ this is: not valid'}, + ]); + expect(res.error).toBeDefined(); + expect(res.error).toContain('broken.yaml'); + expect(res.models.map(m => m.model.name)).toEqual(['sales']); + }); + + test('applies defaultProject when a source omits its project', () => { + const res = loadSemanticModels( + [{name: 'sales.yaml', text: doc('sales', 'ds.tbl')}], + {defaultProject: 'scope-proj'}); + expect(res.error).toBeUndefined(); + expect(res.models[0].model.entities[0].dataSource) + .toBe('scope-proj.ds.tbl'); + }); + + test('no documents yields no models and no error', () => { + const res = loadSemanticModels([]); + expect(res.error).toBeUndefined(); + expect(res.models).toEqual([]); + expect(res.warnings).toEqual([]); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/loader.test.ts b/toolbox/mdcode/tests/libts/semantic/loader.test.ts index 5f0eef71..12d9c91e 100644 --- a/toolbox/mdcode/tests/libts/semantic/loader.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/loader.test.ts @@ -720,7 +720,7 @@ function fixture(name: string): string { describe('gold fixtures parse from disk (real YAML files)', () => { test('star_orders_customer.yaml: happy path (ai_context, time dimension, metrics)', () => { - const { models, warnings } = loadModels(fixture('star_orders_customer.yaml')); + const { models } = loadModels(fixture('star_orders_customer.yaml')); expect(models).toHaveLength(1); const m = models[0]; expect(m.name).toBe('sales'); @@ -741,10 +741,11 @@ describe('gold fixtures parse from disk (real YAML files)', () => { expect(revenue.expression).toBe('SUM(orders.o_totalprice)'); expect(revenue.entity).toBe('orders'); expect(revenue.aiContext?.synonyms).toEqual(['revenue', 'sales']); - // COUNT(*) references no entity -> warned, attach entity omitted. + // order_count is entity-scoped (COUNT(orders.o_orderkey)) -> attach entity + // inferred from the qualifier, so it is a valid single-entity measure. const count = m.metrics.find(mt => mt.name === 'order_count')!; - expect(count.entity).toBeUndefined(); - expect(warnings.some(w => w.includes("metric 'order_count'"))).toBe(true); + expect(count.entity).toBe('orders'); + expect(count.expression).toBe('COUNT(orders.o_orderkey)'); }); test('vendor_dialects.yaml: non-target dialects kept as imported_expression', () => { diff --git a/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts new file mode 100644 index 00000000..82943898 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts @@ -0,0 +1,334 @@ +// Behavior specification for the OSI converter's serialize direction +// (serializeModel in src/libts/semantic/osi_converter.ts). +// +// The serializer is the inverse of loader.ts: IR -> open-format YAML. The +// strongest guarantee is a round trip through the loader -- load a fixture to +// the IR, serialize it, load the serialized text again, and assert the two IRs +// are identical. That pins IR-level fidelity across every feature the loader +// produces (datasets, fields, datatypes, dimensions, labels, ai_context, +// custom_extensions, relationships, metrics, imported expressions) without +// hard-coding YAML text. Targeted structural tests cover the mapping details a +// round trip cannot isolate. + +import {describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as yaml from 'yaml'; + +import {Field, Metric, Relationship, SemanticModel} from '../../../src/libts/semantic/ir'; +import {loadModels} from '../../../src/libts/semantic/loader'; +import {modelDocument, serializeModel} from '../../../src/libts/semantic/osi_converter'; + +const FIXTURES = path.join(__dirname, 'fixtures'); + +function loadFixture(name: string): SemanticModel[] { + const text = fs.readFileSync(path.join(FIXTURES, name), 'utf8'); + return loadModels(text).models; +} + + +describe('loader <-> serialize round trip is IR-stable', () => { + // Each fixture exercises a different slice of the format: relationships + + // ai_context + synonyms + label + dimension; the full tpc-ds corpus with + // custom_extensions + unique_keys; explicit datatypes; and imported + // (vendor-dialect) expressions. + const fixtures = [ + 'star_orders_customer.yaml', + 'tpcds_retail.yaml', + 'sales_google_ext.yaml', + 'vendor_dialects.yaml', + 'lineitem_databricks_ext.yaml', + 'sales_bq_graph_target.yaml', + ]; + + for (const fixture of fixtures) { + test(`${fixture} survives IR -> YAML -> IR unchanged`, () => { + const original = loadFixture(fixture); + expect(original.length).toBeGreaterThan(0); + + for (const model of original) { + const {yaml: text} = serializeModel(model); + const reloaded = loadModels(text).models; + expect(reloaded).toHaveLength(1); + // IR-level equality: every field the loader keeps must match exactly. + expect(reloaded[0]).toEqual(model); + } + }); + } +}); + + +describe('serialized document structure', () => { + const model = loadFixture('star_orders_customer.yaml')[0]; + const doc = modelDocument(model) as any; + const sm = doc.semantic_model[0]; + + test('emits the supported version and a single model', () => { + expect(doc.version).toBe('0.2.0.dev0'); + expect(doc.semantic_model).toHaveLength(1); + expect(sm.name).toBe(model.name); + }); + + test('a dataset source is the opaque dataSource string, verbatim', () => { + const orders = sm.datasets.find((d: any) => d.name === 'orders'); + const entity = model.entities.find(e => e.name === 'orders')!; + expect(orders.source).toBe(entity.dataSource); + expect(typeof orders.source).toBe('string'); + }); + + test('primary_key mirrors the entity keys', () => { + const orders = sm.datasets.find((d: any) => d.name === 'orders'); + const entity = model.entities.find(e => e.name === 'orders')!; + expect(orders.primary_key).toEqual(entity.keys); + }); + + test('ai_context is emitted structurally (synonyms round-trip)', () => { + // The fixture annotates a field (o_orderdate) with synonyms; find it and + // assert the structured ai_context is emitted under that field. + const entity = model.entities.find( + e => e.fields.some(f => f.aiContext?.synonyms?.length))!; + const field = entity.fields.find(f => f.aiContext?.synonyms?.length)!; + const dsDoc = sm.datasets.find((d: any) => d.name === entity.name); + const fieldDoc = dsDoc.fields.find((f: any) => f.name === field.name); + expect(fieldDoc.ai_context.synonyms).toEqual(field.aiContext!.synonyms); + }); + + test('a relationship maps to from/to + positional columns', () => { + expect(sm.relationships.length).toBeGreaterThan(0); + const rel = model.relationships[0]; + const relDoc = sm.relationships[0]; + expect(relDoc.from).toBe(rel.source.entity); + expect(relDoc.to).toBe(rel.destination.entity); + expect(relDoc.from_columns).toEqual(rel.source.columns); + expect(relDoc.to_columns).toEqual(rel.destination.columns); + }); +}); + + +describe('expression + datatype + dimension mapping', () => { + test('an explicit datatype round-trips as `datatype`', () => { + const model = loadFixture('sales_google_ext.yaml')[0]; + const typed = model.entities.flatMap(e => e.fields).find(f => f.type); + expect(typed).toBeDefined(); + const {yaml: text} = serializeModel(model); + const reloaded = loadModels(text).models[0]; + const back = reloaded.entities.flatMap(e => e.fields) + .find(f => f.name === typed!.name)!; + expect(back.type).toBe(typed!.type); + }); + + test('a bare dimension marker survives as `dimension: {}`', () => { + const field: + Field = {name: 'ship_date', expression: 'e.ship_date', dimension: {}}; + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: ['k'], fields: [field]}], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const fieldDoc = doc.semantic_model[0].datasets[0].fields[0]; + expect(fieldDoc.dimension).toEqual({}); + // And it reloads back to a dimension field. + const reloaded = loadModels(serializeModel(model).yaml).models[0]; + expect(reloaded.entities[0].fields[0].dimension).toEqual({}); + }); + + test('an imported vendor expression is emitted under its own dialect', () => { + const field: Field = { + name: 'amt', + expression: 'e.amt', + importedExpression: 'e.amt::NUMBER', + importedDialect: 'SNOWFLAKE', + }; + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: ['k'], fields: [field]}], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const dialects = + doc.semantic_model[0].datasets[0].fields[0].expression.dialects; + const labels = dialects.map((d: any) => d.dialect); + expect(labels).toContain('SNOWFLAKE'); + // The canonical form is labeled BIGQUERY so the loader re-picks it exactly. + expect(labels).toContain('BIGQUERY'); + }); + + test('a metric does not emit its derived attach entity', () => { + const metric: Metric = { + name: 'total', + expression: 'SUM(orders.amt)', + entity: 'orders', + }; + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: ['k'], + fields: [{name: 'amt', expression: 'orders.amt'}], + }], + relationships: [], + metrics: [metric], + }; + const metricDoc = + (modelDocument(model) as any).semantic_model[0].metrics[0]; + expect(metricDoc).not.toHaveProperty('entity'); + // The loader re-derives it on reload. + const reloaded = loadModels(serializeModel(model).yaml).models[0]; + expect(reloaded.metrics[0].entity).toBe('orders'); + }); +}); + + +describe('lossy edges are flagged', () => { + test( + 'an association relationship warns and drops the junction detail', () => { + const rel: Relationship = { + name: 'enrollment', + source: {entity: 'student', columns: ['id']}, + destination: {entity: 'course', columns: ['id']}, + association: { + dataSource: 'p.d.enrollment', + keys: ['student_id', 'course_id'], + sourceColumns: ['student_id'], + destinationColumns: ['course_id'], + }, + }; + const model: SemanticModel = { + name: 'school', + entities: [ + { + name: 'student', + dataSource: 'p.d.student', + keys: ['id'], + fields: [] + }, + { + name: 'course', + dataSource: 'p.d.course', + keys: ['id'], + fields: [] + }, + ], + relationships: [rel], + metrics: [], + }; + const {yaml: text, warnings} = serializeModel(model); + expect(warnings.some(w => /association/i.test(w))).toBe(true); + // The direct-FK view is still emitted (from/to + columns), so it + // reloads. + const relDoc = yaml.parse(text).semantic_model[0].relationships[0]; + expect(relDoc.from).toBe('student'); + expect(relDoc.to).toBe('course'); + expect(relDoc.from_columns).toEqual(['id']); + }); +}); + + +describe('serialize flags loader-invalid reconstructions', () => { + test('a model with no entities warns that datasets is required', () => { + const model: SemanticModel = { + name: 'empty', + entities: [], + relationships: [], + metrics: [], + }; + const {warnings} = serializeModel(model); + expect(warnings.some(w => /no datasets/i.test(w))).toBe(true); + }); + + test('a field with no expression warns that one is required', () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'orphan'}], + }], + relationships: [], + metrics: [], + }; + const {warnings} = serializeModel(model); + expect(warnings.some(w => /field 'orphan'.*no expression/i.test(w))) + .toBe(true); + }); + + test( + 'an imported dialect colliding with the canonical label is relabeled', + () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: [], + fields: [{ + name: 'amt', + expression: 'orders.amt', + importedExpression: 'orders.AMT', + importedDialect: 'BIGQUERY', + }], + }], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const labels: string[] = + doc.semantic_model[0].datasets[0].fields[0].expression.dialects.map( + (d: any) => d.dialect); + // No duplicate dialect label: the imported form is relabeled off + // BIGQUERY so the loader does not pick between two BIGQUERY entries. + expect(new Set(labels).size).toBe(labels.length); + expect(labels).toContain('BIGQUERY'); + }); +}); + + +// -- Golden corpus: the whole IR -> OSI YAML output, reviewable as a file. -- +// +// The round-trip tests above prove IR-level stability but never pin the exact +// YAML text. These goldens capture the full serialized document for a small +// corpus so a reviewer can open a `.yaml` next to its `.osi.golden.yaml` and see +// exactly what the serializer emits. The same corpus backs kc_converter's +// `.pull.golden.yaml`, so diffing the two shows what a Knowledge Catalog round +// trip loses relative to the authored source. +// +// Regenerate after an intentional serializer change: +// UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/osi_converter.test.ts +describe('golden OSI document: each corpus fixture serializes to its exact YAML', + () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + // Same load defaults as the KC e2e/pull goldens, so the OSI golden + // and the pull golden are directly comparable. + const LOAD = {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}; + const osiGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.osi.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + const models = loadModels(text, LOAD).models; + const actual = models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = osiGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename(golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts new file mode 100644 index 00000000..83f5aee0 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -0,0 +1,352 @@ +// Tests for the semantic-model Knowledge Catalog pull leg +// (pullKnowledgeCatalog in src/libts/semantic/pull_kc.ts). +// +// pullKnowledgeCatalog is the orchestration around the pure reader: enumerate +// the entry group, hydrate each semantic entry's aspect data, and reconstruct +// the IR. The catalog client is stubbed so no network call is made. The entries +// the fake serves are produced by the real emitter, so this exercises the true +// list -> hydrate -> read path end to end (an entity is re-fetched with BOTH +// its semantic-entity and schema aspects, and a second pass fetches each +// entity's schema-join links). The reader's own mapping is covered in +// kc_converter.test.ts; the focus here is the fetch SEQUENCE: aspect hydration, +// the relationship-link fetch (per-entry, deduped across endpoints), the +// --model filter, skipped entries, and ignoring foreign entries. + +import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; + +import {ApiResult} from '../../../src/libts/gcp/api'; +import {CatalogClient, Entry, EntryLink} from '../../../src/libts/gcp/dataplex'; +import {SemanticModel} from '../../../src/libts/semantic/ir'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {pullKnowledgeCatalog} from '../../../src/libts/semantic/pull_kc'; + +const OPTS = { + project: 'dest', + location: 'us', + entryGroup: 'eg' +}; + +const SALES: SemanticModel = { + name: 'sales', + entities: [{ + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], + fields: [ + {name: 'o_totalprice', expression: 'orders.o_totalprice', type: 'Decimal'} + ], + }], + relationships: [], + metrics: [{ + name: 'total_revenue', + expression: 'SUM(orders.o_totalprice)', + entity: 'orders', + // Metrics carry a datatype through KC (semantic-metric.dataType is + // required); Decimal -> NUMERIC on emit, NUMERIC -> Decimal on read. + type: 'Decimal' + }], +}; + +// The entries the emitter would have written for a model. Emitted with +// expressions on (a `--emit-expressions` push) so a model that carries field / +// metric expressions reconstructs exactly; the default push omits them (the +// converter tests in kc_converter.test.ts pin that gating). +function entriesFor(model: SemanticModel): Entry[] { + return generateCatalogResources(model, {...OPTS, emitExpressions: true}) + .entries; +} + +function ok(result?: T): ApiResult { + return {status: 200, result}; +} +function err(status: number, message: string): ApiResult { + return {status, message}; +} + +// Stubs listEntries (yields `listed`) and lookupEntry (serves `served` by name, +// or 404s an unknown name). `lookupFail` forces a failure for one entry name. +// `links` are served by lookupEntryLinks per referenced entry (so an undirected +// link comes back once from each endpoint, exercising the puller's dedup); +// `linkFail` forces a link-lookup failure for one entry name. +function stubClient( + listed: Entry[], served: Entry[], lookupFail?: string, + links: EntryLink[] = [], + linkFail?: string): {list: any; lookup: any; lookupLinks: any} { + const byName = new Map(served.map(e => [e.name, e])); + const list = spyOn(CatalogClient.prototype, 'listEntries') + .mockImplementation(async function*() { + for (const e of listed) yield e; + } as any); + const lookup = spyOn(CatalogClient.prototype, 'lookupEntry') + .mockImplementation(async (_p, _l, name) => { + if (name === lookupFail) return err(500, 'boom'); + const e = byName.get(name); + return e ? ok(e) : err(404, 'not found'); + }); + const lookupLinks = + spyOn(CatalogClient.prototype, 'lookupEntryLinks') + .mockImplementation(async (_p: any, _l: any, opts: any) => { + if (opts.entry === linkFail) return err(500, 'boom'); + const forEntry = links.filter( + l => + (l.entryReferences ?? []).some(r => r.name === opts.entry)); + return ok(forEntry); + }); + return {list, lookup, lookupLinks}; +} + +afterEach(() => { + mock.restore(); +}); + + +describe('pullKnowledgeCatalog: happy path', () => { + test('reconstructs the model from listed + hydrated entries', async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models).toHaveLength(1); + expect(models[0]).toEqual(SALES); + expect(warnings).toHaveLength(0); + // Every listed semantic entry (model + entity + metric) was hydrated. + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test( + 'an entity is hydrated with BOTH its semantic-entity and schema aspects', + async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + await pullKnowledgeCatalog(cat, OPTS); + + // Find the lookup call for the entity entry and inspect its requested + // aspect types (the 4th arg). + const entityEntry = + entries.find(e => e.entryType.endsWith('/semantic-entity'))!; + const call = + lookup.mock.calls.find((c: any[]) => c[2] === entityEntry.name)!; + const aspectTypes: string[] = call[3]; + expect( + aspectTypes.some(t => t.endsWith('/aspectTypes/semantic-entity'))) + .toBe(true); + expect(aspectTypes.some(t => t.endsWith('/aspectTypes/schema'))) + .toBe(true); + }); +}); + + +describe('pullKnowledgeCatalog: relationship links', () => { + // A 1:N model: orders.o_custkey (the FK side) references customer.c_custkey. + const SALES_REL: SemanticModel = { + name: 'sales', + entities: [ + { + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], + fields: [{name: 'o_custkey', expression: 'orders.o_custkey'}], + }, + { + name: 'customer', + dataSource: 'demo.sales.customer', + keys: [], + fields: [{name: 'c_custkey', expression: 'customer.c_custkey'}], + }, + ], + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + + test( + 'a schema-join link reconstructs into a relationship exactly once', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + const {lookupLinks} = + stubClient(entries, entries, undefined, entryLinks); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models[0].relationships).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + expect(warnings).toHaveLength(0); + // Links are fetched per entity (2 entities), and the single undirected + // link -- returned from both endpoints -- is deduped to one edge. + expect(lookupLinks).toHaveBeenCalledTimes(2); + }); + + test( + 'an unnamed link returned from both endpoints is deduped to one edge', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + // A nameless link can't dedup by name; the puller must fall back to the + // endpoint pair, or the per-endpoint fan-out would yield it twice. + const nameless = entryLinks.map(l => ({...l, name: undefined})); + const {lookupLinks} = stubClient(entries, entries, undefined, nameless); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, OPTS); + + // Deduped to a single edge (a nameless link has no id, so the name + // itself can't be recovered -- but it must not be counted twice). + expect(models[0].relationships).toHaveLength(1); + expect(models[0].relationships[0].source.entity).toBe('orders'); + expect(lookupLinks).toHaveBeenCalledTimes(2); + }); + + test( + 'a failed link lookup on one endpoint still recovers the edge from the ' + + 'other, and warns', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + const ordersEntry = + entries.find(e => (e.entrySource?.displayName) === 'orders')!; + // Fail the link lookup for the orders entity; the link still comes back + // from the customer endpoint. + stubClient(entries, entries, undefined, entryLinks, ordersEntry.name); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models[0].relationships.map(r => r.name)).toEqual(['places']); + expect(warnings.some( + w => /failed to fetch entry links/i.test(w) && + w.includes(ordersEntry.name))) + .toBe(true); + }); +}); + + +describe('pullKnowledgeCatalog: filtering and robustness', () => { + test('--model keeps only the named model', async () => { + const other: SemanticModel = { + name: 'inventory', + entities: + [{name: 'items', dataSource: 'p.d.items', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + const entries = [...entriesFor(SALES), ...entriesFor(other)]; + stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + expect(models.map(m => m.name)).toEqual(['sales']); + }); + + test('--model with no match returns nothing and warns', async () => { + const entries = entriesFor(SALES); + stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'nope'}); + expect(models).toHaveLength(0); + expect(warnings.some(w => /no semantic model named 'nope'/i.test(w))) + .toBe(true); + }); + + test( + 'a non-semantic entry in the group is ignored, not fetched', async () => { + const entries = entriesFor(SALES); + const foreign: Entry = { + name: `projects/dest/locations/us/entryGroups/eg/entries/foreign`, + entryType: 'projects/x/locations/us/entryTypes/some-other-type', + }; + const {lookup} = stubClient([...entries, foreign], entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, OPTS); + expect(models).toHaveLength(1); + // The foreign entry is never hydrated (only the 3 semantic entries + // are). + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test('a failed hydration is skipped with a warning', async () => { + const entries = entriesFor(SALES); + const metricEntry = + entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + stubClient(entries, entries, metricEntry.name); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + // The model + entity still reconstruct; only the metric is dropped. + expect(models).toHaveLength(1); + expect(models[0].metrics).toHaveLength(0); + expect(warnings.some( + w => /failed to fetch/i.test(w) && w.includes(metricEntry.name))) + .toBe(true); + }); + + test('--model hydrates only the target model\'s entries', async () => { + const other: SemanticModel = { + name: 'inventory', + entities: + [{name: 'items', dataSource: 'p.d.items', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + const entries = [...entriesFor(SALES), ...entriesFor(other)]; + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + expect(models.map(m => m.name)).toEqual(['sales']); + // SALES has 3 entries (model + entity + metric); inventory's are never + // fetched -- the flag scopes hydration, not just the final result. + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test( + '--model keeps a child whose parentEntry does not resolve (sole-anchor ' + + 'fallback)', + async () => { + const entries = entriesFor(SALES); + // Simulate a project-id normalization mismatch: the metric points at an + // anchor name that differs from the emitted one. With a single model in + // the group a full pull still attaches it via the reader's sole-anchor + // fallback, so a scoped pull must keep it too (else --model silently + // drops a child a full pull returns). + const metricEntry = + entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + metricEntry.parentEntry = metricEntry.parentEntry!.replace( + 'projects/dest/', 'projects/12345/'); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + + expect(models).toHaveLength(1); + expect(models[0].metrics.map(m => m.name)).toEqual(['total_revenue']); + // The metric was hydrated despite the parent mismatch (3 entries). + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test('--model with no match fetches nothing and warns', async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'nope'}); + expect(models).toHaveLength(0); + expect(lookup).not.toHaveBeenCalled(); + expect(warnings.some(w => /no semantic model named 'nope'/i.test(w))) + .toBe(true); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts new file mode 100644 index 00000000..ac4c7bc0 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts @@ -0,0 +1,75 @@ +// Tests for the SemanticModel layout's pull write-path +// (src/libts/layouts/semantic-model.ts): modelPath / hasModel / +// writeModelDocument. These are the sink `pull` writes reconstructed models to; +// the push-side discovery (modelDocuments) is exercised via the deploy tests. + +import {afterEach, beforeEach, describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import {SemanticModelLayout} from '../../../src/libts/layouts/semantic-model'; + +let root: string; +let catalogPath: string; + +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'kcmd-layout-')); + catalogPath = path.join(root, 'catalog'); +}); + +afterEach(() => { + fs.rmSync(root, {recursive: true, force: true}); +}); + +async function layout(entryGroup?: string): Promise { + const l = new SemanticModelLayout(catalogPath, entryGroup); + await l.init(); + return l; +} + + +describe('SemanticModelLayout write path', () => { + test('modelPath maps to EntryGroups//.yaml', async () => { + const l = await layout('eg'); + expect(l.modelPath('sales')) + .toBe(path.join(catalogPath, 'EntryGroups', 'eg', 'sales.yaml')); + }); + + test('modelPath sanitizes path separators in the model name', async () => { + const l = await layout('eg'); + expect(l.modelPath('a/b')) + .toBe(path.join(catalogPath, 'EntryGroups', 'eg', 'a_b.yaml')); + }); + + test('modelPath throws without an entry group', async () => { + const l = await layout(undefined); + expect(() => l.modelPath('sales')).toThrow(/entry group/i); + }); + + test( + 'writeModelDocument creates the file, dirs, and indexes it', async () => { + const l = await layout('eg'); + expect(l.hasModel('sales')).toBe(false); + + l.writeModelDocument('sales', 'version: x\n'); + + const p = l.modelPath('sales'); + expect(fs.existsSync(p)).toBe(true); + expect(fs.readFileSync(p, 'utf8')).toBe('version: x\n'); + expect(l.hasModel('sales')).toBe(true); + // Indexed, so a subsequent read surfaces it as a model document. + expect(l.modelDocuments()).toEqual([ + {name: 'sales', text: 'version: x\n'} + ]); + }); + + test( + 'writeModelDocument overwrites an existing document (last-write-wins)', + async () => { + const l = await layout('eg'); + l.writeModelDocument('sales', 'first\n'); + l.writeModelDocument('sales', 'second\n'); + expect(fs.readFileSync(l.modelPath('sales'), 'utf8')).toBe('second\n'); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/target.test.ts b/toolbox/mdcode/tests/libts/semantic/target.test.ts new file mode 100644 index 00000000..476a2200 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/target.test.ts @@ -0,0 +1,45 @@ +// Tests --target resolution for the semantic-model push (src/tool/commands.ts). +// +// The two deploy legs are covered end to end elsewhere +// (deploy_bigquery.test.ts for BigQuery, deploy_knowledge_catalog.test.ts for +// Knowledge Catalog); this pins only the flag-to-destinations mapping and its +// ordering, which drives the dispatch in push(). + +import {describe, expect, test} from 'bun:test'; + +import {resolveTargets} from '../../../src/tool/commands'; + +describe('resolveTargets', () => { + test('defaults to every destination when no target is given', () => { + expect(resolveTargets(undefined)).toEqual(['bigquery', 'kc']); + }); + test('\'bq\' and \'bigquery\' both select BigQuery', () => { + expect(resolveTargets('bq')).toEqual(['bigquery']); + expect(resolveTargets('bigquery')).toEqual(['bigquery']); + }); + test('\'kc\' selects Knowledge Catalog', () => { + expect(resolveTargets('kc')).toEqual(['kc']); + }); + test('a comma-separated list selects each destination', () => { + expect(resolveTargets('bq,kc')).toEqual(['bigquery', 'kc']); + }); + test('\'all\' selects every destination', () => { + expect(resolveTargets('all')).toEqual(['bigquery', 'kc']); + }); + test('the result is always in canonical order (BigQuery first)', () => { + // Regardless of how the user orders the flag, fail-fast runs BigQuery + // first. + expect(resolveTargets('kc,bq')).toEqual(['bigquery', 'kc']); + }); + test('duplicate tokens are de-duplicated', () => { + expect(resolveTargets('bq,bq,kc')).toEqual(['bigquery', 'kc']); + expect(resolveTargets('all,kc')).toEqual(['bigquery', 'kc']); + }); + test('is case-insensitive and tolerates surrounding whitespace', () => { + expect(resolveTargets(' BQ , KC ')).toEqual(['bigquery', 'kc']); + }); + test('an unknown token anywhere in the list resolves to undefined', () => { + expect(resolveTargets('snowflake')).toBeUndefined(); + expect(resolveTargets('bq,snowflake')).toBeUndefined(); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/validate.test.ts b/toolbox/mdcode/tests/libts/semantic/validate.test.ts new file mode 100644 index 00000000..048482c2 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/validate.test.ts @@ -0,0 +1,185 @@ +// Behavior spec for the push-time validation gate +// (src/libts/semantic/validate.ts). + +import {describe, expect, test} from 'bun:test'; + +import {CustomExtension, Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; +import {LoadedModel} from '../../../src/libts/semantic/loader'; +import {validateBigQueryDataSources, validatePushRequirements} from '../../../src/libts/semantic/validate'; +import {BigQueryClientMock} from '../mocks'; + +// A parsed BigQuery Graph deployment target the strict matcher accepts. +const BQ_TARGET = + '//bigquery.googleapis.com/projects/p/datasets/d/propertyGraphs/g'; + +function googleExt(targets: string[]): CustomExtension { + return {vendorName: 'GOOGLE', data: JSON.stringify({deploymentTargets: targets})}; +} + +function loaded(model: SemanticModel, document = 'doc'): LoadedModel { + return {document, model}; +} + +function model( + over: Partial = {}, + exts?: CustomExtension[]): SemanticModel { + return { + name: 'm', + entities: [], + relationships: [], + metrics: [], + ...(exts ? {customExtensions: exts} : {}), + ...over, + }; +} + +describe('validatePushRequirements', () => { + test('a model with a deployment target and resolved metrics passes', () => { + const m = model( + {metrics: [{name: 'rev', expression: 'SUM(o.p)', entity: 'o'} as Metric]}, + [googleExt([BQ_TARGET])]); + expect(validatePushRequirements([loaded(m)])).toEqual([]); + }); + + test('a model with no deployment target is rejected', () => { + const errs = validatePushRequirements([loaded(model())]); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('exactly one'); + expect(errs[0]).toContain('doc'); + }); + + test('a BigQuery-target metric that resolves to no entity is rejected', () => { + const m = model( + {metrics: [{name: 'cnt', expression: 'COUNT(*)'} as Metric]}, + [googleExt([BQ_TARGET])]); + const errs = validatePushRequirements([loaded(m)]); + expect(errs.length).toBe(1); + expect(errs[0]).toContain("metric 'cnt'"); + expect(errs[0]).toContain('single entity'); + }); + + test('a non-BigQuery Graph deployment target is rejected as malformed', () => { + // We only support BigQuery Graph targets; a single, otherwise well-formed + // non-BigQuery URI still fails because it does not parse as one. + const m = model( + {}, [googleExt(['//dataplex.googleapis.com/projects/p/locations/us'])]); + const errs = validatePushRequirements([loaded(m)]); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('not a valid BigQuery Graph URI'); + }); + + test('more than one deployment target is rejected', () => { + const other = + '//bigquery.googleapis.com/projects/p/datasets/d/propertyGraphs/g2'; + const m = model({}, [googleExt([BQ_TARGET, other])]); + const errs = validatePushRequirements([loaded(m)]); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('exactly one'); + }); + + test('malformed GOOGLE extension JSON is reported, not thrown', () => { + const m = model({}, [{vendorName: 'GOOGLE', data: '{not json'}]); + const errs = validatePushRequirements([loaded(m)]); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('not valid JSON'); + }); + + test('errors accumulate across models', () => { + const a = loaded(model({name: 'a'}), 'a'); + const b = loaded(model({name: 'b'}), 'b'); + const errs = validatePushRequirements([a, b]); + expect(errs.length).toBe(2); + }); +}); + + +// Helpers for the live data-source check. +function entity(name: string, dataSource: string): Entity { + return {name, dataSource, keys: [], fields: []} as Entity; +} + +function mockTable(project: string, dataset: string, tableId: string) { + return {id: tableId, tableReference: {projectId: project, datasetId: dataset, tableId}}; +} + +describe('validateBigQueryDataSources', () => { + test('passes when every entity source table is reachable', async () => { + const bq = new BigQueryClientMock(); + bq.addMockTable(mockTable('p', 'd', 'orders')); + bq.addMockTable(mockTable('p', 'd', 'customer')); + const m = model( + {entities: [entity('orders', 'p.d.orders'), + entity('customer', 'p.d.customer')]}); + expect(await validateBigQueryDataSources([loaded(m)], bq, 'p')).toEqual([]); + }); + + test('reports a missing source table, naming the entity/model/document', + async () => { + const bq = new BigQueryClientMock(); + bq.addMockTable(mockTable('p', 'd', 'orders')); + const m = model({ + entities: [entity('orders', 'p.d.orders'), + entity('gone', 'p.d.ghost')], + }); + const errs = await validateBigQueryDataSources([loaded(m)], bq, 'p'); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('p.d.ghost'); + expect(errs[0]).toContain('does not exist'); + expect(errs[0]).toContain("entity 'gone'"); + expect(errs[0]).toContain('doc'); + }); + + test('covers a four-part REST-catalog / Iceberg source', async () => { + // A federated REST-catalog table (e.g. Iceberg via BigLake) is a four-part + // name that tables.get cannot address; the dry-run resolves it as the deploy + // will. A reachable one passes; a missing one is reported by name. + const bq = new BigQueryClientMock(); + bq.addMockSource('ice_cat.db.sales.orders'); + const ok = model({entities: [entity('orders', 'ice_cat.db.sales.orders')]}); + expect(await validateBigQueryDataSources([loaded(ok)], bq, 'p')).toEqual([]); + + const missing = + model({entities: [entity('lost', 'ice_cat.db.sales.ghost')]}); + const errs = await validateBigQueryDataSources([loaded(missing)], bq, 'p'); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('ice_cat.db.sales.ghost'); + expect(errs[0]).toContain('does not exist'); + }); + + test('reports a permission-denied source table', async () => { + const bq = new BigQueryClientMock(); + bq.query = + (async () => ({status: 403, message: 'Access Denied: no permission'})) as + any; + const m = model({entities: [entity('o', 'p.d.o')]}); + const errs = await validateBigQueryDataSources([loaded(m)], bq, 'p'); + expect(errs.length).toBe(1); + expect(errs[0]).toContain('permission denied'); + }); + + test('skips a source that is a query, not a table', async () => { + // A query source (contains whitespace) is not a table and cannot be probed, + // so it is not checked (nothing is mocked for it). + const bq = new BigQueryClientMock(); + const m = model({entities: [entity('q', 'SELECT * FROM x')]}); + expect(await validateBigQueryDataSources([loaded(m)], bq, 'p')).toEqual([]); + }); + + test('probes each distinct table once across entities and models', + async () => { + const bq = new BigQueryClientMock(); + bq.addMockTable(mockTable('p', 'd', 'shared')); + let calls = 0; + const orig = bq.query.bind(bq); + bq.query = ((p: string, sql: string, loc?: string, dry?: boolean) => { + calls++; + return orig(p, sql, loc, dry); + }) as any; + const m1 = model({name: 'm1', entities: [entity('a', 'p.d.shared')]}); + const m2 = model({name: 'm2', entities: [entity('b', 'p.d.shared')]}); + const errs = await validateBigQueryDataSources( + [loaded(m1, 'd1'), loaded(m2, 'd2')], bq, 'p'); + expect(errs).toEqual([]); + expect(calls).toBe(1); + }); +}); diff --git a/toolbox/mdcode/tests/tool/init_semantic_model.test.ts b/toolbox/mdcode/tests/tool/init_semantic_model.test.ts new file mode 100644 index 00000000..9191b7a3 --- /dev/null +++ b/toolbox/mdcode/tests/tool/init_semantic_model.test.ts @@ -0,0 +1,100 @@ +// Tests that `kcmd init --semantic-model` provisions the destination entry +// group (src/tool/commands.ts, init()). +// +// The entry group is created at init -- not on push -- so a semantic-model push +// writes only entries, matching how the standard layout operates (its push +// creates entries, never the entry group). These tests spy on the catalog +// client so no network call is made and run init inside a temp working +// directory (it writes catalog.yaml + the layout dirs relative to cwd). + +import {afterEach, beforeEach, describe, expect, mock, spyOn, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import {ApiResult} from '../../src/libts/gcp/api'; +import {ApiContext} from '../../src/libts/gcp/context'; +import {CatalogClient} from '../../src/libts/gcp/dataplex'; +import {init} from '../../src/tool/commands'; + +const CTX = new ApiContext('test-project', 'us', 'test-token'); + +function ok(result?: T): ApiResult { + return {status: 200, result}; +} +function err(status: number, message: string): ApiResult { + return {status, message}; +} + +let dir = ''; +let cwd = ''; + +beforeEach(() => { + cwd = process.cwd(); + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'kcmd-init-')); + process.chdir(dir); + // init() resolves its context from gcloud; pin it to a hermetic value. + spyOn(ApiContext, 'default').mockReturnValue(CTX); + // init() prints the generated catalog.yaml; keep test output quiet. + spyOn(console, 'log').mockImplementation(() => {}); + spyOn(console, 'error').mockImplementation(() => {}); +}); + +afterEach(() => { + process.chdir(cwd); + if (dir) fs.rmSync(dir, {recursive: true, force: true}); + dir = ''; + mock.restore(); +}); + + +describe('init --semantic-model: entry-group provisioning', () => { + test( + 'creates the destination entry group and the local layout dir', + async () => { + const group = + spyOn(CatalogClient.prototype, 'createEntryGroup') + .mockImplementation(async () => ok({name: 'sales-group'})); + + const code = await init({semanticModel: 'proj.us.sales-group'}); + + expect(code).toBe(0); + expect(group).toHaveBeenCalledTimes(1); + const [project, location, entryGroupId] = group.mock.calls[0]; + expect(project).toBe('proj'); + expect(location).toBe('us'); + expect(entryGroupId).toBe('sales-group'); + expect( + fs.existsSync(path.join('catalog', 'EntryGroups', 'sales-group'))) + .toBe(true); + expect(fs.readFileSync('catalog.yaml', 'utf8')) + .toContain('scope: semantic-model.proj.us.sales-group'); + }); + + test('an already-existing entry group (409) is success', async () => { + const group = + spyOn(CatalogClient.prototype, 'createEntryGroup') + .mockImplementation(async () => err(409, 'already exists')); + + const code = await init({semanticModel: 'proj.us.sales-group'}); + + expect(code).toBe(0); + expect(group).toHaveBeenCalledTimes(1); + expect(fs.existsSync(path.join('catalog', 'EntryGroups', 'sales-group'))) + .toBe(true); + }); + + test('a fatal entry-group error fails init', async () => { + const group = + spyOn(CatalogClient.prototype, 'createEntryGroup') + .mockImplementation(async () => err(403, 'permission denied')); + + const code = await init({semanticModel: 'proj.us.sales-group'}); + + expect(code).toBe(1); + expect(group).toHaveBeenCalledTimes(1); + // The layout dir is not created when provisioning fails. + expect(fs.existsSync(path.join('catalog', 'EntryGroups', 'sales-group'))) + .toBe(false); + }); +});