mdcode: add Knowledge Catalog pull for the semantic model

toolbox/mdcode/README.md

@@ -183,6 +183,14 @@ declared project (the `<projectId>` in the init scope), not the ambient
 `gcloud` project; write a fully-qualified `project.dataset.table` source when
 the tables live elsewhere.
 
+`kcmd push` also deploys the model to Knowledge Catalog (entries for the model,
+its entities and metrics, and `schema-join` links for its relationships). Use
+`--target bq|kc|all` (default `all`) to choose destinations, `--print` to dump
+each destination's generated artifact, and `--force-remove` to delete models
+left in the entry group that the push no longer includes. See
+[docs/semantic-model.md](docs/semantic-model.md) for the full guide, including
+how re-push reconciles removed entities, metrics, relationships, and models.
+
 NOTE: The CLI uses `gcloud` to obtain authentication tokens, so ensure you are authenticated via `gcloud auth application-default login`.
 
 ### MCP Server

toolbox/mdcode/demo/README.md

@@ -183,7 +183,10 @@ bun push.ts
 
 **Cleanup**
 
-* Deletes the Dataplex EntryGroup and the custom `okf` aspect type.
+* Deletes the Dataplex EntryGroup. The custom `okf` aspect type is left in
+  place: it is scoped to the project and location rather than to this demo, so
+  other OKF bundles in the same project carry their signal layer on it. The
+  command to remove it manually is printed at the end.
 
 ```bash
 bun cleanup.ts

toolbox/mdcode/demo/okf/cleanup.ts

@@ -14,10 +14,11 @@ function dataplex(cmd: string, data: string|null=null) {
 dataplex(`entry-groups delete ${entryGroup}`);
 console.log(`Deleted entry group ${entryGroup}`);
 
-try {
-  dataplex(`aspect-types delete okf`);
-  console.log('Deleted custom aspect type okf');
-}
-catch {
-  // Might not exist, or still referenced
-}
+// The okf aspect type is scoped to the project and location, not to this demo's
+// entry group, so any other OKF bundle in the project attaches its signal layer
+// to the same type. Deleting it here would strip that signal from bundles this
+// demo does not own.
+console.log(
+  `Left aspect type ${project}.${location}.okf in place (shared across OKF bundles). ` +
+  `To remove it: gcloud dataplex aspect-types delete okf --project ${project} --location ${location}`
+);

toolbox/mdcode/src/libts/gcp/api.ts

@@ -46,6 +46,12 @@ export class ApiClient {
     return this._requestRetry('PATCH', url, queryParams, body);
   }
 
+  async _delete<T>(resourceName: string,
+                   queryParams?: Record<string, any>): Promise<ApiResult<T>> {
+    const url = `${this._endpoint}/${this._pathPrefix}/${resourceName}`;
+    return this._requestRetry('DELETE', url, queryParams);
+  }
+
   private async _requestRetry<T>(method: string,
                                  url: string,
                                  queryParams?: Record<string, any>,

toolbox/mdcode/src/libts/gcp/bigquery.ts

@@ -60,6 +60,17 @@ export class BigQueryClient extends api.ApiClient {
     return await this._get(name, params);
   }
 
+  // Fetches a single table's metadata. Used as a cheap existence/access probe
+  // before deploy: a 200 means the table is reachable, a 404 that it does not
+  // exist, a 403 that the caller cannot see it. `selectedFields` trims the
+  // response to the reference alone, since only the status is consulted.
+  async getTable(project: string, dataset: string, table: string): Promise<api.ApiResult<Table>> {
+    const name = `projects/${project}/datasets/${dataset}/tables/${table}`;
+    const params: Record<string, any> = { selectedFields: 'tableReference' };
+
+    return await this._get<Table>(name, params);
+  }
+
   async *listTables(project: string, dataset: string): AsyncGenerator<Table> {
     const name = `projects/${project}/datasets/${dataset}/tables`;
 
@@ -87,13 +98,18 @@ export class BigQueryClient extends api.ApiClient {
   // Executes a SQL statement (including DDL) synchronously via jobs.query.
   // When `location` is set it pins the job's processing location so it agrees
   // with getQueryResults/getJob; otherwise BigQuery infers it from the
-  // referenced tables.
-  async query(project: string, sql: string, location?: string): Promise<api.ApiResult<QueryResponse>> {
+  // referenced tables. With `dryRun` the statement is validated -- name
+  // resolution, table existence and access -- but not executed, so it serves as
+  // a cheap pre-flight probe over any reference form BigQuery can resolve.
+  async query(project: string, sql: string, location?: string, dryRun?: boolean): Promise<api.ApiResult<QueryResponse>> {
     const name = `projects/${project}/queries`;
     const body: Record<string, any> = { query: sql, useLegacySql: false };
     if (location) {
       body.location = location;
     }
+    if (dryRun) {
+      body.dryRun = true;
+    }
     return await this._post<QueryResponse>(name, body);
   }
 

toolbox/mdcode/src/libts/gcp/dataplex.ts

@@ -49,11 +49,38 @@ export interface Entry {
   aspects?: Record<string, Aspect>;
 }
 
+export interface EntryReference {
+  // Full resource name of the referenced entry.
+  name: string;
+  // Path within the entry that is referenced; empty means the entry itself.
+  path?: string;
+  // SOURCE / TARGET for a directed link; UNSPECIFIED for an undirected one.
+  type: 'UNSPECIFIED' | 'SOURCE' | 'TARGET';
+}
+
+export interface EntryLink {
+  // Server-assigned (output only); set by the emitter to the destination name so
+  // the publisher can address it for an in-place update.
+  name?: string;
+  entryLinkType: string;
+  // Exactly two references.
+  entryReferences: EntryReference[];
+  // At most one Dataplex-owned aspect (e.g. schema-join), keyed as
+  // `project.location.aspectType`.
+  aspects?: Record<string, Aspect>;
+}
+
 interface EntryList {
   entries: Entry[];
   nextPageToken?: string;
 }
 
+interface EntryLinkList {
+  // Absent when the referenced entry has no links of the requested type(s).
+  entryLinks?: EntryLink[];
+  nextPageToken?: string;
+}
+
 
 export class CatalogClient extends api.ApiClient {
 
@@ -193,6 +220,12 @@ export class CatalogClient extends api.ApiClient {
     return res;
   }
 
+  async deleteEntry(project: string, location: string, entryGroup: string,
+                    entry: string): Promise<api.ApiResult<Entry>> {
+    const name = `${catalogContainer(project, location, entryGroup)}/entries/${entry}`;
+    return await this._delete<Entry>(name);
+  }
+
   async createEntryGroup(project: string, location: string, 
                          entryGroupId: string, entryGroup?: EntryGroup): Promise<api.ApiResult<EntryGroup>> {
     const parent = catalogContainer(project, location);
@@ -205,6 +238,74 @@ export class CatalogClient extends api.ApiClient {
     return res;
   }
 
+  async createEntryLink(project: string, location: string, entryGroup: string,
+                        entryLinkId: string,
+                        entryLink: EntryLink): Promise<api.ApiResult<EntryLink>> {
+    const parent = catalogContainer(project, location, entryGroup);
+    const resourceName = `${parent}/entryLinks`;
+
+    const params: Record<string, any> = { entryLinkId };
+
+    return await this._post<EntryLink>(resourceName, entryLink, params);
+  }
+
+  // Patches an existing entry link. UpdateEntryLink has no update mask: the
+  // aspects present in the request body are the ones written, narrowed to
+  // `aspectKeys` (each `project.location.aspectType`) when given. The entry
+  // references and link type are immutable server-side.
+  async updateEntryLink(entryLink: EntryLink,
+                        aspectKeys?: string[]): Promise<api.ApiResult<EntryLink>> {
+    const params: Record<string, any> = {};
+    if (aspectKeys && aspectKeys.length) {
+      params.aspectKeys = aspectKeys;
+    }
+    return await this._patch<EntryLink>(entryLink.name!, entryLink, params);
+  }
+
+  async deleteEntryLink(project: string, location: string, entryGroup: string,
+                        entryLinkId: string): Promise<api.ApiResult<EntryLink>> {
+    const name = `${
+        catalogContainer(project, location, entryGroup)}/entryLinks/${
+        entryLinkId}`;
+    return await this._delete<EntryLink>(name);
+  }
+
+  // Returns every entry link that references `entry` (its full resource name),
+  // draining all pages. There is no list-entry-links collection API; the server
+  // only exposes links per referenced entry, via the location-scoped
+  // :lookupEntryLinks custom verb (mirroring :lookupEntry). `entryLinkTypes`
+  // optionally filters to specific link types (server caps it at 10); `entryMode`
+  // filters by the entry's role in the link (SOURCE/TARGET). The server caps a
+  // page at 10 links, so this follows nextPageToken until exhausted and returns
+  // the flat list. A non-200 on any page aborts and is returned as-is.
+  async lookupEntryLinks(
+      project: string, location: string,
+      opts: {entry: string; entryLinkTypes?: string[];
+             entryMode?: 'UNSPECIFIED' | 'SOURCE' | 'TARGET'}):
+      Promise<api.ApiResult<EntryLink[]>> {
+    const container = `${catalogContainer(project, location)}:lookupEntryLinks`;
+    const links: EntryLink[] = [];
+    let pageToken: string | undefined = undefined;
+    do {
+      const params: Record<string, any> = {
+        entry: opts.entry,
+        entryLinkTypes: opts.entryLinkTypes,
+        entryMode: opts.entryMode,
+        pageSize: 10,
+        pageToken,
+      };
+      const res = await this._get<EntryLinkList>(container, params);
+      if (res.status != 200) {
+        return { status: res.status, message: res.message };
+      }
+      for (const link of res.result?.entryLinks ?? []) {
+        links.push(link);
+      }
+      pageToken = res.result?.nextPageToken;
+    } while (pageToken);
+    return { status: 200, result: links };
+  }
+
 }
 
 

toolbox/mdcode/src/libts/layouts/semantic-model.ts

@@ -91,6 +91,36 @@ export class SemanticModelLayout implements CatalogLayout {
     return docs;
   }
 
+  // True when a model document with this handle already exists on disk. `pull`
+  // uses it to report which files it would overwrite vs. create.
+  hasModel(name: string): boolean {
+    return fs.existsSync(this.modelPath(name));
+  }
+
+  // The absolute path a model document with this handle maps to:
+  // `<catalog>/EntryGroups/<entryGroup>/<name>.yaml`. Path separators in the
+  // model name are replaced so a name still yields a single flat file. Requires
+  // the layout to be scoped to an entry group (the semantic-model source always
+  // is).
+  modelPath(name: string): string {
+    if (!this._entryGroup) {
+      throw new Error(
+          'SemanticModel layout has no entry group; cannot resolve a model path.');
+    }
+    const file = `${name.replace(/[/\\]/g, '_')}.yaml`;
+    return path.join(this._catalogPath, 'EntryGroups', this._entryGroup, file);
+  }
+
+  // Writes a model's serialized document to its path, creating the EntryGroup
+  // directory if needed, and indexes it so a later modelDocuments() sees it.
+  // This is the sink `pull` writes reconstructed models to.
+  writeModelDocument(name: string, text: string): void {
+    const localPath = this.modelPath(name);
+    fs.mkdirSync(path.dirname(localPath), {recursive: true});
+    fs.writeFileSync(localPath, text);
+    this._index.set(name, localPath);
+  }
+
   // The Knowledge Catalog entry-level members are not applicable to this
   // push-only layout; the model is authored as a single Ossie document, not as
   // per-entry Knowledge Catalog files. These are wired when KC-resource emit