diff --git a/migrations/cassandra/keyspaces/README.md b/migrations/cassandra/keyspaces/README.md index 9859dc55f8..58fe61fca7 100644 --- a/migrations/cassandra/keyspaces/README.md +++ b/migrations/cassandra/keyspaces/README.md @@ -9,11 +9,13 @@ Migrations are applied in filename order and follow the naming convention: NN_description.up.sql ``` -The schemas in this directory follow a clean-slate model. There are no delta -`ALTER TABLE` migrations. Each `03_init_tables.up.sql` represents the complete, -canonical schema for its keyspace at the pinned upstream service version. When the -upstream schema changes, `03_init_tables.up.sql` is updated in place and the -pinned version reference is bumped accordingly. +Each `03_init_tables.up.sql` is the baseline schema for its keyspace at the +pinned version in the Schema Sources table. Fresh installations apply that file, +then every later migration in filename order. Existing clusters apply only the +migrations newer than their recorded version. + +Do not edit `03_init_tables.up.sql` when the schema changes. Add the next +numbered migration instead. --- @@ -42,19 +44,25 @@ pinned version reference is bumped accordingly. |---------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `01_init_keyspace.up.sql` | Creates the keyspace with `NetworkTopologyStrategy` replication. Uses `${REPLICA_COUNT}`, which the entrypoint substitutes before migration. | | `02_init_roles.up.sql` | Creates the application role, grants privileges, and sets the service login password via `${SERVICE_ROLE_PASSWORD}`. | -| `03_init_tables.up.sql` | Complete canonical schema with all UDTs, tables, and indexes at the pinned upstream version. | -| `04_*` and later | Incremental deltas for rolling upgrades. These add tables/columns that are not in `03_init_tables.up.sql` at the version that was applied on existing clusters. `ess_api/04_*` is a data seed (deployment-specific values). The `sis_api` and `nvcf_api` deltas are DDL. | +| `03_init_tables.up.sql` | Baseline schema (UDTs, tables, and indexes) at the pinned version. Later schema changes belong in a new migration file, not in this file. | +| `04_*` and later | Incremental deltas for rolling upgrades. These add tables/columns that are not in `03_init_tables.up.sql` at the version that was applied on existing clusters. `ess_api/04_*` is a data seed (deployment-specific values). The `api_keys_api`, `sis_api`, and `nvcf_api` deltas are DDL. | --- ## Conventions and Rules -### Clean-Slate Model +### Baseline and later migrations + +`03_init_tables.up.sql` is the baseline. Fresh installations run it, then apply +`04_*` and later in filename order. Existing clusters skip migrations they have +already applied. + +Put each later schema change in a new migration. Leave `03_init_tables.up.sql` +unchanged so a cluster already at version 3 still receives the change. -This repo does not use incremental `ALTER TABLE` migrations for fresh -installations. The `03_init_tables.up.sql` file always reflects the full desired -schema. This avoids the complexity of replaying a long chain of deltas on new -clusters. +DDL migrations create or alter tables, types, and indexes. The `api_keys_api`, +`sis_api`, and `nvcf_api` deltas are DDL. Deployment-specific data seeds, such +as `ess_api/04_*`, load values for this deployment and stay in their own files. ### Upstream vs. Our Values @@ -79,10 +87,16 @@ upstream: -o /tmp/upstream_schema.cql ``` -3. Diff against the current `03_init_tables.up.sql`. Identify: +3. Diff against `03_init_tables.up.sql` and the migrations that follow it. + Identify: - Net-new tables or columns - Dropped tables or columns - - Any data/config values that must use our deployment's values -4. Update `03_init_tables.up.sql` in place. -5. Update the Schema Sources table in this README with the new version and - commit SHA. + - Data or config values that must use this deployment's values +4. Add the next `NN_description.up.sql`. Do not edit `03_init_tables.up.sql` + in place. + - DDL (tables, types, columns, indexes) goes in a DDL migration. + - Deployment-specific data stays in a separate seed migration. Do not mix + seed values into a DDL file. +5. Update the Schema Sources table in this README only when the baseline + `03_init_tables.up.sql` pin changes. A later migration does not change that + pin by itself. diff --git a/migrations/cassandra/keyspaces/api_keys_api/04_add_multi_tenant_schema.up.sql b/migrations/cassandra/keyspaces/api_keys_api/04_add_multi_tenant_schema.up.sql new file mode 100644 index 0000000000..6f8065217a --- /dev/null +++ b/migrations/cassandra/keyspaces/api_keys_api/04_add_multi_tenant_schema.up.sql @@ -0,0 +1,90 @@ +-- Multi-tenant API Keys tables. 03_init_tables.up.sql stays the original schema. +-- New and existing clusters apply this file. Statements are idempotent. +-- Keep keys_by_owner_and_service until the dual-write migration stops using it. +-- Hash lookup stays keyed by api_key_hash. nca_id is a regular column. + +ALTER TABLE api_keys_api.keys ADD IF NOT EXISTS nca_id TEXT; + +CREATE TABLE IF NOT EXISTS api_keys_api.keys_by_account_owner_and_service +( + nca_id TEXT, + owner_type TEXT, + owner_id TEXT, + issuer_service_id TEXT, + key_id TEXT, + key_status TEXT, + created_at TIMESTAMP, + expires_at TIMESTAMP, + deletes_at TIMESTAMP, + key_details TEXT, + PRIMARY KEY ((nca_id, owner_type, owner_id), issuer_service_id, key_id) +); + +CREATE CUSTOM INDEX IF NOT EXISTS keys_by_scope_nca_idx + ON api_keys_api.keys_by_account_owner_and_service (nca_id) + USING 'StorageAttachedIndex'; + +CREATE CUSTOM INDEX IF NOT EXISTS keys_by_scope_owner_idx + ON api_keys_api.keys_by_account_owner_and_service (owner_id) + USING 'StorageAttachedIndex'; + +CREATE CUSTOM INDEX IF NOT EXISTS keys_by_scope_owner_type_idx + ON api_keys_api.keys_by_account_owner_and_service (owner_type) + USING 'StorageAttachedIndex'; + +CREATE CUSTOM INDEX IF NOT EXISTS keys_by_scope_service_idx + ON api_keys_api.keys_by_account_owner_and_service (issuer_service_id) + USING 'StorageAttachedIndex'; + +CREATE CUSTOM INDEX IF NOT EXISTS keys_by_scope_status_idx + ON api_keys_api.keys_by_account_owner_and_service (key_status) + USING 'StorageAttachedIndex'; + +CREATE CUSTOM INDEX IF NOT EXISTS keys_by_scope_created_at_idx + ON api_keys_api.keys_by_account_owner_and_service (created_at) + USING 'StorageAttachedIndex'; + +CREATE TABLE IF NOT EXISTS api_keys_api.owner_status_by_account +( + nca_id TEXT, + owner_type TEXT, + owner_id TEXT, + owner_status TEXT, + created_at TIMESTAMP, + updated_at TIMESTAMP, + PRIMARY KEY ((nca_id, owner_type, owner_id)) +); + +CREATE TABLE IF NOT EXISTS api_keys_api.owner_status_by_account_and_service +( + nca_id TEXT, + owner_type TEXT, + owner_id TEXT, + issuer_service_id TEXT, + owner_status TEXT, + created_at TIMESTAMP, + updated_at TIMESTAMP, + PRIMARY KEY ((nca_id, owner_type, owner_id, issuer_service_id)) +); + +CREATE TABLE IF NOT EXISTS api_keys_api.key_operations_by_id +( + operation_id UUID, + actor_type TEXT, + actor_id TEXT, + operation TEXT, + nca_ids FROZEN>, + issuer_service_ids FROZEN>, + user_ids FROZEN>, + operation_status TEXT, + matched_count BIGINT, + completed_count BIGINT, + failed_count BIGINT, + selection_state TEXT, + paging_state TEXT, + reason TEXT, + cutoff_at TIMESTAMP, + created_at TIMESTAMP, + updated_at TIMESTAMP, + PRIMARY KEY ((operation_id)) +); diff --git a/migrations/cassandra/tests/test-execute-sqls.sh b/migrations/cassandra/tests/test-execute-sqls.sh index e0eb252556..db53040749 100755 --- a/migrations/cassandra/tests/test-execute-sqls.sh +++ b/migrations/cassandra/tests/test-execute-sqls.sh @@ -290,4 +290,17 @@ if ! grep -F -q 'ALTER TABLE nvct_api.tasks_v2 ADD IF NOT EXISTS health TEXT;' \ fail "NVCT upgrade migration does not add tasks_v2.health idempotently" fi +api_keys_schema="${keyspaces}/api_keys_api/03_init_tables.up.sql" +if grep -F -q 'keys_by_account_owner_and_service' "${api_keys_schema}"; then + fail "api_keys_api 03_init_tables.up.sql must stay the original single-tenant schema" +fi + +api_keys_mt_migration="${keyspaces}/api_keys_api/04_add_multi_tenant_schema.up.sql" +if ! grep -F -q 'ALTER TABLE api_keys_api.keys ADD IF NOT EXISTS nca_id TEXT;' \ + "${api_keys_mt_migration}" || + ! grep -F -q 'CREATE TABLE IF NOT EXISTS api_keys_api.keys_by_account_owner_and_service' \ + "${api_keys_mt_migration}"; then + fail "api_keys_api upgrade migration does not add the multi-tenant schema idempotently" +fi + exit "${status}" diff --git a/src/control-plane-services/api-keys/AGENTS.md b/src/control-plane-services/api-keys/AGENTS.md index 77a03a8014..6b4442c19c 100644 --- a/src/control-plane-services/api-keys/AGENTS.md +++ b/src/control-plane-services/api-keys/AGENTS.md @@ -28,6 +28,25 @@ bazel --output_user_root="${BAZEL_OUTPUT_USER_ROOT}" \ The test target starts Cassandra through Testcontainers and Docker Compose. It is tagged `requires-docker` and runs in the GitHub `docker-host` lane. +## Cassandra schema + +Local and Testcontainers CQL lives in `local_env/cassandra/schema/`. The +deployed keyspace is `api_keys_api`, applied from +`migrations/cassandra/keyspaces/api_keys_api/`. Keep those copies aligned. + +`0001_initial_schema.cql` follows the clean-slate model. It holds the full +local schema, including the multi-tenant tables. Update it in place instead of +adding local delta files. + +`03_init_tables.up.sql` stays the original single-tenant tables. Do not add +multi-tenant objects to `03`. `04_add_multi_tenant_schema.up.sql` is the +deployed delta for new and existing clusters. Keep `keys_by_owner_and_service` +until the dual-write migration stops using it. + +Integration tests bind each `.cql` file in `local_env/docker-compose.test.yml` +because Bazel runfiles are symlinks. Local Compose mounts the whole schema +directory. + ## Dependencies The root `MODULE.bazel` and `maven_install.json` own diff --git a/src/control-plane-services/api-keys/local_env/cassandra/schema/0001_initial_schema.cql b/src/control-plane-services/api-keys/local_env/cassandra/schema/0001_initial_schema.cql index ecc79e609b..0f917d7ec8 100644 --- a/src/control-plane-services/api-keys/local_env/cassandra/schema/0001_initial_schema.cql +++ b/src/control-plane-services/api-keys/local_env/cassandra/schema/0001_initial_schema.cql @@ -17,9 +17,12 @@ CREATE KEYSPACE IF NOT EXISTS nvcf_api_keys with replication = {'class':'SimpleS USE nvcf_api_keys; +-- Hash lookup stays keyed by api_key_hash. nca_id is a regular column because +-- evaluate starts from the presented secret and does not know the account yet. CREATE TABLE IF NOT EXISTS keys ( api_key_hash TEXT, + nca_id TEXT, status TEXT, expires_at TIMESTAMP, deletes_at TIMESTAMP, @@ -58,3 +61,93 @@ CREATE TABLE IF NOT EXISTS row_update_lock updated_at TIMESTAMP, PRIMARY KEY ((table_name, record_key)) ); + +-- Account-scoped management index. Keep keys_by_owner_and_service until the +-- dual-write migration stops using it. +CREATE TABLE IF NOT EXISTS keys_by_account_owner_and_service +( + nca_id TEXT, + owner_type TEXT, + owner_id TEXT, + issuer_service_id TEXT, + key_id TEXT, + key_status TEXT, + created_at TIMESTAMP, + expires_at TIMESTAMP, + deletes_at TIMESTAMP, + key_details TEXT, + PRIMARY KEY ((nca_id, owner_type, owner_id), issuer_service_id, key_id) +); + +-- SAI on the management table for admin and bulk selection that does not +-- have the full partition key. These indexes select rows only. Bulk updates +-- still write each key in keys and keys_by_account_owner_and_service. + +CREATE CUSTOM INDEX IF NOT EXISTS keys_by_scope_nca_idx + ON keys_by_account_owner_and_service (nca_id) + USING 'StorageAttachedIndex'; + +CREATE CUSTOM INDEX IF NOT EXISTS keys_by_scope_owner_idx + ON keys_by_account_owner_and_service (owner_id) + USING 'StorageAttachedIndex'; + +CREATE CUSTOM INDEX IF NOT EXISTS keys_by_scope_owner_type_idx + ON keys_by_account_owner_and_service (owner_type) + USING 'StorageAttachedIndex'; + +CREATE CUSTOM INDEX IF NOT EXISTS keys_by_scope_service_idx + ON keys_by_account_owner_and_service (issuer_service_id) + USING 'StorageAttachedIndex'; + +CREATE CUSTOM INDEX IF NOT EXISTS keys_by_scope_status_idx + ON keys_by_account_owner_and_service (key_status) + USING 'StorageAttachedIndex'; + +CREATE CUSTOM INDEX IF NOT EXISTS keys_by_scope_created_at_idx + ON keys_by_account_owner_and_service (created_at) + USING 'StorageAttachedIndex'; + +CREATE TABLE IF NOT EXISTS owner_status_by_account +( + nca_id TEXT, + owner_type TEXT, + owner_id TEXT, + owner_status TEXT, + created_at TIMESTAMP, + updated_at TIMESTAMP, + PRIMARY KEY ((nca_id, owner_type, owner_id)) +); + +CREATE TABLE IF NOT EXISTS owner_status_by_account_and_service +( + nca_id TEXT, + owner_type TEXT, + owner_id TEXT, + issuer_service_id TEXT, + owner_status TEXT, + created_at TIMESTAMP, + updated_at TIMESTAMP, + PRIMARY KEY ((nca_id, owner_type, owner_id, issuer_service_id)) +); + +CREATE TABLE IF NOT EXISTS key_operations_by_id +( + operation_id UUID, + actor_type TEXT, + actor_id TEXT, + operation TEXT, + nca_ids FROZEN>, + issuer_service_ids FROZEN>, + user_ids FROZEN>, + operation_status TEXT, + matched_count BIGINT, + completed_count BIGINT, + failed_count BIGINT, + selection_state TEXT, + paging_state TEXT, + reason TEXT, + cutoff_at TIMESTAMP, + created_at TIMESTAMP, + updated_at TIMESTAMP, + PRIMARY KEY ((operation_id)) +); diff --git a/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/persistance/MultiTenantSchemaIntegrationTest.java b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/persistance/MultiTenantSchemaIntegrationTest.java new file mode 100644 index 0000000000..fdbf5fc6b5 --- /dev/null +++ b/src/control-plane-services/api-keys/src/test/java/com/nvidia/apikeys/persistance/MultiTenantSchemaIntegrationTest.java @@ -0,0 +1,152 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nvidia.apikeys.persistance; + +import static com.nvidia.apikeys.config.IntegrationTestConfiguration.KEY_SPACE; +import static org.assertj.core.api.Assertions.assertThat; + +import com.datastax.driver.core.Row; +import com.datastax.driver.core.Session; +import com.nvidia.apikeys.App; +import com.nvidia.apikeys.config.IntegrationTestConfiguration; +import com.nvidia.apikeys.config.IntegrationTestConfiguration.TestCleanerExtension; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.resttestclient.autoconfigure.AutoConfigureTestRestTemplate; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ContextConfiguration; + +@ExtendWith(TestCleanerExtension.class) +@AutoConfigureTestRestTemplate +@SpringBootTest( + classes = App.class, + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = "spring.profiles.active=integrationtest") +@ContextConfiguration(initializers = IntegrationTestConfiguration.Initializer.class) +class MultiTenantSchemaIntegrationTest { + + private static final String MANAGEMENT_TABLE = "keys_by_account_owner_and_service"; + + private static final List SAI_INDEXES = List.of( + "keys_by_scope_nca_idx", + "keys_by_scope_owner_idx", + "keys_by_scope_owner_type_idx", + "keys_by_scope_service_idx", + "keys_by_scope_status_idx", + "keys_by_scope_created_at_idx"); + + @Test + void keysTableKeepsHashPartitionKeyAndAddsNcaId() { + assertThat(partitionKeyColumns("keys")).containsExactly("api_key_hash"); + assertThat(clusteringColumns("keys")).isEmpty(); + assertThat(regularColumns("keys")).contains("nca_id", "status", "expires_at", "deletes_at", + "key_details"); + } + + @Test + void tenantAwareManagementTableUsesAccountOwnerPartition() { + assertThat(partitionKeyColumns(MANAGEMENT_TABLE)) + .containsExactly("nca_id", "owner_type", "owner_id"); + assertThat(clusteringColumns(MANAGEMENT_TABLE)) + .containsExactly("issuer_service_id", "key_id"); + assertThat(regularColumns(MANAGEMENT_TABLE)) + .contains("key_status", "created_at", "expires_at", "deletes_at", "key_details"); + } + + @Test + void ownerStatusTablesAreKeyedByAccountAndOptionalIssuer() { + assertThat(partitionKeyColumns("owner_status_by_account")) + .containsExactly("nca_id", "owner_type", "owner_id"); + assertThat(clusteringColumns("owner_status_by_account")).isEmpty(); + + assertThat(partitionKeyColumns("owner_status_by_account_and_service")) + .containsExactly("nca_id", "owner_type", "owner_id", "issuer_service_id"); + assertThat(clusteringColumns("owner_status_by_account_and_service")).isEmpty(); + } + + @Test + void bulkOperationTableIsKeyedByOperationId() { + assertThat(partitionKeyColumns("key_operations_by_id")) + .containsExactly("operation_id"); + assertThat(clusteringColumns("key_operations_by_id")).isEmpty(); + assertThat(regularColumns("key_operations_by_id")) + .contains("nca_ids", "issuer_service_ids", "user_ids", "operation_status", + "paging_state", "cutoff_at"); + } + + @Test + void managementTableHasStorageAttachedIndexes() { + Map> indexes = indexesOn(MANAGEMENT_TABLE); + + assertThat(indexes.keySet()).containsExactlyInAnyOrderElementsOf(SAI_INDEXES); + indexes.values().forEach(options -> assertThat(options.toString()) + .contains("StorageAttachedIndex")); + } + + @Test + void legacyOwnerIndexTableRemainsForDualWrite() { + assertThat(partitionKeyColumns("keys_by_owner_and_service")) + .containsExactly("owner_type", "owner_id"); + assertThat(clusteringColumns("keys_by_owner_and_service")) + .containsExactly("issuer_service_id", "key_id"); + } + + private static List partitionKeyColumns(String table) { + return columns(table, "partition_key"); + } + + private static List clusteringColumns(String table) { + return columns(table, "clustering"); + } + + private static List regularColumns(String table) { + return columns(table, "regular"); + } + + private static List columns(String table, String kind) { + Session session = IntegrationTestConfiguration.CQL_SESSION; + var result = session.execute( + "SELECT column_name, kind, position FROM system_schema.columns " + + "WHERE keyspace_name = ? AND table_name = ?", + KEY_SPACE, table); + return result.all().stream() + .filter(row -> kind.equals(row.getString("kind"))) + .sorted((left, right) -> Integer.compare( + left.getInt("position"), right.getInt("position"))) + .map(row -> row.getString("column_name")) + .toList(); + } + + private static Map> indexesOn(String table) { + Session session = IntegrationTestConfiguration.CQL_SESSION; + var result = session.execute( + "SELECT index_name, options FROM system_schema.indexes " + + "WHERE keyspace_name = ? AND table_name = ?", + KEY_SPACE, table); + return result.all().stream().collect(Collectors.toMap( + row -> row.getString("index_name"), + MultiTenantSchemaIntegrationTest::indexOptions)); + } + + private static Map indexOptions(Row row) { + return row.getMap("options", String.class, String.class); + } +}