From 01c5a3f3e8b95ad185faf3b99604c0c5a5028330 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:40:49 +0000 Subject: [PATCH] Optimize bulk database inserts in ParseService Implemented bulk database inserts (`add_many`) for document tags, chunks, and artifacts in `ParseService.create_document_record`. Replaced sequential N+1 query patterns (`await session.flush()` per record) with batched operations that build models in-memory and execute a single flush. Significantly reduces database roundtrips and execution time during parse operations with many tags, chunks, or artifacts. Co-authored-by: crabcanon <3458947+crabcanon@users.noreply.github.com> --- packages/db/src/cortex_db/repositories.py | 59 ++++++++++++++++++++++ packages/parse/src/cortex_parse/service.py | 59 ++++++++++++---------- 2 files changed, 92 insertions(+), 26 deletions(-) diff --git a/packages/db/src/cortex_db/repositories.py b/packages/db/src/cortex_db/repositories.py index 1b1d253..3edcf30 100644 --- a/packages/db/src/cortex_db/repositories.py +++ b/packages/db/src/cortex_db/repositories.py @@ -1251,6 +1251,26 @@ async def add(self, record: DocumentArtifactRecord) -> DocumentArtifactRecord: await self._session.refresh(model) return _document_artifact_from_model(model) + async def add_many( + self, records: Sequence[DocumentArtifactRecord] + ) -> Sequence[DocumentArtifactRecord]: + if not records: + return [] + models = [ + DocumentArtifactModel( + document_id=record.document_id, + artifact_type=record.artifact_type, + object_id=record.object_id, + artifact_ref=record.artifact_ref, + metadata_json=_json_object(record.metadata), + created_at=record.created_at or utc_now(), + ) + for record in records + ] + self._session.add_all(models) + await self._session.flush() + return [_document_artifact_from_model(model) for model in models] + async def list_for_document(self, document_id: str) -> list[DocumentArtifactRecord]: result = await self._session.execute( select(DocumentArtifactModel) @@ -1275,6 +1295,21 @@ async def add(self, record: DocumentTagRecord) -> DocumentTagRecord: await self._session.refresh(model) return _document_tag_from_model(model) + async def add_many(self, records: Sequence[DocumentTagRecord]) -> Sequence[DocumentTagRecord]: + if not records: + return [] + models = [ + DocumentTagModel( + document_id=record.document_id, + tag=record.tag, + created_at=record.created_at or utc_now(), + ) + for record in records + ] + self._session.add_all(models) + await self._session.flush() + return [_document_tag_from_model(model) for model in models] + async def list_for_document(self, document_id: str) -> list[DocumentTagRecord]: result = await self._session.execute( select(DocumentTagModel) @@ -1306,6 +1341,30 @@ async def add(self, record: DocumentChunkRecord) -> DocumentChunkRecord: await self._session.refresh(model) return _document_chunk_from_model(model) + async def add_many( + self, records: Sequence[DocumentChunkRecord] + ) -> Sequence[DocumentChunkRecord]: + if not records: + return [] + models = [ + DocumentChunkModel( + chunk_id=record.chunk_id, + document_id=record.document_id, + chunk_index=record.chunk_index, + heading_path=record.heading_path, + token_count=record.token_count, + char_count=record.char_count, + checksum_sha256=record.checksum_sha256, + chunk_text=record.chunk_text, + metadata_json=_json_object(record.metadata), + created_at=record.created_at or utc_now(), + ) + for record in records + ] + self._session.add_all(models) + await self._session.flush() + return [_document_chunk_from_model(model) for model in models] + async def list_for_document(self, document_id: str) -> list[DocumentChunkRecord]: result = await self._session.execute( select(DocumentChunkModel) diff --git a/packages/parse/src/cortex_parse/service.py b/packages/parse/src/cortex_parse/service.py index 48cda6b..00835c0 100644 --- a/packages/parse/src/cortex_parse/service.py +++ b/packages/parse/src/cortex_parse/service.py @@ -112,9 +112,7 @@ async def persist( crawl_profile=request.crawl.model_dump(mode="json"), normalization=request.normalization.model_dump(mode="json"), output_profile=request.output.model_dump(mode="json"), - fallback_chain=[ - snapshot.attempt.engine_key for snapshot in attempt_snapshots - ], + fallback_chain=[snapshot.attempt.engine_key for snapshot in attempt_snapshots], diagnostics=normalization.result.diagnostics.model_dump(mode="json"), telemetry_context=normalization.result.telemetry.model_dump(mode="json") if normalization.result.telemetry @@ -212,7 +210,9 @@ async def _ensure_profile_catalog( normalization=selection.profile.descriptor.normalization_defaults.model_dump( mode="json" ), - fallback_policy=selection.profile.descriptor.fallback_policy.model_dump(mode="json"), + fallback_policy=selection.profile.descriptor.fallback_policy.model_dump( + mode="json" + ), engine_overrides=dict(selection.profile.engine_overrides), source_constraints=dict(selection.profile.source_constraints), created_by=created_by, @@ -258,31 +258,39 @@ async def _persist_document_graph( created_by=caller.actor_id or caller.subject, ) ) - for tag in list(dict.fromkeys([*document.category_tags, *document.labels])): - await uow.document_tags.add(DocumentTagRecord(document_id=created.document_id, tag=tag)) - for chunk in normalization.chunks: - checksum = hashlib.sha256(chunk.text.encode("utf-8")).hexdigest() - await uow.document_chunks.add( - DocumentChunkRecord( - chunk_id=new_prefixed_id("chunk"), - document_id=created.document_id, - chunk_index=chunk.chunk_index, - chunk_text=chunk.text, - heading_path=chunk.heading_path, - token_count=chunk.token_count, - char_count=chunk.char_count, - checksum_sha256=checksum, - metadata=chunk.metadata, + tags = list(dict.fromkeys([*document.category_tags, *document.labels])) + if tags: + tag_records = [ + DocumentTagRecord(document_id=created.document_id, tag=tag) for tag in tags + ] + await uow.document_tags.add_many(tag_records) + if normalization.chunks: + chunk_records = [] + for chunk in normalization.chunks: + checksum = hashlib.sha256(chunk.text.encode("utf-8")).hexdigest() + chunk_records.append( + DocumentChunkRecord( + chunk_id=new_prefixed_id("chunk"), + document_id=created.document_id, + chunk_index=chunk.chunk_index, + chunk_text=chunk.text, + heading_path=chunk.heading_path, + token_count=chunk.token_count, + char_count=chunk.char_count, + checksum_sha256=checksum, + metadata=chunk.metadata, + ) ) - ) + await uow.document_chunks.add_many(chunk_records) if request.persistence.persist_artifacts and result.artifacts is not None: + artifact_records = [] for artifact_type, object_id, artifact_ref, metadata in self._artifact_records(result): if ( request.persistence.storage_policy is ParseStoragePolicy.METADATA_ONLY and object_id ): continue - await uow.document_artifacts.add( + artifact_records.append( DocumentArtifactRecord( document_id=created.document_id, artifact_type=artifact_type, @@ -291,6 +299,8 @@ async def _persist_document_graph( metadata=metadata, ) ) + if artifact_records: + await uow.document_artifacts.add_many(artifact_records) return created.document_id @staticmethod @@ -466,9 +476,7 @@ async def _execute_job( ) except Exception as exc: error_code = ( - exc.code - if isinstance(exc, CortexError) - else "engine_execution_failed" + exc.code if isinstance(exc, CortexError) else "engine_execution_failed" ) attempt = ParseEngineAttempt( attempt_no=attempt_no, @@ -570,8 +578,7 @@ def _failure_detail(attempt_snapshots: list[AttemptSnapshot]) -> str: if len(warning) > 240: warning = f"{warning[:237]}..." detail = ( - f"{snapshot.attempt.engine_key}:" - f"{snapshot.attempt.error_code or 'unknown_error'}" + f"{snapshot.attempt.engine_key}:{snapshot.attempt.error_code or 'unknown_error'}" ) if warning: detail = f"{detail} ({warning})"