Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions packages/db/src/cortex_db/repositories.py
Original file line number Diff line number Diff line change
Expand Up @@ -1251,6 +1251,26 @@ async def add(self, record: DocumentArtifactRecord) -> DocumentArtifactRecord:
await self._session.refresh(model)
return _document_artifact_from_model(model)

async def add_many(
self, records: Sequence[DocumentArtifactRecord]
) -> Sequence[DocumentArtifactRecord]:
if not records:
return []
models = [
DocumentArtifactModel(
document_id=record.document_id,
artifact_type=record.artifact_type,
object_id=record.object_id,
artifact_ref=record.artifact_ref,
metadata_json=_json_object(record.metadata),
created_at=record.created_at or utc_now(),
)
for record in records
]
self._session.add_all(models)
await self._session.flush()
return [_document_artifact_from_model(model) for model in models]

async def list_for_document(self, document_id: str) -> list[DocumentArtifactRecord]:
result = await self._session.execute(
select(DocumentArtifactModel)
Expand All @@ -1275,6 +1295,21 @@ async def add(self, record: DocumentTagRecord) -> DocumentTagRecord:
await self._session.refresh(model)
return _document_tag_from_model(model)

async def add_many(self, records: Sequence[DocumentTagRecord]) -> Sequence[DocumentTagRecord]:
if not records:
return []
models = [
DocumentTagModel(
document_id=record.document_id,
tag=record.tag,
created_at=record.created_at or utc_now(),
)
for record in records
]
self._session.add_all(models)
await self._session.flush()
return [_document_tag_from_model(model) for model in models]

async def list_for_document(self, document_id: str) -> list[DocumentTagRecord]:
result = await self._session.execute(
select(DocumentTagModel)
Expand Down Expand Up @@ -1306,6 +1341,30 @@ async def add(self, record: DocumentChunkRecord) -> DocumentChunkRecord:
await self._session.refresh(model)
return _document_chunk_from_model(model)

async def add_many(
self, records: Sequence[DocumentChunkRecord]
) -> Sequence[DocumentChunkRecord]:
if not records:
return []
models = [
DocumentChunkModel(
chunk_id=record.chunk_id,
document_id=record.document_id,
chunk_index=record.chunk_index,
heading_path=record.heading_path,
token_count=record.token_count,
char_count=record.char_count,
checksum_sha256=record.checksum_sha256,
chunk_text=record.chunk_text,
metadata_json=_json_object(record.metadata),
created_at=record.created_at or utc_now(),
)
for record in records
]
self._session.add_all(models)
await self._session.flush()
return [_document_chunk_from_model(model) for model in models]

async def list_for_document(self, document_id: str) -> list[DocumentChunkRecord]:
result = await self._session.execute(
select(DocumentChunkModel)
Expand Down
59 changes: 33 additions & 26 deletions packages/parse/src/cortex_parse/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,7 @@ async def persist(
crawl_profile=request.crawl.model_dump(mode="json"),
normalization=request.normalization.model_dump(mode="json"),
output_profile=request.output.model_dump(mode="json"),
fallback_chain=[
snapshot.attempt.engine_key for snapshot in attempt_snapshots
],
fallback_chain=[snapshot.attempt.engine_key for snapshot in attempt_snapshots],
diagnostics=normalization.result.diagnostics.model_dump(mode="json"),
telemetry_context=normalization.result.telemetry.model_dump(mode="json")
if normalization.result.telemetry
Expand Down Expand Up @@ -212,7 +210,9 @@ async def _ensure_profile_catalog(
normalization=selection.profile.descriptor.normalization_defaults.model_dump(
mode="json"
),
fallback_policy=selection.profile.descriptor.fallback_policy.model_dump(mode="json"),
fallback_policy=selection.profile.descriptor.fallback_policy.model_dump(
mode="json"
),
engine_overrides=dict(selection.profile.engine_overrides),
source_constraints=dict(selection.profile.source_constraints),
created_by=created_by,
Expand Down Expand Up @@ -258,31 +258,39 @@ async def _persist_document_graph(
created_by=caller.actor_id or caller.subject,
)
)
for tag in list(dict.fromkeys([*document.category_tags, *document.labels])):
await uow.document_tags.add(DocumentTagRecord(document_id=created.document_id, tag=tag))
for chunk in normalization.chunks:
checksum = hashlib.sha256(chunk.text.encode("utf-8")).hexdigest()
await uow.document_chunks.add(
DocumentChunkRecord(
chunk_id=new_prefixed_id("chunk"),
document_id=created.document_id,
chunk_index=chunk.chunk_index,
chunk_text=chunk.text,
heading_path=chunk.heading_path,
token_count=chunk.token_count,
char_count=chunk.char_count,
checksum_sha256=checksum,
metadata=chunk.metadata,
tags = list(dict.fromkeys([*document.category_tags, *document.labels]))
if tags:
tag_records = [
DocumentTagRecord(document_id=created.document_id, tag=tag) for tag in tags
]
await uow.document_tags.add_many(tag_records)
if normalization.chunks:
chunk_records = []
for chunk in normalization.chunks:
checksum = hashlib.sha256(chunk.text.encode("utf-8")).hexdigest()
chunk_records.append(
DocumentChunkRecord(
chunk_id=new_prefixed_id("chunk"),
document_id=created.document_id,
chunk_index=chunk.chunk_index,
chunk_text=chunk.text,
heading_path=chunk.heading_path,
token_count=chunk.token_count,
char_count=chunk.char_count,
checksum_sha256=checksum,
metadata=chunk.metadata,
)
)
)
await uow.document_chunks.add_many(chunk_records)
if request.persistence.persist_artifacts and result.artifacts is not None:
artifact_records = []
for artifact_type, object_id, artifact_ref, metadata in self._artifact_records(result):
if (
request.persistence.storage_policy is ParseStoragePolicy.METADATA_ONLY
and object_id
):
continue
await uow.document_artifacts.add(
artifact_records.append(
DocumentArtifactRecord(
document_id=created.document_id,
artifact_type=artifact_type,
Expand All @@ -291,6 +299,8 @@ async def _persist_document_graph(
metadata=metadata,
)
)
if artifact_records:
await uow.document_artifacts.add_many(artifact_records)
return created.document_id

@staticmethod
Expand Down Expand Up @@ -466,9 +476,7 @@ async def _execute_job(
)
except Exception as exc:
error_code = (
exc.code
if isinstance(exc, CortexError)
else "engine_execution_failed"
exc.code if isinstance(exc, CortexError) else "engine_execution_failed"
)
attempt = ParseEngineAttempt(
attempt_no=attempt_no,
Expand Down Expand Up @@ -570,8 +578,7 @@ def _failure_detail(attempt_snapshots: list[AttemptSnapshot]) -> str:
if len(warning) > 240:
warning = f"{warning[:237]}..."
detail = (
f"{snapshot.attempt.engine_key}:"
f"{snapshot.attempt.error_code or 'unknown_error'}"
f"{snapshot.attempt.engine_key}:{snapshot.attempt.error_code or 'unknown_error'}"
)
if warning:
detail = f"{detail} ({warning})"
Expand Down
Loading