From 4d6301c9000235c2cfe15dacd534ec63b1ac7f1e Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 15 Jul 2026 17:28:15 -0700 Subject: [PATCH 01/25] feat(pii): regex-only block-output redaction + drop GLiNER/GPU image (#5697) * chore(pii): remove GLiNER/GPU image + add spaCy-skip fast path to CPU server * feat(pii): restrict block-output redaction to regex-only entities * fix(pii): derive spaCy-NER set from registry + skip fast path when score_threshold set * fix(pii): include ORGANIZATION in app-side NER set (align with server) --- apps/pii/engines.py | 267 --------------- apps/pii/requirements-dev.txt | 5 - apps/pii/requirements-gliner.txt | 10 - apps/pii/scripts/bench_engines.py | 206 ------------ apps/pii/scripts/bench_payload.json | 97 ------ apps/pii/server.py | 317 ++++++++++++++---- apps/pii/tests/conftest.py | 5 - apps/pii/tests/test_engines.py | 105 ------ apps/pii/tests/test_integration.py | 102 ------ .../components/data-retention-settings.tsx | 16 +- .../lib/api/contracts/data-retention.test.ts | 35 ++ apps/sim/lib/api/contracts/primitives.ts | 33 +- apps/sim/lib/billing/retention.test.ts | 39 +++ apps/sim/lib/billing/retention.ts | 20 +- apps/sim/lib/guardrails/pii-entities.test.ts | 91 +++++ apps/sim/lib/guardrails/pii-entities.ts | 48 ++- docker/pii.Dockerfile | 84 +---- helm/sim/templates/deployment-pii.yaml | 8 +- helm/sim/tests/pii_test.yaml | 54 +-- helm/sim/values.schema.json | 9 - helm/sim/values.yaml | 12 - 21 files changed, 531 insertions(+), 1032 deletions(-) delete mode 100644 apps/pii/engines.py delete mode 100644 apps/pii/requirements-dev.txt delete mode 100644 apps/pii/requirements-gliner.txt delete mode 100644 apps/pii/scripts/bench_engines.py delete mode 100644 apps/pii/scripts/bench_payload.json delete mode 100644 apps/pii/tests/conftest.py delete mode 100644 apps/pii/tests/test_engines.py delete mode 100644 apps/pii/tests/test_integration.py create mode 100644 apps/sim/lib/guardrails/pii-entities.test.ts diff --git a/apps/pii/engines.py b/apps/pii/engines.py deleted file mode 100644 index c8edf982ff1..00000000000 --- a/apps/pii/engines.py +++ /dev/null @@ -1,267 +0,0 @@ -"""Analyzer engine builders for the PII service. - -Two NER engines share one recognizer surface: - -- spacy (default): the 5 large spaCy models do NER (PERSON/LOCATION/NRP/ - DATE_TIME) and tokenization. -- gliner (opt-in): one multilingual GLiNER model does NER on CPU or GPU; - small spaCy models remain only for tokenization + lemmas. - -Both engines register the identical regex/checksum recognizer set (Presidio -defaults, EXTRA_RECOGNIZERS, VIN) — only the source of the 4 NER entity types -differs. Side-effect free: importing this module loads no models. -""" - -import importlib.util - -import spacy.util -from presidio_analyzer import AnalyzerEngine, Pattern, PatternRecognizer -from presidio_analyzer.nlp_engine import NlpEngineProvider -from presidio_analyzer.predefined_recognizers import ( - AuAbnRecognizer, - AuAcnRecognizer, - AuMedicareRecognizer, - AuTfnRecognizer, - EsNieRecognizer, - EsNifRecognizer, - FiPersonalIdentityCodeRecognizer, - GLiNERRecognizer, - InAadhaarRecognizer, - InPanRecognizer, - InPassportRecognizer, - InVehicleRegistrationRecognizer, - InVoterRecognizer, - ItDriverLicenseRecognizer, - ItFiscalCodeRecognizer, - ItIdentityCardRecognizer, - ItPassportRecognizer, - ItVatCodeRecognizer, - PlPeselRecognizer, - SgFinRecognizer, - SgUenRecognizer, - UkNinoRecognizer, -) - -# Languages served. Each needs its spaCy model installed in the image; the -# es/it/pl/fi predefined recognizers (ES_NIF, IT_FISCAL_CODE, PL_PESEL, ...) -# auto-load once their NLP engine is present. -NLP_CONFIGURATION = { - "nlp_engine_name": "spacy", - "models": [ - {"lang_code": "en", "model_name": "en_core_web_lg"}, - {"lang_code": "es", "model_name": "es_core_news_lg"}, - {"lang_code": "it", "model_name": "it_core_news_lg"}, - {"lang_code": "pl", "model_name": "pl_core_news_lg"}, - {"lang_code": "fi", "model_name": "fi_core_news_lg"}, - ], -} -SUPPORTED_LANGUAGES = [m["lang_code"] for m in NLP_CONFIGURATION["models"]] - -# The gliner engine still needs a spaCy pipeline per language: the regex -# recognizers consume NlpArtifacts and the LemmaContextAwareEnhancer boosts -# scores from surrounding lemmas. The small models (~12-40MB each vs ~400MB -# large) keep tokenization + lemmas intact while GLiNER owns NER. Blank -# pipelines ("blank:xx") are not an option: Presidio's SpacyNlpEngine treats -# unknown model names as pip packages and tries to download them. -# labels_to_ignore strips the small models' NER output from NlpArtifacts — -# correctness comes from removing SpacyRecognizer in build_gliner_analyzer; -# this only silences unmapped-label noise. -GLINER_NLP_CONFIGURATION = { - "nlp_engine_name": "spacy", - "models": [ - {"lang_code": "en", "model_name": "en_core_web_sm"}, - {"lang_code": "es", "model_name": "es_core_news_sm"}, - {"lang_code": "it", "model_name": "it_core_news_sm"}, - {"lang_code": "pl", "model_name": "pl_core_news_sm"}, - {"lang_code": "fi", "model_name": "fi_core_news_sm"}, - ], - "ner_model_configuration": { - "labels_to_ignore": [ - "CARDINAL", "DATE", "EVENT", "FAC", "GPE", "LANGUAGE", "LAW", - "LOC", "MISC", "MONEY", "NORP", "ORDINAL", "ORG", "PER", - "PERCENT", "PERSON", "PRODUCT", "QUANTITY", "TIME", "WORK_OF_ART", - ], - }, -} - -# Zero-shot label prompts -> the 4 Presidio NER entities GLiNER owns. Multiple -# prompts per entity trade a little inference cost for recall; tune against -# scripts/bench_engines.py output. -GLINER_ENTITY_MAPPING = { - "person": "PERSON", - "name": "PERSON", - "location": "LOCATION", - "address": "LOCATION", - "date": "DATE_TIME", - "time": "DATE_TIME", - "nationality": "NRP", - "religious group": "NRP", - "political group": "NRP", - "ethnic group": "NRP", -} - -# Predefined recognizers Presidio ships but does NOT load into the default -# registry — they must be added explicitly. Each carries its own -# supported_language, so it fires under that language once its NLP model is -# loaded. en: UK/AU/IN/SG locale ids; es/it/pl/fi: national ids. -EXTRA_RECOGNIZERS = [ - UkNinoRecognizer, - AuAbnRecognizer, - AuAcnRecognizer, - AuTfnRecognizer, - AuMedicareRecognizer, - InPanRecognizer, - InAadhaarRecognizer, - InVehicleRegistrationRecognizer, - InVoterRecognizer, - InPassportRecognizer, - SgFinRecognizer, - SgUenRecognizer, - EsNifRecognizer, - EsNieRecognizer, - ItFiscalCodeRecognizer, - ItDriverLicenseRecognizer, - ItVatCodeRecognizer, - ItPassportRecognizer, - ItIdentityCardRecognizer, - PlPeselRecognizer, - FiPersonalIdentityCodeRecognizer, -] - - -class VinRecognizer(PatternRecognizer): - """VIN (17 chars, A-Z/0-9 excluding I/O/Q) with ISO 3779 check-digit - validation (position 9). Validation makes accidental matches on arbitrary - 17-char codes (request ids, SKUs, tokens) extremely unlikely. Some - non-North-American VINs omit the check digit and are skipped — an - intentional bias toward precision. - """ - - _TRANSLIT = { - **{str(d): d for d in range(10)}, - "A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "H": 8, - "J": 1, "K": 2, "L": 3, "M": 4, "N": 5, "P": 7, "R": 9, - "S": 2, "T": 3, "U": 4, "V": 5, "W": 6, "X": 7, "Y": 8, "Z": 9, - } - _WEIGHTS = [8, 7, 6, 5, 4, 3, 2, 10, 0, 9, 8, 7, 6, 5, 4, 3, 2] - - def validate_result(self, pattern_text: str): - vin = pattern_text.upper() - if len(vin) != 17: - return False - try: - total = sum(self._TRANSLIT[c] * w for c, w in zip(vin, self._WEIGHTS)) - except KeyError: - return False - check = total % 11 - expected = "X" if check == 10 else str(check) - return vin[8] == expected - - -class SharedModelGLiNERRecognizer(GLiNERRecognizer): - """Per-language GLiNER recognizer sharing ONE loaded model. - - Presidio routes recognizers by supported_language, so the registry holds - one instance per served language — but each instance's load() would pull - its own ~1.2GB model copy. The first instance loads (an ImportError from - a missing gliner package propagates — fail fast in the lean image); the - rest reuse the cached model. - """ - - _shared_models: dict = {} - - def load(self) -> None: - key = (self.model_name, self.map_location) - cached = self._shared_models.get(key) - if cached is None: - super().load() - self._shared_models[key] = self.gliner - else: - self.gliner = cached - - def analyze(self, text, entities, nlp_artifacts=None): - """GLiNERRecognizer appends any requested entity it doesn't know as an - ad-hoc zero-shot label and returns its hits. The analyzer passes ALL - supported entities (~40) when a request doesn't narrow them, which - would prompt GLiNER for CREDIT_CARD/VIN/ES_NIF/... — wrong scope, and - inference cost scales with label count. Restrict to the NER entities - this recognizer owns.""" - requested = [e for e in (entities or self.supported_entities) if e in self.supported_entities] - if not requested: - return [] - return super().analyze(text, requested, nlp_artifacts) - - -def _register_common_recognizers(analyzer: AnalyzerEngine) -> None: - """Regex/checksum recognizers shared by both engines.""" - # VIN is language-agnostic, so register it under every served language — - # a recognizer only fires for the language the caller routes to. - vin_pattern = Pattern(name="vin", regex=r"\b[A-HJ-NPR-Z0-9]{17}\b", score=0.7) - for language in SUPPORTED_LANGUAGES: - analyzer.registry.add_recognizer( - VinRecognizer( - supported_entity="VIN", - patterns=[vin_pattern], - context=["vin", "vehicle", "chassis"], - supported_language=language, - ) - ) - for recognizer_cls in EXTRA_RECOGNIZERS: - analyzer.registry.add_recognizer(recognizer_cls()) - - -def build_spacy_analyzer() -> AnalyzerEngine: - nlp_engine = NlpEngineProvider(nlp_configuration=NLP_CONFIGURATION).create_engine() - analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=SUPPORTED_LANGUAGES) - _register_common_recognizers(analyzer) - return analyzer - - -def build_gliner_analyzer(model_name: str, device: str | None) -> AnalyzerEngine: - """GLiNER engine: one multilingual zero-shot model replaces spaCy NER for - PERSON/LOCATION/NRP/DATE_TIME; everything else is unchanged. - - :param model_name: HuggingFace id of the GLiNER model. - :param device: torch device ("cpu", "cuda", "cuda:0"); None auto-detects - via Presidio's device_detector (cuda when available, else cpu). - """ - # Fail fast with an actionable message when gliner deps are missing (e.g. - # a custom-built image without them). Without these checks Presidio would - # try to pip-download the missing spaCy models at startup (a silent - # network fallback that dies with an unrelated pip permission error), and - # the gliner ImportError would surface only later. - if importlib.util.find_spec("gliner") is None: - raise RuntimeError( - "PII_ENGINE=gliner but the gliner package is not installed; " - "use the stock pii image (docker/pii.Dockerfile ships torch + gliner)" - ) - missing = [ - m["model_name"] - for m in GLINER_NLP_CONFIGURATION["models"] - if not spacy.util.is_package(m["model_name"]) - ] - if missing: - raise RuntimeError( - f"PII_ENGINE=gliner needs spaCy models {missing}; " - "use the stock pii image (docker/pii.Dockerfile ships them)" - ) - nlp_engine = NlpEngineProvider(nlp_configuration=GLINER_NLP_CONFIGURATION).create_engine() - analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=SUPPORTED_LANGUAGES) - # The default registry wires SpacyRecognizer per language; with GLiNER - # owning the NER entities it would emit duplicate/competing spans from the - # small models' ner pipe. remove_recognizer only logs when nothing matched, - # so assert the removal actually happened. - analyzer.registry.remove_recognizer("SpacyRecognizer") - if any(r.name == "SpacyRecognizer" for r in analyzer.registry.recognizers): - raise RuntimeError("SpacyRecognizer removal failed; Presidio registry layout changed") - for language in SUPPORTED_LANGUAGES: - analyzer.registry.add_recognizer( - SharedModelGLiNERRecognizer( - entity_mapping=GLINER_ENTITY_MAPPING, - model_name=model_name, - map_location=device, - supported_language=language, - ) - ) - _register_common_recognizers(analyzer) - return analyzer diff --git a/apps/pii/requirements-dev.txt b/apps/pii/requirements-dev.txt deleted file mode 100644 index 68a538f5f1d..00000000000 --- a/apps/pii/requirements-dev.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Test-only deps. Unit tests need requirements.txt + this file (no models); -# integration tests additionally need the models baked into the docker images -# (see tests/test_integration.py). -pytest==9.0.3 -httpx==0.28.1 diff --git a/apps/pii/requirements-gliner.txt b/apps/pii/requirements-gliner.txt deleted file mode 100644 index a3d002f3850..00000000000 --- a/apps/pii/requirements-gliner.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Extras for the opt-in GLiNER engine — installed ONLY in the `gliner` -# Dockerfile target, on top of requirements.txt. Pinned for reproducible image -# builds; bump deliberately. presidio-analyzer 2.2.362 requires -# gliner >=0.2.13,<1.0.0. -# -# torch is pinned in the Dockerfile instead: the CPU and CUDA targets install -# the same version from different wheel indexes. -gliner==0.2.27 -transformers==5.3.0 -huggingface_hub==1.3.0 diff --git a/apps/pii/scripts/bench_engines.py b/apps/pii/scripts/bench_engines.py deleted file mode 100644 index b7ea70ca764..00000000000 --- a/apps/pii/scripts/bench_engines.py +++ /dev/null @@ -1,206 +0,0 @@ -"""Benchmark + parity harness for the spacy vs gliner NER engines. - -Runs the same payload through both engines and reports per-engine throughput -(batch analyze, the production /redact_batch path) and per-text latency, plus -an accuracy diff over the 4 NER entity types (PERSON/LOCATION/NRP/DATE_TIME). -Non-NER (regex/checksum) results must be identical between engines — both -register the same recognizers — so any mismatch there is a wiring bug and the -script exits non-zero. - -Meant to run inside the pii image (both engines ship in it): - - docker run --rm python scripts/bench_engines.py - docker run --rm -v $PWD/texts.json:/data.json \\ - python scripts/bench_engines.py --payload /data.json - -Payload format: JSON list of {"text": str, "language": str} objects. -This doubles as the tuning harness for GLINER_ENTITY_MAPPING label prompts. -""" - -import argparse -import json -import statistics -import sys -import time -from collections import defaultdict -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -import engines # noqa: E402 - -# Entities sourced from the NER models rather than regex/checksum patterns. -# ORGANIZATION is emitted by the spacy engine's NER on unfiltered requests but -# is not in the app's supported set and has no GLiNER mapping — it shows up in -# the NER diff (spacy-only) rather than failing the regex-parity gate. -NER_ENTITIES = {"PERSON", "LOCATION", "NRP", "DATE_TIME", "ORGANIZATION"} -DEFAULT_PAYLOAD = Path(__file__).resolve().parent / "bench_payload.json" - - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--payload", type=Path, default=DEFAULT_PAYLOAD) - parser.add_argument("--engines", default="spacy,gliner") - parser.add_argument("--runs", type=int, default=3) - parser.add_argument("--warmup", type=int, default=1) - parser.add_argument("--device", default=None, help="torch device for gliner (default: auto)") - parser.add_argument("--gliner-model", default="urchade/gliner_multi_pii-v1") - parser.add_argument("--max-examples", type=int, default=10) - parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") - return parser.parse_args() - - -def build(engine: str, args) -> tuple: - started = time.perf_counter() - if engine == "spacy": - analyzer = engines.build_spacy_analyzer() - elif engine == "gliner": - analyzer = engines.build_gliner_analyzer(model_name=args.gliner_model, device=args.device) - else: - raise ValueError(f"Unknown engine {engine!r}") - return analyzer, time.perf_counter() - started - - -def analyze_all(analyzer, items) -> list[list]: - """One analyze() call per text, in payload order.""" - return [analyzer.analyze(text=item["text"], language=item["language"]) for item in items] - - -def bench(analyzer, items, runs: int, warmup: int) -> dict: - for _ in range(warmup): - analyze_all(analyzer, items) - run_times = [] - latencies = [] - for _ in range(runs): - run_started = time.perf_counter() - for item in items: - text_started = time.perf_counter() - analyzer.analyze(text=item["text"], language=item["language"]) - latencies.append(time.perf_counter() - text_started) - run_times.append(time.perf_counter() - run_started) - total_chars = sum(len(item["text"]) for item in items) - avg_run = statistics.mean(run_times) - return { - "texts_per_sec": len(items) / avg_run, - "chars_per_sec": total_chars / avg_run, - "latency_p50_ms": statistics.median(latencies) * 1000, - "latency_p95_ms": statistics.quantiles(latencies, n=20)[18] * 1000, - } - - -def spans(results, keep_ner: bool) -> set: - return { - (r.entity_type, r.start, r.end) - for r in results - if (r.entity_type in NER_ENTITIES) == keep_ner - } - - -def iou(a: tuple, b: tuple) -> float: - inter = max(0, min(a[2], b[2]) - max(a[1], b[1])) - union = max(a[2], b[2]) - min(a[1], b[1]) - return inter / union if union else 0.0 - - -def diff_ner(items, results_a, results_b, max_examples: int) -> dict: - """Per-entity-type agreement between two engines (span IoU >= 0.5).""" - per_type = defaultdict(lambda: {"a_total": 0, "b_total": 0, "matched": 0}) - examples = [] - for item, res_a, res_b in zip(items, results_a, results_b): - a = sorted(spans(res_a, keep_ner=True)) - b = sorted(spans(res_b, keep_ner=True)) - unmatched_b = set(b) - for span_a in a: - per_type[span_a[0]]["a_total"] += 1 - match = next( - (s for s in unmatched_b if s[0] == span_a[0] and iou(span_a, s) >= 0.5), None - ) - if match: - per_type[span_a[0]]["matched"] += 1 - unmatched_b.discard(match) - for span_b in b: - per_type[span_b[0]]["b_total"] += 1 - only_a = [s for s in a if not any(s[0] == t[0] and iou(s, t) >= 0.5 for t in b)] - only_b = sorted(unmatched_b) - if (only_a or only_b) and len(examples) < max_examples: - examples.append( - { - "text": item["text"], - "language": item["language"], - "only_a": [f"{t}[{s}:{e}]={item['text'][s:e]!r}" for t, s, e in only_a], - "only_b": [f"{t}[{s}:{e}]={item['text'][s:e]!r}" for t, s, e in only_b], - } - ) - return {"per_type": dict(per_type), "examples": examples} - - -def diff_regex(items, results_a, results_b) -> list: - """Non-NER results must be identical: same recognizers on both engines.""" - mismatches = [] - for item, res_a, res_b in zip(items, results_a, results_b): - a = spans(res_a, keep_ner=False) - b = spans(res_b, keep_ner=False) - if a != b: - mismatches.append({"text": item["text"], "only_a": sorted(a - b), "only_b": sorted(b - a)}) - return mismatches - - -def main() -> int: - args = parse_args() - items = json.loads(args.payload.read_text()) - engine_names = [e.strip() for e in args.engines.split(",") if e.strip()] - - report = {"payload": str(args.payload), "texts": len(items), "engines": {}} - results_by_engine = {} - for name in engine_names: - analyzer, build_secs = build(name, args) - stats = bench(analyzer, items, runs=args.runs, warmup=args.warmup) - stats["build_secs"] = build_secs - report["engines"][name] = stats - results_by_engine[name] = analyze_all(analyzer, items) - - exit_code = 0 - if set(engine_names) >= {"spacy", "gliner"}: - report["ner_diff"] = diff_ner( - items, results_by_engine["spacy"], results_by_engine["gliner"], args.max_examples - ) - regex_mismatches = diff_regex( - items, results_by_engine["spacy"], results_by_engine["gliner"] - ) - report["regex_mismatches"] = regex_mismatches - if regex_mismatches: - exit_code = 1 - - if args.json: - print(json.dumps(report, indent=2, default=str)) - return exit_code - - for name, stats in report["engines"].items(): - print(f"\n== {name} ==") - print(f" build: {stats['build_secs']:.1f}s") - print(f" throughput: {stats['texts_per_sec']:.2f} texts/s ({stats['chars_per_sec']:.0f} chars/s)") - print(f" latency: p50 {stats['latency_p50_ms']:.1f}ms p95 {stats['latency_p95_ms']:.1f}ms") - if "ner_diff" in report: - print("\n== NER parity (spacy=a vs gliner=b, span IoU>=0.5) ==") - for entity, counts in sorted(report["ner_diff"]["per_type"].items()): - print( - f" {entity:<10} spacy={counts['a_total']:<4} gliner={counts['b_total']:<4} " - f"matched={counts['matched']}" - ) - for example in report["ner_diff"]["examples"]: - print(f"\n [{example['language']}] {example['text']}") - if example["only_a"]: - print(f" spacy only: {', '.join(example['only_a'])}") - if example["only_b"]: - print(f" gliner only: {', '.join(example['only_b'])}") - if report["regex_mismatches"]: - print("\n!! REGEX MISMATCHES (wiring bug — engines must agree on non-NER):") - for mismatch in report["regex_mismatches"]: - print(f" {mismatch}") - else: - print("\n regex/checksum entities: identical across engines ✓") - return exit_code - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/apps/pii/scripts/bench_payload.json b/apps/pii/scripts/bench_payload.json deleted file mode 100644 index fd0f207164d..00000000000 --- a/apps/pii/scripts/bench_payload.json +++ /dev/null @@ -1,97 +0,0 @@ -[ - { "text": "My name is John Smith and I live in Paris with my wife Marie.", "language": "en" }, - { - "text": "Dr. Angela Rodriguez will see you on March 14, 2026 at 3:30 PM in the Boston clinic.", - "language": "en" - }, - { - "text": "Contact Sarah O'Connor at sarah.oconnor@example.com or call (212) 555-0123.", - "language": "en" - }, - { "text": "The package ships to 45 Queen Street, Toronto, next Tuesday.", "language": "en" }, - { - "text": "Ahmed is a practicing Muslim from Egypt who moved to Berlin in 2019.", - "language": "en" - }, - { "text": "She is a Catholic Norwegian citizen born on 12/05/1988.", "language": "en" }, - { - "text": "Payment with card 4111111111111111 was declined yesterday at 10am.", - "language": "en" - }, - { - "text": "The vehicle VIN 1HGCM82633A004352 was registered to James Wilson in Ohio.", - "language": "en" - }, - { - "text": "His National Insurance number is AB123456C and he lives in Manchester.", - "language": "en" - }, - { - "text": "Meeting rescheduled: Friday, 9 January 2026, 14:00, with Priya Natarajan from Mumbai.", - "language": "en" - }, - { "text": "Send the invoice to accounts@acme.io before the end of Q1 2026.", "language": "en" }, - { - "text": "Klaus, a German engineer, and his Buddhist colleague Mei flew from Munich to Osaka.", - "language": "en" - }, - { - "text": "The Democrats and Republicans debated in Washington on election night.", - "language": "en" - }, - { - "text": "No PII here: the quarterly revenue grew 14% and margins held steady.", - "language": "en" - }, - { - "text": "Server request id a7f3k2m9x1q8w5z2b is not a VIN and not a person.", - "language": "en" - }, - { - "text": "Me llamo María García y vivo en Madrid desde el 3 de mayo de 2020.", - "language": "es" - }, - { "text": "Mi NIF es 12345678Z y mi correo es maria.garcia@ejemplo.es.", "language": "es" }, - { - "text": "El señor Javier Morales, ciudadano mexicano, llegó a Barcelona el lunes.", - "language": "es" - }, - { "text": "La reunión con Carmen será el 15 de junio de 2026 en Sevilla.", "language": "es" }, - { - "text": "Los musulmanes y los católicos convivieron durante siglos en Córdoba.", - "language": "es" - }, - { "text": "Mi chiamo Marco Rossi e abito a Roma vicino al Colosseo.", "language": "it" }, - { - "text": "Il codice fiscale di Maria Rossi è RSSMRA85T10A562S, nata il 10 dicembre 1985.", - "language": "it" - }, - { - "text": "Giulia Bianchi, cittadina italiana, si trasferì a Milano nel gennaio 2021.", - "language": "it" - }, - { - "text": "L'appuntamento con il dottor Ferrari è fissato per il 20 marzo 2026 a Torino.", - "language": "it" - }, - { "text": "Nazywam się Jan Kowalski i mieszkam w Warszawie od 2015 roku.", "language": "pl" }, - { - "text": "Mój numer PESEL to 44051401359, urodziłem się 14 maja 1944 w Krakowie.", - "language": "pl" - }, - { - "text": "Anna Nowak, obywatelka polska, spotka się z nami we wtorek 12 maja 2026.", - "language": "pl" - }, - { "text": "Katolicy i protestanci wspólnie świętowali w Gdańsku.", "language": "pl" }, - { - "text": "Nimeni on Matti Virtanen ja asun Helsingissä Töölön kaupunginosassa.", - "language": "fi" - }, - { - "text": "Henkilötunnukseni on 131052-308T ja synnyin Tampereella lokakuussa 1952.", - "language": "fi" - }, - { "text": "Liisa Korhonen muutti Ouluun maanantaina 5. tammikuuta 2026.", "language": "fi" }, - { "text": "Suomalaiset ja ruotsalaiset kilpailevat jääkiekossa joka vuosi.", "language": "fi" } -] diff --git a/apps/pii/server.py b/apps/pii/server.py index 3f59cc540aa..b60368eb58c 100644 --- a/apps/pii/server.py +++ b/apps/pii/server.py @@ -1,54 +1,270 @@ """Combined Presidio REST service: analyzer + anonymizer on one port. -Constructs one warm AnalyzerEngine (multi-language NLP + a native check-digit -VIN recognizer) and one AnonymizerEngine at startup, exposing stock-compatible -endpoints so a single PRESIDIO_URL serves both. - -NER engine selection (see engines.py): -- PII_ENGINE=spacy (default): the 5 large spaCy models, unchanged behavior. -- PII_ENGINE=gliner: one multilingual GLiNER model for PERSON/LOCATION/NRP/ - DATE_TIME. The stock image ships both engines, so this is a pure env flip. - PII_DEVICE picks cpu/cuda (unset = auto-detect), PII_GLINER_MODEL overrides - the model id. The same code runs on CPU and GPU. Each uvicorn worker - (PII_WORKERS) loads its own GLiNER model copy — into GPU memory when on - cuda — so GPU deployments generally want PII_WORKERS=1 per GPU, unlike the - CPU/spacy path where workers scale with vCPUs. +Constructs one warm AnalyzerEngine (5 large spaCy models for NER + +regex/checksum pattern recognizers, incl. a native check-digit VIN recognizer) +and one AnonymizerEngine at startup, exposing stock-compatible endpoints so a +single PII_URL serves both. """ import logging -import os import time from typing import Any -from engines import build_gliner_analyzer, build_spacy_analyzer from fastapi import FastAPI -from presidio_analyzer import AnalyzerEngine, BatchAnalyzerEngine, RecognizerResult +from presidio_analyzer import ( + AnalyzerEngine, + BatchAnalyzerEngine, + Pattern, + PatternRecognizer, + RecognizerResult, +) +from presidio_analyzer.nlp_engine import NlpEngineProvider +from presidio_analyzer.predefined_recognizers import ( + AuAbnRecognizer, + AuAcnRecognizer, + AuMedicareRecognizer, + AuTfnRecognizer, + EsNieRecognizer, + EsNifRecognizer, + FiPersonalIdentityCodeRecognizer, + InAadhaarRecognizer, + InPanRecognizer, + InPassportRecognizer, + InVehicleRegistrationRecognizer, + InVoterRecognizer, + ItDriverLicenseRecognizer, + ItFiscalCodeRecognizer, + ItIdentityCardRecognizer, + ItPassportRecognizer, + ItVatCodeRecognizer, + PlPeselRecognizer, + SgFinRecognizer, + SgUenRecognizer, + SpacyRecognizer, + UkNinoRecognizer, +) from presidio_anonymizer import AnonymizerEngine from presidio_anonymizer.entities import OperatorConfig from pydantic import BaseModel -PII_ENGINE = os.environ.get("PII_ENGINE", "spacy") -if PII_ENGINE not in ("spacy", "gliner"): - raise ValueError(f"Invalid PII_ENGINE={PII_ENGINE!r}; expected 'spacy' or 'gliner'") -# Empty/unset -> None -> auto-detect (cuda when torch sees a GPU, else cpu). -PII_DEVICE = os.environ.get("PII_DEVICE") or None -PII_GLINER_MODEL = os.environ.get("PII_GLINER_MODEL", "urchade/gliner_multi_pii-v1") - -# Propagates to uvicorn's root handler, so timing lands in the container log stream. -logger = logging.getLogger("sim.pii") +# Languages served. Each needs its spaCy model installed in the image; the +# es/it/pl/fi predefined recognizers (ES_NIF, IT_FISCAL_CODE, PL_PESEL, ...) +# auto-load once their NLP engine is present. +NLP_CONFIGURATION = { + "nlp_engine_name": "spacy", + "models": [ + {"lang_code": "en", "model_name": "en_core_web_lg"}, + {"lang_code": "es", "model_name": "es_core_news_lg"}, + {"lang_code": "it", "model_name": "it_core_news_lg"}, + {"lang_code": "pl", "model_name": "pl_core_news_lg"}, + {"lang_code": "fi", "model_name": "fi_core_news_lg"}, + ], +} +SUPPORTED_LANGUAGES = [m["lang_code"] for m in NLP_CONFIGURATION["models"]] + +# spaCy pipeline components PII detection does not use. NER depends only on +# `tok2vec` + `ner`; the parser (the most expensive stage), tagger, morphologizer, +# attribute_ruler and lemmatizer are dead weight here. Disabling them is a ~2x +# throughput win with NO change to the detected entities. The only side effect is +# that Presidio's lemma-based context score-boosting weakens slightly — an +# acceptable trade for the speed on this CPU-bound service. +NER_ONLY_DISABLE = ("parser", "tagger", "morphologizer", "attribute_ruler", "lemmatizer") + +# Predefined recognizers Presidio ships but does NOT load into the default +# registry — they must be added explicitly. Each carries its own +# supported_language, so it fires under that language once its NLP model is +# loaded. en: UK/AU/IN/SG locale ids; es/it/pl/fi: national ids. +EXTRA_RECOGNIZERS = [ + UkNinoRecognizer, + AuAbnRecognizer, + AuAcnRecognizer, + AuTfnRecognizer, + AuMedicareRecognizer, + InPanRecognizer, + InAadhaarRecognizer, + InVehicleRegistrationRecognizer, + InVoterRecognizer, + InPassportRecognizer, + SgFinRecognizer, + SgUenRecognizer, + EsNifRecognizer, + EsNieRecognizer, + ItFiscalCodeRecognizer, + ItDriverLicenseRecognizer, + ItVatCodeRecognizer, + ItPassportRecognizer, + ItIdentityCardRecognizer, + PlPeselRecognizer, + FiPersonalIdentityCodeRecognizer, +] + + +class VinRecognizer(PatternRecognizer): + """VIN (17 chars, A-Z/0-9 excluding I/O/Q) with ISO 3779 check-digit + validation (position 9). Validation makes accidental matches on arbitrary + 17-char codes (request ids, SKUs, tokens) extremely unlikely. Some + non-North-American VINs omit the check digit and are skipped — an + intentional bias toward precision. + """ + + _TRANSLIT = { + **{str(d): d for d in range(10)}, + "A": 1, "B": 2, "C": 3, "D": 4, "E": 5, "F": 6, "G": 7, "H": 8, + "J": 1, "K": 2, "L": 3, "M": 4, "N": 5, "P": 7, "R": 9, + "S": 2, "T": 3, "U": 4, "V": 5, "W": 6, "X": 7, "Y": 8, "Z": 9, + } + _WEIGHTS = [8, 7, 6, 5, 4, 3, 2, 10, 0, 9, 8, 7, 6, 5, 4, 3, 2] + + def validate_result(self, pattern_text: str): + vin = pattern_text.upper() + if len(vin) != 17: + return False + try: + total = sum(self._TRANSLIT[c] * w for c, w in zip(vin, self._WEIGHTS)) + except KeyError: + return False + check = total % 11 + expected = "X" if check == 10 else str(check) + return vin[8] == expected + + +def _register_common_recognizers(analyzer: AnalyzerEngine) -> None: + """Regex/checksum recognizers on top of spaCy NER + the Presidio defaults.""" + # VIN is language-agnostic, so register it under every served language — + # a recognizer only fires for the language the caller routes to. + vin_pattern = Pattern(name="vin", regex=r"\b[A-HJ-NPR-Z0-9]{17}\b", score=0.7) + for language in SUPPORTED_LANGUAGES: + analyzer.registry.add_recognizer( + VinRecognizer( + supported_entity="VIN", + patterns=[vin_pattern], + context=["vin", "vehicle", "chassis"], + supported_language=language, + ) + ) + for recognizer_cls in EXTRA_RECOGNIZERS: + analyzer.registry.add_recognizer(recognizer_cls()) def build_analyzer() -> AnalyzerEngine: - if PII_ENGINE == "gliner": - return build_gliner_analyzer(model_name=PII_GLINER_MODEL, device=PII_DEVICE) - return build_spacy_analyzer() - - -logger.info("building analyzer engine=%s device=%s", PII_ENGINE, PII_DEVICE or "auto") + nlp_engine = NlpEngineProvider(nlp_configuration=NLP_CONFIGURATION).create_engine() + for nlp in getattr(nlp_engine, "nlp", {}).values(): + for pipe in NER_ONLY_DISABLE: + if pipe in nlp.pipe_names: + nlp.disable_pipe(pipe) + analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=SUPPORTED_LANGUAGES) + _register_common_recognizers(analyzer) + return analyzer + + +# Own handler at INFO. uvicorn configures only its own loggers, not the root, so a +# bare getLogger propagates to a handler-less root at the default WARNING level and +# every info() (the per-request timing lines) is silently dropped. Attach a stream +# handler directly and stop propagation so timing lands in the container log stream +# regardless of uvicorn's config or worker count. +logger = logging.getLogger("sim.pii") +logger.setLevel(logging.INFO) +if not logger.handlers: + _log_handler = logging.StreamHandler() + _log_handler.setFormatter(logging.Formatter("%(levelname)s: %(name)s: %(message)s")) + logger.addHandler(_log_handler) + logger.propagate = False + +logger.info("building analyzer (spacy)") analyzer = build_analyzer() batch_analyzer = BatchAnalyzerEngine(analyzer_engine=analyzer) anonymizer = AnonymizerEngine() +# Every entity the spaCy NER recognizers can produce. A request touching any of +# these must run spaCy; a request naming only non-NER (regex/checksum) entities can +# skip it. Derived from the live registry so it stays authoritative if Presidio's +# default entity set changes (e.g. ORGANIZATION), unioned with a known floor so an +# unexpectedly empty derivation can never let an NER request skip the NLP pass. +_SPACY_NER_FLOOR = frozenset({"PERSON", "LOCATION", "NRP", "DATE_TIME", "ORGANIZATION"}) +NER_ENTITIES = _SPACY_NER_FLOOR | frozenset( + entity + for recognizer in analyzer.registry.recognizers + if isinstance(recognizer, SpacyRecognizer) + for entity in recognizer.supported_entities +) + +# One blank NlpArtifacts per language, built once at startup. Passing these to +# analyze() skips nlp_engine.process_text (the spaCy tok2vec+ner pass) entirely: +# the pattern recognizers still match on the raw text, SpacyRecognizer is excluded +# by the entity filter, and score_threshold is unset so detection is identical. +# Only context-based score boosting (which needs real tokens) is unavailable — an +# accepted trade for skipping NER on the hot block-output path. Read-only, so it +# is safe to share across requests and workers. +_BLANK_ARTIFACTS = { + language: analyzer.nlp_engine.process_text("", language) + for language in SUPPORTED_LANGUAGES +} + + +def _regex_only(entities: list[str] | None, score_threshold: float | None) -> bool: + """True when the spaCy NLP pass can be skipped: the request names entities, none + require spaCy NER, and no positive score_threshold is set. The blank-artifacts + fast path drops context-based score boosting, which can only change what is + returned when a threshold gates a match between its base and context-boosted + score — so fall back to the full path whenever a threshold is in play.""" + return ( + bool(entities) + and NER_ENTITIES.isdisjoint(entities) + and (score_threshold is None or score_threshold <= 0) + ) + + +def _analyze_one( + text: str, + language: str, + entities: list[str] | None, + score_threshold: float | None, + return_decision_process: bool = False, +): + # Regex-only requests reuse a blank NlpArtifacts to skip the spaCy NLP pass; + # otherwise analyze() computes artifacts (runs spaCy) as usual. + nlp_artifacts = ( + _BLANK_ARTIFACTS.get(language) if _regex_only(entities, score_threshold) else None + ) + return analyzer.analyze( + text=text, + language=language, + entities=entities or None, + score_threshold=score_threshold, + return_decision_process=return_decision_process, + nlp_artifacts=nlp_artifacts, + ) + + +def _analyze_many( + texts: list[str], + language: str, + entities: list[str] | None, + score_threshold: float | None, +): + """Analyze many texts, skipping the spaCy pass for regex-only requests.""" + if _regex_only(entities, score_threshold): + blank = _BLANK_ARTIFACTS.get(language) + return [ + analyzer.analyze( + text=text, + language=language, + entities=entities, + score_threshold=score_threshold, + nlp_artifacts=blank, + ) + for text in texts + ] + return list( + batch_analyzer.analyze_iterator( + texts=texts, + language=language, + entities=entities or None, + score_threshold=score_threshold, + ) + ) + + app = FastAPI(title="Sim Presidio", docs_url=None, redoc_url=None) @@ -150,12 +366,12 @@ def supported_entities(language: str = "en") -> list[str]: @app.post("/analyze") def analyze(req: AnalyzeRequest) -> list[dict[str, Any]]: started = time.perf_counter() - results = analyzer.analyze( - text=req.text, - language=req.language, - entities=req.entities or None, - score_threshold=req.score_threshold, - return_decision_process=req.return_decision_process, + results = _analyze_one( + req.text, + req.language, + req.entities, + req.score_threshold, + req.return_decision_process, ) logger.info( "analyze lang=%s chars=%d entities=%d duration_ms=%.1f", @@ -171,12 +387,7 @@ def analyze(req: AnalyzeRequest) -> list[dict[str, Any]]: def analyze_batch(req: AnalyzeBatchRequest) -> list[list[dict[str, Any]]]: """Analyze many texts in one pass (spaCy nlp.pipe), returning one span list per input in request order — the batched counterpart to /analyze.""" - results = batch_analyzer.analyze_iterator( - texts=req.texts, - language=req.language, - entities=req.entities or None, - score_threshold=req.score_threshold, - ) + results = _analyze_many(req.texts, req.language, req.entities, req.score_threshold) return [[r.to_dict() for r in per_text] for per_text in results] @@ -228,12 +439,7 @@ def redact(req: RedactRequest) -> dict[str, str]: anonymizer directly (no dict round-trip).""" started = time.perf_counter() operators = build_operators(req.anonymizers or req.operators) - results = analyzer.analyze( - text=req.text, - language=req.language, - entities=req.entities or None, - score_threshold=req.score_threshold, - ) + results = _analyze_one(req.text, req.language, req.entities, req.score_threshold) text = ( req.text if not results @@ -261,14 +467,7 @@ def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]: texts that actually matched.""" started = time.perf_counter() operators = build_operators(req.anonymizers or req.operators) - analyzed = list( - batch_analyzer.analyze_iterator( - texts=req.texts, - language=req.language, - entities=req.entities or None, - score_threshold=req.score_threshold, - ) - ) + analyzed = _analyze_many(req.texts, req.language, req.entities, req.score_threshold) masked: list[str] = [] total_spans = 0 for text, per_text in zip(req.texts, analyzed): @@ -282,9 +481,11 @@ def redact_batch(req: RedactBatchRequest) -> dict[str, list[str]]: ).text ) logger.info( - "redact_batch lang=%s texts=%d spans=%d duration_ms=%.1f", + "redact_batch lang=%s texts=%d entities=%s nlp=%s spans=%d duration_ms=%.1f", req.language, len(req.texts), + len(req.entities) if req.entities else "all", + "skip" if _regex_only(req.entities, req.score_threshold) else "full", total_spans, (time.perf_counter() - started) * 1000, ) diff --git a/apps/pii/tests/conftest.py b/apps/pii/tests/conftest.py deleted file mode 100644 index c9787a309d9..00000000000 --- a/apps/pii/tests/conftest.py +++ /dev/null @@ -1,5 +0,0 @@ -import sys -from pathlib import Path - -# server.py / engines.py live one level up (repo: apps/pii, image: /app). -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) diff --git a/apps/pii/tests/test_engines.py b/apps/pii/tests/test_engines.py deleted file mode 100644 index 1e6c1b8840c..00000000000 --- a/apps/pii/tests/test_engines.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Unit tests for engines.py — no models, no downloads, no network. - -Run: pip install -r requirements.txt -r requirements-dev.txt && python -m pytest tests -""" - -import importlib.util -import os -import subprocess -import sys -from pathlib import Path - -import pytest -from presidio_analyzer.predefined_recognizers.ner import gliner_recognizer - -import engines - -PII_DIR = Path(__file__).resolve().parent.parent - - -class FakeModel: - def __init__(self): - self.seen_labels: list[list[str]] = [] - - def predict_entities(self, text, labels, flat_ner=True, threshold=0.3, multi_label=False): - self.seen_labels.append(list(labels)) - return [{"label": "person", "score": 0.92, "start": 0, "end": 4, "text": text[0:4]}] - - -class FakeGLiNER: - calls = 0 - - @classmethod - def from_pretrained(cls, model_name, **kwargs): - cls.calls += 1 - return FakeModel() - - -@pytest.fixture -def fake_gliner(monkeypatch): - monkeypatch.setattr(gliner_recognizer, "GLiNER", FakeGLiNER) - engines.SharedModelGLiNERRecognizer._shared_models.clear() - FakeGLiNER.calls = 0 - yield FakeGLiNER - engines.SharedModelGLiNERRecognizer._shared_models.clear() - - -def make_recognizer(language: str): - return engines.SharedModelGLiNERRecognizer( - entity_mapping=engines.GLINER_ENTITY_MAPPING, - model_name="fake/model", - map_location="cpu", - supported_language=language, - ) - - -def test_invalid_pii_engine_fails_import(): - result = subprocess.run( - [sys.executable, "-c", "import server"], - cwd=PII_DIR, - env={**os.environ, "PII_ENGINE": "bogus"}, - capture_output=True, - text=True, - ) - assert result.returncode != 0 - assert "Invalid PII_ENGINE" in result.stderr - - -@pytest.mark.skipif( - importlib.util.find_spec("gliner") is not None, - reason="fail-fast path only exists when gliner is not installed", -) -def test_build_gliner_analyzer_fails_fast_without_gliner(): - with pytest.raises(RuntimeError, match="gliner package is not installed"): - engines.build_gliner_analyzer(model_name="fake/model", device="cpu") - - -def test_shared_model_loads_once_across_languages(fake_gliner): - first = make_recognizer("en") - second = make_recognizer("es") - assert fake_gliner.calls == 1 - assert first.gliner is second.gliner - - -def test_analyze_never_prompts_gliner_with_foreign_entities(fake_gliner): - recognizer = make_recognizer("en") - all_supported = ["PERSON", "LOCATION", "NRP", "DATE_TIME", "CREDIT_CARD", "VIN", "ES_NIF"] - results = recognizer.analyze("John went home", entities=all_supported) - for labels in recognizer.gliner.seen_labels: - assert set(labels) <= set(engines.GLINER_ENTITY_MAPPING) - assert results and results[0].entity_type == "PERSON" - - -def test_analyze_skips_inference_when_no_owned_entity_requested(fake_gliner): - recognizer = make_recognizer("en") - assert recognizer.analyze("4111111111111111", entities=["CREDIT_CARD"]) == [] - assert recognizer.gliner.seen_labels == [] - - -def test_entity_mapping_targets_exactly_the_ner_entities(): - assert set(engines.GLINER_ENTITY_MAPPING.values()) == { - "PERSON", - "LOCATION", - "NRP", - "DATE_TIME", - } diff --git a/apps/pii/tests/test_integration.py b/apps/pii/tests/test_integration.py deleted file mode 100644 index 40c97975754..00000000000 --- a/apps/pii/tests/test_integration.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Integration tests — exercise the real engines end-to-end via the FastAPI app. - -Requires the models present, so run inside the built images (gated behind -RUN_PII_INTEGRATION to keep plain `pytest` runs model-free): - - # spacy regression (default engine) - docker run --rm -e RUN_PII_INTEGRATION=1 python -m pytest tests - - # gliner engine - docker run --rm -e RUN_PII_INTEGRATION=1 -e PII_ENGINE=gliner \ - python -m pytest tests/test_integration.py - -The suite adapts to PII_ENGINE: shared assertions always run, engine-specific -ones only for the active engine. -""" - -import os - -import pytest - -if not os.environ.get("RUN_PII_INTEGRATION"): - pytest.skip( - "integration tests need the built image (RUN_PII_INTEGRATION=1)", - allow_module_level=True, - ) - -from fastapi.testclient import TestClient - -import server - -ENGINE = server.PII_ENGINE -client = TestClient(server.app) - - -def redact_batch(texts, language="en"): - response = client.post("/redact_batch", json={"texts": texts, "language": language}) - assert response.status_code == 200 - return response.json()["texts"] - - -def test_health(): - assert client.get("/health").json() == {"status": "ok"} - - -def test_masks_person_and_email(): - [masked] = redact_batch(["My name is John Smith, email john.smith@example.com."]) - assert "" in masked - assert "" in masked - assert "John Smith" not in masked - assert "john.smith@example.com" not in masked - - -def test_masks_location_and_phone(): - [masked] = redact_batch(["I live in Paris, call me at (212) 555-0123."]) - assert "" in masked - assert "" in masked - assert "Paris" not in masked - - -def test_regex_recognizers_fire_in_non_english_languages(): - [masked] = redact_batch(["Mi NIF es 12345678Z."], language="es") - assert "" in masked - # On the spacy engine the it_core_news_lg NER tags the fiscal code as - # ORGANIZATION and outscores the pattern recognizer, so only assert the - # value is masked; the exact label is checked on the gliner engine where - # spaCy NER can't compete. - [masked] = redact_batch(["Il codice fiscale è RSSMRA85T10A562S."], language="it") - assert "RSSMRA85T10A562S" not in masked - if ENGINE == "gliner": - assert "" in masked - - -def test_vin_checksum_recognizer_fires(): - [masked] = redact_batch(["The car VIN is 1HGCM82633A004352."]) - assert "" in masked - - -def test_no_pii_passes_through_unchanged(): - # NB: "Quarterly" would be tagged DATE_TIME by the spacy engine — keep - # this text free of anything either engine considers an entity. - text = "Revenue grew and margins held steady." - assert redact_batch([text]) == [text] - - -@pytest.mark.skipif(ENGINE != "gliner", reason="gliner-only wiring assertions") -def test_gliner_registry_has_no_spacy_recognizer(): - names = {r.name for r in server.analyzer.registry.recognizers} - assert "SpacyRecognizer" not in names - assert "GLiNERRecognizer" in names - - -@pytest.mark.skipif(ENGINE != "gliner", reason="gliner-only wiring assertions") -def test_gliner_supported_entities_keep_ner_types(): - supported = set(server.analyzer.get_supported_entities("en")) - assert {"PERSON", "LOCATION", "NRP", "DATE_TIME"} <= supported - - -@pytest.mark.skipif(ENGINE != "spacy", reason="spacy-only wiring assertions") -def test_spacy_registry_unchanged(): - names = {r.name for r in server.analyzer.registry.recognizers} - assert "SpacyRecognizer" in names - assert "GLiNERRecognizer" not in names diff --git a/apps/sim/ee/data-retention/components/data-retention-settings.tsx b/apps/sim/ee/data-retention/components/data-retention-settings.tsx index d115bfc33e0..99ca1070a16 100644 --- a/apps/sim/ee/data-retention/components/data-retention-settings.tsx +++ b/apps/sim/ee/data-retention/components/data-retention-settings.tsx @@ -35,6 +35,7 @@ import { type PiiStageKey, type PiiStagePolicy, type PiiStages, + stripNerEntities, } from '@/lib/guardrails/pii-entities' import { UnsavedChangesModal } from '@/app/workspace/[workspaceId]/components/credential-detail' import { saveDiscardActions } from '@/app/workspace/[workspaceId]/settings/components/save-discard-actions/save-discard-actions' @@ -172,7 +173,10 @@ function anyStageHasContent(stages: PiiStages): boolean { /** Persist-time guarantee that `enabled` mirrors "has entity types" for every stage. */ function withSyncedEnabled(stages: PiiStages): PiiStages { return PII_STAGES.reduce((acc, key) => { - acc[key] = { ...stages[key], enabled: stages[key].entityTypes.length > 0 } + // Block outputs are regex-only — strip any NER before persisting. + const entityTypes = + key === 'blockOutputs' ? stripNerEntities(stages[key].entityTypes) : stages[key].entityTypes + acc[key] = { ...stages[key], entityTypes, enabled: entityTypes.length > 0 } return acc }, {} as PiiStages) } @@ -321,6 +325,7 @@ function PiiLanguageSelect({ value, onChange }: PiiLanguageSelectProps) { } interface PiiStagePanelProps { + stageKey: PiiStageKey description: string value: PiiStagePolicy onChange: (next: PiiStagePolicy) => void @@ -331,8 +336,12 @@ interface PiiStagePanelProps { * stage is "on" purely by virtue of having entity types selected — `enabled` is * kept in sync with that, so there is no separate toggle. */ -function PiiStagePanel({ description, value, onChange }: PiiStagePanelProps) { - const groups = getEntityGroupsForLanguage(value.language) +function PiiStagePanel({ stageKey, description, value, onChange }: PiiStagePanelProps) { + // Block outputs run in-flight on large payloads, so they are restricted to the + // regex/checksum recognizers (no spaCy NER) — see the server fast path. + const groups = getEntityGroupsForLanguage(value.language, { + regexOnly: stageKey === 'blockOutputs', + }) function update(entityTypes: string[], language = value.language) { onChange({ ...value, language, entityTypes, enabled: entityTypes.length > 0 }) @@ -570,6 +579,7 @@ function PolicyDetail({ /> )} diff --git a/apps/sim/lib/api/contracts/data-retention.test.ts b/apps/sim/lib/api/contracts/data-retention.test.ts index 10fdcb9e720..d5969732426 100644 --- a/apps/sim/lib/api/contracts/data-retention.test.ts +++ b/apps/sim/lib/api/contracts/data-retention.test.ts @@ -145,6 +145,41 @@ describe('piiRedactionRuleSchema', () => { expect(result.success).toBe(false) }) + it('strips spaCy-NER entities from the blockOutputs stage only (regex-only)', () => { + const result = piiRedactionRuleSchema.safeParse({ + id: 'r-1', + workspaceId: null, + stages: { + input: stage(true, ['PERSON', 'EMAIL_ADDRESS']), + blockOutputs: stage(true, ['PERSON', 'EMAIL_ADDRESS']), + logs: stage(true, ['DATE_TIME']), + }, + }) + expect(result.success).toBe(true) + if (!result.success) return + expect(result.data.stages?.blockOutputs.entityTypes).toEqual(['EMAIL_ADDRESS']) + expect(result.data.stages?.blockOutputs.enabled).toBe(true) + // input and logs keep their NER entities. + expect(result.data.stages?.input.entityTypes).toEqual(['PERSON', 'EMAIL_ADDRESS']) + expect(result.data.stages?.logs.entityTypes).toEqual(['DATE_TIME']) + }) + + it('disables blockOutputs when the NER strip leaves it empty (migration-safe, no lockout)', () => { + const result = piiRedactionRuleSchema.safeParse({ + id: 'r-1', + workspaceId: null, + stages: { + input: stage(false, []), + blockOutputs: stage(true, ['PERSON', 'LOCATION']), + logs: stage(false, []), + }, + }) + expect(result.success).toBe(true) + if (!result.success) return + expect(result.data.stages?.blockOutputs.entityTypes).toEqual([]) + expect(result.data.stages?.blockOutputs.enabled).toBe(false) + }) + it('enforces one rule per scope (uniqueness refine still applies)', () => { const result = piiRedactionSettingsSchema.safeParse({ rules: [ diff --git a/apps/sim/lib/api/contracts/primitives.ts b/apps/sim/lib/api/contracts/primitives.ts index 6acf45a4238..1c6e1e28eb3 100644 --- a/apps/sim/lib/api/contracts/primitives.ts +++ b/apps/sim/lib/api/contracts/primitives.ts @@ -1,5 +1,5 @@ import { z } from 'zod' -import { PII_LANGUAGE_CODES } from '@/lib/guardrails/pii-entities' +import { PII_LANGUAGE_CODES, stripNerEntities } from '@/lib/guardrails/pii-entities' export const unknownRecordSchema = z.record(z.string(), z.unknown()) @@ -134,12 +134,31 @@ export const piiStagePolicySchema = z export type PiiStagePolicy = z.output -/** The three redaction stages, each independently configured. */ -export const piiStagesSchema = z.object({ - input: piiStagePolicySchema, - blockOutputs: piiStagePolicySchema, - logs: piiStagePolicySchema, -}) +/** + * The three redaction stages, each independently configured. + * + * Block outputs are regex-only: they run in-flight on Presidio's spaCy-free fast + * path, so the spaCy-NER entities (PERSON/LOCATION/NRP/DATE_TIME) are stripped + * here rather than rejected — a stored rule that still selects NER stays saveable + * (migration-safe), and a blockOutputs stage left empty by the strip is disabled. + */ +export const piiStagesSchema = z + .object({ + input: piiStagePolicySchema, + blockOutputs: piiStagePolicySchema, + logs: piiStagePolicySchema, + }) + .transform((stages) => { + const entityTypes = stripNerEntities(stages.blockOutputs.entityTypes) + return { + ...stages, + blockOutputs: { + ...stages.blockOutputs, + entityTypes, + enabled: stages.blockOutputs.enabled && entityTypes.length > 0, + }, + } + }) export type PiiStages = z.output diff --git a/apps/sim/lib/billing/retention.test.ts b/apps/sim/lib/billing/retention.test.ts index f14e343fdfc..506e4c118ec 100644 --- a/apps/sim/lib/billing/retention.test.ts +++ b/apps/sim/lib/billing/retention.test.ts @@ -128,6 +128,45 @@ describe('resolveEffectivePiiRedaction', () => { expect(result.logs).toEqual({ enabled: true, entityTypes: ['PERSON'], language: 'en' }) }) + it('strips spaCy-NER entities from blockOutputs at resolve time (regex-only)', () => { + const result = resolveEffectivePiiRedaction({ + orgSettings: settings([ + { + id: 'r-1', + workspaceId: 'ws-1', + stages: { + input: stage(true, ['PERSON', 'EMAIL_ADDRESS']), + blockOutputs: stage(true, ['PERSON', 'EMAIL_ADDRESS']), + logs: stage(true, ['DATE_TIME']), + }, + }, + ]), + workspaceId: 'ws-1', + }) + // input + logs keep NER; blockOutputs drops it (regex-only execution path). + expect(result.input.entityTypes).toEqual(['PERSON', 'EMAIL_ADDRESS']) + expect(result.blockOutputs.entityTypes).toEqual(['EMAIL_ADDRESS']) + expect(result.logs.entityTypes).toEqual(['DATE_TIME']) + }) + + it('disables blockOutputs when only NER was stored (un-migrated rule)', () => { + const result = resolveEffectivePiiRedaction({ + orgSettings: settings([ + { + id: 'r-1', + workspaceId: 'ws-1', + stages: { + input: stage(false, []), + blockOutputs: stage(true, ['PERSON']), + logs: stage(false, []), + }, + }, + ]), + workspaceId: 'ws-1', + }) + expect(result.blockOutputs).toEqual(DISABLED) + }) + it('selects the whole workspace rule over the all rule (no per-stage merge)', () => { const result = resolveEffectivePiiRedaction({ orgSettings: settings([ diff --git a/apps/sim/lib/billing/retention.ts b/apps/sim/lib/billing/retention.ts index afc8e0c0c76..08eecafe26a 100644 --- a/apps/sim/lib/billing/retention.ts +++ b/apps/sim/lib/billing/retention.ts @@ -1,5 +1,9 @@ import type { DataRetentionSettings, PiiStagePolicy } from '@sim/db/schema' -import { coercePiiLanguage, DEFAULT_PII_LANGUAGE } from '@/lib/guardrails/pii-entities' +import { + coercePiiLanguage, + DEFAULT_PII_LANGUAGE, + stripNerEntities, +} from '@/lib/guardrails/pii-entities' /** Resolved policy for one redaction stage. */ export interface EffectivePiiStage { @@ -44,8 +48,16 @@ function sanitizeEntityTypes(value: unknown): string[] { * rejects enabled-with-no-types), so an empty entity list always means "off" — * consistent across the UI, the contract, and the masking layer. */ -function toEffectiveStage(policy: PiiStagePolicy | undefined): EffectivePiiStage { - const types = sanitizeEntityTypes(policy?.entityTypes) +function toEffectiveStage( + policy: PiiStagePolicy | undefined, + opts?: { regexOnly?: boolean } +): EffectivePiiStage { + // Block outputs are regex-only. Strip any spaCy-NER entity defensively here so + // execution never masks block outputs with NER, even for rules stored before + // the restriction (the write path already strips via the API contract). + const types = opts?.regexOnly + ? stripNerEntities(sanitizeEntityTypes(policy?.entityTypes)) + : sanitizeEntityTypes(policy?.entityTypes) if (!policy?.enabled || types.length === 0) return DISABLED_STAGE return { enabled: true, @@ -93,7 +105,7 @@ export function resolveEffectivePiiRedaction(params: { return { input: toEffectiveStage(rule.stages.input), - blockOutputs: toEffectiveStage(rule.stages.blockOutputs), + blockOutputs: toEffectiveStage(rule.stages.blockOutputs, { regexOnly: true }), logs: toEffectiveStage(rule.stages.logs), } } diff --git a/apps/sim/lib/guardrails/pii-entities.test.ts b/apps/sim/lib/guardrails/pii-entities.test.ts new file mode 100644 index 00000000000..6c0d67debc2 --- /dev/null +++ b/apps/sim/lib/guardrails/pii-entities.test.ts @@ -0,0 +1,91 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + getEntityGroupsForLanguage, + NER_PII_ENTITIES, + normalizeRuleStages, + stripNerEntities, +} from '@/lib/guardrails/pii-entities' + +describe('NER_PII_ENTITIES', () => { + it('covers the spaCy-NER entities including ORGANIZATION', () => { + expect(new Set(NER_PII_ENTITIES)).toEqual( + new Set(['PERSON', 'LOCATION', 'NRP', 'DATE_TIME', 'ORGANIZATION']) + ) + }) +}) + +describe('stripNerEntities', () => { + it('drops NER entities and keeps regex/checksum ones (order preserved)', () => { + expect( + stripNerEntities([ + 'PERSON', + 'EMAIL_ADDRESS', + 'DATE_TIME', + 'US_SSN', + 'ORGANIZATION', + 'LOCATION', + 'PHONE_NUMBER', + ]) + ).toEqual(['EMAIL_ADDRESS', 'US_SSN', 'PHONE_NUMBER']) + }) + + it('returns an empty list when only NER was selected', () => { + expect(stripNerEntities(['PERSON', 'NRP'])).toEqual([]) + }) + + it('is a no-op for a regex-only list', () => { + expect(stripNerEntities(['EMAIL_ADDRESS', 'US_SSN'])).toEqual(['EMAIL_ADDRESS', 'US_SSN']) + }) +}) + +describe('getEntityGroupsForLanguage', () => { + const flatten = (groups: Array<{ entities: Array<{ value: string }> }>) => + groups.flatMap((g) => g.entities.map((e) => e.value)) + + it('includes NER entities by default', () => { + const values = flatten(getEntityGroupsForLanguage('en')) + expect(values).toContain('PERSON') + expect(values).toContain('EMAIL_ADDRESS') + }) + + it('excludes the spaCy-NER entities when regexOnly', () => { + const values = flatten(getEntityGroupsForLanguage('en', { regexOnly: true })) + for (const ner of ['PERSON', 'LOCATION', 'NRP', 'DATE_TIME']) { + expect(values).not.toContain(ner) + } + // Regex/checksum entities remain selectable. + expect(values).toContain('EMAIL_ADDRESS') + expect(values).toContain('US_SSN') + }) +}) + +describe('normalizeRuleStages', () => { + it('strips NER from a stored blockOutputs stage (input/logs keep it)', () => { + const stages = normalizeRuleStages({ + stages: { + input: { enabled: true, entityTypes: ['PERSON', 'EMAIL_ADDRESS'], language: 'en' }, + blockOutputs: { enabled: true, entityTypes: ['PERSON', 'EMAIL_ADDRESS'], language: 'en' }, + logs: { enabled: true, entityTypes: ['DATE_TIME'], language: 'en' }, + }, + }) + expect(stages.blockOutputs.entityTypes).toEqual(['EMAIL_ADDRESS']) + expect(stages.blockOutputs.enabled).toBe(true) + expect(stages.input.entityTypes).toEqual(['PERSON', 'EMAIL_ADDRESS']) + expect(stages.logs.entityTypes).toEqual(['DATE_TIME']) + }) + + it('disables blockOutputs when the NER strip empties it', () => { + const stages = normalizeRuleStages({ + stages: { + input: { enabled: false, entityTypes: [] }, + blockOutputs: { enabled: true, entityTypes: ['PERSON', 'LOCATION'] }, + logs: { enabled: false, entityTypes: [] }, + }, + }) + expect(stages.blockOutputs.entityTypes).toEqual([]) + expect(stages.blockOutputs.enabled).toBe(false) + }) +}) diff --git a/apps/sim/lib/guardrails/pii-entities.ts b/apps/sim/lib/guardrails/pii-entities.ts index 9fd57eee8a8..cb7f5504636 100644 --- a/apps/sim/lib/guardrails/pii-entities.ts +++ b/apps/sim/lib/guardrails/pii-entities.ts @@ -229,11 +229,43 @@ export function isEntitySupportedForLanguage( return PII_ENTITIES_BY_LANGUAGE[language].has(entity) } -/** {@link PII_ENTITY_GROUPS} filtered to entities the language recognizes (empty groups dropped). */ -export function getEntityGroupsForLanguage(language: PIILanguage) { +/** + * Entity types produced by the spaCy NER model (vs the regex/checksum pattern + * recognizers). The block-output redaction stage is restricted to the non-NER + * (regex) entities so it runs on the Presidio spaCy-free fast path without the + * per-leaf NER cost. Includes ORGANIZATION — which Presidio's spaCy recognizer + * emits but the user-facing catalog above does not list — so it too is stripped + * from block-output selections, keeping this in sync with the derived + * `NER_ENTITIES` in `apps/pii/server.py`. Typed as strings because ORGANIZATION + * isn't a catalog `PIIEntityType`. + */ +export const NER_PII_ENTITIES: ReadonlySet = new Set([ + 'PERSON', + 'LOCATION', + 'NRP', + 'DATE_TIME', + 'ORGANIZATION', +]) + +/** Drop the spaCy-NER entities ({@link NER_PII_ENTITIES}) from a selection. */ +export function stripNerEntities(entities: readonly string[]): string[] { + return entities.filter((e) => !NER_PII_ENTITIES.has(e)) +} + +/** + * {@link PII_ENTITY_GROUPS} filtered to entities the language recognizes (empty + * groups dropped). With `regexOnly`, the spaCy-NER entities are also excluded — + * used for the block-output stage. + */ +export function getEntityGroupsForLanguage(language: PIILanguage, opts?: { regexOnly?: boolean }) { + const regexOnly = opts?.regexOnly ?? false return PII_ENTITY_GROUPS.map((group) => ({ label: group.label, - entities: group.entities.filter((e) => isEntitySupportedForLanguage(e.value, language)), + entities: group.entities.filter( + (e) => + isEntitySupportedForLanguage(e.value, language) && + (!regexOnly || !NER_PII_ENTITIES.has(e.value)) + ), })).filter((group) => group.entities.length > 0) } @@ -319,9 +351,17 @@ export function normalizeRuleStages(rule: { }) if (rule.stages) { + // Block outputs are regex-only (no spaCy NER) — strip NER from any stored + // rule so hydrated drafts never carry it; a stage left empty becomes disabled. + const blockOutputs = sanitize(rule.stages.blockOutputs) + const blockOutputsEntities = stripNerEntities(blockOutputs.entityTypes) return { input: sanitize(rule.stages.input), - blockOutputs: sanitize(rule.stages.blockOutputs), + blockOutputs: { + ...blockOutputs, + entityTypes: blockOutputsEntities, + enabled: blockOutputs.enabled && blockOutputsEntities.length > 0, + }, logs: sanitize(rule.stages.logs), } } diff --git a/docker/pii.Dockerfile b/docker/pii.Dockerfile index 205462bad3f..4c688ac6e55 100644 --- a/docker/pii.Dockerfile +++ b/docker/pii.Dockerfile @@ -1,15 +1,6 @@ # ======================================== -# Combined Presidio service (analyzer + anonymizer) on a single port (5001) -# -# ONE image serves both NER engines — the engine is a pure runtime choice via -# PII_ENGINE (spacy default | gliner). spaCy large models, torch (CPU), the -# gliner package, and the baked GLiNER weights all ship in it, so flipping -# engines never requires an image swap. -# -# ONE image also serves both fleets: the amd64 build ships CUDA torch, which -# falls back to CPU when no GPU is present, so the Fargate CPU tasks and the -# EC2-GPU tasks pull the same tag. (torch CUDA wheels bundle their own CUDA -# libs; the host only needs the nvidia driver + container runtime.) +# Combined Presidio service (analyzer + anonymizer) on a single port (5001). +# CPU spaCy NER + regex/checksum pattern recognizers. # # Source files are COPY'd last so code edits never re-download deps or models. # ======================================== @@ -43,78 +34,11 @@ RUN --mount=type=cache,target=/root/.cache/pip \ pip install /tmp/*.whl && \ rm /tmp/*.whl -# --- GLiNER engine deps ------------------------------------------------------- -# torch is pinned here (not requirements-gliner.txt) because the CPU and CUDA -# builds install the same version from different wheel indexes. 2.11.0 is the -# newest release published on both the cpu and cu128 indexes for py312. -# -# cu128's arch list keeps sm_75, the compute capability of the GPU fleet's T4s. -# cu121 could not serve this pin anyway — that index stops at torch 2.5.1. -# CUDA 12.8 needs an NVIDIA driver >=525 via minor-version compatibility, which -# the ECS GPU AMI's nvidia-driver-latest-dkms satisfies. -# -# arm64 takes the cpu index: cu128 publishes no aarch64 wheel at 2.11.0, and no -# arm64 target has a GPU. -ARG TORCH_VERSION=2.11.0 -ARG TORCH_CUDA_INDEX_URL=https://download.pytorch.org/whl/cu128 -ARG TORCH_CPU_INDEX_URL=https://download.pytorch.org/whl/cpu -ARG TARGETARCH -RUN --mount=type=cache,target=/root/.cache/pip \ - case "${TARGETARCH}" in \ - amd64) torch_index="${TORCH_CUDA_INDEX_URL}" ;; \ - arm64) torch_index="${TORCH_CPU_INDEX_URL}" ;; \ - *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ - esac && \ - pip install torch==${TORCH_VERSION} --index-url "${torch_index}" - -COPY apps/pii/requirements-gliner.txt ./requirements-gliner.txt -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install -r requirements-gliner.txt - -# Small spaCy models (~60MB total) give the gliner engine tokenization + -# lemmas for the regex recognizers; GLiNER does the NER (see engines.py). -ARG SPACY_SM_MODELS="en_core_web_sm-3.8.0 es_core_news_sm-3.8.0 it_core_news_sm-3.8.0 pl_core_news_sm-3.8.0 fi_core_news_sm-3.8.0" -RUN --mount=type=cache,target=/root/.cache/pip \ - for model in ${SPACY_SM_MODELS}; do \ - whl="${model}-py3-none-any.whl"; \ - curl -fL --retry 5 --retry-delay 5 --retry-all-errors -C - \ - -o "/tmp/${whl}" \ - "https://github.com/explosion/spacy-models/releases/download/${model}/${whl}" || exit 1; \ - done && \ - pip install /tmp/*.whl && \ - rm /tmp/*.whl - -# Bake the GLiNER weights at build time (cached layer) so startup never -# touches the network. HF_HUB_OFFLINE makes a missing/overridden -# PII_GLINER_MODEL fail fast at startup instead of silently downloading. -ENV HF_HOME=/opt/hf-cache -ARG GLINER_MODEL=urchade/gliner_multi_pii-v1 -RUN python -c "from gliner import GLiNER; GLiNER.from_pretrained('${GLINER_MODEL}')" && \ - chmod -R a+rX /opt/hf-cache -ENV HF_HUB_OFFLINE=1 - -# pytest/httpx for the in-image test suites (tests/) — baked in because the -# runtime user has no writable HOME for pip install --user. -COPY apps/pii/requirements-dev.txt ./requirements-dev.txt -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install -r requirements-dev.txt - -# Runs after every pip install, because the requirements above resolve against -# PyPI and could swap the wheel torch_index chose. A cpu-only torch on amd64 -# otherwise surfaces only as "torch.cuda.is_available() is False" once GLiNER -# loads on a GPU host. -RUN python -c "import torch; \ -have = torch.version.cuda is not None; \ -want = '${TARGETARCH}' == 'amd64'; \ -assert have == want, f'{torch.__version__}: cuda build={have}, expected={want}'" - RUN groupadd -g 1001 pii && \ useradd -u 1001 -g pii pii && \ chown -R pii:pii /app -COPY --chown=pii:pii apps/pii/server.py apps/pii/engines.py ./ -COPY --chown=pii:pii apps/pii/scripts ./scripts -COPY --chown=pii:pii apps/pii/tests ./tests +COPY --chown=pii:pii apps/pii/server.py ./ USER pii @@ -134,6 +58,4 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=300s --retries=3 \ # `sh -c exec` expands the env var while keeping uvicorn as PID 1 for clean SIGTERM. # Quote the expansion so a malformed PII_WORKERS fails uvicorn arg-parsing rather # than being interpreted by the shell. -# NB for the gliner engine: EACH worker loads its own GLiNER model copy (into GPU -# memory when on cuda), so GPU deployments generally want PII_WORKERS=1 per GPU. CMD ["sh", "-c", "exec uvicorn server:app --host 0.0.0.0 --port 5001 --workers \"${PII_WORKERS:-1}\""] diff --git a/helm/sim/templates/deployment-pii.yaml b/helm/sim/templates/deployment-pii.yaml index 2dd4b8b22c6..033d87725a1 100644 --- a/helm/sim/templates/deployment-pii.yaml +++ b/helm/sim/templates/deployment-pii.yaml @@ -45,13 +45,7 @@ spec: containerPort: {{ .Values.pii.service.targetPort }} protocol: TCP env: - - name: PII_ENGINE - value: {{ .Values.pii.engine | quote }} - {{- if .Values.pii.device }} - - name: PII_DEVICE - value: {{ .Values.pii.device | quote }} - {{- end }} - {{- range $key, $value := omit (.Values.pii.env | default dict) "PII_ENGINE" "PII_DEVICE" }} + {{- range $key, $value := .Values.pii.env | default dict }} - name: {{ $key }} value: {{ $value | quote }} {{- end }} diff --git a/helm/sim/tests/pii_test.yaml b/helm/sim/tests/pii_test.yaml index f35c87a5759..8c03f193c30 100644 --- a/helm/sim/tests/pii_test.yaml +++ b/helm/sim/tests/pii_test.yaml @@ -30,64 +30,18 @@ tests: path: spec.template.spec.containers[0].ports[0].containerPort value: 5001 - - it: pii pod defaults to the spacy engine - template: deployment-pii.yaml - set: - pii.enabled: true - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: PII_ENGINE - value: "spacy" - - notContains: - path: spec.template.spec.containers[0].env - content: - name: PII_DEVICE - value: "cpu" - - - it: pii.engine and pii.device render through to the container env - template: deployment-pii.yaml - set: - pii.enabled: true - pii.engine: gliner - pii.device: cuda - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: PII_ENGINE - value: "gliner" - - contains: - path: spec.template.spec.containers[0].env - content: - name: PII_DEVICE - value: "cuda" - - - it: user-set pii.env cannot override the chart-owned engine keys + - it: pii.env renders through to the container env template: deployment-pii.yaml set: pii.enabled: true pii.env: - PII_ENGINE: evil - PII_DEVICE: evil - PII_GLINER_MODEL: acme/custom-model + PII_WORKERS: "4" asserts: - - notContains: - path: spec.template.spec.containers[0].env - content: - name: PII_ENGINE - value: "evil" - - notContains: - path: spec.template.spec.containers[0].env - content: - name: PII_DEVICE - value: "evil" - contains: path: spec.template.spec.containers[0].env content: - name: PII_GLINER_MODEL - value: "acme/custom-model" + name: PII_WORKERS + value: "4" - it: app pod gets chart-computed PII_URL pointing at the in-cluster service template: deployment-app.yaml diff --git a/helm/sim/values.schema.json b/helm/sim/values.schema.json index 60bdf101dcb..9ba8ac8f657 100644 --- a/helm/sim/values.schema.json +++ b/helm/sim/values.schema.json @@ -759,15 +759,6 @@ "type": "boolean", "description": "Enable the Presidio PII redaction service" }, - "engine": { - "type": "string", - "enum": ["spacy", "gliner"], - "description": "NER engine (spacy default, gliner opt-in; both ship in the image)" - }, - "device": { - "type": "string", - "description": "Torch device for the gliner engine (cpu, cuda, cuda:N); empty auto-detects" - }, "replicaCount": { "type": "integer", "minimum": 1, diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 8361b45e034..a3d4c94085e 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -836,18 +836,6 @@ pii: digest: "" # sha256: pin overrides tag pullPolicy: IfNotPresent - # NER engine: "spacy" (default) or "gliner" (opt-in zero-shot transformer NER - # for PERSON/LOCATION/NRP/DATE_TIME; regex/checksum recognizers are identical - # on both engines). The image ships both engines, so this is a pure env flip - # — no image change needed. Note: gliner on CPU is orders of magnitude slower - # than spacy; it is intended for GPU nodes. - engine: "spacy" - - # Torch device for the gliner engine: "cpu", "cuda", or "cuda:N". Empty - # auto-detects (cuda when a GPU is visible, else cpu). GPU resource requests - # (nvidia.com/gpu) are not wired yet — GPU scheduling is an infra follow-up. - device: "" - # Number of replicas replicaCount: 1 From 54b35a4f0e696737b80612d1d2b4500afbf4d830 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Wed, 15 Jul 2026 18:01:53 -0700 Subject: [PATCH 02/25] improvement(deployments): bugfixes for run-block, airtable + external sub management (#5680) * improvement(webhooks): external subscription management * ui/ux * remove test file * fix tests * address comments * address comments * update to grain v2 api * improvement(grain): hide auto-registered webhook URL on v2 triggers * Revert "improvement(grain): hide auto-registered webhook URL on v2 triggers" This reverts commit c89660cc3e4a46ad66bfe9139c9cdef0c97b0774. * address comments * address comments * address rollback * fix grain v2 * fix more comments --- apps/docs/components/ui/icon-mapping.ts | 1 + .../content/docs/en/integrations/airtable.mdx | 3 + .../content/docs/en/integrations/grain.mdx | 182 +- .../app/api/chat/manage/[id]/route.test.ts | 67 +- apps/sim/app/api/chat/manage/[id]/route.ts | 81 +- .../app/api/cron/renew-subscriptions/route.ts | 3 +- .../api/mcp/serve/[serverId]/route.test.ts | 33 +- .../sim/app/api/mcp/serve/[serverId]/route.ts | 29 +- apps/sim/app/api/schedules/execute/route.ts | 2 + .../app/api/tools/deployments/deploy/route.ts | 6 +- .../api/tools/deployments/promote/route.ts | 6 +- .../app/api/tools/deployments/routes.test.ts | 20 + apps/sim/app/api/v1/admin/types.ts | 12 - .../v1/admin/workflows/[id]/deploy/route.ts | 23 +- .../versions/[versionId]/activate/route.ts | 8 +- .../v1/workflows/[id]/deploy/route.test.ts | 29 +- .../app/api/v1/workflows/[id]/deploy/route.ts | 16 +- .../v1/workflows/[id]/rollback/route.test.ts | 18 + .../api/v1/workflows/[id]/rollback/route.ts | 14 +- .../app/api/workflows/[id]/deploy/route.ts | 54 +- .../[id]/deployments/[version]/route.ts | 25 +- .../app/api/workflows/[id]/execute/route.ts | 17 +- .../workspaces/[id]/fork/rollback/route.ts | 11 +- .../general/components/versions.tsx | 39 +- .../components/deploy-modal/deploy-modal.tsx | 211 +- .../condition-input/condition-input.tsx | 24 +- .../async-preprocessing-correlation.test.ts | 1 + apps/sim/background/schedule-execution.ts | 23 +- apps/sim/background/tiktok-webhook-targets.ts | 4 +- apps/sim/background/webhook-execution.test.ts | 69 +- apps/sim/background/webhook-execution.ts | 37 +- apps/sim/blocks/blocks/airtable.test.ts | 109 + apps/sim/blocks/blocks/airtable.ts | 12 +- apps/sim/blocks/blocks/grain.ts | 299 +- apps/sim/blocks/registry-maps.ts | 4 +- .../ee/workspace-forking/components/forks.tsx | 10 +- .../lib/promote/reactivate-in-tx.ts | 82 +- .../lib/promote/rollback.test.ts | 107 +- .../workspace-forking/lib/promote/rollback.ts | 93 +- apps/sim/executor/constants.ts | 5 - .../condition/condition-handler.test.ts | 35 + .../handlers/condition/condition-handler.ts | 19 +- apps/sim/hooks/queries/deployments.ts | 63 +- apps/sim/lib/api/contracts/deployments.ts | 52 +- .../lib/api/contracts/v1/admin/workflows.ts | 16 +- apps/sim/lib/api/contracts/v1/workflows.ts | 22 +- apps/sim/lib/api/contracts/workflows.ts | 2 + apps/sim/lib/api/contracts/workspace-fork.ts | 5 + .../tools/handlers/deployment/deploy.ts | 19 +- .../tools/handlers/deployment/manage.test.ts | 36 +- .../tools/handlers/deployment/manage.ts | 31 +- .../workflow/edit-workflow/builders.test.ts | 13 +- .../server/workflow/edit-workflow/builders.ts | 4 +- apps/sim/lib/core/outbox/service.test.ts | 24 + apps/sim/lib/core/outbox/service.ts | 27 +- apps/sim/lib/integrations/icon-mapping.ts | 1 + apps/sim/lib/integrations/integrations.json | 79 +- .../lib/webhooks/delivery-predicate.test.ts | 55 + apps/sim/lib/webhooks/delivery-predicate.ts | 21 + apps/sim/lib/webhooks/deploy.ts | 509 +- apps/sim/lib/webhooks/path-claims.test.ts | 109 + apps/sim/lib/webhooks/path-claims.ts | 117 + apps/sim/lib/webhooks/polling/utils.ts | 4 +- apps/sim/lib/webhooks/processor.test.ts | 38 +- apps/sim/lib/webhooks/processor.ts | 65 +- .../webhooks/provider-subscriptions.test.ts | 46 +- .../lib/webhooks/provider-subscriptions.ts | 50 +- apps/sim/lib/webhooks/providers/gmail.ts | 47 +- apps/sim/lib/webhooks/providers/grain.test.ts | 236 + apps/sim/lib/webhooks/providers/grain.ts | 447 +- apps/sim/lib/webhooks/providers/imap.ts | 42 +- apps/sim/lib/webhooks/providers/outlook.ts | 59 +- apps/sim/lib/webhooks/providers/rss.ts | 32 +- apps/sim/lib/webhooks/providers/types.ts | 6 +- .../webhooks/registration-identity.test.ts | 132 + .../sim/lib/webhooks/registration-identity.ts | 99 + .../registration-reconciliation.test.ts | 158 + .../webhooks/registration-reconciliation.ts | 110 + .../lib/webhooks/registration-service.test.ts | 349 + apps/sim/lib/webhooks/registration-service.ts | 390 + .../lib/webhooks/registration-store.test.ts | 337 + apps/sim/lib/webhooks/registration-store.ts | 591 + apps/sim/lib/webhooks/utils.server.test.ts | 109 + apps/sim/lib/webhooks/utils.server.ts | 36 +- apps/sim/lib/workflows/conditions.test.ts | 23 + apps/sim/lib/workflows/conditions.ts | 9 + .../workflows/deployment-lifecycle.test.ts | 62 + .../sim/lib/workflows/deployment-lifecycle.ts | 215 + .../lib/workflows/deployment-outbox.test.ts | 514 + apps/sim/lib/workflows/deployment-outbox.ts | 796 +- .../executor/execution-state.test.ts | 173 + .../lib/workflows/executor/execution-state.ts | 92 +- apps/sim/lib/workflows/lifecycle.test.ts | 11 +- apps/sim/lib/workflows/lifecycle.ts | 5 + .../workflows/orchestration/chat-deploy.ts | 61 +- .../workflows/orchestration/deploy.test.ts | 449 +- .../sim/lib/workflows/orchestration/deploy.ts | 575 +- apps/sim/lib/workflows/orchestration/index.ts | 1 + .../lib/workflows/orchestration/types.test.ts | 17 + apps/sim/lib/workflows/orchestration/types.ts | 1 + .../persistence/deployment-operations.test.ts | 491 + .../persistence/deployment-operations.ts | 912 + .../lib/workflows/persistence/utils.test.ts | 85 +- apps/sim/lib/workflows/persistence/utils.ts | 571 +- .../lib/workflows/schedules/deploy.test.ts | 4 +- apps/sim/lib/workflows/schedules/deploy.ts | 37 +- apps/sim/lib/workspace-events/no-activity.ts | 4 +- .../sim/lib/workspace-events/subscriptions.ts | 4 +- .../stores/workflows/registry/store.test.ts | 37 + apps/sim/stores/workflows/registry/store.ts | 7 + apps/sim/tools/airtable/airtable.test.ts | 92 + apps/sim/tools/airtable/create_records.ts | 11 +- apps/sim/tools/airtable/types.ts | 34 +- .../tools/airtable/update_multiple_records.ts | 11 +- apps/sim/tools/airtable/update_record.ts | 11 +- apps/sim/tools/grain/create_hook_v2.ts | 104 + apps/sim/tools/grain/delete_hook.ts | 2 +- apps/sim/tools/grain/delete_hook_v2.ts | 56 + apps/sim/tools/grain/get_recording.ts | 19 +- apps/sim/tools/grain/get_transcript.ts | 2 +- apps/sim/tools/grain/index.ts | 22 +- apps/sim/tools/grain/list_hooks_v2.ts | 84 + apps/sim/tools/grain/list_meeting_types.ts | 2 +- apps/sim/tools/grain/list_recordings.ts | 15 +- apps/sim/tools/grain/list_teams.ts | 2 +- apps/sim/tools/grain/types.ts | 80 +- apps/sim/tools/registry.ts | 6 + apps/sim/triggers/grain/events_v2.ts | 200 + apps/sim/triggers/grain/highlight_created.ts | 1 + apps/sim/triggers/grain/highlight_updated.ts | 1 + apps/sim/triggers/grain/index.ts | 13 + apps/sim/triggers/grain/item_added.ts | 1 + apps/sim/triggers/grain/item_updated.ts | 1 + apps/sim/triggers/grain/recording_created.ts | 1 + apps/sim/triggers/grain/recording_updated.ts | 1 + apps/sim/triggers/grain/story_created.ts | 1 + apps/sim/triggers/grain/utils.ts | 102 + apps/sim/triggers/grain/webhook.ts | 1 + apps/sim/triggers/registry.ts | 22 + apps/sim/triggers/types.ts | 8 + .../migrations/0261_tranquil_donald_blake.sql | 87 + .../db/migrations/meta/0261_snapshot.json | 17352 ++++++++++++++++ packages/db/migrations/meta/_journal.json | 7 + packages/db/schema.ts | 125 + ...space-storage-accounting-migration.test.ts | 73 - packages/testing/src/mocks/schema.mock.ts | 32 + .../src/mocks/workflows-orchestration.mock.ts | 2 + scripts/generate-docs.ts | 30 +- 148 files changed, 28338 insertions(+), 1863 deletions(-) create mode 100644 apps/sim/blocks/blocks/airtable.test.ts create mode 100644 apps/sim/lib/webhooks/delivery-predicate.test.ts create mode 100644 apps/sim/lib/webhooks/delivery-predicate.ts create mode 100644 apps/sim/lib/webhooks/path-claims.test.ts create mode 100644 apps/sim/lib/webhooks/path-claims.ts create mode 100644 apps/sim/lib/webhooks/providers/grain.test.ts create mode 100644 apps/sim/lib/webhooks/registration-identity.test.ts create mode 100644 apps/sim/lib/webhooks/registration-identity.ts create mode 100644 apps/sim/lib/webhooks/registration-reconciliation.test.ts create mode 100644 apps/sim/lib/webhooks/registration-reconciliation.ts create mode 100644 apps/sim/lib/webhooks/registration-service.test.ts create mode 100644 apps/sim/lib/webhooks/registration-service.ts create mode 100644 apps/sim/lib/webhooks/registration-store.test.ts create mode 100644 apps/sim/lib/webhooks/registration-store.ts create mode 100644 apps/sim/lib/webhooks/utils.server.test.ts create mode 100644 apps/sim/lib/workflows/conditions.test.ts create mode 100644 apps/sim/lib/workflows/conditions.ts create mode 100644 apps/sim/lib/workflows/deployment-lifecycle.test.ts create mode 100644 apps/sim/lib/workflows/deployment-lifecycle.ts create mode 100644 apps/sim/lib/workflows/deployment-outbox.test.ts create mode 100644 apps/sim/lib/workflows/executor/execution-state.test.ts create mode 100644 apps/sim/lib/workflows/orchestration/types.test.ts create mode 100644 apps/sim/lib/workflows/persistence/deployment-operations.test.ts create mode 100644 apps/sim/lib/workflows/persistence/deployment-operations.ts create mode 100644 apps/sim/tools/airtable/airtable.test.ts create mode 100644 apps/sim/tools/grain/create_hook_v2.ts create mode 100644 apps/sim/tools/grain/delete_hook_v2.ts create mode 100644 apps/sim/tools/grain/list_hooks_v2.ts create mode 100644 apps/sim/triggers/grain/events_v2.ts create mode 100644 packages/db/migrations/0261_tranquil_donald_blake.sql create mode 100644 packages/db/migrations/meta/0261_snapshot.json delete mode 100644 packages/db/workspace-storage-accounting-migration.test.ts diff --git a/apps/docs/components/ui/icon-mapping.ts b/apps/docs/components/ui/icon-mapping.ts index a091d2c4827..a8e939b23c1 100644 --- a/apps/docs/components/ui/icon-mapping.ts +++ b/apps/docs/components/ui/icon-mapping.ts @@ -350,6 +350,7 @@ export const blockTypeToIconMap: Record = { google_vault: GoogleVaultIcon, grafana: GrafanaIcon, grain: GrainIcon, + grain_v2: GrainIcon, granola: GranolaIcon, greenhouse: GreenhouseIcon, greptile: GreptileIcon, diff --git a/apps/docs/content/docs/en/integrations/airtable.mdx b/apps/docs/content/docs/en/integrations/airtable.mdx index 63314699c05..ee76053d391 100644 --- a/apps/docs/content/docs/en/integrations/airtable.mdx +++ b/apps/docs/content/docs/en/integrations/airtable.mdx @@ -142,6 +142,7 @@ Write new records to an Airtable table | `baseId` | string | Yes | Airtable base ID \(starts with "app", e.g., "appXXXXXXXXXXXXXX"\) | | `tableId` | string | Yes | Table ID \(starts with "tbl"\) or table name | | `records` | json | Yes | Array of records to create, each with a `fields` object | +| `typecast` | boolean | No | When true, Airtable automatically converts string values to the field type | #### Output @@ -166,6 +167,7 @@ Update an existing record in an Airtable table by ID | `tableId` | string | Yes | Table ID \(starts with "tbl"\) or table name | | `recordId` | string | Yes | Record ID to update \(starts with "rec", e.g., "recXXXXXXXXXXXXXX"\) | | `fields` | json | Yes | An object containing the field names and their new values | +| `typecast` | boolean | No | When true, Airtable automatically converts string values to the field type | #### Output @@ -190,6 +192,7 @@ Update multiple existing records in an Airtable table | `baseId` | string | Yes | Airtable base ID \(starts with "app", e.g., "appXXXXXXXXXXXXXX"\) | | `tableId` | string | Yes | Table ID \(starts with "tbl"\) or table name | | `records` | json | Yes | Array of records to update, each with an `id` and a `fields` object | +| `typecast` | boolean | No | When true, Airtable automatically converts string values to the field type | #### Output diff --git a/apps/docs/content/docs/en/integrations/grain.mdx b/apps/docs/content/docs/en/integrations/grain.mdx index f027f2c9575..c54010f70c1 100644 --- a/apps/docs/content/docs/en/integrations/grain.mdx +++ b/apps/docs/content/docs/en/integrations/grain.mdx @@ -6,7 +6,7 @@ description: Access meeting recordings, transcripts, and AI summaries import { BlockInfoCard } from "@/components/ui/block-info-card" @@ -58,6 +58,7 @@ List recordings from Grain with optional filters and pagination | `includeHighlights` | boolean | No | Include highlights/clips in response | | `includeParticipants` | boolean | No | Include participant list in response | | `includeAiSummary` | boolean | No | Include AI-generated summary | +| `includeAiActionItems` | boolean | No | Include AI-detected action items | #### Output @@ -91,6 +92,7 @@ Get details of a single recording by ID | `includeHighlights` | boolean | No | Include highlights/clips | | `includeParticipants` | boolean | No | Include participant list | | `includeAiSummary` | boolean | No | Include AI summary | +| `includeAiActionItems` | boolean | No | Include AI-detected action items | | `includeCalendarEvent` | boolean | No | Include calendar event data | | `includeHubspot` | boolean | No | Include HubSpot associations | @@ -113,6 +115,7 @@ Get details of a single recording by ID | `highlights` | array | Highlights \(if included\) | | `participants` | array | Participants \(if included\) | | `ai_summary` | object | AI summary text \(if included\) | +| `ai_action_items` | array | AI-detected action items with status, text, and assignee \(if included\) | | `calendar_event` | object | Calendar event data \(if included\) | | `hubspot` | object | HubSpot associations \(if included\) | @@ -138,26 +141,6 @@ Get the full transcript of a recording | ↳ `end` | number | End timestamp in ms | | ↳ `text` | string | Transcript text | -### `grain_list_views` - -List available Grain views for webhook subscriptions - -#### Input - -| Parameter | Type | Required | Description | -| --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Grain API key \(Personal Access Token\) | -| `typeFilter` | string | No | Optional view type filter: recordings, highlights, or stories | - -#### Output - -| Parameter | Type | Description | -| --------- | ---- | ----------- | -| `views` | array | Array of Grain views | -| ↳ `id` | string | View UUID | -| ↳ `name` | string | View name | -| ↳ `type` | string | View type: recordings, highlights, or stories | - ### `grain_list_teams` List all teams in the workspace @@ -197,16 +180,16 @@ List all meeting types in the workspace ### `grain_create_hook` -Create a webhook to receive recording events +Create a webhook for a specific Grain event type (v2 API) #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Grain API key \(Personal Access Token\) | -| `hookUrl` | string | Yes | Webhook endpoint URL \(e.g., "https://example.com/webhooks/grain"\) | -| `viewId` | string | Yes | Grain view ID from GET /_/public-api/views | -| `actions` | array | No | Optional list of actions to subscribe to: added, updated, removed | +| `apiKey` | string | Yes | Grain API key \(Personal or Workspace Access Token\) | +| `hookUrl` | string | Yes | Webhook endpoint URL. Grain performs a reachability test on creation — the endpoint must respond 2xx. | +| `hookType` | string | Yes | Event type the hook subscribes to. One of: recording_added, recording_updated, recording_deleted, highlight_added, highlight_updated, highlight_deleted, story_added, story_updated, story_deleted, upload_status | +| `include` | json | No | Optional include object controlling payload richness. For recording hooks: \{"participants": true, "highlights": true, "ai_summary": true\}. For highlight hooks: \{"transcript": true, "speakers": true\}. | #### Output @@ -215,19 +198,21 @@ Create a webhook to receive recording events | `id` | string | Hook UUID | | `enabled` | boolean | Whether hook is active | | `hook_url` | string | The webhook URL | -| `view_id` | string | Grain view ID for the webhook | -| `actions` | array | Configured actions for the webhook | +| `hook_type` | string | Event type the hook subscribes to | +| `include` | json | Include object the hook was created with | | `inserted_at` | string | ISO8601 creation timestamp | ### `grain_list_hooks` -List all webhooks for the account +List webhooks for the account (v2 API) #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Grain API key \(Personal Access Token\) | +| `apiKey` | string | Yes | Grain API key \(Personal or Workspace Access Token\) | +| `hookType` | string | No | Only return hooks with this event type. One of: recording_added, recording_updated, recording_deleted, highlight_added, highlight_updated, highlight_deleted, story_added, story_updated, story_deleted, upload_status | +| `state` | string | No | Only return hooks that are "enabled" or "disabled" | #### Output @@ -237,19 +222,19 @@ List all webhooks for the account | ↳ `id` | string | Hook UUID | | ↳ `enabled` | boolean | Whether hook is active | | ↳ `hook_url` | string | Webhook URL | -| ↳ `view_id` | string | Grain view ID | -| ↳ `actions` | array | Configured actions | +| ↳ `hook_type` | string | Event type the hook subscribes to | +| ↳ `include` | object | Include object the hook was created with | | ↳ `inserted_at` | string | Creation timestamp | ### `grain_delete_hook` -Delete a webhook by ID +Delete a webhook by ID (v2 API) #### Input | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | -| `apiKey` | string | Yes | Grain API key \(Personal Access Token\) | +| `apiKey` | string | Yes | Grain API key \(Personal or Workspace Access Token\) | | `hookId` | string | Yes | The hook UUID to delete \(e.g., "a1b2c3d4-e5f6-7890-abcd-ef1234567890"\) | #### Output @@ -266,14 +251,13 @@ A **Trigger** is a block that starts a workflow when an event happens in this se ### Grain All Events -Trigger on all actions (added, updated, removed) in a Grain view +Trigger on every Grain event (recordings, highlights, stories, uploads) #### Configuration | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Required to create the webhook in Grain. | -| `viewId` | string | Yes | The view determines which content type fires events \(recordings, highlights, or stories\). | #### Output @@ -286,16 +270,15 @@ Trigger on all actions (added, updated, removed) in a Grain view --- -### Grain Highlight Created +### Grain Highlight Added -Trigger workflow when a new highlight/clip is created in Grain +Trigger when a new highlight/clip is created in Grain #### Configuration | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Required to create the webhook in Grain. | -| `viewId` | string | Yes | Required by Grain to create the webhook subscription. | #### Output @@ -317,18 +300,38 @@ Trigger workflow when a new highlight/clip is created in Grain | ↳ `created_datetime` | string | ISO8601 creation timestamp | +--- + +### Grain Highlight Deleted + +Trigger when a highlight/clip is deleted in Grain + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Required to create the webhook in Grain. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `type` | string | Event type \(e.g., recording_added\) | +| `user_id` | string | User UUID who triggered the event | +| `data` | object | Event data object \(recording, highlight, etc.\) | + + --- ### Grain Highlight Updated -Trigger workflow when a highlight/clip is updated in Grain +Trigger when a highlight/clip is updated in Grain #### Configuration | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Required to create the webhook in Grain. | -| `viewId` | string | Yes | Required by Grain to create the webhook subscription. | #### Output @@ -352,38 +355,48 @@ Trigger workflow when a highlight/clip is updated in Grain --- -### Grain Item Added +### Grain Recording Added -Trigger when a new item is added to a Grain view (recording, highlight, or story) +Trigger when a new recording is added in Grain #### Configuration | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Required to create the webhook in Grain. | -| `viewId` | string | Yes | The view determines which content type fires events \(recordings, highlights, or stories\). | #### Output | Parameter | Type | Description | | --------- | ---- | ----------- | -| `type` | string | Event type \(e.g., recording_added\) | +| `type` | string | Event type | | `user_id` | string | User UUID who triggered the event | -| `data` | object | Event data object \(recording, highlight, etc.\) | +| `data` | object | data output from the tool | +| ↳ `id` | string | Recording UUID | +| ↳ `title` | string | Recording title | +| ↳ `start_datetime` | string | ISO8601 start timestamp | +| ↳ `end_datetime` | string | ISO8601 end timestamp | +| ↳ `duration_ms` | number | Duration in milliseconds | +| ↳ `media_type` | string | audio, transcript, or video | +| ↳ `source` | string | Recording source \(zoom, meet, local_capture, etc.\) | +| ↳ `url` | string | URL to view in Grain | +| ↳ `thumbnail_url` | string | Thumbnail URL \(nullable\) | +| ↳ `tags` | array | Array of tag strings | +| ↳ `teams` | array | Array of team objects | +| ↳ `meeting_type` | object | Meeting type info with id, name, scope \(nullable\) | --- -### Grain Item Updated +### Grain Recording Deleted -Trigger when an item is updated in a Grain view (recording, highlight, or story) +Trigger when a recording is deleted in Grain #### Configuration | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Required to create the webhook in Grain. | -| `viewId` | string | Yes | The view determines which content type fires events \(recordings, highlights, or stories\). | #### Output @@ -396,16 +409,15 @@ Trigger when an item is updated in a Grain view (recording, highlight, or story) --- -### Grain Recording Created +### Grain Recording Updated -Trigger workflow when a new recording is added in Grain +Trigger when a recording is updated in Grain #### Configuration | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Required to create the webhook in Grain. | -| `viewId` | string | Yes | Required by Grain to create the webhook subscription. | #### Output @@ -430,16 +442,15 @@ Trigger workflow when a new recording is added in Grain --- -### Grain Recording Updated +### Grain Story Added -Trigger workflow when a recording is updated in Grain +Trigger when a new story is created in Grain #### Configuration | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Required to create the webhook in Grain. | -| `viewId` | string | Yes | Required by Grain to create the webhook subscription. | #### Output @@ -448,32 +459,44 @@ Trigger workflow when a recording is updated in Grain | `type` | string | Event type | | `user_id` | string | User UUID who triggered the event | | `data` | object | data output from the tool | -| ↳ `id` | string | Recording UUID | -| ↳ `title` | string | Recording title | -| ↳ `start_datetime` | string | ISO8601 start timestamp | -| ↳ `end_datetime` | string | ISO8601 end timestamp | -| ↳ `duration_ms` | number | Duration in milliseconds | -| ↳ `media_type` | string | audio, transcript, or video | -| ↳ `source` | string | Recording source \(zoom, meet, local_capture, etc.\) | +| ↳ `id` | string | Story UUID | +| ↳ `title` | string | Story title | | ↳ `url` | string | URL to view in Grain | -| ↳ `thumbnail_url` | string | Thumbnail URL \(nullable\) | -| ↳ `tags` | array | Array of tag strings | -| ↳ `teams` | array | Array of team objects | -| ↳ `meeting_type` | object | Meeting type info with id, name, scope \(nullable\) | +| ↳ `created_datetime` | string | ISO8601 creation timestamp | + + +--- + +### Grain Story Deleted + +Trigger when a story is deleted in Grain + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Required to create the webhook in Grain. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `type` | string | Event type \(e.g., recording_added\) | +| `user_id` | string | User UUID who triggered the event | +| `data` | object | Event data object \(recording, highlight, etc.\) | --- -### Grain Story Created +### Grain Story Updated -Trigger workflow when a new story is created in Grain +Trigger when a story is updated in Grain #### Configuration | Parameter | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `apiKey` | string | Yes | Required to create the webhook in Grain. | -| `viewId` | string | Yes | Required by Grain to create the webhook subscription. | #### Output @@ -487,3 +510,24 @@ Trigger workflow when a new story is created in Grain | ↳ `url` | string | URL to view in Grain | | ↳ `created_datetime` | string | ISO8601 creation timestamp | + +--- + +### Grain Upload Status + +Trigger on progress updates for recordings uploaded to Grain + +#### Configuration + +| Parameter | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `apiKey` | string | Yes | Required to create the webhook in Grain. | + +#### Output + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `type` | string | Event type \(e.g., recording_added\) | +| `user_id` | string | User UUID who triggered the event | +| `data` | object | Event data object \(recording, highlight, etc.\) | + diff --git a/apps/sim/app/api/chat/manage/[id]/route.test.ts b/apps/sim/app/api/chat/manage/[id]/route.test.ts index 38b24348167..399caeb3b66 100644 --- a/apps/sim/app/api/chat/manage/[id]/route.test.ts +++ b/apps/sim/app/api/chat/manage/[id]/route.test.ts @@ -26,9 +26,12 @@ const { mockCheckChatAccess } = vi.hoisted(() => ({ const mockCreateSuccessResponse = workflowsApiUtilsMockFns.mockCreateSuccessResponse const mockCreateErrorResponse = workflowsApiUtilsMockFns.mockCreateErrorResponse +const mockCheckNeedsRedeployment = workflowsApiUtilsMockFns.mockCheckNeedsRedeployment const mockEncryptSecret = encryptionMockFns.mockEncryptSecret const mockPerformFullDeploy = workflowsOrchestrationMockFns.mockPerformFullDeploy const mockPerformChatUndeploy = workflowsOrchestrationMockFns.mockPerformChatUndeploy +const mockGetWorkflowDeploymentSummary = + workflowsOrchestrationMockFns.mockGetWorkflowDeploymentSummary const mockNotifySocketDeploymentChanged = workflowsOrchestrationMockFns.mockNotifySocketDeploymentChanged @@ -72,7 +75,17 @@ describe('Chat Edit API Route', () => { }) mockEncryptSecret.mockResolvedValue({ encrypted: 'encrypted-password' }) - mockPerformFullDeploy.mockResolvedValue({ success: true, version: 1 }) + mockGetWorkflowDeploymentSummary.mockResolvedValue({ + activeDeployment: null, + latestDeploymentAttempt: null, + warnings: [], + }) + mockCheckNeedsRedeployment.mockResolvedValue(false) + mockPerformFullDeploy.mockResolvedValue({ + success: true, + version: 1, + latestDeploymentAttempt: { status: 'active' }, + }) mockNotifySocketDeploymentChanged.mockResolvedValue(undefined) }) @@ -200,6 +213,58 @@ describe('Chat Edit API Route', () => { expect(data.message).toBe('Chat deployment updated successfully') }) + it('rejects the update without admitting a new deploy while an attempt is in flight', async () => { + authMockFns.mockGetSession.mockResolvedValue({ user: { id: 'user-id' } }) + mockCheckChatAccess.mockResolvedValue({ + hasAccess: true, + chat: { id: 'chat-123', identifier: 'test-chat', workflowId: 'workflow-123' }, + workspaceId: 'workspace-123', + }) + mockGetWorkflowDeploymentSummary.mockResolvedValue({ + activeDeployment: null, + latestDeploymentAttempt: { status: 'preparing' }, + warnings: [], + }) + + const req = new NextRequest('http://localhost:3000/api/chat/manage/chat-123', { + method: 'PATCH', + body: JSON.stringify({ title: 'Updated Chat' }), + }) + const response = await PATCH(req, { params: Promise.resolve({ id: 'chat-123' }) }) + + expect(response.status).toBe(409) + expect(mockPerformFullDeploy).not.toHaveBeenCalled() + }) + + it('skips redeploying when the active version already matches the draft', async () => { + authMockFns.mockGetSession.mockResolvedValue({ user: { id: 'user-id' } }) + mockCheckChatAccess.mockResolvedValue({ + hasAccess: true, + chat: { id: 'chat-123', identifier: 'test-chat', workflowId: 'workflow-123' }, + workspaceId: 'workspace-123', + }) + mockGetWorkflowDeploymentSummary.mockResolvedValue({ + activeDeployment: { + deploymentVersionId: 'dv-1', + version: 3, + deployedAt: '2026-07-15T00:00:00.000Z', + }, + latestDeploymentAttempt: { status: 'active' }, + warnings: [], + }) + mockCheckNeedsRedeployment.mockResolvedValue(false) + + const req = new NextRequest('http://localhost:3000/api/chat/manage/chat-123', { + method: 'PATCH', + body: JSON.stringify({ title: 'Updated Chat' }), + }) + const response = await PATCH(req, { params: Promise.resolve({ id: 'chat-123' }) }) + + expect(response.status).toBe(200) + expect(mockPerformFullDeploy).not.toHaveBeenCalled() + expect(dbChainMockFns.update).toHaveBeenCalled() + }) + it('should handle identifier conflicts', async () => { authMockFns.mockGetSession.mockResolvedValue({ user: { id: 'user-id' }, diff --git a/apps/sim/app/api/chat/manage/[id]/route.ts b/apps/sim/app/api/chat/manage/[id]/route.ts index 707051d7e9f..0afb00b152a 100644 --- a/apps/sim/app/api/chat/manage/[id]/route.ts +++ b/apps/sim/app/api/chat/manage/[id]/route.ts @@ -12,9 +12,17 @@ import { isDev } from '@/lib/core/config/env-flags' import { encryptSecret } from '@/lib/core/security/encryption' import { getEmailDomain } from '@/lib/core/utils/urls' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { performChatUndeploy, performFullDeploy } from '@/lib/workflows/orchestration' +import { + getWorkflowDeploymentSummary, + performChatUndeploy, + performFullDeploy, +} from '@/lib/workflows/orchestration' import { checkChatAccess } from '@/app/api/chat/utils' -import { createErrorResponse, createSuccessResponse } from '@/app/api/workflows/utils' +import { + checkNeedsRedeployment, + createErrorResponse, + createSuccessResponse, +} from '@/app/api/workflows/utils' export const dynamic = 'force-dynamic' export const maxDuration = 120 @@ -136,26 +144,59 @@ export const PATCH = withRouteHandler( logger.info('Keeping existing password') } - // Redeploy the workflow to ensure latest version is active - const deployResult = await performFullDeploy({ - workflowId: existingChat[0].workflowId, - userId: session.user.id, - request, - }) + /** + * A settings update only redeploys when the draft actually drifted from + * the active version, and never while another attempt is in flight — + * otherwise each blocked retry would admit a fresh deployment version + * on top of the pending one. + */ + const deploymentSummary = await getWorkflowDeploymentSummary(existingChat[0].workflowId) + const attemptStatus = deploymentSummary.latestDeploymentAttempt?.status + if (attemptStatus === 'preparing' || attemptStatus === 'activating') { + return createErrorResponse( + 'A workflow deployment is still preparing. Retry the chat update after it becomes active.', + 409 + ) + } - if (!deployResult.success) { - logger.warn(`Failed to redeploy workflow for chat update: ${deployResult.error}`) - const status = - deployResult.errorCode === 'validation' - ? 400 - : deployResult.errorCode === 'not_found' - ? 404 - : 500 - return createErrorResponse(deployResult.error || 'Failed to redeploy workflow', status) + const needsRedeploy = + !deploymentSummary.activeDeployment || + (await checkNeedsRedeployment(existingChat[0].workflowId)) + + if (needsRedeploy) { + const deployResult = await performFullDeploy({ + workflowId: existingChat[0].workflowId, + userId: session.user.id, + }) + + if (!deployResult.success) { + logger.warn(`Failed to redeploy workflow for chat update: ${deployResult.error}`) + const status = + deployResult.errorCode === 'validation' + ? 400 + : deployResult.errorCode === 'not_found' + ? 404 + : 500 + return createErrorResponse(deployResult.error || 'Failed to redeploy workflow', status) + } + /** + * Deploys settle asynchronously: `success` only admits the attempt. + * The chat record must not advance until cutover finished, otherwise + * a later preparation failure strands the chat on the previous + * version with no error. A blocked retry lands in the in-flight gate + * above without admitting another version. Mirrors performChatDeploy. + */ + if (deployResult.latestDeploymentAttempt?.status !== 'active') { + return createErrorResponse( + deployResult.warnings?.[0] ?? + 'Workflow deployment is still preparing. Retry the chat update after it becomes active.', + 409 + ) + } + logger.info( + `Redeployed workflow ${existingChat[0].workflowId} for chat update (v${deployResult.version})` + ) } - logger.info( - `Redeployed workflow ${existingChat[0].workflowId} for chat update (v${deployResult.version})` - ) const updateData: Record = { updatedAt: new Date(), diff --git a/apps/sim/app/api/cron/renew-subscriptions/route.ts b/apps/sim/app/api/cron/renew-subscriptions/route.ts index 4728f0fd666..97f00a26ff2 100644 --- a/apps/sim/app/api/cron/renew-subscriptions/route.ts +++ b/apps/sim/app/api/cron/renew-subscriptions/route.ts @@ -8,6 +8,7 @@ import { verifyCronAuth } from '@/lib/auth/internal' import { acquireLock, releaseLock } from '@/lib/core/config/redis' import { runDetached } from '@/lib/core/utils/background' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { deliverableWebhookPredicate } from '@/lib/webhooks/delivery-predicate' import { getCredentialOwner, getNotificationUrl } from '@/lib/webhooks/provider-subscription-utils' import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' @@ -96,7 +97,7 @@ async function renewExpiringSubscriptions(): Promise<{ .from(webhookTable) .where( and( - eq(webhookTable.isActive, true), + deliverableWebhookPredicate(webhookTable, 'active_only'), or( eq(webhookTable.provider, 'microsoft-teams'), eq(webhookTable.provider, 'microsoftteams') diff --git a/apps/sim/app/api/mcp/serve/[serverId]/route.test.ts b/apps/sim/app/api/mcp/serve/[serverId]/route.test.ts index cd8ec829cae..a67cb62ba37 100644 --- a/apps/sim/app/api/mcp/serve/[serverId]/route.test.ts +++ b/apps/sim/app/api/mcp/serve/[serverId]/route.test.ts @@ -251,7 +251,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) hybridAuthMockFns.mockCheckHybridAuth.mockResolvedValueOnce({ success: true, @@ -307,7 +307,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) hybridAuthMockFns.mockCheckHybridAuth.mockResolvedValueOnce({ success: true, @@ -361,7 +361,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) mockGenerateInternalToken.mockResolvedValueOnce('internal-token-owner-1') fetchMock.mockResolvedValueOnce( new Response(JSON.stringify({ output: { ok: true } }), { @@ -411,7 +411,9 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: workflowWorkspaceId }]) + .mockResolvedValueOnce([ + { workspaceId: workflowWorkspaceId, deploymentVersionId: 'deployment-1' }, + ]) const req = new NextRequest('http://localhost:3000/api/mcp/serve/server-1', { method: 'POST', @@ -442,7 +444,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) mockResolveBillingAttribution.mockResolvedValueOnce( createBillingAttribution('different-actor', 'ws-1') ) @@ -565,7 +567,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) fetchMock.mockResolvedValueOnce( new Response( new ReadableStream({ @@ -609,7 +611,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) fetchMock.mockResolvedValueOnce( new Response( new ReadableStream({ @@ -656,7 +658,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) fetchMock.mockResolvedValueOnce( new Response( JSON.stringify({ @@ -689,6 +691,9 @@ describe('MCP Serve Route', () => { const fetchOptions = fetchMock.mock.calls[0][1] as RequestInit const headers = fetchOptions.headers as Record expect(headers['X-Sim-MCP-Tool-Call']).toBe('true') + expect(JSON.parse(fetchOptions.body as string)).toMatchObject({ + deploymentVersionId: 'deployment-1', + }) }) it('preserves downstream attributed usage admission rejections', async () => { @@ -703,7 +708,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) fetchMock.mockResolvedValueOnce( new Response( JSON.stringify({ @@ -747,7 +752,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) fetchMock.mockResolvedValueOnce(new Response('gateway timeout', { status: 408 })) const req = new NextRequest('http://localhost:3000/api/mcp/serve/server-1', { @@ -780,7 +785,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) fetchMock.mockResolvedValueOnce( new Response(JSON.stringify({ success: true, output: false }), { status: 200, @@ -817,7 +822,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) fetchMock.mockResolvedValueOnce( new Response(JSON.stringify({ success: true }), { status: 200, @@ -854,7 +859,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) hybridAuthMockFns.mockCheckHybridAuth.mockResolvedValueOnce({ success: true, userId: 'user-1', @@ -934,7 +939,7 @@ describe('MCP Serve Route', () => { }, ]) .mockResolvedValueOnce([{ toolName: 'tool_a', workflowId: 'wf-1' }]) - .mockResolvedValueOnce([{ isDeployed: true, workspaceId: 'ws-1' }]) + .mockResolvedValueOnce([{ workspaceId: 'ws-1', deploymentVersionId: 'deployment-1' }]) fetchMock.mockImplementationOnce((_url, init: RequestInit) => { const signal = init.signal as AbortSignal return new Promise((_resolve, reject) => { diff --git a/apps/sim/app/api/mcp/serve/[serverId]/route.ts b/apps/sim/app/api/mcp/serve/[serverId]/route.ts index a5650de1042..e342f35e343 100644 --- a/apps/sim/app/api/mcp/serve/[serverId]/route.ts +++ b/apps/sim/app/api/mcp/serve/[serverId]/route.ts @@ -18,7 +18,13 @@ import { type Tool, } from '@modelcontextprotocol/sdk/types.js' import { db } from '@sim/db' -import { workflow, workflowMcpServer, workflowMcpTool, workspace } from '@sim/db/schema' +import { + workflow, + workflowDeploymentVersion, + workflowMcpServer, + workflowMcpTool, + workspace, +} from '@sim/db/schema' import { createLogger } from '@sim/logger' import { and, asc, eq, gt, isNull, sql } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' @@ -751,14 +757,30 @@ async function handleToolsCall( } const [wf] = await db - .select({ isDeployed: workflow.isDeployed, workspaceId: workflow.workspaceId }) + .select({ + workspaceId: workflow.workspaceId, + deploymentVersionId: workflowDeploymentVersion.id, + }) .from(workflow) + .leftJoin( + workflowDeploymentVersion, + and( + eq(workflowDeploymentVersion.workflowId, workflow.id), + eq(workflowDeploymentVersion.isActive, true) + ) + ) .where(and(eq(workflow.id, tool.workflowId), isNull(workflow.archivedAt))) .limit(1) const abortedAfterWorkflowLookup = callerAbortedJsonRpcResponse(id, abortSignal) if (abortedAfterWorkflowLookup) return abortedAfterWorkflowLookup - if (!wf?.isDeployed) { + /** + * Deployed means an active version snapshot exists — the legacy + * `workflow.isDeployed` flag is not consulted because when the two + * disagree the workflow cannot serve traffic anyway. Same definition as + * the deploy status GET route. + */ + if (!wf?.deploymentVersionId) { return NextResponse.json( createError(id, ErrorCode.InternalError, 'Workflow is not deployed'), { @@ -813,6 +835,7 @@ async function handleToolsCall( input: params.arguments || {}, triggerType: 'mcp', includeFileBase64: false, + ...(wf.deploymentVersionId ? { deploymentVersionId: wf.deploymentVersionId } : {}), }) assertKnownSizeWithinLimit( Buffer.byteLength(workflowRequestBody, 'utf-8'), diff --git a/apps/sim/app/api/schedules/execute/route.ts b/apps/sim/app/api/schedules/execute/route.ts index 63a9258211d..13f87274810 100644 --- a/apps/sim/app/api/schedules/execute/route.ts +++ b/apps/sim/app/api/schedules/execute/route.ts @@ -165,6 +165,7 @@ async function claimWorkflowSchedules(queuedAt: Date, limit: number) { lastQueuedAt: workflowSchedule.lastQueuedAt, timezone: workflowSchedule.timezone, deploymentVersionId: workflowSchedule.deploymentVersionId, + deploymentOperationId: workflowSchedule.deploymentOperationId, sourceType: workflowSchedule.sourceType, }) @@ -864,6 +865,7 @@ async function processScheduleItem( workspaceId, billingAttribution, deploymentVersionId: schedule.deploymentVersionId || undefined, + deploymentOperationId: schedule.deploymentOperationId || undefined, cronExpression: schedule.cronExpression || undefined, timezone: schedule.timezone || undefined, lastRanAt: schedule.lastRanAt?.toISOString(), diff --git a/apps/sim/app/api/tools/deployments/deploy/route.ts b/apps/sim/app/api/tools/deployments/deploy/route.ts index bf0ea376a84..5f35c91da86 100644 --- a/apps/sim/app/api/tools/deployments/deploy/route.ts +++ b/apps/sim/app/api/tools/deployments/deploy/route.ts @@ -51,11 +51,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const result = await performFullDeploy({ workflowId, userId: auth.userId, - workflowName: access.workflow.name || undefined, versionName: name, versionDescription: description ?? undefined, requestId, - request, }) if (!result.success) { @@ -69,9 +67,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => { success: true, output: { workflowId, - isDeployed: true, + isDeployed: Boolean(result.activeDeployment), deployedAt: result.deployedAt?.toISOString() ?? null, version: result.version, + activeDeployment: result.activeDeployment, + latestDeploymentAttempt: result.latestDeploymentAttempt, warnings: result.warnings ?? [], }, }) diff --git a/apps/sim/app/api/tools/deployments/promote/route.ts b/apps/sim/app/api/tools/deployments/promote/route.ts index 406b08b0b9d..a126c3dbd57 100644 --- a/apps/sim/app/api/tools/deployments/promote/route.ts +++ b/apps/sim/app/api/tools/deployments/promote/route.ts @@ -53,9 +53,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { workflowId, version, userId: auth.userId, - workflow: access.workflow as Record, requestId, - request, }) if (!result.success) { @@ -69,9 +67,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => { success: true, output: { workflowId, - isDeployed: true, + isDeployed: Boolean(result.activeDeployment), deployedAt: result.deployedAt?.toISOString() ?? null, version, + activeDeployment: result.activeDeployment, + latestDeploymentAttempt: result.latestDeploymentAttempt, warnings: result.warnings ?? [], }, }) diff --git a/apps/sim/app/api/tools/deployments/routes.test.ts b/apps/sim/app/api/tools/deployments/routes.test.ts index 63698c3a8b2..f0313a779b3 100644 --- a/apps/sim/app/api/tools/deployments/routes.test.ts +++ b/apps/sim/app/api/tools/deployments/routes.test.ts @@ -90,6 +90,11 @@ describe('POST /api/tools/deployments/deploy', () => { success: true, deployedAt: new Date('2026-06-12T00:00:00Z'), version: 4, + activeDeployment: { + deploymentVersionId: 'dv-4', + version: 4, + deployedAt: '2026-06-12T00:00:00.000Z', + }, }) }) @@ -157,6 +162,11 @@ describe('POST /api/tools/deployments/deploy', () => { isDeployed: true, deployedAt: '2026-06-12T00:00:00.000Z', version: 4, + activeDeployment: { + deploymentVersionId: 'dv-4', + version: 4, + deployedAt: '2026-06-12T00:00:00.000Z', + }, warnings: [], }, }) @@ -236,6 +246,11 @@ describe('POST /api/tools/deployments/promote', () => { mockPerformActivateVersion.mockResolvedValue({ success: true, deployedAt: new Date('2026-06-12T00:00:00Z'), + activeDeployment: { + deploymentVersionId: 'dv-3', + version: 3, + deployedAt: '2026-06-12T00:00:00.000Z', + }, }) }) @@ -255,6 +270,11 @@ describe('POST /api/tools/deployments/promote', () => { isDeployed: true, deployedAt: '2026-06-12T00:00:00.000Z', version: 3, + activeDeployment: { + deploymentVersionId: 'dv-3', + version: 3, + deployedAt: '2026-06-12T00:00:00.000Z', + }, warnings: [], }) }) diff --git a/apps/sim/app/api/v1/admin/types.ts b/apps/sim/app/api/v1/admin/types.ts index 04f61a8a964..4754de31054 100644 --- a/apps/sim/app/api/v1/admin/types.ts +++ b/apps/sim/app/api/v1/admin/types.ts @@ -629,18 +629,6 @@ export interface AdminDeploymentVersion { deployedByName: string | null } -export interface AdminDeployResult { - isDeployed: boolean - version: number - deployedAt: string - warnings?: string[] -} - -export interface AdminUndeployResult { - isDeployed: boolean - warnings?: string[] -} - // ============================================================================= // Audit Log Types // ============================================================================= diff --git a/apps/sim/app/api/v1/admin/workflows/[id]/deploy/route.ts b/apps/sim/app/api/v1/admin/workflows/[id]/deploy/route.ts index 62fe6405bfd..5e844a00188 100644 --- a/apps/sim/app/api/v1/admin/workflows/[id]/deploy/route.ts +++ b/apps/sim/app/api/v1/admin/workflows/[id]/deploy/route.ts @@ -1,6 +1,8 @@ import { createLogger } from '@sim/logger' import { getActiveWorkflowRecord } from '@sim/platform-authz/workflow' import { + type AdminV1DeployResult, + type AdminV1UndeployResult, adminV1DeployWorkflowContract, adminV1UndeployWorkflowContract, } from '@/lib/api/contracts/v1/admin' @@ -15,7 +17,6 @@ import { notFoundResponse, singleResponse, } from '@/app/api/v1/admin/responses' -import type { AdminDeployResult, AdminUndeployResult } from '@/app/api/v1/admin/types' const logger = createLogger('AdminWorkflowDeployAPI') export const maxDuration = 120 @@ -51,9 +52,7 @@ export const POST = withRouteHandler( const result = await performFullDeploy({ workflowId, userId: workflowRecord.userId, - workflowName: workflowRecord.name, requestId, - request, actorId: 'admin-api', }) @@ -63,13 +62,19 @@ export const POST = withRouteHandler( return internalErrorResponse(result.error || 'Failed to deploy workflow') } - logger.info(`[${requestId}] Admin API: Deployed workflow ${workflowId} as v${result.version}`) + const isDeployed = Boolean(result.activeDeployment) + const attemptActivated = result.latestDeploymentAttempt?.status === 'active' + logger.info( + `[${requestId}] Admin API: Deployment ${attemptActivated ? 'activated' : 'accepted'} for workflow ${workflowId}` + ) - const response: AdminDeployResult = { - isDeployed: true, - version: result.version!, - deployedAt: result.deployedAt!.toISOString(), + const response: AdminV1DeployResult = { + isDeployed, + version: result.version ?? null, + deployedAt: result.deployedAt?.toISOString() ?? null, warnings: result.warnings, + activeDeployment: result.activeDeployment, + latestDeploymentAttempt: result.latestDeploymentAttempt, } return singleResponse(response) @@ -108,7 +113,7 @@ export const DELETE = withRouteHandler( logger.info(`Admin API: Undeployed workflow ${workflowId}`) - const response: AdminUndeployResult = { + const response: AdminV1UndeployResult = { isDeployed: false, warnings: result.warnings, } diff --git a/apps/sim/app/api/v1/admin/workflows/[id]/versions/[versionId]/activate/route.ts b/apps/sim/app/api/v1/admin/workflows/[id]/versions/[versionId]/activate/route.ts index 360a817c5b6..4f5922c3745 100644 --- a/apps/sim/app/api/v1/admin/workflows/[id]/versions/[versionId]/activate/route.ts +++ b/apps/sim/app/api/v1/admin/workflows/[id]/versions/[versionId]/activate/route.ts @@ -40,9 +40,7 @@ export const POST = withRouteHandler( workflowId, version: versionNum, userId: workflowRecord.userId, - workflow: workflowRecord as Record, requestId, - request, actorId: 'admin-api', }) @@ -53,14 +51,16 @@ export const POST = withRouteHandler( } logger.info( - `[${requestId}] Admin API: Activated version ${versionNum} for workflow ${workflowId}` + `[${requestId}] Admin API: ${result.latestDeploymentAttempt?.status === 'active' ? 'Activated' : 'Accepted activation for'} version ${versionNum} on workflow ${workflowId}` ) return singleResponse({ success: true, version: versionNum, - deployedAt: result.deployedAt!.toISOString(), + deployedAt: result.deployedAt?.toISOString() ?? null, warnings: result.warnings, + activeDeployment: result.activeDeployment, + latestDeploymentAttempt: result.latestDeploymentAttempt, }) } catch (error) { logger.error( diff --git a/apps/sim/app/api/v1/workflows/[id]/deploy/route.test.ts b/apps/sim/app/api/v1/workflows/[id]/deploy/route.test.ts index dd78899e2ac..42e977bdfe2 100644 --- a/apps/sim/app/api/v1/workflows/[id]/deploy/route.test.ts +++ b/apps/sim/app/api/v1/workflows/[id]/deploy/route.test.ts @@ -82,6 +82,22 @@ describe('POST /api/v1/workflows/[id]/deploy', () => { deployedAt: new Date('2026-06-12T00:00:00Z'), version: 4, warnings: undefined, + activeDeployment: { + deploymentVersionId: 'dv-4', + version: 4, + deployedAt: '2026-06-12T00:00:00.000Z', + }, + latestDeploymentAttempt: { + id: 'op-1', + deploymentVersionId: 'dv-4', + version: 4, + action: 'deploy', + status: 'active', + readiness: { webhooks: 'ready', schedules: 'ready', mcp: 'ready' }, + requestedAt: '2026-06-12T00:00:00.000Z', + activatedAt: '2026-06-12T00:00:00.000Z', + error: null, + }, }) }) @@ -163,6 +179,8 @@ describe('POST /api/v1/workflows/[id]/deploy', () => { deployedAt: '2026-06-12T00:00:00.000Z', version: 4, warnings: [], + activeDeployment: expect.objectContaining({ deploymentVersionId: 'dv-4', version: 4 }), + latestDeploymentAttempt: expect.objectContaining({ id: 'op-1', status: 'active' }), }) }) @@ -179,12 +197,11 @@ describe('POST /api/v1/workflows/[id]/deploy', () => { versionDescription: 'Fixes the agent prompt', }) ) - expect(mockCaptureServerEvent).toHaveBeenCalledWith( - 'user-1', - 'workflow_deployed', - expect.objectContaining({ workflow_id: WORKFLOW_ID }), - expect.anything() - ) + /** + * The workflow_deployed analytics event is emitted by the activation + * side effects in the deployment outbox, not by this route. + */ + expect(mockCaptureServerEvent).not.toHaveBeenCalled() }) it('maps validation failures from the orchestration to 400', async () => { diff --git a/apps/sim/app/api/v1/workflows/[id]/deploy/route.ts b/apps/sim/app/api/v1/workflows/[id]/deploy/route.ts index 008a45a9820..822e00ab2ca 100644 --- a/apps/sim/app/api/v1/workflows/[id]/deploy/route.ts +++ b/apps/sim/app/api/v1/workflows/[id]/deploy/route.ts @@ -60,11 +60,9 @@ export const POST = withRouteHandler( const result = await performFullDeploy({ workflowId: id, userId, - workflowName: workflow.name || undefined, versionName: body.data.name, versionDescription: body.data.description ?? undefined, requestId, - request, }) if (!result.success) { @@ -74,25 +72,19 @@ export const POST = withRouteHandler( ) } - captureServerEvent( - userId, - 'workflow_deployed', - { workflow_id: id, workspace_id: workspaceId }, - { - groups: { workspace: workspaceId }, - setOnce: { first_workflow_deployed_at: new Date().toISOString() }, - } - ) + const isDeployed = Boolean(result.activeDeployment) const limits = await getUserLimits(userId) const apiResponse = createApiResponse( { data: { id, - isDeployed: true, + isDeployed, deployedAt: result.deployedAt?.toISOString() ?? null, version: result.version, warnings: result.warnings ?? [], + activeDeployment: result.activeDeployment ?? null, + latestDeploymentAttempt: result.latestDeploymentAttempt ?? null, }, }, limits, diff --git a/apps/sim/app/api/v1/workflows/[id]/rollback/route.test.ts b/apps/sim/app/api/v1/workflows/[id]/rollback/route.test.ts index cdebe9e35af..8ee6ec2940b 100644 --- a/apps/sim/app/api/v1/workflows/[id]/rollback/route.test.ts +++ b/apps/sim/app/api/v1/workflows/[id]/rollback/route.test.ts @@ -81,6 +81,22 @@ describe('POST /api/v1/workflows/[id]/rollback', () => { mockPerformActivateVersion.mockResolvedValue({ success: true, deployedAt: new Date('2026-06-12T00:00:00Z'), + activeDeployment: { + deploymentVersionId: 'dv-4', + version: 4, + deployedAt: '2026-06-12T00:00:00.000Z', + }, + latestDeploymentAttempt: { + id: 'op-1', + deploymentVersionId: 'dv-4', + version: 4, + action: 'activate', + status: 'active', + readiness: { webhooks: 'ready', schedules: 'ready', mcp: 'ready' }, + requestedAt: '2026-06-12T00:00:00.000Z', + activatedAt: '2026-06-12T00:00:00.000Z', + error: null, + }, }) }) @@ -128,6 +144,8 @@ describe('POST /api/v1/workflows/[id]/rollback', () => { deployedAt: '2026-06-12T00:00:00.000Z', version: 4, warnings: [], + activeDeployment: expect.objectContaining({ deploymentVersionId: 'dv-4', version: 4 }), + latestDeploymentAttempt: expect.objectContaining({ id: 'op-1', status: 'active' }), }) }) diff --git a/apps/sim/app/api/v1/workflows/[id]/rollback/route.ts b/apps/sim/app/api/v1/workflows/[id]/rollback/route.ts index 534e7fd89de..63ca166cdf1 100644 --- a/apps/sim/app/api/v1/workflows/[id]/rollback/route.ts +++ b/apps/sim/app/api/v1/workflows/[id]/rollback/route.ts @@ -9,7 +9,6 @@ import { import { parseOptionalJsonBody, parseRequest, validationErrorResponse } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { captureServerEvent } from '@/lib/posthog/server' import { performActivateVersion } from '@/lib/workflows/orchestration' import { statusForOrchestrationError } from '@/lib/workflows/orchestration/types' import { findPreviousDeploymentVersion } from '@/lib/workflows/persistence/utils' @@ -81,9 +80,7 @@ export const POST = withRouteHandler( workflowId: id, version: targetVersion, userId, - workflow: workflow as Record, requestId, - request, }) if (!result.success) { @@ -93,22 +90,17 @@ export const POST = withRouteHandler( ) } - captureServerEvent( - userId, - 'deployment_version_activated', - { workflow_id: id, workspace_id: workspaceId, version: targetVersion }, - { groups: { workspace: workspaceId } } - ) - const limits = await getUserLimits(userId) const apiResponse = createApiResponse( { data: { id, - isDeployed: true, + isDeployed: Boolean(result.activeDeployment), deployedAt: result.deployedAt?.toISOString() ?? null, version: targetVersion, warnings: result.warnings ?? [], + activeDeployment: result.activeDeployment ?? null, + latestDeploymentAttempt: result.latestDeploymentAttempt ?? null, }, }, limits, diff --git a/apps/sim/app/api/workflows/[id]/deploy/route.ts b/apps/sim/app/api/workflows/[id]/deploy/route.ts index 440636186e1..4c7e1027161 100644 --- a/apps/sim/app/api/workflows/[id]/deploy/route.ts +++ b/apps/sim/app/api/workflows/[id]/deploy/route.ts @@ -10,7 +10,11 @@ import { parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { captureServerEvent } from '@/lib/posthog/server' -import { performFullDeploy, performFullUndeploy } from '@/lib/workflows/orchestration' +import { + getWorkflowDeploymentSummary, + performFullDeploy, + performFullUndeploy, +} from '@/lib/workflows/orchestration' import { statusForOrchestrationError } from '@/lib/workflows/orchestration/types' import { validateWorkflowPermissions } from '@/lib/workflows/utils' import { @@ -44,7 +48,17 @@ export const GET = withRouteHandler( return createErrorResponse(error.message, error.status) } - if (!workflowData.isDeployed) { + /** + * A workflow is deployed only when an active version snapshot exists — + * the same definition POST and the v1 routes use. The legacy + * `workflow.isDeployed` flag is deliberately not consulted: when it + * disagrees with the version table the workflow cannot actually serve + * traffic, so reporting it as live would be untruthful. + */ + const deploymentSummary = await getWorkflowDeploymentSummary(id) + const isDeployed = deploymentSummary.activeDeployment !== null + + if (!isDeployed) { logger.info(`[${requestId}] Workflow is not deployed: ${id}`) return createSuccessResponse({ isDeployed: false, @@ -52,10 +66,17 @@ export const GET = withRouteHandler( apiKey: null, needsRedeployment: false, isPublicApi: workflowData.isPublicApi ?? false, + activeDeployment: deploymentSummary.activeDeployment, + latestDeploymentAttempt: deploymentSummary.latestDeploymentAttempt, + warnings: deploymentSummary.warnings, }) } - const needsRedeployment = await checkNeedsRedeployment(id) + const attemptStatus = deploymentSummary.latestDeploymentAttempt?.status + const needsRedeployment = + attemptStatus === 'preparing' || attemptStatus === 'activating' + ? false + : await checkNeedsRedeployment(id) logger.info(`[${requestId}] Successfully retrieved deployment info: ${id}`) @@ -65,10 +86,13 @@ export const GET = withRouteHandler( return createSuccessResponse({ apiKey: responseApiKeyInfo, - isDeployed: workflowData.isDeployed, - deployedAt: workflowData.deployedAt, + isDeployed, + deployedAt: deploymentSummary.activeDeployment?.deployedAt ?? workflowData.deployedAt, needsRedeployment, isPublicApi: workflowData.isPublicApi ?? false, + activeDeployment: deploymentSummary.activeDeployment, + latestDeploymentAttempt: deploymentSummary.latestDeploymentAttempt, + warnings: deploymentSummary.warnings, }) } catch (error: any) { logger.error(`[${requestId}] Error fetching deployment info: ${id}`, error) @@ -102,9 +126,7 @@ export const POST = withRouteHandler( const result = await performFullDeploy({ workflowId: id, userId: actorUserId, - workflowName: workflowData!.name || undefined, requestId, - request, }) if (!result.success) { @@ -114,16 +136,10 @@ export const POST = withRouteHandler( ) } - logger.info(`[${requestId}] Workflow deployed successfully: ${id}`) - - captureServerEvent( - actorUserId, - 'workflow_deployed', - { workflow_id: id, workspace_id: workflowData!.workspaceId ?? '' }, - { - groups: workflowData!.workspaceId ? { workspace: workflowData!.workspaceId } : undefined, - setOnce: { first_workflow_deployed_at: new Date().toISOString() }, - } + const isDeployed = Boolean(result.activeDeployment) + const attemptActivated = result.latestDeploymentAttempt?.status === 'active' + logger.info( + `[${requestId}] Workflow deployment ${attemptActivated ? 'activated' : 'accepted for preparation'}: ${id}` ) const responseApiKeyInfo = workflowData!.workspaceId @@ -132,9 +148,11 @@ export const POST = withRouteHandler( return createSuccessResponse({ apiKey: responseApiKeyInfo, - isDeployed: true, + isDeployed, deployedAt: result.deployedAt, warnings: result.warnings, + activeDeployment: result.activeDeployment, + latestDeploymentAttempt: result.latestDeploymentAttempt, }) } catch (error: unknown) { if (error instanceof WorkflowLockedError) { diff --git a/apps/sim/app/api/workflows/[id]/deployments/[version]/route.ts b/apps/sim/app/api/workflows/[id]/deployments/[version]/route.ts index 751171b0b3b..d3c3337e62f 100644 --- a/apps/sim/app/api/workflows/[id]/deployments/[version]/route.ts +++ b/apps/sim/app/api/workflows/[id]/deployments/[version]/route.ts @@ -6,7 +6,6 @@ import { updateDeploymentVersionMetadataContract } from '@/lib/api/contracts/dep import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { captureServerEvent } from '@/lib/posthog/server' import { performActivateVersion } from '@/lib/workflows/orchestration' import { statusForOrchestrationError } from '@/lib/workflows/orchestration/types' import { @@ -73,11 +72,11 @@ export const PATCH = withRouteHandler( // Activation requires admin permission, other updates require write const requiredPermission = isActive ? 'admin' : 'write' - const { - error, - session, - workflow: workflowData, - } = await validateWorkflowPermissions(id, requestId, requiredPermission) + const { error, session } = await validateWorkflowPermissions( + id, + requestId, + requiredPermission + ) if (error) { return createErrorResponse(error.message, error.status) } @@ -98,9 +97,7 @@ export const PATCH = withRouteHandler( workflowId: id, version: versionNum, userId: actorUserId, - workflow: workflowData as Record, requestId, - request, }) if (!activateResult.success) { @@ -145,18 +142,12 @@ export const PATCH = withRouteHandler( } } - const wsId = (workflowData as { workspaceId?: string } | null)?.workspaceId - captureServerEvent( - actorUserId, - 'deployment_version_activated', - { workflow_id: id, workspace_id: wsId ?? '', version: versionNum }, - wsId ? { groups: { workspace: wsId } } : undefined - ) - return createSuccessResponse({ success: true, - deployedAt: activateResult.deployedAt, + deployedAt: activateResult.deployedAt ?? null, warnings: activateResult.warnings, + activeDeployment: activateResult.activeDeployment ?? null, + latestDeploymentAttempt: activateResult.latestDeploymentAttempt ?? null, ...(updatedName !== undefined && { name: updatedName }), ...(updatedDescription !== undefined && { description: updatedDescription }), }) diff --git a/apps/sim/app/api/workflows/[id]/execute/route.ts b/apps/sim/app/api/workflows/[id]/execute/route.ts index cbc3ea579c8..49f50b3ae21 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.ts @@ -81,6 +81,7 @@ import { import { handlePostExecutionPauseState } from '@/lib/workflows/executor/pause-persistence' import { loadDeployedWorkflowState, + loadWorkflowDeploymentVersionState, loadWorkflowFromNormalizedTables, } from '@/lib/workflows/persistence/utils' import { createStreamingResponse } from '@/lib/workflows/streaming/streaming' @@ -665,6 +666,7 @@ async function handleExecutePost( includeFileBase64, base64MaxBytes, workflowStateOverride, + deploymentVersionId: admittedDeploymentVersionId, executionId: rawBodyExecutionId, triggerBlockId: parsedTriggerBlockId, startBlockId, @@ -673,6 +675,12 @@ async function handleExecutePost( parentWorkspaceId, } = validation.data const triggerBlockId = parsedTriggerBlockId ?? startBlockId + if (admittedDeploymentVersionId && !isMcpBridgeRequest) { + return NextResponse.json( + { error: 'deploymentVersionId is reserved for internal MCP execution' }, + { status: 400 } + ) + } const headerExecutionId = headerValidation.data[WORKFLOW_EXECUTION_ID_HEADER] let legacyBodyExecutionId: string | undefined if (!headerExecutionId && rawBodyExecutionId !== undefined) { @@ -803,6 +811,7 @@ async function handleExecutePost( includeFileBase64, base64MaxBytes, workflowStateOverride, + deploymentVersionId: _deploymentVersionId, triggerBlockId: _triggerBlockId, stopAfterBlockId: _stopAfterBlockId, runFromBlock: _runFromBlock, @@ -1074,7 +1083,13 @@ async function handleExecutePost( } const workflowData = shouldUseDraftState ? await loadWorkflowFromNormalizedTables(workflowId) - : await loadDeployedWorkflowState(workflowId, workspaceId) + : admittedDeploymentVersionId + ? await loadWorkflowDeploymentVersionState( + workflowId, + admittedDeploymentVersionId, + workspaceId + ) + : await loadDeployedWorkflowState(workflowId, workspaceId) if (req.signal.aborted) { await releaseExecutionSlot(executionId) diff --git a/apps/sim/app/api/workspaces/[id]/fork/rollback/route.ts b/apps/sim/app/api/workspaces/[id]/fork/rollback/route.ts index 21363f21f1d..68575860876 100644 --- a/apps/sim/app/api/workspaces/[id]/fork/rollback/route.ts +++ b/apps/sim/app/api/workspaces/[id]/fork/rollback/route.ts @@ -63,8 +63,14 @@ export const POST = withRouteHandler( await recordBackgroundWork(db, { workspaceId: id, kind: 'fork_rollback', - status: result.skipped > 0 ? 'completed_with_warnings' : 'completed', - message: `Undid the last sync from "${otherName}"`, + status: + result.skipped > 0 || result.pendingActivations.length > 0 + ? 'completed_with_warnings' + : 'completed', + message: + result.pendingActivations.length > 0 + ? `Undid the last sync from "${otherName}" — ${result.pendingActivations.length} deployment(s) still activating` + : `Undid the last sync from "${otherName}"`, metadata: { actorName: session.user.name ?? undefined, otherWorkspaceId, @@ -73,6 +79,7 @@ export const POST = withRouteHandler( removed: result.archived, unarchived: result.unarchived, skipped: result.skipped, + pendingActivations: result.pendingActivations.length, }, }).catch((error) => logger.error(`[${requestId}] Failed to record rollback activity`, { diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/deploy/components/deploy-modal/components/general/components/versions.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/deploy/components/deploy-modal/components/general/components/versions.tsx index 14781118029..868bca830f8 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/deploy/components/deploy-modal/components/general/components/versions.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/deploy/components/deploy-modal/components/general/components/versions.tsx @@ -207,6 +207,20 @@ export function Versions({
{versions.map((v) => { const isSelected = selectedVersion === v.version + const operationStatus = + !v.isActive && v.latestOperationStatus !== 'active' ? v.latestOperationStatus : null + const isOperationPending = + operationStatus === 'preparing' || operationStatus === 'activating' + /** Exactly one parenthetical per row; selection already highlights the row. */ + const rowLabel = v.isActive + ? 'live' + : isOperationPending + ? 'pending' + : operationStatus === 'failed' + ? 'failed' + : isSelected + ? 'selected' + : null return (
{editingVersion === v.version ? ( {v.name && {v.name}} - {v.isActive && ( - (live) - )} - {isSelected && ( - (selected) + {rowLabel && ( + ({rowLabel}) )} )} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/deploy/components/deploy-modal/deploy-modal.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/deploy/components/deploy-modal/deploy-modal.tsx index 68178c9f3c4..3af6d13baf0 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/deploy/components/deploy-modal/deploy-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/deploy/components/deploy-modal/deploy-modal.tsx @@ -17,11 +17,13 @@ import { ModalTabsList, ModalTabsTrigger, Tooltip, + toast, } from '@sim/emcn' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { useQueryClient } from '@tanstack/react-query' import { useParams } from 'next/navigation' +import type { DeploymentOperationSummary } from '@/lib/api/contracts/deployments' import { getBaseUrl } from '@/lib/core/utils/urls' import { getInputFormatExample as getInputFormatExampleUtil } from '@/lib/workflows/operations/deployment-utils' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' @@ -110,7 +112,6 @@ export function DeployModal({ const [activeTab, setActiveTab] = useState('general') const [chatSubmitting, setChatSubmitting] = useState(false) const [deployError, setDeployError] = useState(null) - const [deployWarnings, setDeployWarnings] = useState([]) const [isFinalizingDeploy, setIsFinalizingDeploy] = useState(false) const [isActivatingVersion, setIsActivatingVersion] = useState(false) const [isChatFormValid, setIsChatFormValid] = useState(false) @@ -149,7 +150,7 @@ export function DeployModal({ data: deploymentInfoData, isLoading: isLoadingDeploymentInfo, refetch: refetchDeploymentInfo, - } = useDeploymentInfo(workflowId, { enabled: open && isDeployed }) + } = useDeploymentInfo(workflowId, { enabled: open }) const { data: versionsData, isLoading: versionsLoading } = useDeploymentVersions(workflowId, { enabled: open, @@ -170,30 +171,44 @@ export function DeployModal({ const activateVersionMutation = useActivateDeploymentVersion() const versions = versionsData?.versions ?? [] + const deploymentAttemptStatus = deploymentInfoData?.latestDeploymentAttempt?.status + const attemptErrorMessage = + deploymentInfoData?.latestDeploymentAttempt?.error?.message ?? + (deploymentAttemptStatus === 'failed' ? 'Deployment preparation failed' : null) const isWorkflowStillActive = (targetWorkflowId: string) => { return useWorkflowRegistry.getState().activeWorkflowId === targetWorkflowId } - const syncDraftAfterDeploy = async (): Promise => { - if (!workflowId) return null + const syncDraftAfterDeploy = async (): Promise => { + if (!workflowId) return try { - const syncedActiveWorkflow = await syncLocalDraftFromServer(workflowId) - if (!syncedActiveWorkflow && isWorkflowStillActive(workflowId)) { - return 'Deployment succeeded, but local sync is still catching up. Refresh if the status looks stale.' - } - return null + await syncLocalDraftFromServer(workflowId) } catch (error) { - if (!isWorkflowStillActive(workflowId)) return null + if (!isWorkflowStillActive(workflowId)) return logger.warn('Workflow deployed, but local draft sync failed', { workflowId, error: toError(error).message, }) - return 'Deployment succeeded, but local sync failed. Refresh if the status looks stale.' } } + /** + * Post-activation warnings (dead-lettered or still-queued side effects) + * arrive with an `active` attempt, so the Live badge gives no signal — + * surface them as a toast. Pending/failed attempts are excluded: the + * status badge already covers those. + */ + const toastPostActivationWarnings = ( + title: string, + result: { latestDeploymentAttempt?: { status: string } | null; warnings?: string[] } + ) => { + if (result.latestDeploymentAttempt?.status !== 'active') return + if (!result.warnings?.length) return + toast.warning(title, { description: result.warnings.join(' ') }) + } + useEffect(() => { deployActionIdRef.current += 1 setIsFinalizingDeploy(false) @@ -241,7 +256,6 @@ export function DeployModal({ if (open && workflowId) { setActiveTab('general') setDeployError(null) - setDeployWarnings([]) setChatSuccess(false) const currentOutputs = selectedStreamingOutputsRef.current @@ -297,7 +311,6 @@ export function DeployModal({ deployActionIdRef.current = actionId setIsFinalizingDeploy(true) setDeployError(null) - setDeployWarnings([]) try { if (!(await deployReadiness.waitUntilReady())) { @@ -309,9 +322,12 @@ export function DeployModal({ try { const result = await deployMutation.mutateAsync({ workflowId }) - const syncWarning = await syncDraftAfterDeploy() - if (!isWorkflowStillActive(workflowId) || deployActionIdRef.current !== actionId) return - setDeployWarnings([...(result.warnings || []), ...(syncWarning ? [syncWarning] : [])]) + if (result.latestDeploymentAttempt?.status === 'active') { + await syncDraftAfterDeploy() + } + if (isWorkflowStillActive(workflowId)) { + toastPostActivationWarnings('Workflow deployed', result) + } } finally { if (deployActionIdRef.current === actionId) { setIsFinalizingDeploy(false) @@ -337,18 +353,17 @@ export function DeployModal({ activateVersionInFlightRef.current = true setIsActivatingVersion(true) - setDeployWarnings([]) + setDeployError(null) try { const result = await activateVersionMutation.mutateAsync({ workflowId, version }) - if (!isWorkflowStillActive(workflowId)) return - if (result.warnings && result.warnings.length > 0) { - setDeployWarnings(result.warnings) + if (isWorkflowStillActive(workflowId)) { + toastPostActivationWarnings(`Promoted v${version} to live`, result) } } catch (error) { if (!isWorkflowStillActive(workflowId)) return logger.error('Error promoting version:', { error }) - throw error + setDeployError(toError(error).message || `Failed to promote v${version} to live`) } finally { activateVersionInFlightRef.current = false setIsActivatingVersion(false) @@ -363,20 +378,23 @@ export function DeployModal({ return } - setDeployWarnings([]) - try { const result = await undeployMutation.mutateAsync({ workflowId: targetWorkflowId }) if (!isWorkflowStillActive(targetWorkflowId)) return setUndeployTargetWorkflowId(null) - if (result.warnings && result.warnings.length > 0) { - setDeployWarnings(result.warnings) - return - } onOpenChange(false) + /** + * Partial cleanup warnings (e.g. external subscription teardown left to + * background retries) surface as a toast so closing the modal does not + * silently swallow them. + */ + if (result.warnings?.length) { + toast.warning('Workflow undeployed', { description: result.warnings.join(' ') }) + } } catch (error: unknown) { if (!isWorkflowStillActive(targetWorkflowId)) return logger.error('Error undeploying workflow:', { error }) + toast.error('Failed to undeploy workflow', { description: toError(error).message }) } } @@ -388,7 +406,6 @@ export function DeployModal({ deployActionIdRef.current = actionId setIsFinalizingDeploy(true) setDeployError(null) - setDeployWarnings([]) try { if (!(await deployReadiness.waitUntilReady())) { @@ -414,9 +431,12 @@ export function DeployModal({ try { const result = await deployMutation.mutateAsync({ workflowId }) - const syncWarning = await syncDraftAfterDeploy() - if (!isWorkflowStillActive(workflowId) || deployActionIdRef.current !== actionId) return - setDeployWarnings([...(result.warnings || []), ...(syncWarning ? [syncWarning] : [])]) + if (result.latestDeploymentAttempt?.status === 'active') { + await syncDraftAfterDeploy() + } + if (isWorkflowStillActive(workflowId)) { + toastPostActivationWarnings('Workflow redeployed', result) + } } finally { if (deployActionIdRef.current === actionId) { setIsFinalizingDeploy(false) @@ -442,7 +462,6 @@ export function DeployModal({ if (workflowId) releaseDeployAction(workflowId) setChatSubmitting(false) setDeployError(null) - setDeployWarnings([]) onOpenChange(false) } @@ -514,24 +533,11 @@ export function DeployModal({ Configure and manage workflow deployment settings including API, MCP, and chat options. - {(deployError || deployWarnings.length > 0) && ( -
- {deployError && ( - - {deployError} - - )} - {deployWarnings.map((warning) => ( - - {warning} - - ))} + {deployError && ( +
+ + {deployError} +
)} @@ -604,6 +610,8 @@ export function DeployModal({ isUndeploying={isUndeploying} deployReadiness={deployReadiness} isDeploymentSettling={isDeploymentSettling} + attemptStatus={deploymentAttemptStatus} + attemptErrorMessage={attemptErrorMessage} onDeploy={onDeploy} onRedeploy={handleRedeploy} onUndeploy={() => { @@ -746,15 +754,77 @@ export function DeployModal({ ) } +type DeploymentAttemptStatus = DeploymentOperationSummary['status'] + interface StatusBadgeProps { - isWarning: boolean + isDeployed: boolean + needsRedeployment: boolean + attemptStatus?: DeploymentAttemptStatus + attemptErrorMessage?: string | null } -function StatusBadge({ isWarning }: StatusBadgeProps) { - const label = isWarning ? 'Update deployment' : 'Live' +/** + * Lifecycle-aware deployment status badge. Pending attempts render amber + * (labelled Retrying once an attempt has recorded a transient error), failed + * attempts render red with the failure reason in a tooltip, and a settled + * live deployment falls back to the Live/Update states. + */ +function StatusBadge({ + isDeployed, + needsRedeployment, + attemptStatus, + attemptErrorMessage, +}: StatusBadgeProps) { + if (attemptStatus === 'preparing' || attemptStatus === 'activating') { + const isRetrying = Boolean(attemptErrorMessage) + return ( + + + + {isRetrying ? 'Retrying' : 'Pending'} + + + + {isRetrying &&

{attemptErrorMessage}

} +

+ {isRetrying + ? isDeployed + ? 'Retrying automatically. The current version stays live until cutover completes.' + : 'Retrying automatically. The workflow goes live once activation completes.' + : isDeployed + ? 'A new version is being prepared. The current version stays live until cutover completes.' + : 'Triggers and schedules are being registered. The workflow goes live once activation completes.'} +

+ + + ) + } + + if (attemptStatus === 'failed') { + return ( + + + + Failed + + + +

{attemptErrorMessage || 'Deployment preparation failed.'}

+

+ {isDeployed + ? 'The previously deployed version is still live.' + : 'The workflow remains undeployed.'} +

+ + + ) + } + + if (!isDeployed) return null + return ( - - {label} + + {needsRedeployment ? 'Update deployment' : 'Live'} ) } @@ -766,6 +836,8 @@ interface GeneralFooterProps { isUndeploying: boolean deployReadiness: DeployReadiness isDeploymentSettling: boolean + attemptStatus?: DeploymentAttemptStatus + attemptErrorMessage?: string | null onDeploy: () => Promise onRedeploy: () => Promise onUndeploy: () => void @@ -778,6 +850,8 @@ function GeneralFooter({ isUndeploying, deployReadiness, isDeploymentSettling, + attemptStatus, + attemptErrorMessage, onDeploy, onRedeploy, onUndeploy, @@ -788,12 +862,30 @@ function GeneralFooter({ deployReadiness.isBlocked && !deployReadiness.isSyncing && !isSubmitting && !isUndeploying ? deployReadiness.tooltip : null + const status = ( +
+ + {blockedMessage && ( +
+ {blockedMessage} +
+ )} +
+ ) const deployActionLoading = isSubmitting || isDeploymentSettling if (!isDeployed) { return ( -
{blockedMessage}
+ {status}