Single-cell: interactive master summary, TCRdist3 + TCRi fixes, GIANA clonotype dedup

.cirro/singlecell_full_gex/preprocess.py

@@ -68,22 +68,42 @@ def prepare_vdj_file_params(ds, samples):
     """
     files = ds.files
 
+    # ds.files is Cirro's *indexed* listing, not everything present in S3 - which files get
+    # indexed is set by the upstream process's file-type config. SCRATCH-align commonly
+    # registers only airr_rearrangement.tsv, so a direct match on the three names VDJ_QC
+    # needs finds nothing even though they sit right beside it in the same outs/ directory.
+    # Fall back to deriving the sibling path from whichever file for that sample IS indexed.
+    def sibling_in_outs(sample, fname):
+        rows = files.loc[files['sample'] == sample, 'file'].tolist()
+        for f in rows:
+            if '/outs/' in f:
+                return f.rsplit('/outs/', 1)[0] + '/outs/' + fname
+        return ''
+
     for fname, param_name in VDJ_FILE_PARAMS.items():
         hits = files[files['file'].str.endswith(fname)]
         per_sample = []
         for sample in samples:
             sample_hits = sorted(hits.loc[hits['sample'] == sample, 'file'].tolist())
             if not sample_hits:
-                ds.logger.warning(
-                    f"No {fname} found for sample '{sample}' via ds.files - "
-                    "VDJ_QC will fall back to sample_sheet's own path column for this sample."
-                )
-                per_sample.append('')
+                derived = sibling_in_outs(sample, fname)
+                if derived:
+                    ds.logger.info(f"{fname} not indexed for '{sample}'; derived sibling path {derived}")
+                    per_sample.append(derived)
+                else:
+                    ds.logger.warning(
+                        f"No {fname} for sample '{sample}' and no indexed file under an outs/ "
+                        "directory to derive it from - VDJ_QC will fall back to sample_sheet's path column."
+                    )
+                    per_sample.append('')
             else:
                 if len(sample_hits) > 1:
                     ds.logger.warning(f"Multiple {fname} matches for sample '{sample}', using the first: {sample_hits}")
                 per_sample.append(sample_hits[0])
-        ds.add_param(param_name, ','.join(per_sample))
+        # overwrite=True: process-input.json declares these params (as empty strings) so
+        # they exist in ds.params before preprocessing runs. Without overwrite, add_param
+        # asserts "already exists".
+        ds.add_param(param_name, ','.join(per_sample), overwrite=True)
 
 
 def prepare_sample_sheet(ds):
@@ -105,8 +125,35 @@ def prepare_sample_sheet(ds):
             ds.logger.warning(f"sample_sheet is missing required column '{colname}'. Populating with NaN.")
             sample_sheet[colname] = np.nan
 
+    try:
+        # The PATIENT step (GIANA + GLIPH2) pools samples by params.patient_col. If that column
+        # is absent every sample becomes its own patient and the clustering is silently wrong -
+        # no error, just useless results - so say so loudly here. samplesheet_from_params()'s
+        # fallback frame in particular carries only sample/path.
+        # ds.params may be a plain dict or a params object depending on cirro version, and a
+        # warning must never be the thing that breaks preprocessing - so degrade to the default.
+        try:
+            patient_col = dict(ds.params).get("patient_col") or "patient_id"
+        except Exception:
+            patient_col = "patient_id"
+        if patient_col not in sample_sheet.columns:
+            ds.logger.warning(
+                f"sample_sheet has no '{patient_col}' column. GIANA and GLIPH2 pool samples per "
+                "patient, so each sample will be treated as its own patient and cross-sample "
+                "clustering within a patient will be lost. Add the column to the dataset "
+                "samplesheet, or set 'Patient column' to one that exists."
+            )
+        elif sample_sheet[patient_col].isna().any():
+            missing = sample_sheet.loc[sample_sheet[patient_col].isna(), "sample"].tolist()
+            ds.logger.warning(f"Samples with no {patient_col}: {missing}. These will not pool with any patient.")
+        else:
+            n_pat = sample_sheet[patient_col].nunique()
+            ds.logger.info(f"{len(sample_sheet)} samples across {n_pat} patients (by '{patient_col}')")
+    except Exception as e:
+        ds.logger.warning(f"patient-column check skipped: {e}")
+
     sample_sheet.to_csv('sample_sheet.csv', index=None)
-    ds.add_param('sample_sheet', 'sample_sheet.csv')
+    ds.add_param('sample_sheet', 'sample_sheet.csv', overwrite=True)
     ds.logger.info(sample_sheet.to_dict())
 
 

.cirro/singlecell_full_gex/process-form.json

@@ -5,9 +5,12 @@
         "properties": {
             "input_annotated_object": {
                 "default": "",
-                "description": "S3 path to the annotated GEX Seurat object (.rds) to integrate VDJ contigs onto.",
+                "description": "Annotated GEX Seurat object (.rds) to integrate VDJ contigs onto.",
                 "title": "Annotated GEX object (Seurat RDS)",
-                "type": "string"
+                "type": "string",
+                "multiple": false,
+                "pathType": "dataset",
+                "file": "**/*.{RDS,rds}"
             },
             "project_name": {
                 "default": "",
@@ -66,6 +69,30 @@
                 "description": "Render the master summary report.",
                 "title": "Run master summary report",
                 "type": "boolean"
+            },
+            "patient_col": {
+                "type": "string",
+                "title": "Patient column",
+                "description": "Sample-sheet column holding the patient/subject ID. GIANA and GLIPH2 pool samples per patient using this; if it is wrong, every sample is treated as its own patient.",
+                "default": "patient_id"
+            },
+            "run_tcri": {
+                "type": "boolean",
+                "title": "Run TCRi",
+                "description": "Immunogenicity scoring on the enriched Seurat object. Requires a GEX object. Adds roughly an hour of runtime.",
+                "default": true
+            },
+            "giana_dedup_clonotypes": {
+                "type": "boolean",
+                "title": "Deduplicate clonotypes before GIANA",
+                "description": "Collapse to one row per clonotype before GIANA. Patient pooling stacks per-sample rows, so without this GIANA clusters cross-sample duplicates rather than similar sequences.",
+                "default": true
+            },
+            "tcrdist_radius": {
+                "type": "integer",
+                "title": "TCRdist radius",
+                "description": "Distance threshold for TCRdist3 connected-component clustering.",
+                "default": 24
             }
         }
     },

.cirro/singlecell_full_gex/process-input.json

@@ -15,5 +15,9 @@
     "run_consensus": "$.params.dataset.paramJson.run_consensus",
     "run_repertoire": "$.params.dataset.paramJson.run_repertoire",
     "run_master_summary": "$.params.dataset.paramJson.run_master_summary",
+    "patient_col": "$.params.dataset.paramJson.patient_col",
+    "run_tcri": "$.params.dataset.paramJson.run_tcri",
+    "giana_dedup_clonotypes": "$.params.dataset.paramJson.giana_dedup_clonotypes",
+    "tcrdist_radius": "$.params.dataset.paramJson.tcrdist_radius",
     "outdir": "$.params.dataset.s3|/data/"
 }

.cirro/singlecell_vdjonly/preprocess.py

@@ -68,22 +68,42 @@ def prepare_vdj_file_params(ds, samples):
     """
     files = ds.files
 
+    # ds.files is Cirro's *indexed* listing, not everything present in S3 - which files get
+    # indexed is set by the upstream process's file-type config. SCRATCH-align commonly
+    # registers only airr_rearrangement.tsv, so a direct match on the three names VDJ_QC
+    # needs finds nothing even though they sit right beside it in the same outs/ directory.
+    # Fall back to deriving the sibling path from whichever file for that sample IS indexed.
+    def sibling_in_outs(sample, fname):
+        rows = files.loc[files['sample'] == sample, 'file'].tolist()
+        for f in rows:
+            if '/outs/' in f:
+                return f.rsplit('/outs/', 1)[0] + '/outs/' + fname
+        return ''
+
     for fname, param_name in VDJ_FILE_PARAMS.items():
         hits = files[files['file'].str.endswith(fname)]
         per_sample = []
         for sample in samples:
             sample_hits = sorted(hits.loc[hits['sample'] == sample, 'file'].tolist())
             if not sample_hits:
-                ds.logger.warning(
-                    f"No {fname} found for sample '{sample}' via ds.files - "
-                    "VDJ_QC will fall back to sample_sheet's own path column for this sample."
-                )
-                per_sample.append('')
+                derived = sibling_in_outs(sample, fname)
+                if derived:
+                    ds.logger.info(f"{fname} not indexed for '{sample}'; derived sibling path {derived}")
+                    per_sample.append(derived)
+                else:
+                    ds.logger.warning(
+                        f"No {fname} for sample '{sample}' and no indexed file under an outs/ "
+                        "directory to derive it from - VDJ_QC will fall back to sample_sheet's path column."
+                    )
+                    per_sample.append('')
             else:
                 if len(sample_hits) > 1:
                     ds.logger.warning(f"Multiple {fname} matches for sample '{sample}', using the first: {sample_hits}")
                 per_sample.append(sample_hits[0])
-        ds.add_param(param_name, ','.join(per_sample))
+        # overwrite=True: process-input.json declares these params (as empty strings) so
+        # they exist in ds.params before preprocessing runs. Without overwrite, add_param
+        # asserts "already exists".
+        ds.add_param(param_name, ','.join(per_sample), overwrite=True)
 
 
 def prepare_sample_sheet(ds):
@@ -105,8 +125,35 @@ def prepare_sample_sheet(ds):
             ds.logger.warning(f"sample_sheet is missing required column '{colname}'. Populating with NaN.")
             sample_sheet[colname] = np.nan
 
+    try:
+        # The PATIENT step (GIANA + GLIPH2) pools samples by params.patient_col. If that column
+        # is absent every sample becomes its own patient and the clustering is silently wrong -
+        # no error, just useless results - so say so loudly here. samplesheet_from_params()'s
+        # fallback frame in particular carries only sample/path.
+        # ds.params may be a plain dict or a params object depending on cirro version, and a
+        # warning must never be the thing that breaks preprocessing - so degrade to the default.
+        try:
+            patient_col = dict(ds.params).get("patient_col") or "patient_id"
+        except Exception:
+            patient_col = "patient_id"
+        if patient_col not in sample_sheet.columns:
+            ds.logger.warning(
+                f"sample_sheet has no '{patient_col}' column. GIANA and GLIPH2 pool samples per "
+                "patient, so each sample will be treated as its own patient and cross-sample "
+                "clustering within a patient will be lost. Add the column to the dataset "
+                "samplesheet, or set 'Patient column' to one that exists."
+            )
+        elif sample_sheet[patient_col].isna().any():
+            missing = sample_sheet.loc[sample_sheet[patient_col].isna(), "sample"].tolist()
+            ds.logger.warning(f"Samples with no {patient_col}: {missing}. These will not pool with any patient.")
+        else:
+            n_pat = sample_sheet[patient_col].nunique()
+            ds.logger.info(f"{len(sample_sheet)} samples across {n_pat} patients (by '{patient_col}')")
+    except Exception as e:
+        ds.logger.warning(f"patient-column check skipped: {e}")
+
     sample_sheet.to_csv('sample_sheet.csv', index=None)
-    ds.add_param('sample_sheet', 'sample_sheet.csv')
+    ds.add_param('sample_sheet', 'sample_sheet.csv', overwrite=True)
     ds.logger.info(sample_sheet.to_dict())
 
 

.cirro/singlecell_vdjonly/process-form.json

@@ -30,6 +30,24 @@
                     "drop",
                     "hard_stop"
                 ]
+            },
+            "patient_col": {
+                "type": "string",
+                "title": "Patient column",
+                "description": "Sample-sheet column holding the patient/subject ID. GIANA and GLIPH2 pool samples per patient using this; if it is wrong, every sample is treated as its own patient.",
+                "default": "patient_id"
+            },
+            "giana_dedup_clonotypes": {
+                "type": "boolean",
+                "title": "Deduplicate clonotypes before GIANA",
+                "description": "Collapse to one row per clonotype before GIANA. Patient pooling stacks per-sample rows, so without this GIANA clusters cross-sample duplicates rather than similar sequences.",
+                "default": true
+            },
+            "tcrdist_radius": {
+                "type": "integer",
+                "title": "TCRdist radius",
+                "description": "Distance threshold for TCRdist3 connected-component clustering.",
+                "default": 24
             }
         }
     },

.cirro/singlecell_vdjonly/process-input.json

@@ -9,5 +9,8 @@
     "pseudobulk_qc_min_clones": "$.params.dataset.paramJson.pseudobulk_qc_min_clones",
     "pseudobulk_qc_min_cells": "$.params.dataset.paramJson.pseudobulk_qc_min_cells",
     "pseudobulk_qc_mode": "$.params.dataset.paramJson.pseudobulk_qc_mode",
+    "patient_col": "$.params.dataset.paramJson.patient_col",
+    "giana_dedup_clonotypes": "$.params.dataset.paramJson.giana_dedup_clonotypes",
+    "tcrdist_radius": "$.params.dataset.paramJson.tcrdist_radius",
     "outdir": "$.params.dataset.s3|/data/"
 }

.gitignore

@@ -38,3 +38,16 @@ tmp
 .vscode/*
 
 .e2e_test_tmp/
+
+## Run artifacts — existing rules only match bare `work` and `results*`,
+## so custom -w / --outdir names slip through.
+# NB: not `work*/` - that also matches workflows/, the source directory.
+work/
+work_*/
+test_results*/
+test_v2port*/
+logs/
+
+## Let the report notebooks through: the `notebooks/*` rule above would
+## otherwise silently skip every template_*.qmd, which the bulk route needs.
+!notebooks/template_*.qmd