Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
241 changes: 241 additions & 0 deletions create_runfolder_archive.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,241 @@
#!/usr/bin/env python3
"""Create an uncompressed tar archive of a run directory restricted to one
project's lanes.

Only an allowlist of top-level run folder entries is archived (see
INCLUDED_TOP_LEVEL_ENTRIES); anything else, including Demultiplexing output,
is left out so that data from other projects cannot leak into the archive.
All .csv files (including sample sheets) are excluded as well.
"""

import argparse
import csv
import hashlib
import os
import re
import sys
import tarfile
from pathlib import Path

SAMPLESHEET_DIR_NAME = "Samplesheets"
LANE_SAMPLESHEET_PATTERN = re.compile(r"SampleSheet_lane_(\d+)_settings_\d+")
LANE_DIR_PATTERN = re.compile(r"L\d{3}")

# Top-level run folder entries that are copied into the archive. Everything
# else is skipped (and reported), so new/unknown output never ends up in a
# delivery by accident.
INCLUDED_TOP_LEVEL_ENTRIES = {
# Directories
"Config",
"Data",
"Images",
"InterOp",
"Logs",
"Recipe",
"Thumbnail_Images",
# Files
"CopyComplete.txt",
"RTA3.cfg",
"RTAComplete.txt",
"RunCompletionStatus.xml",
"RunInfo.xml",
"RunParameters.xml",
"runParameters.xml",
"SequenceComplete.txt",
}


def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description=(
"Create an uncompressed tar archive of a run directory for one project, "
"containing only the selected lanes. All samples in the selected lanes' "
"sample sheets must belong to the project. Only standard run folder "
"entries (Data, Images, InterOp, RunInfo.xml, etc.) are included; lane "
"directories for unselected lanes, Demultiplexing output and all .csv "
"files (including sample sheets) are excluded. The archive and its .md5 "
"file are written to the current directory."
)
)
parser.add_argument(
"--run", "-r", required=True, help="Run directory name to archive"
)
parser.add_argument(
"--project", "-p", required=True, help="Project ID to keep, e.g. P12345"
)
parser.add_argument(
"--lanes",
"-l",
Comment thread
ssjunnebo marked this conversation as resolved.
Resolved
type=lambda s: [int(item) for item in s.split(",")],
required=True,
help="Comma-separated list of lane numbers to include, e.g. 1,2,4",
)
return parser.parse_args()


def find_lane_samplesheets(
run_dir: Path, lane_numbers: set[int]
) -> dict[int, list[Path]]:
samplesheet_dir = run_dir / SAMPLESHEET_DIR_NAME
if not samplesheet_dir.is_dir():
raise FileNotFoundError(f"Samplesheets directory not found: {samplesheet_dir}")

lane_samplesheets: dict[int, list[Path]] = {lane: [] for lane in lane_numbers}
for path in sorted(samplesheet_dir.iterdir()):
match = LANE_SAMPLESHEET_PATTERN.match(path.name)
if match and path.is_file() and int(match.group(1)) in lane_numbers:
lane_samplesheets[int(match.group(1))].append(path)

missing = sorted(lane for lane, paths in lane_samplesheets.items() if not paths)
if missing:
found = sorted(p.name for p in samplesheet_dir.iterdir()) or ["<empty>"]
raise FileNotFoundError(
f"No lane sample sheet matching '{LANE_SAMPLESHEET_PATTERN.pattern}' "
f"found for lane(s) {missing} in {samplesheet_dir}. Directory contains:\n "
+ "\n ".join(found)
)
return lane_samplesheets


def read_bclconvert_sample_names(samplesheet_path: Path) -> list[str]:
in_data_section = False
name_index: int | None = None
sample_names: list[str] = []
for line in samplesheet_path.read_text(
encoding="utf-8", errors="replace"
).splitlines():
stripped = line.strip()
if stripped == "[BCLConvert_Data]":
in_data_section = True
continue
if not in_data_section or not stripped:
continue

row = next(csv.reader([stripped]))
if name_index is None:
if "Sample_Name" not in row:
raise ValueError(
f"'Sample_Name' column not found in {samplesheet_path}"
)
name_index = row.index("Sample_Name")
continue

if len(row) <= name_index:
raise ValueError(f"Malformed row in {samplesheet_path}: {stripped}")
sample_names.append(row[name_index].strip())

if name_index is None:
raise ValueError(f"[BCLConvert_Data] section not found in {samplesheet_path}")
if not sample_names:
raise ValueError(f"No samples listed in {samplesheet_path}")
return sample_names


def check_lane_samplesheets_match_project(
run_dir: Path, project_id: str, lane_numbers: set[int]
) -> None:
foreign_samples = []
for lane, samplesheets in find_lane_samplesheets(run_dir, lane_numbers).items():
for samplesheet in samplesheets:
for sample_name in read_bclconvert_sample_names(samplesheet):
if not sample_name.startswith(f"{project_id}_"):
foreign_samples.append(
f"lane {lane} ({samplesheet.name}): {sample_name}"
)

if foreign_samples:
raise ValueError(
f"Samples not belonging to project {project_id} in selected lanes:\n"
+ "\n".join(foreign_samples)
)


def is_unselected_lane_dir(name: str, selected_lane_dirs: set[str]) -> bool:
return (
LANE_DIR_PATTERN.fullmatch(name) is not None and name not in selected_lane_dirs
)


def is_excluded_file(path: Path) -> bool:
"""Exclude all .csv files (sample sheets etc.) and any other sample sheet."""
name = path.name.lower()
return name.endswith(".csv") or name.startswith("samplesheet")


def iter_files_to_archive(run_dir: Path, lane_numbers: set[int]):
"""Yield files under run_dir that belong in the archive."""
selected_lane_dirs = {f"L{lane:03d}" for lane in lane_numbers}
skipped = []

for entry in sorted(run_dir.iterdir()):
if entry.name not in INCLUDED_TOP_LEVEL_ENTRIES:
skipped.append(entry.name)
continue

if entry.is_symlink() or entry.is_file():
if not is_excluded_file(entry):
yield entry
continue

for root, dirs, files in os.walk(entry):
dirs[:] = sorted(
d for d in dirs if not is_unselected_lane_dir(d, selected_lane_dirs)
)
for filename in sorted(files):
src_file = Path(root) / filename
if not is_excluded_file(src_file):
yield src_file

if skipped:
print(
"Skipped top-level entries not in the allowlist: " + ", ".join(skipped),
file=sys.stderr,
)


def build_run_archive(run_dir: Path, lane_numbers: set[int], output_tar: Path) -> None:
with tarfile.open(output_tar, "w") as tar_handle:
for src_file in iter_files_to_archive(run_dir, lane_numbers):
arcname = Path(run_dir.name) / src_file.relative_to(run_dir)
tar_handle.add(src_file, arcname=str(arcname), recursive=False)


def write_md5sum(archive_path: Path) -> Path:
digest = hashlib.md5()
with archive_path.open("rb") as handle:
for chunk in iter(lambda: handle.read(65536), b""):
digest.update(chunk)

md5_path = archive_path.with_name(archive_path.name + ".md5")
md5_path.write_text(
f"{digest.hexdigest()} {archive_path.name}\n", encoding="utf-8"
)
return md5_path


def main() -> None:
args = parse_args()
run_dir = Path(os.path.abspath(args.run))
if not run_dir.exists() or not run_dir.is_dir():
raise FileNotFoundError(f"Run directory not found: {run_dir}")

lane_numbers = set(args.include_lanes)
invalid_lanes = sorted(lane for lane in lane_numbers if lane < 1 or lane > 8)
if invalid_lanes:
raise ValueError(
f"Lane numbers must be positive integers between 1 and 8, not: {invalid_lanes}"
)

check_lane_samplesheets_match_project(run_dir, args.project, lane_numbers)

archive_name = f"{run_dir.name}_{args.project}.tar"
archive_path = Path.cwd() / archive_name

build_run_archive(run_dir, lane_numbers, archive_path)
md5_path = write_md5sum(archive_path)
print(f"Created archive: {archive_path}")
print(f"Created MD5 file: {md5_path}")


if __name__ == "__main__":
main()
Loading