diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9eabbe1..e3e39e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,5 +44,8 @@ jobs: - name: Lint (ruff) run: ruff check ml tests - - name: Tests (pytest) - run: pytest tests/ -v + - name: Tests (pytest + cobertura) + run: pytest tests/ -v --cov=ml --cov-report=xml --cov-report=term + + - name: Health check (config) + run: python -m ml.health --json --skip-s3 --skip-artifacts diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..9957bea --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,104 @@ +name: Security & Quality + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + # Lunes 3am UTC + - cron: '0 3 * * 1' + workflow_dispatch: + +permissions: + contents: read + security-events: write + pull-requests: read + +concurrency: + group: security-${{ github.ref }} + cancel-in-progress: true + +jobs: + snyk: + name: Snyk (SCA + Code) + runs-on: ubuntu-latest + + steps: + - name: Descargar codigo + uses: actions/checkout@v4 + + - name: Configurar Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Instalar Snyk CLI + uses: snyk/actions/setup@v1 + + - name: Snyk Open Source (runtime) + continue-on-error: true + run: > + snyk test --file=requirements.txt + --severity-threshold=high + --sarif-file-output=snyk-sca.sarif + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + + - name: Snyk Open Source (dev) + continue-on-error: true + run: > + snyk test --file=requirements-dev.txt + --severity-threshold=high + --sarif-file-output=snyk-dev.sarif + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + + - name: Snyk Code (SAST) + continue-on-error: true + run: snyk code test --sarif-file-output=snyk-code.sarif + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + + - name: Subir SARIF a GitHub Security + if: always() + continue-on-error: true + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: . + category: snyk + + sonar: + name: SonarCloud + runs-on: ubuntu-latest + + steps: + - name: Descargar codigo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configurar Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: requirements-dev.txt + + - name: Instalar dependencias + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Tests + cobertura + run: pytest --cov=ml --cov-report=xml --cov-report=term + + - name: SonarCloud Scan + uses: SonarSource/sonarqube-scan-action@v8 + env: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + SONAR_HOST_URL: https://sonarcloud.io + with: + args: > + -Dsonar.organization=${{ vars.SONAR_ORGANIZATION || 'sebithaz-dev' }} + -Dsonar.projectKey=${{ vars.SONAR_PROJECT_KEY || 'Sebithaz-dev_Model-Training-S3' }} diff --git a/.github/workflows/train.yml b/.github/workflows/train.yml index 277e6b0..bd9f9a3 100644 --- a/.github/workflows/train.yml +++ b/.github/workflows/train.yml @@ -42,3 +42,10 @@ jobs: env: S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }} S3_MODEL_PREFIX: ${{ vars.S3_MODEL_PREFIX || 'models/' }} + + - name: Health check (artefactos + S3) + run: python -m ml.health --json --strict + env: + AWS_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} + S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }} + S3_MODEL_PREFIX: ${{ vars.S3_MODEL_PREFIX || 'models/' }} diff --git a/.gitignore b/.gitignore index 2f90d9c..8f2597b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ __pycache__/ artefactos/ data/ .DS_Store +.coverage +coverage.xml +htmlcov/ +*.sarif diff --git a/README.md b/README.md index 1115843..7974604 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Model Training — Citi Bike S3 +[![CI](https://github.com/Sebithaz-dev/Model-Training-S3/actions/workflows/ci.yml/badge.svg)](https://github.com/Sebithaz-dev/Model-Training-S3/actions/workflows/ci.yml) +[![Security & Quality](https://github.com/Sebithaz-dev/Model-Training-S3/actions/workflows/security.yml/badge.svg)](https://github.com/Sebithaz-dev/Model-Training-S3/actions/workflows/security.yml) +[![Quality Gate](https://sonarcloud.io/api/project_badges/measure?project=Sebithaz-dev_Model-Training-S3&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=Sebithaz-dev_Model-Training-S3) + Pipeline de entrenamiento automatizado para clasificar la duración de viajes de Citi Bike NYC en **corto** (< 15 min) o **largo** (≥ 15 min), utilizando datos curados desde AWS Glue y subiendo artefactos a S3. --- @@ -79,6 +83,26 @@ python -m ml.predict Ejecuta ejemplos de prueba con un Passenger (`Customer`, 5pm → largo) vs un Suscriptor (`Subscriber`, 8am → corto). +### Health check del pipeline + +```bash +python -m ml.health # config + artefactos + S3 (si esta configurado) +python -m ml.health --json # salida JSON para CI +python -m ml.health --skip-s3 # sin acceso a red +python -m ml.health --strict # falla si hay estado degradado +``` + +Evalúa tres componentes: `config`, `artifacts` y `s3`. Estados posibles: + +| Estado | Significado | +|---|---| +| `ok` | Componente sano | +| `degraded` | Artefactos locales ausentes (esperado en un checkout limpio) | +| `skipped` | Validación omitida o S3 no configurado | +| `error` | Configuración inválida, artefactos corruptos o S3 inaccesible | + +El comando termina con código `0` salvo que exista un `error`, o que haya `degraded` y se use `--strict`. En CI, `ci.yml` valida solo la configuración (`--skip-s3 --skip-artifacts`) y `train.yml` valida artefactos + S3 después de entrenar. + --- ## Pipeline de entrenamiento (`ml/train.py`) @@ -153,18 +177,51 @@ Las métricas se calculan sobre el test set (80/20 split estratificado) y se gua ### `ci.yml` — lint y tests (calidad continua) - **Triggers:** `pull_request` a `main` (gate de merge, obligatorio verde) y `push` a `main` (verificación post-merge del trunk). -- **Pasos:** checkout → setup Python 3.11 (cache de pip) → instalar `requirements-dev.txt` → `ruff check` → `pytest`. -- **Propósito:** garantizar que todo lo que llega a `main` pasa lint y tests. Bloquea el merge si falla. +- **Pasos:** checkout → setup Python 3.11 (cache de pip) → instalar `requirements-dev.txt` → `ruff check` → `pytest` con cobertura → health check de configuración. +- **Propósito:** garantizar que todo lo que llega a `main` pasa lint, tests y el health check básico. Bloquea el merge si falla. > **Nota sobre IE3/IE4:** el enunciado pide disparar la CI en "push a `develop` y pull request a `main`". Como este repositorio usa **TBD puro** (sin rama `develop` — ver [justificación](#estrategia-de-branching-trunk-based-development-tbd)), el trigger `push a develop` se sustituye por `push a main`, que cumple el mismo rol en TBD: verificar el trunk tras cada integración. ### `train.yml` — entrenamiento + subida a S3 - **Trigger:** Semanal (domingo 2am UTC) + `workflow_dispatch` manual -- **Pasos:** checkout → setup python → instalar deps → configurar AWS → descargar datos curados desde S3 → entrenar → subir artefactos +- **Pasos:** checkout → setup python → instalar deps → configurar AWS → descargar datos curados desde S3 → entrenar → subir artefactos → health check de artefactos + S3 - **Secrets requeridos:** `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, `S3_BUCKET_NAME` - **Variables:** `AWS_REGION` (default `us-east-1`), `S3_DATA_KEY` (default `curated/citibike_trips/citibike_curated.snappy.parquet`), `S3_MODEL_PREFIX` (default `models/`) +### `security.yml` — Snyk + SonarCloud + +- **Triggers:** `pull_request` y `push` a `main`, semanal (lunes 3am UTC) y `workflow_dispatch`. +- **Snyk:** escanea dependencias (`requirements.txt` y `requirements-dev.txt`) con Snyk Open Source y código con Snyk Code. Los resultados se suben como SARIF a la pestaña **Security** de GitHub. +- **SonarCloud:** ejecuta `pytest --cov=ml --cov-report=xml` y publica cobertura, bugs, vulnerabilidades y code smells. +- **Secrets requeridos:** `SNYK_TOKEN`, `SONAR_TOKEN`. +- **Variables:** `SONAR_ORGANIZATION` (default `sebithaz-dev`), `SONAR_PROJECT_KEY` (default `Sebithaz-dev_Model-Training-S3`). +- **Bloqueo:** Snyk es informativo al inicio (`continue-on-error`); SonarCloud publica el análisis sin esperar el quality gate. Cuando el equipo lo decida, se puede activar `sonar.qualitygate.wait=true`. + +--- + +## Seguridad y calidad + +### Snyk + +1. Crear una cuenta en [Snyk](https://app.snyk.io) y habilitar **Snyk Code**. +2. Generar un API token en *Account Settings*. +3. Cargarlo como secret `SNYK_TOKEN` en el repositorio. + +El workflow ejecuta: + +- `snyk test` sobre las dependencias de runtime y desarrollo. +- `snyk code test` sobre el código Python. +- Subida de SARIF a GitHub Security. + +### SonarCloud + +1. Importar el repositorio en [SonarCloud](https://sonarcloud.io) bajo la organización `sebithaz-dev`. +2. **Desactivar Automatic Analysis** en *Administration → Analysis Method* para evitar conflictos con el análisis de CI. +3. Generar un token en *My Account → Security* y cargarlo como secret `SONAR_TOKEN`. + +La configuración vive en `sonar-project.properties` y la cobertura se genera con `pytest-cov`. + --- ## Estrategia de branching: Trunk-Based Development (TBD) @@ -278,7 +335,7 @@ Reglas: ## Repositorios relacionados -- [Flask-Dashboard-S3](https://github.com/anomalyco/Flask-Dashboard-S3) — Dashboard Flask + Chart.js que consume los artefactos de S3 +- [Flask-Dashboard-S3](https://github.com/Sebithaz-dev/Flask-Dashboard-S3) — Dashboard Flask + Chart.js que consume los artefactos de S3 --- @@ -288,7 +345,8 @@ Reglas: . ├── .env.example # Variables de entorno (template) ├── .github/workflows/ # CI/CD -│ ├── ci.yml # Lint + tests (push a develop, PR a main) +│ ├── ci.yml # Lint + tests + health (PR/push a main) +│ ├── security.yml # Snyk + SonarCloud (PR/push/semanal) │ └── train.yml # Entrenamiento semanal + subida a S3 ├── config/ │ └── model.yaml # Hiperparámetros y umbrales del modelo @@ -296,11 +354,14 @@ Reglas: │ ├── __init__.py │ ├── config.py # Carga de configuración YAML │ ├── data_validation.py # Validación de schema y calidad de datos +│ ├── health.py # Health check (config, artefactos, S3) │ ├── train.py # Entrenamiento del modelo │ ├── predict.py # Predicción local (CLI demo) │ └── s3_utils.py # Subida/descarga desde S3 ├── tests/ # Tests unitarios (pytest) +│ └── test_health.py # Tests del health check ├── pyproject.toml # Config de ruff y pytest ├── requirements.txt # Dependencias runtime -└── requirements-dev.txt # Dependencias desarrollo (pytest, ruff) +├── requirements-dev.txt # Dependencias desarrollo (pytest, ruff, coverage) +└── sonar-project.properties # Config de SonarCloud ``` diff --git a/ml/health.py b/ml/health.py new file mode 100644 index 0000000..77860f2 --- /dev/null +++ b/ml/health.py @@ -0,0 +1,191 @@ +"""Health check del pipeline de entrenamiento. + +Valida la configuracion, los artefactos locales y la conexion a S3 desde la +linea de comandos. Se usa en CI para verificar que el pipeline quedo sano +despues de entrenar. + +Uso: + python -m ml.health + python -m ml.health --json --skip-s3 --skip-artifacts + python -m ml.health --strict +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from datetime import UTC, datetime +from pathlib import Path + +import joblib + +from ml.config import DEFAULT_CONFIG_PATH, load_config + +SERVICE_NAME = "model-training-s3" +VERSION = "0.1.0" +ARTIFACT_DIR = Path(__file__).resolve().parent / "artefactos" +ARTIFACT_FILES = ("modelo.pkl", "transformers.pkl", "metricas.json") + + +def _result(status: str, detail: str, **extra) -> dict: + data = {"status": status, "detail": detail} + data.update(extra) + return data + + +def check_config() -> dict: + config_path = Path(os.getenv("MODEL_CONFIG") or DEFAULT_CONFIG_PATH) + try: + cfg = load_config(config_path) + except Exception as exc: + return _result("error", f"No se pudo cargar la configuracion: {exc}") + + return _result( + "ok", + "Configuracion cargada correctamente", + path=str(config_path), + model_class=cfg.model_class, + features=len(cfg.feature_cols), + ) + + +def check_artifacts(artifact_dir: Path | None = None) -> dict: + artifact_dir = artifact_dir or ARTIFACT_DIR + missing = [name for name in ARTIFACT_FILES if not (artifact_dir / name).exists()] + if missing: + return _result( + "degraded", + f"Faltan artefactos locales: {', '.join(missing)}", + directory=str(artifact_dir), + missing=missing, + ) + + try: + joblib.load(artifact_dir / "modelo.pkl") + joblib.load(artifact_dir / "transformers.pkl") + with (artifact_dir / "metricas.json").open("r", encoding="utf-8") as f: + metrics = json.load(f) + except Exception as exc: + return _result( + "error", + f"Artefactos corruptos: {exc}", + directory=str(artifact_dir), + ) + + return _result( + "ok", + "Artefactos cargados correctamente", + directory=str(artifact_dir), + metrics_keys=sorted(metrics)[:10], + ) + + +def check_s3() -> dict: + bucket = os.getenv("S3_BUCKET_NAME") + if not bucket: + return _result("skipped", "S3_BUCKET_NAME no configurado") + + prefix = os.getenv("S3_MODEL_PREFIX", "models/") + region = os.getenv("AWS_REGION", "us-east-1") + key = f"{prefix.rstrip('/')}/metricas.json" + + try: + import boto3 + from botocore.config import Config + + s3 = boto3.client( + "s3", + region_name=region, + config=Config(connect_timeout=5, read_timeout=5, retries={"max_attempts": 0}), + ) + account_id = boto3.client("sts", region_name=region).get_caller_identity()["Account"] + s3.head_bucket(Bucket=bucket, ExpectedBucketOwner=account_id) + s3.head_object(Bucket=bucket, Key=key, ExpectedBucketOwner=account_id) + except Exception as exc: + return _result( + "error", + f"No se pudo verificar S3: {exc}", + bucket=bucket, + region=region, + ) + + return _result( + "ok", + "Bucket y artefacto metricas.json accesibles", + bucket=bucket, + key=key, + region=region, + ) + + +def _overall_status(checks: dict) -> str: + statuses = {check["status"] for check in checks.values()} + if "error" in statuses: + return "error" + if "degraded" in statuses: + return "degraded" + return "ok" + + +def build_report(skip_s3: bool = False, skip_artifacts: bool = False) -> dict: + checks = {"config": check_config()} + if skip_artifacts: + checks["artifacts"] = _result("skipped", "Validacion de artefactos omitida") + else: + checks["artifacts"] = check_artifacts() + + if skip_s3: + checks["s3"] = _result("skipped", "Validacion de S3 omitida") + else: + checks["s3"] = check_s3() + + return { + "status": _overall_status(checks), + "service": SERVICE_NAME, + "version": VERSION, + "timestamp": datetime.now(UTC).isoformat(), + "checks": checks, + } + + +def exit_code_for(report: dict, strict: bool = False) -> int: + if report["status"] == "error": + return 1 + if strict and report["status"] == "degraded": + return 1 + return 0 + + +def print_report(report: dict) -> None: + print(f"[{report['status'].upper()}] {report['service']} v{report['version']}") + for name, check in report["checks"].items(): + print(f" - {name}: {check['status']} ({check['detail']})") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Health check del pipeline de entrenamiento") + parser.add_argument("--json", action="store_true", help="Imprime el reporte en JSON") + parser.add_argument("--skip-s3", action="store_true", help="Omite la validacion de S3") + parser.add_argument( + "--skip-artifacts", + action="store_true", + help="Omite la validacion de artefactos locales", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Falla ante cualquier estado degradado", + ) + args = parser.parse_args(argv) + + report = build_report(skip_s3=args.skip_s3, skip_artifacts=args.skip_artifacts) + if args.json: + print(json.dumps(report, indent=2, ensure_ascii=False)) + else: + print_report(report) + return exit_code_for(report, strict=args.strict) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7a3ce19..55b1409 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ -r requirements.txt pytest>=7.4 +pytest-cov>=5 ruff>=0.4 diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..c3f007f --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,11 @@ +sonar.projectKey=Sebithaz-dev_Model-Training-S3 +sonar.organization=sebithaz-dev +sonar.projectName=Model Training S3 +sonar.projectVersion=0.1.0 + +sonar.sources=ml +sonar.tests=tests +sonar.python.version=3.11 +sonar.sourceEncoding=UTF-8 +sonar.python.coverage.reportPaths=coverage.xml +sonar.exclusions=ml/artefactos/**,data/**,Docs/**,**/__pycache__/** diff --git a/tests/test_health.py b/tests/test_health.py new file mode 100644 index 0000000..4e39101 --- /dev/null +++ b/tests/test_health.py @@ -0,0 +1,139 @@ +"""Tests para ml.health.""" +import json + +import joblib +from sklearn.linear_model import LogisticRegression + +from ml import health + + +def test_check_config_ok(): + result = health.check_config() + assert result["status"] == "ok" + assert result["model_class"] == "sklearn.linear_model.LogisticRegression" + + +def test_check_artifacts_missing(tmp_path): + result = health.check_artifacts(tmp_path) + assert result["status"] == "degraded" + assert set(result["missing"]) == set(health.ARTIFACT_FILES) + assert health.exit_code_for({"status": "degraded"}) == 0 + assert health.exit_code_for({"status": "degraded"}, strict=True) == 1 + + +def test_check_artifacts_ok(tmp_path): + joblib.dump(LogisticRegression(), tmp_path / "modelo.pkl") + joblib.dump({"scaler": None}, tmp_path / "transformers.pkl") + (tmp_path / "metricas.json").write_text( + json.dumps({"accuracy": 0.9}), encoding="utf-8" + ) + + result = health.check_artifacts(tmp_path) + assert result["status"] == "ok" + assert result["metrics_keys"] == ["accuracy"] + + +def test_check_artifacts_corrupt(tmp_path): + (tmp_path / "modelo.pkl").write_text("no es un pickle", encoding="utf-8") + (tmp_path / "transformers.pkl").write_text("no es un pickle", encoding="utf-8") + (tmp_path / "metricas.json").write_text("{}", encoding="utf-8") + + result = health.check_artifacts(tmp_path) + assert result["status"] == "error" + assert health.exit_code_for({"status": "error"}) == 1 + + +def test_check_s3_skipped_without_bucket(monkeypatch): + monkeypatch.delenv("S3_BUCKET_NAME", raising=False) + result = health.check_s3() + assert result["status"] == "skipped" + + +def test_build_report_skips_checks(monkeypatch): + monkeypatch.delenv("S3_BUCKET_NAME", raising=False) + report = health.build_report(skip_s3=True, skip_artifacts=True) + assert report["service"] == health.SERVICE_NAME + assert report["checks"]["config"]["status"] == "ok" + assert report["checks"]["artifacts"]["status"] == "skipped" + assert report["checks"]["s3"]["status"] == "skipped" + assert report["status"] == "ok" + + +def test_main_json_output(monkeypatch, capsys): + monkeypatch.delenv("S3_BUCKET_NAME", raising=False) + code = health.main(["--json", "--skip-s3", "--skip-artifacts"]) + assert code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["status"] == "ok" + + +def test_main_text_output(monkeypatch, capsys): + monkeypatch.delenv("S3_BUCKET_NAME", raising=False) + code = health.main(["--skip-s3", "--skip-artifacts"]) + assert code == 0 + assert "[OK]" in capsys.readouterr().out + + +def test_main_strict_degraded(monkeypatch, tmp_path): + monkeypatch.delenv("S3_BUCKET_NAME", raising=False) + monkeypatch.setattr(health, "ARTIFACT_DIR", tmp_path) + code = health.main(["--strict", "--skip-s3"]) + assert code == 1 + + +class _FakeS3: + def __init__(self): + self.bucket_kwargs = {} + self.object_kwargs = {} + + def head_bucket(self, **kwargs): + self.bucket_kwargs = kwargs + + def head_object(self, **kwargs): + self.object_kwargs = kwargs + + +class _FakeSts: + def get_caller_identity(self): + return {"Account": "123456789012"} + + +def test_check_s3_ok(monkeypatch): + monkeypatch.setenv("S3_BUCKET_NAME", "bucket-test") + monkeypatch.setenv("S3_MODEL_PREFIX", "models/") + fake_s3 = _FakeS3() + + def fake_client(service_name, **kwargs): + return _FakeSts() if service_name == "sts" else fake_s3 + + monkeypatch.setattr("boto3.client", fake_client) + result = health.check_s3() + assert result["status"] == "ok" + assert result["bucket"] == "bucket-test" + assert fake_s3.bucket_kwargs["ExpectedBucketOwner"] == "123456789012" + assert fake_s3.object_kwargs["Key"] == "models/metricas.json" + + +def test_check_s3_error(monkeypatch): + monkeypatch.setenv("S3_BUCKET_NAME", "bucket-test") + + def fake_client(service_name, **kwargs): + raise RuntimeError("sin credenciales") + + monkeypatch.setattr("boto3.client", fake_client) + result = health.check_s3() + assert result["status"] == "error" + assert "sin credenciales" in result["detail"] + + +def test_print_report(capsys): + report = { + "status": "ok", + "service": "svc", + "version": "1", + "checks": {"config": {"status": "ok", "detail": "ok"}}, + } + health.print_report(report) + out = capsys.readouterr().out + assert "[OK] svc v1" in out + assert "config: ok" in out