diff --git a/.github/workflows/documentation-checks.yml b/.github/workflows/documentation-checks.yml index 887c436..c77e596 100644 --- a/.github/workflows/documentation-checks.yml +++ b/.github/workflows/documentation-checks.yml @@ -48,6 +48,7 @@ jobs: python -m pytest -q --tb=short shared/tests/test_catalog.py shared/tests/test_cookbook_contract.py + shared/tests/test_countdown_cancellation.py shared/tests/test_dataset_validation.py shared/tests/test_config.py shared/tests/test_logging_naming.py diff --git a/managed_fine_tuning/03-core-rft/01-rft-countdown/.env.template b/managed_fine_tuning/03-core-rft/01-rft-countdown/.env.template index 5e2bdce..2766ae7 100644 --- a/managed_fine_tuning/03-core-rft/01-rft-countdown/.env.template +++ b/managed_fine_tuning/03-core-rft/01-rft-countdown/.env.template @@ -4,8 +4,6 @@ FOUNDRY_PROJECT_ENDPOINT= FOUNDRY_SUBSCRIPTION_ID= FOUNDRY_RESOURCE_GROUP= FOUNDRY_RESOURCE_LOCATION= -# Optional display label, for example East US 2. -FOUNDRY_REGION= # Optional deployment name; blank defaults to gpt-5.4-mini. FOUNDRY_GRADER_DEPLOYMENT= diff --git a/managed_fine_tuning/03-core-rft/01-rft-countdown/CANCELLATION_AND_CLEANUP.md b/managed_fine_tuning/03-core-rft/01-rft-countdown/CANCELLATION_AND_CLEANUP.md new file mode 100644 index 0000000..76ae992 --- /dev/null +++ b/managed_fine_tuning/03-core-rft/01-rft-countdown/CANCELLATION_AND_CLEANUP.md @@ -0,0 +1,81 @@ +# Countdown job cancellation and cleanup + +Countdown saves two jobs. The [first-SFT post-training examples](../../POST_TRAINING.md) +use a different state format and cannot be used unchanged here. + +## Before requesting cancellation + +Closing the notebook or timing out monitoring does **not** cancel training. +Cancellation loses progress and does not undo accrued charges. Obtain approval +for the specific project and job, including remaining cost uncertainty. Check +Foundry for jobs missing from local state after an ambiguous submission. + +From the Countdown demo directory, review the ignored +`outputs/submission-state.json` privately. Its `jobs` map holds `model-grader` +and `python-grader` IDs, but no per-job project endpoint in current runs. Check +the project independently in Foundry; an optional `job_endpoints` entry in +older state is checked but is **not** sufficient proof on its own. Do not +retroactively fill it from `.env`, or share populated state or credentials. + +With the managed environment installed, run this snippet **once** after +approval. It retrieves and confirms **one** selected job before requesting +cancellation; it does not cancel both jobs: + +```python +import json +from dataclasses import replace +from pathlib import Path + +from shared.auth import create_project_context +from shared.config import load_foundry_config + +state = json.loads(Path("outputs/submission-state.json").read_text(encoding="utf-8")) +label = input("Select one job (model-grader or python-grader): ").strip() +if label not in {"model-grader", "python-grader"}: + raise ValueError("Select exactly one known Countdown job label.") +job_id = state.get("jobs", {}).get(label) +if not isinstance(job_id, str) or not job_id.strip(): + raise RuntimeError("No recorded job ID for the selected label; reconcile in Foundry.") + +config = replace(load_foundry_config(".env"), allow_preview=True) +saved_endpoint = state.get("job_endpoints", {}).get(label) +if saved_endpoint is not None and saved_endpoint != config.project_endpoint: + raise RuntimeError("Saved job project does not match the configured project.") + +# Obtain this independently from Foundry, not from state or .env. +verified_endpoint = input("Paste the project endpoint from Foundry: ").strip().rstrip("/") +if verified_endpoint != config.project_endpoint: + raise RuntimeError("Foundry project does not match the configured project.") + +with create_project_context(config) as context: + current = context.openai_client.fine_tuning.jobs.retrieve(job_id) + if current.id != job_id or current.model != "qwen3.6-35b-a3b": + raise RuntimeError("Retrieved job does not match the Countdown state.") + status = str(current.status).casefold() + if status in {"succeeded", "failed", "cancelled", "canceled"}: + raise RuntimeError("The selected job is already terminal; do not cancel it.") + if status in {"cancelling", "canceling"}: + raise RuntimeError("Cancellation is already in progress; inspect status in Foundry.") + + if input("Type the selected job label to confirm: ").strip() != label: + raise RuntimeError("Job label confirmation did not match.") + if input("Paste the exact selected job ID to confirm: ").strip() != job_id: + raise RuntimeError("Job ID confirmation did not match.") + requested = context.openai_client.fine_tuning.jobs.cancel(job_id) + print("Cancellation requested; returned status:", requested.status) +``` + +**Do not blindly retry** a failed or ambiguous request. Check the job in +Foundry until terminal; a cancellation response is not proof it stopped. +Reconcile the other job separately and retain private evidence. + +## Other resources are separate operations + +- **Uploaded files:** the jobs may share them; identify all owners before + separately approving deletion and confirming each file ID. Countdown does + not save these IDs, so the first-SFT deletion example does not apply. +- **Grader deployment:** consult the ignored + `outputs/grader-deployment-state.json`. After relevant jobs are terminal, + separately approve restoring prior capacity or removing a deployment you + own; cancellation does neither. +- Keep final status, retained-file, deployment, and cost evidence private. \ No newline at end of file diff --git a/managed_fine_tuning/03-core-rft/01-rft-countdown/README.md b/managed_fine_tuning/03-core-rft/01-rft-countdown/README.md index 4ea3910..04ee5fd 100644 --- a/managed_fine_tuning/03-core-rft/01-rft-countdown/README.md +++ b/managed_fine_tuning/03-core-rft/01-rft-countdown/README.md @@ -34,4 +34,8 @@ Executing the deployment, upload, and submission cells can incur charges and sub Runtime state and downloaded results are written under ignored `outputs/`. The submission fingerprint includes the project, model, recipe, graders, response schema, and data hashes. A mismatch requires deliberate archival of the previous state. Job-creation POSTs use disabled SDK retries; reconcile an ambiguous submission in Foundry before rerunning. +For stopping either remote job and reconciling shared files and grader deployment +capacity, follow the [Countdown cancellation and cleanup guide](CANCELLATION_AND_CLEANUP.md). +Closing the notebook or stopping monitoring does not cancel a job. + Training reward is service telemetry, not held-out task-quality evidence. Actual results may vary by model version, data, configuration, region availability, and service conditions. diff --git a/managed_fine_tuning/03-core-rft/01-rft-countdown/notebooks/demo.ipynb b/managed_fine_tuning/03-core-rft/01-rft-countdown/notebooks/demo.ipynb index d19ed28..aeb5c3e 100644 --- a/managed_fine_tuning/03-core-rft/01-rft-countdown/notebooks/demo.ipynb +++ b/managed_fine_tuning/03-core-rft/01-rft-countdown/notebooks/demo.ipynb @@ -106,7 +106,6 @@ "RESOURCE_GROUP = os.getenv(\"FOUNDRY_RESOURCE_GROUP\", \"\").strip()\n", "RESOURCE_LOCATION = os.getenv(\"FOUNDRY_RESOURCE_LOCATION\", \"\").strip()\n", "\n", - "REGION = os.getenv(\"FOUNDRY_REGION\", RESOURCE_LOCATION).strip()\n", "MODEL = \"qwen3.6-35b-a3b\"\n", "MODEL_PUBLISHER = \"Alibaba\"\n", "MODEL_VERSION = \"1\"\n", @@ -125,7 +124,7 @@ "\n", "\n", "print(\"Project:\", PROJECT_ENDPOINT)\n", - "print(\"Region:\", REGION)\n", + "print(\"Region:\", RESOURCE_LOCATION)\n", "print(\"Model:\", f\"{MODEL_PUBLISHER}/{MODEL}, version {MODEL_VERSION}\")\n", "print(\"Training type:\", TRAINING_TYPE)\n", "print(\"Grader deployment:\", GRADER_DEPLOYMENT, GRADER_MODEL, GRADER_DEPLOYMENT_SKU)\n", @@ -369,7 +368,7 @@ "id": "countdown-07", "metadata": {}, "source": [ - "## 4. Upload the exact 100/50 training inputs" + "## 4. Check or update the grader deployment, then upload the exact 100/50 training inputs" ] }, { diff --git a/managed_fine_tuning/POST_TRAINING.md b/managed_fine_tuning/POST_TRAINING.md index b04d184..4d9a5bb 100644 --- a/managed_fine_tuning/POST_TRAINING.md +++ b/managed_fine_tuning/POST_TRAINING.md @@ -47,6 +47,9 @@ for model-specific deployment and inference support. Closing a notebook, timing out a polling loop, or closing local SDK clients does not cancel a remote job. Cancellation is a separate mutating operation. +For the two-job Countdown state, follow its +[cancellation and cleanup guide](03-core-rft/01-rft-countdown/CANCELLATION_AND_CLEANUP.md) +instead of the first-SFT example below. Run the following from the first-SFT cookbook directory after reviewing the ignored `outputs/submission-state.json`. It reconstructs a live client, verifies that the saved job belongs to the configured project, and requires you to paste diff --git a/managed_fine_tuning/shared/tests/test_countdown_cancellation.py b/managed_fine_tuning/shared/tests/test_countdown_cancellation.py new file mode 100644 index 0000000..217696f --- /dev/null +++ b/managed_fine_tuning/shared/tests/test_countdown_cancellation.py @@ -0,0 +1,150 @@ +"""Offline checks for the Countdown cancellation operator guide and job state.""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from shared import auth, config + + +DEMO_ROOT = Path(__file__).parents[2] / "03-core-rft" / "01-rft-countdown" +ENDPOINT = "https://example.services.ai.azure.com/api/projects/example" +JOB_ID = "example-job" + + +def test_countdown_submissions_keep_their_original_state_contract(tmp_path: Path) -> None: + notebook = json.loads((DEMO_ROOT / "notebooks/demo.ipynb").read_text(encoding="utf-8")) + source = "\n".join("\n".join(cell["source"]) for cell in notebook["cells"] if cell["cell_type"] == "code") + selected = [ + node for node in ast.parse(source).body + if isinstance(node, ast.FunctionDef) + and node.name in {"load_state", "save_state", "submit_once"} + ] + assert len(selected) == 3 + + class Jobs: + def __init__(self) -> None: + self.created = 0 + + def create(self, **kwargs): + self.created += 1 + return SimpleNamespace(id=JOB_ID) + + def retrieve(self, job_id): + return SimpleNamespace(id=job_id, status="running") + + class Client: + def __init__(self) -> None: + self.jobs = Jobs() + self.fine_tuning = SimpleNamespace(jobs=self.jobs) + + def with_options(self, **kwargs): + assert kwargs == {"max_retries": 0} + return self + + client = Client() + state_path = tmp_path / "submission-state.json" + values = { + "json": json, + "OUTPUT_ROOT": tmp_path, + "STATE_PATH": state_path, + "RUN_FINGERPRINT": "expected-fingerprint", + "PROJECT_ENDPOINT": ENDPOINT, + "openai_client": client, + "HYPERPARAMETERS": {}, + "RESPONSE_FORMAT": {}, + "MODEL": "qwen3.6-35b-a3b", + "TRAINING_TYPE": "globalStandard", + "training_file_id": "train", + "validation_file_id": "validation", + } + exec(compile(ast.Module(body=selected, type_ignores=[]), "countdown", "exec"), values) + values["submit_once"]("model-grader", {"type": "score_model"}, "suffix") + state = json.loads(state_path.read_text(encoding="utf-8")) + assert state["jobs"] == {"model-grader": JOB_ID} + assert "job_endpoints" not in state + + # Reusing an older recorded job must not invent provenance for that job. + legacy = {"fingerprint": "expected-fingerprint", "jobs": {"model-grader": JOB_ID}} + state_path.write_text(json.dumps(legacy), encoding="utf-8") + values["submit_once"]("model-grader", {"type": "score_model"}, "suffix") + assert json.loads(state_path.read_text(encoding="utf-8")) == legacy + assert client.jobs.created == 1 + + +@pytest.mark.parametrize( + ("saved_endpoint", "verified_endpoint", "status", "confirmed_label", "confirmed_id", "cancels"), + [ + (ENDPOINT, ENDPOINT, "running", "model-grader", JOB_ID, True), + ("https://other.services.ai.azure.com/api/projects/other", ENDPOINT, "running", "model-grader", JOB_ID, False), + (None, "https://other.services.ai.azure.com/api/projects/other", "running", "model-grader", JOB_ID, False), + (ENDPOINT, ENDPOINT, "succeeded", "model-grader", JOB_ID, False), + (ENDPOINT, ENDPOINT, "cancelling", "model-grader", JOB_ID, False), + (ENDPOINT, ENDPOINT, "running", "python-grader", JOB_ID, False), + (ENDPOINT, ENDPOINT, "running", "model-grader", "wrong-job", False), + (None, ENDPOINT, "running", "model-grader", JOB_ID, True), + ], +) +def test_countdown_guide_cancels_only_verified_selected_job( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + saved_endpoint: str | None, + verified_endpoint: str, + status: str, + confirmed_label: str, + confirmed_id: str, + cancels: bool, +) -> None: + guide = (DEMO_ROOT / "CANCELLATION_AND_CLEANUP.md").read_text(encoding="utf-8") + code = guide.split("```python\n", 1)[1].split("\n```", 1)[0] + compile(code, "countdown-guide", "exec") + + outputs = tmp_path / "outputs" + outputs.mkdir() + state = {"jobs": {"model-grader": JOB_ID, "python-grader": "other-job"}} + if saved_endpoint is not None: + state["job_endpoints"] = {"model-grader": saved_endpoint} + (outputs / "submission-state.json").write_text(json.dumps(state), encoding="utf-8") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(config, "load_foundry_config", lambda path: config.FoundryConfig(ENDPOINT)) + + class Jobs: + def __init__(self) -> None: + self.cancelled: list[str] = [] + + def retrieve(self, job_id: str): + assert job_id == JOB_ID + return SimpleNamespace(id=job_id, model="qwen3.6-35b-a3b", status=status) + + def cancel(self, job_id: str): + self.cancelled.append(job_id) + return SimpleNamespace(status="cancelling") + + class Context: + def __init__(self) -> None: + self.jobs = Jobs() + self.openai_client = SimpleNamespace(fine_tuning=SimpleNamespace(jobs=self.jobs)) + + def __enter__(self): + return self + + def __exit__(self, *_args): + pass + + context = Context() + monkeypatch.setattr(auth, "create_project_context", lambda selected: context) + inputs = iter(["model-grader", verified_endpoint, confirmed_label, confirmed_id]) + monkeypatch.setattr("builtins.input", lambda _prompt: next(inputs)) + + if cancels: + exec(compile(code, "countdown-guide", "exec"), {}) + assert context.jobs.cancelled == [JOB_ID] + else: + with pytest.raises(RuntimeError): + exec(compile(code, "countdown-guide", "exec"), {}) + assert context.jobs.cancelled == [] \ No newline at end of file