diff --git a/.github/workflows/documentation-checks.yml b/.github/workflows/documentation-checks.yml index c77e596..36f8aba 100644 --- a/.github/workflows/documentation-checks.yml +++ b/.github/workflows/documentation-checks.yml @@ -6,7 +6,11 @@ on: - '*.md' - 'tests/**' - 'managed_fine_tuning/**' - - 'interactive_training/docs/supported_models.md' + - 'Sample_Datasets/**' + - 'interactive_training/README.md' + - 'interactive_training/docs/**' + - 'interactive_training/envs/**/*.md' + - 'interactive_training/interactive_training/recipes/**/*.md' - '.github/requirements-docs.txt' - '.github/workflows/documentation-checks.yml' push: @@ -15,7 +19,11 @@ on: - '*.md' - 'tests/**' - 'managed_fine_tuning/**' - - 'interactive_training/docs/supported_models.md' + - 'Sample_Datasets/**' + - 'interactive_training/README.md' + - 'interactive_training/docs/**' + - 'interactive_training/envs/**/*.md' + - 'interactive_training/interactive_training/recipes/**/*.md' - '.github/requirements-docs.txt' - '.github/workflows/documentation-checks.yml' diff --git a/AGENTS.md b/AGENTS.md index 57f059d..933808c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,7 @@ python -m unittest discover -s tests -p "test_*.py" Use the repository workflow's selected managed tests for the managed contracts. Run interactive tests from `interactive_training` using its test environment. Tests validate local contracts, not cloud eligibility. -Dependency installation, network dataset downloads, credential prompts, or resource reads are not implied by an offline preflight. Explain them and obtain approval where needed. +Dependency installation, network downloads, authentication, and resource reads are not offline checks. When needed for the requested workflow, explain them briefly and proceed within that scope using the selected environment and supported credential providers. Ask for missing information that changes the scope, not repeated approval for routine setup or reads; never request secrets in chat. ## Paid execution: warn, then proceed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6ccc627..3238349 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,26 +51,21 @@ chances of your issue being dealt with quickly: * **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be causing the problem (line of code or commit) -You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/Azure/fine-tuning/issues/new. +You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/microsoft-foundry/fine-tuning/issues/new. ### Submitting a Pull Request (PR) Before you submit your Pull Request (PR) consider the following guidelines: -* Search the repository (https://github.com/Azure/fine-tuning/pulls) for an open or closed PR +* Search the repository (https://github.com/microsoft-foundry/fine-tuning/pulls) for an open or closed PR that relates to your submission. You don't want to duplicate effort. -* Make your changes in a new git fork: +* Create a feature branch from an up-to-date `main` in your fork. * Commit your changes using a descriptive commit message -* Push your fork to GitHub: -* In GitHub, create a pull request +* Push your feature branch to GitHub. +* In GitHub, create a pull request targeting this repository's `main` branch. * If we suggest changes then: * Make the required updates. - * Rebase your fork and force push to your GitHub repository (this will update your Pull Request): - - ```shell - git rebase master -i - git push -f - ``` + * Commit and push the updates to the same feature branch; the existing pull request updates automatically. That's it! Thank you for your contribution! \ No newline at end of file diff --git a/Sample_Datasets/README.md b/Sample_Datasets/README.md index 71772c6..acd288f 100644 --- a/Sample_Datasets/README.md +++ b/Sample_Datasets/README.md @@ -39,7 +39,7 @@ We offer three training techniques to optimize your models: - *When to use:* RFT is ideal for objective domains like mathematics, chemistry, and physics where there are clear right and wrong answers and the model already shows some competency. It works best when lucky guessing is difficult and expert evaluators would consistently agree on an unambiguous, correct answer. Requires more ML expertise to implement effectively. - *Supported Models:* o4-mini - *Sample Datasets*: - - [Clause Matching](./Reinforcement_Fine_Tuning/clause-matching/) - legal contract dataset + - [Clause Matching](./Reinforcement_Fine_Tuning/ClauseMatching/) - legal contract dataset - [Med MCQ](./Reinforcement_Fine_Tuning/MedMCQ/) - Multiple Choice Medical Q&A **Most customers should start with SFT,** as it addresses the broadest number of fine-tuning use cases. \ No newline at end of file diff --git a/Sample_Datasets/Reinforcement_Fine_Tuning/MedMCQ/README.md b/Sample_Datasets/Reinforcement_Fine_Tuning/MedMCQ/README.md index 39ed10f..ed8a981 100644 --- a/Sample_Datasets/Reinforcement_Fine_Tuning/MedMCQ/README.md +++ b/Sample_Datasets/Reinforcement_Fine_Tuning/MedMCQ/README.md @@ -1,6 +1,6 @@ # Notice -This is a sample training and validation data, along with a sample grader. This was curated using [this](https://github.com/medmcqa/medmcqa) as the reference dataset. **This is a very large dataset and may cause time outs using the RFT service**. If you're just getting started, consider using our simple RFT [Countdown Dataset](https://github.com/microsoft-foundry/fine-tuning/tree/main/Demos/RFT_Countdown) +This is a sample training and validation data, along with a sample grader. This was curated using [this](https://github.com/medmcqa/medmcqa) as the reference dataset. **This is a very large dataset and may cause time outs using the RFT service**. If you're just getting started, consider using our simple RFT [Countdown dataset and demo](../../../managed_fine_tuning/03-core-rft/01-rft-countdown/README.md). ## Dataset Summary diff --git a/Sample_Datasets/Supervised_Fine_Tuning/Text-NL-to-Python/README.md b/Sample_Datasets/Supervised_Fine_Tuning/Text-NL-to-Python/README.md index e4a61fc..629b1e5 100644 --- a/Sample_Datasets/Supervised_Fine_Tuning/Text-NL-to-Python/README.md +++ b/Sample_Datasets/Supervised_Fine_Tuning/Text-NL-to-Python/README.md @@ -36,7 +36,7 @@ Standard chat messages JSONL — each line is a JSON object with a `messages` ar ## Related Demo -See the full end-to-end notebook in [Demos/NL_to_Python_Distillation](../../Demos/NL_to_Python_Distillation/) which walks through data generation, fine-tuning, deployment, and evaluation. +See the maintained [code-distillation demo](../../../managed_fine_tuning/05-data-and-distillation/02-code-distillation/README.md) for a training-only workflow. That demo preserves a different, hash-pinned dataset of 1,576 training and 83 validation examples and covers upload, training, and service metrics—not data generation, deployment, or task-quality evaluation. ## Languages diff --git a/interactive_training/README.md b/interactive_training/README.md index 6cf4a01..cda0833 100644 --- a/interactive_training/README.md +++ b/interactive_training/README.md @@ -50,7 +50,7 @@ Python imports use `azure.ai.finetuningsessions`, including the `aio` namespace. No local fine-tuning SDK build, sibling source checkout, or bundled SDK wheel is required. ```bash -git clone https://github.com/johnwu0604/fine-tuning.git +git clone https://github.com/microsoft-foundry/fine-tuning.git cd fine-tuning/interactive_training python -m venv .venv source .venv/bin/activate # Linux / macOS / WSL only diff --git a/interactive_training/docs/README.md b/interactive_training/docs/README.md index d65a6f5..eb25c76 100644 --- a/interactive_training/docs/README.md +++ b/interactive_training/docs/README.md @@ -13,7 +13,7 @@ Follow a complete learning journey, or use the reference pages for an existing r | Did training help, and how do I use the result? | [Evaluation and inference](./evaluation-and-inference.md) | Comparable held-out measurements and sampling from a saved checkpoint | | Where are my artifacts? | [Storage](./storage.md) and [dashboard](./dashboard.md) | Config, metrics, ledger, and sensitive artifacts can be located and interpreted | | How do I recover or free resources? | [Recovery](./training.md#resuming-after-an-interruption), [continual training](./continual-fine-tuning.md), [session management](./session-management.md) | The intended checkpoint/cursor is loaded; owned sessions are inspected and explicitly closed | -| What can a coding agent safely run? | [Repository execution contract](https://github.com/johnwu0604/fine-tuning/blob/main/AGENTS.md) and [contributing](./contributing.md) | Offline checks are separated from downloads and paid/destructive operations | +| What can a coding agent safely run? | [Repository execution contract](https://github.com/microsoft-foundry/fine-tuning/blob/main/AGENTS.md) and [contributing](./contributing.md) | Offline checks are separated from downloads and paid/destructive operations | ## CLI conventions diff --git a/interactive_training/docs/custom-data.md b/interactive_training/docs/custom-data.md index 4575476..e5c2dd6 100644 --- a/interactive_training/docs/custom-data.md +++ b/interactive_training/docs/custom-data.md @@ -80,9 +80,9 @@ example below sets `preflight_dataset=True` and prepares them locally. Preflight reports the split and failing batch when rendering raises a validation error; context and truncation checks only run if the builder forwards `model_context_length` and `fail_on_truncation`. Floor-batched tail rows are not -scanned. Do not assume that accepting these options means a builder enforces -them: the [Tulu3 builder](../interactive_training/recipes/tulu3_sft/README.md#sft-on-tulu3) -currently does not forward them, even though its launcher offers preflight. +scanned. The conversation-file and +[Tulu3 builders](../interactive_training/recipes/tulu3_sft/README.md#sft-on-tulu3) +both forward these checks; custom builders must do the same to enforce them. ### 3. Wire the builder into an SFT launcher diff --git a/interactive_training/docs/quickstart.md b/interactive_training/docs/quickstart.md index 9cf3ecc..7b258b4 100644 --- a/interactive_training/docs/quickstart.md +++ b/interactive_training/docs/quickstart.md @@ -25,7 +25,7 @@ No local GPU or local model-weight download is required for this text recipe. Ke For a new checkout: ```bash -git clone https://github.com/johnwu0604/fine-tuning.git +git clone https://github.com/microsoft-foundry/fine-tuning.git cd fine-tuning/interactive_training ``` diff --git a/interactive_training/docs/training.md b/interactive_training/docs/training.md index 26d509e..73e96a4 100644 --- a/interactive_training/docs/training.md +++ b/interactive_training/docs/training.md @@ -94,7 +94,7 @@ The defaults below are from **math Azure `CLIConfig`** and **Tulu3 Azure `CLICon | `temperature` | `1.0` | Not a training-target parameter | Sampling diversity. Record it for comparisons; a model-specific quality claim requires evaluation. | | `group_size` | `4` | Not applicable | Completions per training prompt; increases rollout cost and changes advantage estimates. | | `groups_per_batch` | `100` | Not applicable | Prompts per RL iteration; smaller examples are easier to inspect. | -| `max_length` | Not applicable | `16384` | **Rendered sequence** budget, not output length. Tulu3 truncates to this budget; its current builder does not forward the fail-fast/context fields ([limitation](../interactive_training/recipes/tulu3_sft/README.md#sft-on-tulu3)). | +| `max_length` | Not applicable | `16384` | **Rendered sequence** budget, not output length. Tulu3 truncates by default; `fail_on_truncation=true` rejects overlength examples and `model_context_length` checks the resulting training input ([preflight](../interactive_training/recipes/tulu3_sft/README.md#sft-on-tulu3)). | | `batch_size` | Not applicable | `128` | SFT rows per full batch. Floor batching can omit tail rows or yield no batches; see [custom data](./custom-data.md#2-render-and-inspect-locally). | | `num_epochs` | One dataset traversal | `1` | Dataset passes; choose using validation rather than a universal epoch recommendation. | | `lr_schedule` | Constant recipe learning rate | `linear` | SFT schedule supports `linear`, `cosine`, or `constant`; a step cap does not redefine its full-dataset schedule denominator. | diff --git a/interactive_training/docs/troubleshooting.md b/interactive_training/docs/troubleshooting.md index c4bae7b..6c1bbf6 100644 --- a/interactive_training/docs/troubleshooting.md +++ b/interactive_training/docs/troubleshooting.md @@ -89,7 +89,7 @@ A typical Azure VM workflow: ```bash # one-time on the VM az login -git clone https://github.com/johnwu0604/fine-tuning.git +git clone https://github.com/microsoft-foundry/fine-tuning.git cd fine-tuning/interactive_training python -m venv .venv && source .venv/bin/activate # Offer CPU Torch for driver utilities; inspect the selected build. @@ -195,7 +195,7 @@ The quickstart's tokenizer check has **no Azure call** but can download files. T ## SFT truncation or empty loss masks -Check which builder actually forwards the context/truncation fields. The current [Tulu3 builder](../interactive_training/recipes/tulu3_sft/README.md#sft-on-tulu3) does not, even though the CLI accepts them. A successful preflight is not proof that labels survived; inspect decoded batches, rendered lengths, and positive `weights` locally before a remote launch. +The [Tulu3 builder](../interactive_training/recipes/tulu3_sft/README.md#sft-on-tulu3) and conversation-file builder enforce `fail_on_truncation` and `model_context_length` when configured. For custom builders, verify that they forward those fields. A successful preflight is not proof that labels survived; inspect decoded batches, rendered lengths, and positive `weights` locally before a remote launch. For builders that enforce the checks, shorten or split overlength examples, or raise `max_length` only within the verified model/service context budget. Lowering `max_length` cannot preserve content that already exceeds it. Image inputs also consume context. Separately, a split smaller than `batch_size` can produce zero full batches; changing length alone does not fix that. Keep fail-fast checks enabled when preservation is required and follow the [custom-data inspection](./custom-data.md#2-render-and-inspect-locally). @@ -209,7 +209,7 @@ If a request is terminally failed, preserve its error code/request ID and follow Check `*/env/all/ac_tokens_frac_at_max`, completion-length tails, `format`, scored-episode coverage, and actual trajectory HTML. `max_tokens` includes reasoning; the math default of five tokens is for toy arithmetic, not GSM8K. Renderer selection must match the model. Changing reasoning mode or budget can change results independently of training, so keep the comparison contract fixed. -Check labels, loss masks, grader correctness, split leakage, constant-reward filtering, learning rate, and number of real updates before spending a larger budget. Training reward, held-out NLL, and task accuracy are different measurements. Use [evaluation and inference](./evaluation-and-inference.md); a workflow smoke test can succeed with zero quality gain, and missing evaluation is not a zero score. +If held-out quality stalls or regresses, inspect labels, loss masks, grader correctness, split leakage, constant-reward filtering, learning rate, and the number of real updates rather than increasing compute blindly. Training reward, held-out NLL, and task accuracy are different measurements. Use [evaluation and inference](./evaluation-and-inference.md); a workflow smoke test can succeed with zero quality gain, and missing evaluation is not a zero score. ## Minimal support evidence diff --git a/interactive_training/interactive_training/recipes/tulu3_sft/README.md b/interactive_training/interactive_training/recipes/tulu3_sft/README.md index 71395bb..1ea2ae7 100644 --- a/interactive_training/interactive_training/recipes/tulu3_sft/README.md +++ b/interactive_training/interactive_training/recipes/tulu3_sft/README.md @@ -28,17 +28,15 @@ allocating a session, or `preflight_only=true` to return after that scan without training. This can download tokenizers/data and write local metadata; floor batching can omit tail source rows ([batching caveat](../../../docs/custom-data.md#2-render-and-inspect-locally)). -> [!WARNING] -> **Existing Tulu3 limitation:** the CLI accepts `fail_on_truncation` and -> `model_context_length`, but [Tulu3Builder](./chat_datasets.py) does not pass -> them to `conversation_to_datum`. Setting those flags does not currently -> reject truncated/over-context rows here. Preflight can succeed after -> truncation to `max_length`, even with no remaining assistant loss tokens; -> it is not a guarantee of complete labels or service-context validity. -> Inspect rendered lengths and positive loss weights locally. The existing -> [conversation-file builder example](../../../docs/custom-data.md#2-render-and-inspect-locally) -> forwards the checks, but requires its own programmatic launcher; it is not -> a replacement Tulu3 CLI flag. This limitation is documented, not changed. +The [Tulu3 builder](./chat_datasets.py) enforces `fail_on_truncation=true` +by rejecting rendered examples longer than `max_length`. Set +`model_context_length` to the verified model/service limit to reject training +inputs exceeding that limit. Both checks apply to training and evaluation +batches; the defaults retain truncation and do not assume a context limit. +Combine them with preflight to catch invalid full batches before allocating a +session. Inspect rendered lengths and positive loss weights locally: length +checks alone do not guarantee assistant loss tokens, and floor-batched tail +rows are not scanned ([data inspection](../../../docs/custom-data.md#2-render-and-inspect-locally)). Keep `max_steps` **positive** when budgeting: zero/`None` disables the SFT cap, not an evaluation-only mode. Compare held-out `test/nll` and task-specific diff --git a/interactive_training/interactive_training/recipes/tulu3_sft/chat_datasets.py b/interactive_training/interactive_training/recipes/tulu3_sft/chat_datasets.py index e2529ae..110cb3a 100644 --- a/interactive_training/interactive_training/recipes/tulu3_sft/chat_datasets.py +++ b/interactive_training/interactive_training/recipes/tulu3_sft/chat_datasets.py @@ -46,10 +46,14 @@ def build(self) -> tuple[SupervisedDataset, SupervisedDataset]: else TrainOnWhat.LAST_ASSISTANT_MESSAGE ) - # take the last 1000 as test, the rest as train def map_fn(row: dict) -> Datum: return conversation_to_datum( - row["messages"], self.renderer, self.common_config.max_length, train_on_what + row["messages"], + self.renderer, + self.common_config.max_length, + train_on_what, + model_context_length=self.common_config.model_context_length, + fail_on_truncation=self.common_config.fail_on_truncation, ) return SupervisedDatasetFromHFDataset( diff --git a/interactive_training/tests/test_tulu3_preflight.py b/interactive_training/tests/test_tulu3_preflight.py index c91c2b3..c643e7a 100644 --- a/interactive_training/tests/test_tulu3_preflight.py +++ b/interactive_training/tests/test_tulu3_preflight.py @@ -1,10 +1,114 @@ """The documented SFT preflight rejects bad data before allocating a session.""" +from types import SimpleNamespace from unittest.mock import AsyncMock, Mock +import datasets import pytest +import torch +from azure.ai.finetuningsessions.models import ModelInput, ModelInputChunk from interactive_training.recipes.tulu3_sft import train_azure as recipe +from interactive_training.supervised.types import ChatDatasetBuilderCommonConfig + + +@pytest.fixture +def synthetic_tulu3(monkeypatch): + """Exercise the real builder without dataset or tokenizer downloads.""" + source = datasets.Dataset.from_list( + [{"messages": [{"role": "assistant", "content": "example"}]}] * 1025 + ) + monkeypatch.setattr( + datasets, + "load_dataset", + Mock(return_value=datasets.DatasetDict({"train": source})), + ) + renderer = SimpleNamespace( + build_supervised_example=Mock( + return_value=( + ModelInput(chunks=[ModelInputChunk(tokens=list(range(10)))]), + torch.ones(10), + ) + ) + ) + monkeypatch.setattr(recipe.Tulu3Builder, "renderer", property(lambda _: renderer)) + + +@pytest.mark.parametrize("split_index", [0, 1], ids=["training", "evaluation"]) +@pytest.mark.parametrize( + ("max_length", "model_context_length", "fail_on_truncation", "message"), + [ + (8, None, True, "training max_length of 8"), + (None, 8, False, "model context length of 8"), + ], +) +def test_tulu3_builder_enforces_configured_limits( + synthetic_tulu3, split_index, max_length, model_context_length, + fail_on_truncation, message, +): + builder = recipe.Tulu3Builder( + common_config=ChatDatasetBuilderCommonConfig( + model_name_for_tokenizer="unused", + renderer_name="unused", + batch_size=1, + max_length=max_length, + model_context_length=model_context_length, + fail_on_truncation=fail_on_truncation, + ), + max_train_examples=1, + max_test_examples=1, + ) + split = builder.build()[split_index] + with pytest.raises(ValueError, match=message): + split.get_batch(0) + + +@pytest.mark.parametrize("split_index", [0, 1], ids=["training", "evaluation"]) +@pytest.mark.parametrize( + ("max_length", "fail_on_truncation", "expected_tokens"), + [(10, True, 9), (8, False, 7)], + ids=["exact-boundary", "legacy-truncation"], +) +def test_tulu3_builder_preserves_valid_and_default_behavior( + synthetic_tulu3, split_index, max_length, fail_on_truncation, expected_tokens, +): + builder = recipe.Tulu3Builder( + common_config=ChatDatasetBuilderCommonConfig( + model_name_for_tokenizer="unused", + renderer_name="unused", + batch_size=1, + max_length=max_length, + model_context_length=9, + fail_on_truncation=fail_on_truncation, + ), + max_train_examples=1, + max_test_examples=1, + ) + datum = builder.build()[split_index].get_batch(0)[0] + assert sum(len(chunk.tokens) for chunk in datum.model_input.chunks) == expected_tokens + + +@pytest.mark.asyncio +async def test_real_tulu3_preflight_rejects_truncation_before_session( + synthetic_tulu3, monkeypatch, tmp_path, +): + monkeypatch.delenv("INTERACTIVE_POST_TRAINING_API_ENDPOINT", raising=False) + create = Mock(side_effect=AssertionError("must not allocate a session")) + monkeypatch.setattr("azure.ai.finetuningsessions.FineTuningSession.create", create) + with pytest.raises(ValueError, match="Training dataset preflight failed"): + await recipe.cli_main( + recipe.CLIConfig( + project_endpoint="https://example.invalid/api/projects/test", + log_path=str(tmp_path / "run"), + preflight_only=True, + fail_on_truncation=True, + max_length=8, + batch_size=1, + max_train_examples=1, + max_test_examples=1, + ) + ) + create.assert_not_called() @pytest.mark.asyncio diff --git a/managed_fine_tuning/05-data-and-distillation/01-teacher-student-basics/data/README.md b/managed_fine_tuning/05-data-and-distillation/01-teacher-student-basics/data/README.md index 8988bdc..d386642 100644 --- a/managed_fine_tuning/05-data-and-distillation/01-teacher-student-basics/data/README.md +++ b/managed_fine_tuning/05-data-and-distillation/01-teacher-student-basics/data/README.md @@ -13,6 +13,7 @@ accepted example uses `Clippy is a factual chatbot that is also sarcastic.` as its system prompt. Labels are neither regenerated nor normalized. The notebook checks hashes, unique accepted questions, membership in the 40 -candidates and separation from the 10 held-out questions. The original source -workflow remains in `Demos/DistillingSarcasm/sarcasm.ipynb`; no source-demo runtime -files are required to execute the cookbook. +candidates and separation from the 10 held-out questions. The historical +teacher/judge workflow came from `Demos/DistillingSarcasm/sarcasm.ipynb`; +that source notebook is not included in this cookbook. Only the preserved data +and maintained training notebook are required here. diff --git a/managed_fine_tuning/05-data-and-distillation/02-code-distillation/data/README.md b/managed_fine_tuning/05-data-and-distillation/02-code-distillation/data/README.md index c918b4b..9240762 100644 --- a/managed_fine_tuning/05-data-and-distillation/02-code-distillation/data/README.md +++ b/managed_fine_tuning/05-data-and-distillation/02-code-distillation/data/README.md @@ -8,8 +8,9 @@ their source paths and byte hashes. These exact source files, including generated code examples, are the training contract. The cookbook does not substitute a smaller sample or rerun generation, filtering or judging. It checks JSON/chat shape and exact row overlap before -upload. The original workflow remains in -`Demos/NL_to_Python_Distillation/Text_to_Python_Fine_Tuning.ipynb`. +upload. The historical generation/filtering workflow came from +`Demos/NL_to_Python_Distillation/Text_to_Python_Fine_Tuning.ipynb`; +that source notebook is not included in this cookbook. The preserved source training recipe is 1 epoch, batch size 1 and learning-rate multiplier 1.3 with supervised `GlobalStandard` training. diff --git a/managed_fine_tuning/05-data-and-distillation/03-synthetic-tool-use/data/README.md b/managed_fine_tuning/05-data-and-distillation/03-synthetic-tool-use/data/README.md index b77d553..c084d40 100644 --- a/managed_fine_tuning/05-data-and-distillation/03-synthetic-tool-use/data/README.md +++ b/managed_fine_tuning/05-data-and-distillation/03-synthetic-tool-use/data/README.md @@ -16,8 +16,9 @@ Python `random.Random(42)` shuffles row indices; training receives The service returned 30 unique rows in this exact dataset: 24 training, 3 validation and 3 test rows. Requested maximum sample counts are not actual returned counts. All bytes are retained without generation, semantic filtering -or renderer normalization. The original generation/split implementation remains -in `Demos/SyntheticDatagen-ToolUse/notebook.ipynb`. +or renderer normalization. The historical generation/split implementation came +from `Demos/SyntheticDatagen-ToolUse/notebook.ipynb`; +that source notebook is not included in this cookbook. Only training and validation are uploaded. The supervised training recipe is 3 epochs and learning-rate multiplier 1.0 with `GlobalStandard`. diff --git a/managed_fine_tuning/shared/retry.py b/managed_fine_tuning/shared/retry.py index 6cc7f35..5e292ec 100644 --- a/managed_fine_tuning/shared/retry.py +++ b/managed_fine_tuning/shared/retry.py @@ -97,7 +97,7 @@ def retry_call( clock: Callable[[], float] = time.monotonic, random_values: Iterable[float] | None = None, ) -> T: - """Execute an operation with bounded exponential backoff and jitter.""" + """Bound retry attempts and backoff, without interrupting an in-flight operation.""" started = clock() random_iterator = iter(random_values) if random_values is not None else None for attempt in range(1, policy.max_attempts + 1): @@ -131,4 +131,9 @@ def retry_call( if on_retry: on_retry(attempt, delay, error) sleep(delay) + if clock() - started >= policy.timeout_seconds: + raise RetryError( + f"{operation_name} cannot retry within its " + f"{policy.timeout_seconds:g}-second timeout" + ) from error raise AssertionError("retry loop ended unexpectedly") diff --git a/managed_fine_tuning/shared/tests/test_retry.py b/managed_fine_tuning/shared/tests/test_retry.py index def0209..03daa86 100644 --- a/managed_fine_tuning/shared/tests/test_retry.py +++ b/managed_fine_tuning/shared/tests/test_retry.py @@ -69,3 +69,40 @@ def test_raises_explicit_error_when_exhausted() -> None: ), sleep=lambda _: None, ) + + +@pytest.mark.parametrize("elapsed_after_sleep", [1.0, 1.5]) +def test_does_not_start_another_attempt_after_retry_deadline( + elapsed_after_sleep: float, +) -> None: + attempts = 0 + now = 0.0 + transient = ServiceError("temporary", 503) + + def operation() -> str: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise transient + return "must not run after the deadline" + + def sleep(delay: float) -> None: + nonlocal now + assert delay == 1.0 + now = elapsed_after_sleep + + with pytest.raises(RetryError, match="cannot retry within") as error: + retry_call( + operation, + policy=RetryPolicy( + max_attempts=2, + initial_delay_seconds=1, + jitter_ratio=0, + timeout_seconds=1, + ), + sleep=sleep, + clock=lambda: now, + random_values=[0.5], + ) + assert attempts == 1 + assert error.value.__cause__ is transient diff --git a/tests/test_root_documentation.py b/tests/test_root_documentation.py index 6c47e3a..206c618 100644 --- a/tests/test_root_documentation.py +++ b/tests/test_root_documentation.py @@ -80,6 +80,33 @@ def test_root_navigation_and_anchors(self): with self.subTest(document=document.name): check_links(document) + def test_sample_dataset_navigation_and_anchors(self): + documents = list((ROOT / "Sample_Datasets").rglob("*.md")) + self.assertTrue(documents) + for document in documents: + with self.subTest(document=document.relative_to(ROOT).as_posix()): + check_links(document) + + def test_repository_links_use_canonical_upstream(self): + documents = list(ROOT.glob("*.md")) + for directory in ( + "Sample_Datasets", "managed_fine_tuning", "interactive_training/docs", + "interactive_training/envs", "interactive_training/interactive_training/recipes", + ): + documents.extend((ROOT / directory).rglob("*.md")) + documents.append(ROOT / "interactive_training/README.md") + references = 0 + for document in documents: + text = document.read_text(encoding="utf-8") + for owner in re.findall( + r"https://github\.com/([^/\s]+)/fine-tuning(?=\.git\b|[/#\s)`]|$)", + text, + ): + references += 1 + with self.subTest(document=document.relative_to(ROOT).as_posix()): + self.assertEqual(owner, "microsoft-foundry") + self.assertGreater(references, 0, "Repository URL check must not pass vacuously") + def test_checker_rejects_missing_paths_and_fragments(self): with TemporaryDirectory() as directory: document = Path(directory) / "README.md"