diff --git a/.github/TESTING.md b/.github/TESTING.md index 7b89fc50..019dfe34 100644 --- a/.github/TESTING.md +++ b/.github/TESTING.md @@ -22,23 +22,24 @@ Other users' comments, edited comments and other text are ignored. How it works: `run_tests_command.yml` (triggered by the comment, never checking out PR code) validates the request and dispatches `model_tests.yml` **on the pull request's -branch**, pinned to the PR head commit at that moment. A dispatched run is attached to -that commit, so every test job appears in the PR's checks list as -**Model tests / ...**, alongside the automatic checks. The comment gets a 🚀 reaction and -a reply with the tested commit. +branch**, pinned to the PR head commit at that moment. GitHub's PR checks box does not +list the jobs of a dispatched run, so the run publishes every test job as a **commit +status** on the tested commit (`Model tests / ...`): pending when the run starts, then +pass / fail when it ends, each linking to the job's log. They appear in the checks box +alongside the automatic checks. The comment gets a 🚀 reaction and a reply with the tested +commit. - A later push is not tested automatically: comment `/run-tests` again. A new request on the same PR cancels the previous run. - The branch must contain the current `model_tests.yml` (with its `workflow_dispatch` trigger); otherwise the reply asks to merge or rebase `main`. - **Pull requests from forks**: a fork's branch cannot be dispatched, so the tests run - from `main` on the PR head commit. They are not attached to the PR; the reply links - the run. + from `main` on the PR head commit; their statuses still appear in the PR's checks. - `run_tests_command.yml` only takes effect once it is on `main` (comment events always use the default branch's workflow files). **Model tests / Requested model, tokenizer & processor tests** passes only when every -test job passes. To make the suite block merging, require that check in the branch +test job passes. To make the suite block merging, require that status in the branch protection rules or ruleset. ## Releases @@ -50,4 +51,5 @@ pip caching and Codecov upload (both disabled for pull request runs). The comment workflow has `actions: write` (to dispatch) and comment permissions, and never checks out or executes PR code. The test jobs that run PR code have read-only repository -access, do not persist checkout credentials, and use no secrets or shared caches. +access, do not persist checkout credentials, and use no secrets or shared caches. Only the +two status-publishing jobs (which run no PR code) can write commit statuses. diff --git a/.github/workflows/model_tests.yml b/.github/workflows/model_tests.yml index 2a928306..785fcb9f 100644 --- a/.github/workflows/model_tests.yml +++ b/.github/workflows/model_tests.yml @@ -269,3 +269,72 @@ jobs: run: | echo "$RESULTS" | jq -r 'to_entries[] | "\(.key): \(.value.result)"' echo "$RESULTS" | jq -e 'all(.[]; .result == "success")' > /dev/null + + # GitHub's PR checks box does not list the jobs of a workflow_dispatch run, but it + # always lists commit statuses. These two jobs mirror every test job of a /run-tests run + # as a commit status on the tested PR commit: pending when the run starts, then the + # job's result, each linking to its log. Neither job checks out or runs PR code. + publish-pending-statuses: + name: Publish pending PR statuses + if: github.event_name == 'workflow_dispatch' && inputs.pr_sha != '' + runs-on: ubuntu-latest + permissions: + actions: read + statuses: write + steps: + - name: Mark every test job pending on the PR commit + uses: actions/github-script@v9 + env: + PR_SHA: ${{ inputs.pr_sha }} + with: + script: | + const internal = new Set([ + 'Lint & Format', 'Publish pending PR statuses', 'Publish PR statuses', + ]); + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + for (const job of jobs.filter(job => !internal.has(job.name))) { + await github.rest.repos.createCommitStatus({ + ...context.repo, + sha: process.env.PR_SHA, + state: 'pending', + context: `Model tests / ${job.name}`, + description: 'Running', + target_url: job.html_url, + }); + } + + publish-statuses: + name: Publish PR statuses + needs: [test-code, test-tokenizers, test-image-processors, test-processors, all-tests] + if: always() && github.event_name == 'workflow_dispatch' && inputs.pr_sha != '' + runs-on: ubuntu-latest + permissions: + actions: read + statuses: write + steps: + - name: Publish every test job's result on the PR commit + uses: actions/github-script@v9 + env: + PR_SHA: ${{ inputs.pr_sha }} + with: + script: | + const internal = new Set([ + 'Lint & Format', 'Publish pending PR statuses', 'Publish PR statuses', + ]); + const states = { success: 'success', failure: 'failure', timed_out: 'failure' }; + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + ...context.repo, run_id: context.runId, per_page: 100, + }); + for (const job of jobs.filter(job => !internal.has(job.name))) { + const conclusion = job.conclusion || 'cancelled'; + await github.rest.repos.createCommitStatus({ + ...context.repo, + sha: process.env.PR_SHA, + state: states[conclusion] || 'error', + context: `Model tests / ${job.name}`, + description: conclusion === 'success' ? 'Passed' : `Result: ${conclusion}`, + target_url: job.html_url, + }); + } diff --git a/.github/workflows/run_tests_command.yml b/.github/workflows/run_tests_command.yml index 29c4f22c..e4a2bf69 100644 --- a/.github/workflows/run_tests_command.yml +++ b/.github/workflows/run_tests_command.yml @@ -1,10 +1,10 @@ name: Run tests command # `/run-tests` from IMvision12 on a pull request starts the model, tokenizer and processor -# tests (model_tests.yml). A comment-triggered run is attached to the default branch, so -# its jobs would never appear in the PR's checks; instead this workflow dispatches -# model_tests.yml ON THE PR BRANCH, whose run is attached to the PR head commit and so -# lists every test job in the PR's checks. This job never checks out or runs PR code. +# tests: this workflow validates the request and dispatches model_tests.yml on the PR +# branch (a fork's PR: on the default branch), pinned to the PR head commit. That run +# publishes every test job as a commit status on the commit, which the PR's checks box +# lists. This job never checks out or runs PR code. on: issue_comment: types: [created] @@ -78,7 +78,7 @@ jobs: }); const where = sameRepo ? 'Every test job is listed in this PR\'s checks.' - : `This PR comes from a fork, so the run is not attached to the PR; follow it at ${runsUrl}.`; + : `Every test job is listed in this PR's checks (the run itself is at ${runsUrl}).`; await github.rest.issues.createComment({ ...context.repo, issue_number: issue.number, diff --git a/README.md b/README.md index 96b0675e..9f7089cf 100644 --- a/README.md +++ b/README.md @@ -44,19 +44,19 @@ Documentation sources are also available in [`docs/`](docs/). | 🏷️ Model Name | 📜 Reference Paper | 📦 Source of Weights | |---------------|-------------------|---------------------| -| [all-mpnet-base-v2](https://huggingface.co/zeromodels/all-mpnet-base-v2) | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | -| [multi-qa-mpnet-base-dot-v1](https://huggingface.co/zeromodels/multi-qa-mpnet-base-dot-v1) | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | -| [all-distilroberta-v1](https://huggingface.co/zeromodels/all-distilroberta-v1) | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | -| [all-MiniLM-L12-v2](https://huggingface.co/zeromodels/all-MiniLM-L12-v2) | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | -| [multi-qa-distilbert-cos-v1](https://huggingface.co/zeromodels/multi-qa-distilbert-cos-v1) | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | -| [all-MiniLM-L6-v2](https://huggingface.co/zeromodels/all-MiniLM-L6-v2) | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | -| [multi-qa-MiniLM-L6-cos-v1](https://huggingface.co/zeromodels/multi-qa-MiniLM-L6-cos-v1) | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | -| [paraphrase-multilingual-mpnet-base-v2](https://huggingface.co/zeromodels/paraphrase-multilingual-mpnet-base-v2) | [Making Monolingual Sentence Embeddings Multilingual using Knowledge Distillation](https://arxiv.org/abs/2004.09813) | `sentence-transformers` | -| [paraphrase-albert-small-v2](https://huggingface.co/zeromodels/paraphrase-albert-small-v2) | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | -| [paraphrase-multilingual-MiniLM-L12-v2](https://huggingface.co/zeromodels/paraphrase-multilingual-MiniLM-L12-v2) | [Making Monolingual Sentence Embeddings Multilingual using Knowledge Distillation](https://arxiv.org/abs/2004.09813) | `sentence-transformers` | -| [paraphrase-MiniLM-L3-v2](https://huggingface.co/zeromodels/paraphrase-MiniLM-L3-v2) | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | -| [distiluse-base-multilingual-cased-v1](https://huggingface.co/zeromodels/distiluse-base-multilingual-cased-v1) | [Making Monolingual Sentence Embeddings Multilingual using Knowledge Distillation](https://arxiv.org/abs/2004.09813) | `sentence-transformers` | -| [distiluse-base-multilingual-cased-v2](https://huggingface.co/zeromodels/distiluse-base-multilingual-cased-v2) | [Making Monolingual Sentence Embeddings Multilingual using Knowledge Distillation](https://arxiv.org/abs/2004.09813) | `sentence-transformers` | +| all-mpnet-base-v2 | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | +| multi-qa-mpnet-base-dot-v1 | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | +| all-distilroberta-v1 | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | +| all-MiniLM-L12-v2 | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | +| multi-qa-distilbert-cos-v1 | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | +| all-MiniLM-L6-v2 | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | +| multi-qa-MiniLM-L6-cos-v1 | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | +| paraphrase-multilingual-mpnet-base-v2 | [Making Monolingual Sentence Embeddings Multilingual using Knowledge Distillation](https://arxiv.org/abs/2004.09813) | `sentence-transformers` | +| paraphrase-albert-small-v2 | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | +| paraphrase-multilingual-MiniLM-L12-v2 | [Making Monolingual Sentence Embeddings Multilingual using Knowledge Distillation](https://arxiv.org/abs/2004.09813) | `sentence-transformers` | +| paraphrase-MiniLM-L3-v2 | [Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks](https://arxiv.org/abs/1908.10084) | `sentence-transformers` | +| distiluse-base-multilingual-cased-v1 | [Making Monolingual Sentence Embeddings Multilingual using Knowledge Distillation](https://arxiv.org/abs/2004.09813) | `sentence-transformers` | +| distiluse-base-multilingual-cased-v2 | [Making Monolingual Sentence Embeddings Multilingual using Knowledge Distillation](https://arxiv.org/abs/2004.09813) | `sentence-transformers` |
diff --git a/zeromodels/__init__.py b/zeromodels/__init__.py index 7e0bfa61..765584f1 100644 --- a/zeromodels/__init__.py +++ b/zeromodels/__init__.py @@ -16,7 +16,7 @@ ) from zeromodels.version import version -__version__ = "1.3.5" +__version__ = "1.3.6" __all__ = [ "auto", diff --git a/zeromodels/version.py b/zeromodels/version.py index 4ec97a1d..f47b1bfa 100644 --- a/zeromodels/version.py +++ b/zeromodels/version.py @@ -1,4 +1,4 @@ -__version__ = "1.3.5" +__version__ = "1.3.6" def version():