Replace serviceFamily sharding with a single traversal per price type

.github/workflows/opendata-ci.yml

@@ -8,7 +8,7 @@ on:
       - 'src/open-data/*.json'
       # Internal operational baselines for the eligibility completeness guard, not
       # reference data; Build-OpenData ignores them, so they shouldn't trigger CI.
-      - '!src/open-data/*.shardcounts.json'
+      - '!src/open-data/*.familycounts.json'
 permissions: 
   contents: write
   pull-requests: write
@@ -45,6 +45,15 @@ jobs:
           git commit -a -m "${{ env.CI_COMMIT_MESSAGE }}"
           git push
         }
+    # Build-OpenData.ps1 -Test calls Test-PowerShell.ps1, which requires Pester 6 (#2254):
+    # the suite uses -AllowNullOrEmptyForEach, which Pester 5 rejects. The ubuntu runner
+    # image ships Pester 5.9.0, so the minimum has to be installed explicitly. This is the
+    # same command Init-Repo.ps1 runs and the one Test-PowerShell.ps1 names in its error.
+    # dev.yml pins the same floor through psmodulecache, which this repo only uses on
+    # Windows runners.
+    - name: Install Pester
+      shell: pwsh
+      run: Install-Module -Name Pester -MinimumVersion 6.0.0 -Scope CurrentUser -Repository PSGallery -Force
     - name: Test Open Data
       id: test
       shell: pwsh

.github/workflows/opendata-commitment-eligibility.yml

@@ -2,20 +2,62 @@ name: Update Commitment Discount Eligibility
 
 on:
   schedule:
-    - cron: '0 6 * * 1'  # Every Monday at 06:00 UTC
+    - cron: '0 6 * * 1' # Every Monday at 06:00 UTC
   workflow_dispatch:
+    inputs:
+      ref:
+        # The scheduled run always uses dev. This input exists so a change to the fetch
+        # script can be exercised end-to-end from its PR branch BEFORE merging, which
+        # matters for a job that only runs weekly and takes ~44 minutes to fail. Only
+        # users with write access can dispatch a workflow, so this does not widen who
+        # can run code in the repo.
+        #
+        # A non-dev ref is TEST-ONLY and is enforced as such by the guard step below:
+        # pushing a data branch built from a feature ref would carry that ref's code
+        # commits into the data PR, so the combination is rejected rather than merely
+        # discouraged.
+        description: 'Branch to check out and run the script from (non-dev requires dry_run)'
+        required: false
+        default: 'dev'
+        type: string
+      dry_run:
+        # Defaults to true so the safe path is also the default one. A real data push is
+        # therefore always an explicit choice, and the scheduled run (where this input is
+        # absent entirely, not false) is unaffected -- see the DRY_RUN env note below.
+        description: 'Fetch and detect changes only; do not push a branch'
+        required: false
+        default: true
+        type: boolean
 
 permissions:
   contents: write
 
 jobs:
   update:
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    # Two full traversals per price type (fetch + verification) measured 44 minutes on
+    # run 31710007476. The headroom absorbs Retry-After backoff on a throttled run.
+    timeout-minutes: 90
     steps:
+      # Fail before the ~44-minute fetch rather than after it, so an invalid input
+      # combination costs seconds instead of most of an hour.
+      - name: Validate dispatch inputs
+        env:
+          DRY_RUN: ${{ github.event.inputs.dry_run }}
+          RUN_REF: ${{ github.event.inputs.ref || 'dev' }}
+        run: |
+          if [ "$RUN_REF" != "dev" ] && [ "$DRY_RUN" != "true" ]; then
+            echo "::error::ref='$RUN_REF' is not 'dev' and dry_run is not true. A push from a" \
+                 "non-dev ref would base the opendata/* branch on that ref, so the data PR" \
+                 "would carry its code commits as well as the data update. Re-run with" \
+                 "dry_run enabled to test a branch, or with ref='dev' to publish data."
+            exit 1
+          fi
+
+      # github.event.inputs is null for the scheduled run, so this falls back to dev.
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
-          ref: dev
+          ref: ${{ github.event.inputs.ref || 'dev' }}
 
       - name: Fetch eligibility data from Azure Retail Prices API
         shell: pwsh
@@ -25,29 +67,53 @@ jobs:
       # a branch and surface a one-click "create PR" link in the job summary for a
       # maintainer to open. Opening the PR triggers Open Data CI and normal review.
       - name: Push branch and surface PR link if data changed
+        env:
+          # Read as a string and compared as one below. A `type: boolean` input arrives
+          # here as the literal "true"/"false", and in a GitHub `if:` expression the
+          # string "false" is truthy -- comparing in bash sidesteps that trap. For the
+          # SCHEDULED run there is no inputs object at all, so this is empty (not the
+          # "true" default), and the run therefore takes the normal push path.
+          DRY_RUN: ${{ github.event.inputs.dry_run }}
+          RUN_REF: ${{ github.event.inputs.ref || 'dev' }}
         run: |
-          # Check for changes to the CSV or the per-shard baseline (handles both
+          # Check for changes to the CSV or the per-family baseline (handles both
           # modified and newly-created/untracked files). The baseline is checked too
           # so a counts-only update (no eligibility change) is still surfaced.
-          if git diff --quiet --exit-code -- src/open-data/CommitmentDiscountEligibility.csv src/open-data/CommitmentDiscountEligibility.shardcounts.json 2>/dev/null && \
-             ! git ls-files --others --exclude-standard -- src/open-data/ | grep -qE 'CommitmentDiscountEligibility\.(csv|shardcounts\.json)'; then
+          if git diff --quiet --exit-code -- src/open-data/CommitmentDiscountEligibility.csv src/open-data/CommitmentDiscountEligibility.familycounts.json 2>/dev/null && \
+             ! git ls-files --others --exclude-standard -- src/open-data/ | grep -qE 'CommitmentDiscountEligibility\.(csv|familycounts\.json)'; then
             echo "No changes detected"
             exit 0
           fi
 
+          if [ "$DRY_RUN" = "true" ]; then
+            {
+              echo "### Dry run: data changed, nothing pushed"
+              echo ""
+              echo "Ran the fetch against \`${RUN_REF}\` and detected changes, but \`dry_run\` was set, so no branch was pushed."
+              echo ""
+              echo '```'
+              git --no-pager diff --stat -- src/open-data/CommitmentDiscountEligibility.csv src/open-data/CommitmentDiscountEligibility.familycounts.json || true
+              # --stat covers tracked files only, so list any newly created ones too.
+              git ls-files --others --exclude-standard -- src/open-data/ | grep -E 'CommitmentDiscountEligibility\.(csv|familycounts\.json)' | sed 's/^/new file: /' || true
+              echo '```'
+            } >> "$GITHUB_STEP_SUMMARY"
+            echo "Dry run: changes detected, not pushing."
+            exit 0
+          fi
+
           BRANCH="opendata/commitment-eligibility-$(date +%Y%m%d)-${{ github.run_number }}"
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
           git checkout -b "$BRANCH"
-          git add src/open-data/CommitmentDiscountEligibility.csv src/open-data/CommitmentDiscountEligibility.shardcounts.json
+          git add src/open-data/CommitmentDiscountEligibility.csv src/open-data/CommitmentDiscountEligibility.familycounts.json
           git commit -m "chore: Update commitment discount eligibility data"
           git push origin "$BRANCH"
 
           PR_URL="${{ github.server_url }}/${{ github.repository }}/compare/dev...${BRANCH}?expand=1"
           {
             echo "### Commitment discount eligibility data updated"
             echo ""
-            echo "Pushed branch \`${BRANCH}\`. GitHub Actions cannot open PRs in this repo, so open it manually:"
+            echo "Pushed branch \`${BRANCH}\` (fetched from \`${RUN_REF}\`). GitHub Actions cannot open PRs in this repo, so open it manually:"
             echo ""
             echo "[**Create pull request →**](${PR_URL})"
             echo ""

docs-mslearn/toolkit/changelog.md

@@ -3,7 +3,7 @@ title: FinOps toolkit changelog
 description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more.
 author: MSBrett
 ms.author: brettwil
-ms.date: 08/13/2026
+ms.date: 08/17/2026
 ms.topic: reference
 ms.service: finops
 ms.subservice: finops-toolkit
@@ -70,6 +70,7 @@ The following section lists features and enhancements that are currently in deve
 
 - **Fixed**
   - Fixed the commitment discount eligibility dataset refresh so it is reproducible and complete; retired meters now age out and previously missed meters are included ([#2164](https://github.com/microsoft/finops-toolkit/pull/2164)).
+  - Fixed the weekly commitment discount eligibility refresh timing out before it could publish, which left the dataset unchanged since it first shipped in v14. The refresh now walks each price type directly instead of sharding by service family, and verifies completeness by comparing two independent traversals before writing ([#2251](https://github.com/microsoft/finops-toolkit/pull/2251)).
 
 -->
 

src/open-data/CommitmentDiscountEligibility.shardcounts.json → src/open-data/CommitmentDiscountEligibility.familycounts.json

@@ -1,44 +1,44 @@
 {
   "Reservation": {
-    "Management and Governance": 1,
-    "Security": 1,
     "AI + Machine Learning": 82,
-    "Compute": 61368,
-    "Developer Tools": 2,
     "Analytics": 131,
-    "Storage": 1673,
+    "Compute": 61368,
+    "Data": 45,
     "Databases": 5873,
-    "Data": 45
+    "Developer Tools": 2,
+    "Management and Governance": 1,
+    "Security": 1,
+    "Storage": 1673
   },
   "Consumption": {
+    "AI + Machine Learning": 0,
+    "Analytics": 0,
+    "Azure Arc": 0,
+    "Azure Communication Services": 0,
     "Azure Security": 0,
-    "Other": 99,
-    "Windows Virtual Desktop": 0,
-    "Blockchain": 0,
-    "Storage": 0,
-    "Power Platform": 0,
-    "Internet of Things": 0,
     "Azure Stack": 0,
-    "Integration": 0,
+    "Blockchain": 0,
+    "Compute": 63558,
+    "Containers": 168,
+    "Data": 0,
     "Databases": 18303,
-    "Management and Governance": 0,
-    "Azure Communication Services": 0,
-    "Azure Arc": 0,
-    "AI + Machine Learning": 0,
     "Developer Tools": 0,
-    "Security": 0,
+    "Gaming": 0,
+    "Integration": 0,
+    "Internet of Things": 0,
+    "Management and Governance": 0,
     "Microsoft 365 Copilot": 0,
-    "Compute": 63558,
-    "Telecommunications": 0,
-    "Windows 365": 0,
-    "Networking": 0,
-    "Containers": 168,
-    "Web": 0,
-    "Mixed Reality": 0,
     "Microsoft Syntex": 0,
+    "Mixed Reality": 0,
+    "Networking": 0,
+    "Other": 99,
+    "Power Platform": 0,
     "Quantum Computing": 0,
-    "Gaming": 0,
-    "Data": 0,
-    "Analytics": 0
+    "Security": 0,
+    "Storage": 0,
+    "Telecommunications": 0,
+    "Web": 0,
+    "Windows 365": 0,
+    "Windows Virtual Desktop": 0
   }
 }