diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 8849f945f..a79209228 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 08/12/2026 +ms.date: 08/17/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -36,6 +36,8 @@ The following section lists features and enhancements that are currently in deve - Fixed the `ContractedCost` recompute guard to compare with a null-safe tolerance instead of exact float equality, eliminating millions of no-op rewrites that polluted the `x_SourceValues` audit trail while preserving the null-cost backfill and no longer overwriting an existing cost when the unit price is missing ([#2216](https://github.com/microsoft/finops-toolkit/issues/2216)). - Fixed the SQL VMs without Azure Hybrid Benefit recommendation query to join on the SQL VM `virtualMachineResourceId` instead of a case-sensitive VM name match that skipped VMs with uppercase names and dropped duplicate names, and made all Azure Resource Graph join kinds explicit so no query relies on the `innerunique` default ([#2225](https://github.com/microsoft/finops-toolkit/pull/2225)). - Switched dimension enrichment in the v1_0/v1_2 ingestion transforms (`PricingUnits`, `Regions`, `ResourceTypes`, `Services`) from `join` to the broadcast-optimized `lookup` operator and deduplicated the `Services` mapping per resource type to prevent cost row fan-out ([#2225](https://github.com/microsoft/finops-toolkit/pull/2225)). + - Fixed the v1_0/v1_2 price transforms creating more rows than were ingested: the savings plan price `lookup` deduplicated its Consumption dimension side with `distinct` over columns that vary by region/currency, so a meter with multiple regional prices fanned out every matching savings plan row; switched to `summarize take_any(...) by tmp_SavingsPlanKey` for a guaranteed one-row-per-key dimension side ([#1736](https://github.com/microsoft/finops-toolkit/issues/1736)). + - Fixed commitment discount eligibility (`x_CommitmentDiscountSpendEligibility`, `x_CommitmentDiscountUsageEligibility`) being computed incorrectly when a pricesheet export lands as multiple parquet files: because `Prices_transform_v1_0`/`v1_2` are Data Explorer update policy functions, each file triggers a separate invocation that only sees that file's rows of `Prices_raw`, so a meter's Reservation/SavingsPlan row and its Consumption row could each be visible in different invocations and eligibility would be computed against a partial view. Eligibility is now sourced from the [Commitment discount eligibility](open-data.md#commitment-discount-eligibility) open-data table, ingested into a new `CommitmentDiscountEligibility` ADX table, which isn't affected by per-invocation partitioning ([#1625](https://github.com/microsoft/finops-toolkit/issues/1625)). Both sides of the eligibility lookup are normalized to lowercase before joining, since the pricesheet's meter ID casing isn't guaranteed to match the open-data table's. ### [FinOps workbooks](workbooks/finops-workbooks-overview.md) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep index 84a29d95c..8eabba68f 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/app.bicep @@ -1092,12 +1092,43 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } } } + { // Update CommitmentDiscountEligibility in ADX + name: 'Update CommitmentDiscountEligibility in ADX' + type: 'AzureDataExplorerCommand' + dependsOn: [ + { + activity: 'Update Services in ADX' + dependencyConditions: [ + 'Succeeded' + ] + } + ] + policy: { + timeout: '0.12:00:00' + retry: 2 // CommitmentDiscountEligibility.csv is ~8x larger than the next biggest reference CSV (ResourceTypes); retry transient network failures instead of failing the whole pipeline + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + command: '.set-or-replace CommitmentDiscountEligibility <| externaldata(MeterId: string, x_CommitmentDiscountSpendEligibility: string, x_CommitmentDiscountUsageEligibility: string)[@"${ftkReleaseUri}/CommitmentDiscountEligibility.csv"] with (format="csv", ignoreFirstRecord=true)' + commandTimeout: '00:30:00' + } + linkedServiceName: { + referenceName: linkedService_dataExplorer.name + type: 'LinkedServiceReference' + parameters: { + database: INGESTION_DB // Do not use dynamic reference since that won't work with Fabric + } + } + } { // Ingestion Complete name: 'Ingestion Complete' type: 'SetVariable' dependsOn: [ { - activity: 'Update Services in ADX' + activity: 'Update CommitmentDiscountEligibility in ADX' dependencyConditions: [ 'Succeeded' ] diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql index 678ca9700..19257d1c0 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_HubInfra.kql @@ -88,6 +88,13 @@ HubScopes() x_ServiceModel: string ) +// CommitmentDiscountEligibility +.create-merge table CommitmentDiscountEligibility( + MeterId: string, + x_CommitmentDiscountSpendEligibility: string, + x_CommitmentDiscountUsageEligibility: string +) + //---------------------------------------------------------------------------------------------------------------------- // parse_resourceid diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql index 50401ebb7..9a647da31 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql @@ -78,26 +78,45 @@ Prices_transform_v1_0() | extend x_IngestionTime = ingestion_time() ); // - // Meters for reservations and savings plans to identify commitment eligibility - let riMeters = prices | where x_SkuPriceType == 'ReservedInstance' | distinct x_SkuMeterId; - let spMeters = prices | where x_SkuPriceType == 'SavingsPlan' | distinct x_SkuMeterId; + // Commitment discount eligibility per meter, used below. Sourced from the + // CommitmentDiscountEligibility open-data table (a snapshot of the Azure Retail Prices API) + // rather than derived from `prices` itself: Prices_raw can land as multiple parquet-snappy + // files per export, each triggering a SEPARATE update policy invocation (see #1625), so a + // meter's Reservation/SavingsPlan row and its Consumption row can be visible in different + // invocations. Deriving eligibility from `prices` only sees whichever rows happen to be in the + // current invocation; CommitmentDiscountEligibility is ingested as a whole table and doesn't + // have that gap. Dedupe with take_any() even though MeterId is expected to be unique in the + // source data -- defends against a future duplicate row silently fanning out the lookup below. + // MeterId is lowercased by the generator (Update-CommitmentDiscountEligibility.ps1); the + // pricesheet's MeterId/MeterID casing isn't documented/guaranteed, and `lookup on` only + // supports case-sensitive equality, so both sides are normalized to lowercase for the join. + let commitmentEligibility = CommitmentDiscountEligibility + | extend tmp_MeterId = tolower(MeterId) + | summarize take_any(x_CommitmentDiscountSpendEligibility), take_any(x_CommitmentDiscountUsageEligibility) by tmp_MeterId; // // Copy list/base/contracted prices from on-demand SKUs prices | where x_SkuPriceType == 'SavingsPlan' // If we use join, specify the shuffle key - // TODO: Compare join vs. lookup perf -- | join kind=leftouter hint.strategy=shuffle (prices | where x_SkuPriceType == 'Consumption' | where x_SkuMeterId in (spMeters) | distinct tmp_SavingsPlanKey, ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice) on tmp_SavingsPlanKey - | lookup kind=leftouter (prices | where x_SkuPriceType == 'Consumption' | where x_SkuMeterId in (spMeters) | distinct tmp_SavingsPlanKey, ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice) on tmp_SavingsPlanKey + // TODO: Compare join vs. lookup perf -- | join kind=leftouter hint.strategy=shuffle (prices | where x_SkuPriceType == 'Consumption' | summarize take_any(ListUnitPrice), take_any(ContractedUnitPrice), take_any(x_BaseUnitPrice) by tmp_SavingsPlanKey) on tmp_SavingsPlanKey + // The dimension side must be unique per tmp_SavingsPlanKey (meter+product+SKU+tier+offer, no region/currency): + // `distinct` over the price columns does not guarantee that when the same key has rows with + // different prices (e.g. multi-region exports), so it can fan out matching SavingsPlan rows. + // `summarize take_any(...) by tmp_SavingsPlanKey` guarantees exactly one row per key. + | lookup kind=leftouter (prices | where x_SkuPriceType == 'Consumption' | summarize take_any(ListUnitPrice), take_any(ContractedUnitPrice), take_any(x_BaseUnitPrice) by tmp_SavingsPlanKey) on tmp_SavingsPlanKey | extend ListUnitPrice = coalesce(ListUnitPrice, ListUnitPrice1) | extend ContractedUnitPrice = coalesce(ContractedUnitPrice, ContractedUnitPrice1) | extend x_BaseUnitPrice = coalesce(x_BaseUnitPrice, x_BaseUnitPrice1) | project-away ListUnitPrice1, ContractedUnitPrice1, x_BaseUnitPrice1, tmp_SavingsPlanKey | union ((prices | where x_SkuPriceType != 'SavingsPlan')) // - // Calculate commitment discount elgibility + // Calculate commitment discount eligibility from the open-data snapshot; unmatched meters default to not eligible // TODO: Would a join be faster? - | extend x_CommitmentDiscountSpendEligibility = iff(x_SkuMeterId in (riMeters) and x_SkuPriceType != 'ReservedInstance', 'Eligible', 'Not Eligible') - | extend x_CommitmentDiscountUsageEligibility = iff(x_SkuMeterId in (spMeters), 'Eligible', 'Not Eligible') + | extend tmp_MeterId = tolower(x_SkuMeterId) + | lookup kind=leftouter (commitmentEligibility) on tmp_MeterId + | extend x_CommitmentDiscountSpendEligibility = iff(isnotempty(x_CommitmentDiscountSpendEligibility) and x_SkuPriceType != 'ReservedInstance', x_CommitmentDiscountSpendEligibility, 'Not Eligible') + | extend x_CommitmentDiscountUsageEligibility = coalesce(x_CommitmentDiscountUsageEligibility, 'Not Eligible') + | project-away tmp_MeterId // // Add PricingUnit and x_PricingBlockSize // TODO: Compare join vs. lookup perf -- | join kind=leftouter (PricingUnits) on x_PricingUnitDescription | project-away x_PricingUnitDescription1 diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql index da41cafa4..84167bdcb 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql @@ -69,16 +69,32 @@ Prices_transform_v1_2() | extend x_IngestionTime = ingestion_time() ); // - // Meters for reservations and savings plans to identify commitment eligibility - let riMeters = prices | where x_SkuPriceType == 'ReservedInstance' | distinct x_SkuMeterId; - let spMeters = prices | where x_SkuPriceType == 'SavingsPlan' | distinct x_SkuMeterId; + // Commitment discount eligibility per meter, used below. Sourced from the + // CommitmentDiscountEligibility open-data table (a snapshot of the Azure Retail Prices API) + // rather than derived from `prices` itself: Prices_raw can land as multiple parquet-snappy + // files per export, each triggering a SEPARATE update policy invocation (see #1625), so a + // meter's Reservation/SavingsPlan row and its Consumption row can be visible in different + // invocations. Deriving eligibility from `prices` only sees whichever rows happen to be in the + // current invocation; CommitmentDiscountEligibility is ingested as a whole table and doesn't + // have that gap. Dedupe with take_any() even though MeterId is expected to be unique in the + // source data -- defends against a future duplicate row silently fanning out the lookup below. + // MeterId is lowercased by the generator (Update-CommitmentDiscountEligibility.ps1); the + // pricesheet's MeterId/MeterID casing isn't documented/guaranteed, and `lookup on` only + // supports case-sensitive equality, so both sides are normalized to lowercase for the join. + let commitmentEligibility = CommitmentDiscountEligibility + | extend tmp_MeterId = tolower(MeterId) + | summarize take_any(x_CommitmentDiscountSpendEligibility), take_any(x_CommitmentDiscountUsageEligibility) by tmp_MeterId; // // Copy list/base/contracted prices from on-demand SKUs prices | where x_SkuPriceType == 'SavingsPlan' // If we use join, specify the shuffle key - // TODO: Compare join vs. lookup perf -- | join kind=leftouter hint.strategy=shuffle (prices | where x_SkuPriceType == 'Consumption' | where x_SkuMeterId in (spMeters) | distinct tmp_SavingsPlanKey, ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice) on tmp_SavingsPlanKey - | lookup kind=leftouter (prices | where x_SkuPriceType == 'Consumption' | where x_SkuMeterId in (spMeters) | distinct tmp_SavingsPlanKey, ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice) on tmp_SavingsPlanKey + // TODO: Compare join vs. lookup perf -- | join kind=leftouter hint.strategy=shuffle (prices | where x_SkuPriceType == 'Consumption' | summarize take_any(ListUnitPrice), take_any(ContractedUnitPrice), take_any(x_BaseUnitPrice) by tmp_SavingsPlanKey) on tmp_SavingsPlanKey + // The dimension side must be unique per tmp_SavingsPlanKey (meter+product+SKU+tier+offer, no region/currency): + // `distinct` over the price columns does not guarantee that when the same key has rows with + // different prices (e.g. multi-region exports), so it can fan out matching SavingsPlan rows. + // `summarize take_any(...) by tmp_SavingsPlanKey` guarantees exactly one row per key. + | lookup kind=leftouter (prices | where x_SkuPriceType == 'Consumption' | summarize take_any(ListUnitPrice), take_any(ContractedUnitPrice), take_any(x_BaseUnitPrice) by tmp_SavingsPlanKey) on tmp_SavingsPlanKey | extend ListUnitPrice = coalesce(ListUnitPrice, ListUnitPrice1) | extend ContractedUnitPrice = coalesce(ContractedUnitPrice, ContractedUnitPrice1) | extend x_BaseUnitPrice = coalesce(x_BaseUnitPrice, x_BaseUnitPrice1) @@ -92,11 +108,12 @@ Prices_transform_v1_2() '' ) // - // Calculate commitment discount eligibility - // TODO: Would a join be faster? - // TODO: Check this to ensure it's correct - | extend x_CommitmentDiscountSpendEligibility = iff(x_SkuMeterId in (riMeters) and x_SkuPriceType != 'ReservedInstance', 'Eligible', 'Not Eligible') - | extend x_CommitmentDiscountUsageEligibility = iff(x_SkuMeterId in (spMeters), 'Eligible', 'Not Eligible') + // Calculate commitment discount eligibility from the open-data snapshot; unmatched meters default to not eligible + | extend tmp_MeterId = tolower(x_SkuMeterId) + | lookup kind=leftouter (commitmentEligibility) on tmp_MeterId + | extend x_CommitmentDiscountSpendEligibility = iff(isnotempty(x_CommitmentDiscountSpendEligibility) and x_SkuPriceType != 'ReservedInstance', x_CommitmentDiscountSpendEligibility, 'Not Eligible') + | extend x_CommitmentDiscountUsageEligibility = coalesce(x_CommitmentDiscountUsageEligibility, 'Not Eligible') + | project-away tmp_MeterId // // TODO: Implement x_CommitmentDiscountNormalizedRatio | extend x_CommitmentDiscountNormalizedRatio = real(null)