From a2ba9c3e0637e2734e8ee3ce3ba5e09aaeacd11d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C5=A1per=20Grom?= Date: Tue, 4 Aug 2026 10:02:19 +0100 Subject: [PATCH 1/3] feat: add separate v2 health score sink to snowflake (IN-1212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gašper Grom --- .../tinybird/pipes/health_score_v2_sink.pipe | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 services/libs/tinybird/pipes/health_score_v2_sink.pipe diff --git a/services/libs/tinybird/pipes/health_score_v2_sink.pipe b/services/libs/tinybird/pipes/health_score_v2_sink.pipe new file mode 100644 index 0000000000..5f70c94fb1 --- /dev/null +++ b/services/libs/tinybird/pipes/health_score_v2_sink.pipe @@ -0,0 +1,23 @@ +NODE health_score_v2_select_fields +SQL > + -- Scheduled at 03:30 UTC, after project_insights_copy_ds's own 03:00 refresh, so this + -- export reflects the current day's data (v1's 00:30 schedule predates its source refresh). + SELECT + id, + slug, + healthScoreV2, + healthLabel, + lifecycleLabel, + impactScore, + impactLabel, + toStartOfDay(now()) as date + FROM project_insights_copy_ds + WHERE type = 'project' + +TYPE SINK +EXPORT_SERVICE kafka +EXPORT_CONNECTION_NAME lfx-oracle-kafka-streaming +EXPORT_SCHEDULE 30 3 * * * +EXPORT_FORMAT csv +EXPORT_STRATEGY @new +EXPORT_KAFKA_TOPIC health_score_v2_sink From 39ab1ab4beb212e8633746e1e1797d3f6123c651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C5=A1per=20Grom?= Date: Tue, 4 Aug 2026 17:57:55 +0100 Subject: [PATCH 2/3] feat: sink health + impact breakdown fields, disable schedule (IN-1212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gašper Grom --- .../tinybird/pipes/health_score_v2_sink.pipe | 48 ++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_sink.pipe b/services/libs/tinybird/pipes/health_score_v2_sink.pipe index 5f70c94fb1..c8bd5c3c48 100644 --- a/services/libs/tinybird/pipes/health_score_v2_sink.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_sink.pipe @@ -1,23 +1,47 @@ NODE health_score_v2_select_fields SQL > - -- Scheduled at 03:30 UTC, after project_insights_copy_ds's own 03:00 refresh, so this - -- export reflects the current day's data (v1's 00:30 schedule predates its source refresh). + -- No EXPORT_SCHEDULE (disabled on purpose): pushed inert per product's request, only + -- fireable on-demand via `tb pipe sink run` until external-stakeholder communication + -- is confirmed. Add an EXPORT_SCHEDULE line to activate the recurring export. + -- + -- Joins in the Health breakdown (maintainerHealthScoreV2/securitySupplyChainScoreV2/ + -- developmentActivityScoreV2, already columns on project_insights_copy_ds) and the + -- Impact breakdown (directDependents/transitiveDependents/downloads/centrality + + -- their percentile bands, from the separate project_insights_impact_breakdown_ds -- + -- materialized on its own 02:40 UTC schedule, see that datasource's pipe for why it's + -- a separate table) via a LEFT JOIN on id, so a project missing impact data still sinks + -- its health fields rather than being dropped. SELECT - id, - slug, - healthScoreV2, - healthLabel, - lifecycleLabel, - impactScore, - impactLabel, + p.id AS id, + p.slug AS slug, + p.healthScoreV2 AS healthScoreV2, + p.healthLabel AS healthLabel, + p.lifecycleLabel AS lifecycleLabel, + p.impactScore AS impactScore, + p.impactLabel AS impactLabel, + p.maintainerHealthScoreV2 AS maintainerHealthScoreV2, + p.securitySupplyChainScoreV2 AS securitySupplyChainScoreV2, + p.developmentActivityScoreV2 AS developmentActivityScoreV2, + i.directDependents AS directDependents, + i.directDependentsTopPct AS directDependentsTopPct, + i.directDependentsBand AS directDependentsBand, + i.transitiveDependents AS transitiveDependents, + i.transitiveDependentsTopPct AS transitiveDependentsTopPct, + i.transitiveDependentsBand AS transitiveDependentsBand, + i.downloads AS downloads, + i.downloadsTopPct AS downloadsTopPct, + i.downloadsBand AS downloadsBand, + i.centrality AS centrality, + i.centralityTopPct AS centralityTopPct, + i.centralityBand AS centralityBand, toStartOfDay(now()) as date - FROM project_insights_copy_ds - WHERE type = 'project' + FROM project_insights_copy_ds AS p + LEFT JOIN project_insights_impact_breakdown_ds AS i ON p.id = i.id + WHERE p.type = 'project' TYPE SINK EXPORT_SERVICE kafka EXPORT_CONNECTION_NAME lfx-oracle-kafka-streaming -EXPORT_SCHEDULE 30 3 * * * EXPORT_FORMAT csv EXPORT_STRATEGY @new EXPORT_KAFKA_TOPIC health_score_v2_sink From efcfa7da115a373880bc78f2eb18da75e9fb4092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C5=A1per=20Grom?= Date: Tue, 4 Aug 2026 18:40:14 +0100 Subject: [PATCH 3/3] fix: use explicit EXPORT_SCHEDULE @on-demand per tb fmt (IN-1212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gašper Grom --- services/libs/tinybird/pipes/health_score_v2_sink.pipe | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/libs/tinybird/pipes/health_score_v2_sink.pipe b/services/libs/tinybird/pipes/health_score_v2_sink.pipe index c8bd5c3c48..c107f65fab 100644 --- a/services/libs/tinybird/pipes/health_score_v2_sink.pipe +++ b/services/libs/tinybird/pipes/health_score_v2_sink.pipe @@ -1,8 +1,8 @@ NODE health_score_v2_select_fields SQL > - -- No EXPORT_SCHEDULE (disabled on purpose): pushed inert per product's request, only - -- fireable on-demand via `tb pipe sink run` until external-stakeholder communication - -- is confirmed. Add an EXPORT_SCHEDULE line to activate the recurring export. + -- EXPORT_SCHEDULE @on-demand (disabled on purpose): pushed inert per product's request, + -- only fireable on-demand via `tb pipe sink run` until external-stakeholder communication + -- is confirmed. Replace @on-demand with a cron expression to activate the recurring export. -- -- Joins in the Health breakdown (maintainerHealthScoreV2/securitySupplyChainScoreV2/ -- developmentActivityScoreV2, already columns on project_insights_copy_ds) and the @@ -42,6 +42,7 @@ SQL > TYPE SINK EXPORT_SERVICE kafka EXPORT_CONNECTION_NAME lfx-oracle-kafka-streaming +EXPORT_SCHEDULE @on-demand EXPORT_FORMAT csv EXPORT_STRATEGY @new EXPORT_KAFKA_TOPIC health_score_v2_sink