From 41b6b30c68529b68dc57e253a4bbdf89f21762dd Mon Sep 17 00:00:00 2001 From: Jayapriya Pai Date: Fri, 24 Jul 2026 14:06:59 +0530 Subject: [PATCH] MON-4059: update TelemeterClientFailures alert Replace deprecated federate_requests_* based alert with two focused alerts using metricsclient_http_requests_total{status_code!~"2.."}: - TelemeterClientSendErrors (client="federate_to") - TelemeterClientRetrieveErrors (client="federate_from") Each alert description distinguishes 4xx (user-actionable) from 5xx (Red Hat service issue). Threshold: >20% error rate for 1h. Also extend check-rules to validate alert rules and add promtool unit tests in rulestests.yaml. Signed-off-by: Jayapriya Pai --- Makefile | 7 +- jsonnet/telemeter/client/kubernetes.libsonnet | 45 ++++++-- manifests/client/prometheusRule.yaml | 42 +++++-- test/rulestests.yaml | 106 ++++++++++++++++++ 4 files changed, 178 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index d35fc24d..81b87333 100644 --- a/Makefile +++ b/Makefile @@ -151,10 +151,13 @@ tmp: tmp/rules.yaml: $(JSONNET_LOCAL_OR_INSTALLED) jsonnet/telemeter/rules.libsonnet tmp $(JSONNET_LOCAL_OR_INSTALLED) -e "(import 'jsonnet/telemeter/rules.libsonnet')['prometheus']['recordingrules']" > tmp/rules.yaml +tmp/alert-rules.yaml: $(JSONNET_LOCAL_OR_INSTALLED) $(JSONNET_SRC) $(JSONNET_VENDOR) $(GOJSONTOYAML_BIN) tmp + $(JSONNET_LOCAL_OR_INSTALLED) -e "{groups: (import 'telemeter/client.libsonnet').telemeterClient.prometheusRule.spec.groups}" -J jsonnet/vendor | $(GOJSONTOYAML_BIN) > tmp/alert-rules.yaml + .PHONY: check-rules -check-rules: $(PROMTOOL_BIN) tmp/rules.yaml +check-rules: $(PROMTOOL_BIN) tmp/rules.yaml tmp/alert-rules.yaml rm -f tmp/"$@".out - $(PROMTOOL_BIN) check rules tmp/rules.yaml | tee "tmp/$@.out" + $(PROMTOOL_BIN) check rules tmp/rules.yaml tmp/alert-rules.yaml | tee "tmp/$@.out" .PHONY: test-rules test-rules: check-rules diff --git a/jsonnet/telemeter/client/kubernetes.libsonnet b/jsonnet/telemeter/client/kubernetes.libsonnet index 445c39a4..30fc3f35 100644 --- a/jsonnet/telemeter/client/kubernetes.libsonnet +++ b/jsonnet/telemeter/client/kubernetes.libsonnet @@ -252,23 +252,52 @@ local securePort = 8443; expr: 'max(federate_samples - federate_filtered_samples)', }, { + alert: 'TelemeterClientSendErrors', expr: ||| - sum by (namespace) ( - rate(federate_requests_failed_total{job="telemeter-client"}[15m]) - ) / - sum by (namespace) ( - rate(federate_requests_total{job="telemeter-client"}[15m]) + ( + sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code!~"2.."}[15m])) + / + sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_to"}[15m])) ) > 0.2 |||, + 'for': '1h', labels: { severity: 'warning', }, annotations: { - description: 'The telemeter client in namespace {{ $labels.namespace }} fails {{ $value | humanize }} of the requests to the telemeter service.\nCheck the logs of the telemeter-client pod with the following command:\noc logs -n openshift-monitoring deployment.apps/telemeter-client -c telemeter-client\nIf the telemeter client fails to authenticate with the telemeter service, make sure that the global pull secret is up to date, see https://docs.openshift.com/container-platform/latest/openshift_images/managing_images/using-image-pull-secrets.html#images-update-global-pull-secret_using-image-pull-secrets for more details.', - summary: 'Telemeter client fails to send metrics', + summary: 'Telemeter client is failing to send metrics.', + description: ||| + The telemeter client in namespace {{ $labels.namespace }} has {{ $value | humanizePercentage }} error rate when sending metrics to the telemeter service. + 4xx errors typically indicate an authentication or authorization issue — check that the global pull secret is valid and up to date: + oc get secret pull-secret -n openshift-config -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d + 5xx errors indicate a server-side issue on Red Hat telemeter service and are not actionable by the cluster administrator. + See https://docs.openshift.com/container-platform/latest/openshift_images/managing_images/using-image-pull-secrets.html#images-update-global-pull-secret_using-image-pull-secrets for more details. + |||, }, - alert: 'TelemeterClientFailures', + }, + { + alert: 'TelemeterClientRetrieveErrors', + expr: ||| + ( + sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code!~"2.."}[15m])) + / + sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_from"}[15m])) + ) > 0.2 + |||, 'for': '1h', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'Telemeter client is failing to retrieve metrics from Prometheus.', + description: ||| + The telemeter client in namespace {{ $labels.namespace }} has {{ $value | humanizePercentage }} error rate when retrieving metrics from the in-cluster Prometheus. + 4xx errors may indicate an RBAC issue or misconfigured service account token. + 5xx errors may indicate that Prometheus is overloaded or unhealthy. + Check the telemeter-client logs: + oc logs -n openshift-monitoring deployment/telemeter-client -c telemeter-client + |||, + }, }, ], }, diff --git a/manifests/client/prometheusRule.yaml b/manifests/client/prometheusRule.yaml index 87a8761f..0e5cb6dd 100644 --- a/manifests/client/prometheusRule.yaml +++ b/manifests/client/prometheusRule.yaml @@ -9,20 +9,38 @@ spec: rules: - expr: max(federate_samples - federate_filtered_samples) record: cluster:telemetry_selected_series:count - - alert: TelemeterClientFailures + - alert: TelemeterClientSendErrors annotations: - description: |- - The telemeter client in namespace {{ $labels.namespace }} fails {{ $value | humanize }} of the requests to the telemeter service. - Check the logs of the telemeter-client pod with the following command: - oc logs -n openshift-monitoring deployment.apps/telemeter-client -c telemeter-client - If the telemeter client fails to authenticate with the telemeter service, make sure that the global pull secret is up to date, see https://docs.openshift.com/container-platform/latest/openshift_images/managing_images/using-image-pull-secrets.html#images-update-global-pull-secret_using-image-pull-secrets for more details. - summary: Telemeter client fails to send metrics + description: | + The telemeter client in namespace {{ $labels.namespace }} has {{ $value | humanizePercentage }} error rate when sending metrics to the telemeter service. + 4xx errors typically indicate an authentication or authorization issue — check that the global pull secret is valid and up to date: + oc get secret pull-secret -n openshift-config -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d + 5xx errors indicate a server-side issue on Red Hat telemeter service and are not actionable by the cluster administrator. + See https://docs.openshift.com/container-platform/latest/openshift_images/managing_images/using-image-pull-secrets.html#images-update-global-pull-secret_using-image-pull-secrets for more details. + summary: Telemeter client is failing to send metrics. expr: | - sum by (namespace) ( - rate(federate_requests_failed_total{job="telemeter-client"}[15m]) - ) / - sum by (namespace) ( - rate(federate_requests_total{job="telemeter-client"}[15m]) + ( + sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code!~"2.."}[15m])) + / + sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_to"}[15m])) + ) > 0.2 + for: 1h + labels: + severity: warning + - alert: TelemeterClientRetrieveErrors + annotations: + description: | + The telemeter client in namespace {{ $labels.namespace }} has {{ $value | humanizePercentage }} error rate when retrieving metrics from the in-cluster Prometheus. + 4xx errors may indicate an RBAC issue or misconfigured service account token. + 5xx errors may indicate that Prometheus is overloaded or unhealthy. + Check the telemeter-client logs: + oc logs -n openshift-monitoring deployment/telemeter-client -c telemeter-client + summary: Telemeter client is failing to retrieve metrics from Prometheus. + expr: | + ( + sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code!~"2.."}[15m])) + / + sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_from"}[15m])) ) > 0.2 for: 1h labels: diff --git a/test/rulestests.yaml b/test/rulestests.yaml index a1bb9af9..fca5aaf8 100644 --- a/test/rulestests.yaml +++ b/test/rulestests.yaml @@ -1,5 +1,6 @@ rule_files: - ../tmp/rules.yaml + - ../tmp/alert-rules.yaml evaluation_interval: 1m @@ -354,3 +355,108 @@ tests: value: 72 - labels: 'acm:managed_cluster_worker_cores:managed:sum{_id="another_hub_cluster"}' value: 80 + + # TelemeterClientSendErrors fires when >20% of federate_to requests are non-2xx + - interval: 1m + input_series: + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="401",namespace="openshift-monitoring"}' + values: '0+1x120' + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="200",namespace="openshift-monitoring"}' + values: '0+1x120' + alert_rule_test: + - alertname: TelemeterClientSendErrors + eval_time: 60m + exp_alerts: [] + - alertname: TelemeterClientSendErrors + eval_time: 61m + exp_alerts: + - exp_labels: + severity: warning + namespace: openshift-monitoring + exp_annotations: + summary: Telemeter client is failing to send metrics. + description: "The telemeter client in namespace openshift-monitoring has 50% error rate when sending metrics to the telemeter service.\n4xx errors typically indicate an authentication or authorization issue \u2014 check that the global pull secret is valid and up to date:\n oc get secret pull-secret -n openshift-config -o jsonpath='{.data.\\.dockerconfigjson}' | base64 -d\n5xx errors indicate a server-side issue on Red Hat telemeter service and are not actionable by the cluster administrator.\nSee https://docs.openshift.com/container-platform/latest/openshift_images/managing_images/using-image-pull-secrets.html#images-update-global-pull-secret_using-image-pull-secrets for more details.\n" + + # TelemeterClientSendErrors does NOT fire when error rate < 20% + - interval: 1m + input_series: + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="503",namespace="openshift-monitoring"}' + values: '0+1x120' + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="200",namespace="openshift-monitoring"}' + values: '0+9x120' + alert_rule_test: + - alertname: TelemeterClientSendErrors + eval_time: 75m + exp_alerts: [] + + # TelemeterClientSendErrors does NOT fire at exactly 20% error rate (> 0.2) + - interval: 1m + input_series: + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="401",namespace="openshift-monitoring"}' + values: '0+1x120' + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="200",namespace="openshift-monitoring"}' + values: '0+4x120' + alert_rule_test: + - alertname: TelemeterClientSendErrors + eval_time: 75m + exp_alerts: [] + + # TelemeterClientRetrieveErrors fires when >20% of federate_from requests are non-2xx + - interval: 1m + input_series: + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="403",namespace="openshift-monitoring"}' + values: '0+1x120' + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="200",namespace="openshift-monitoring"}' + values: '0+1x120' + alert_rule_test: + - alertname: TelemeterClientRetrieveErrors + eval_time: 60m + exp_alerts: [] + - alertname: TelemeterClientRetrieveErrors + eval_time: 61m + exp_alerts: + - exp_labels: + severity: warning + namespace: openshift-monitoring + exp_annotations: + summary: Telemeter client is failing to retrieve metrics from Prometheus. + description: "The telemeter client in namespace openshift-monitoring has 50% error rate when retrieving metrics from the in-cluster Prometheus.\n4xx errors may indicate an RBAC issue or misconfigured service account token.\n5xx errors may indicate that Prometheus is overloaded or unhealthy.\nCheck the telemeter-client logs:\n oc logs -n openshift-monitoring deployment/telemeter-client -c telemeter-client\n" + + # TelemeterClientRetrieveErrors does NOT fire when error rate < 20% + - interval: 1m + input_series: + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="500",namespace="openshift-monitoring"}' + values: '0+1x120' + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="200",namespace="openshift-monitoring"}' + values: '0+9x120' + alert_rule_test: + - alertname: TelemeterClientRetrieveErrors + eval_time: 75m + exp_alerts: [] + + # TelemeterClientRetrieveErrors does NOT fire at exactly 20% error rate (> 0.2) + - interval: 1m + input_series: + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="500",namespace="openshift-monitoring"}' + values: '0+1x120' + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="200",namespace="openshift-monitoring"}' + values: '0+4x120' + alert_rule_test: + - alertname: TelemeterClientRetrieveErrors + eval_time: 75m + exp_alerts: [] + + # No alerts fire when all requests are successful + - interval: 1m + input_series: + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="200",namespace="openshift-monitoring"}' + values: '0+10x120' + - series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="200",namespace="openshift-monitoring"}' + values: '0+10x120' + alert_rule_test: + - alertname: TelemeterClientSendErrors + eval_time: 75m + exp_alerts: [] + - alertname: TelemeterClientRetrieveErrors + eval_time: 75m + exp_alerts: []