Skip to content

MON-4059: split TelemeterClientFailures into send/retrieve alerts #550

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
slashpai wants to merge 1 commit into
base: main
Choose a base branch
from
+178 −22
Open

MON-4059: split TelemeterClientFailures into send/retrieve alerts #550

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,10 +151,13 @@ tmp:
tmp/rules.yaml: $(JSONNET_LOCAL_OR_INSTALLED) jsonnet/telemeter/rules.libsonnet tmp
$(JSONNET_LOCAL_OR_INSTALLED) -e "(import 'jsonnet/telemeter/rules.libsonnet')['prometheus']['recordingrules']" > tmp/rules.yaml

tmp/alert-rules.yaml: $(JSONNET_LOCAL_OR_INSTALLED) $(JSONNET_SRC) $(JSONNET_VENDOR) $(GOJSONTOYAML_BIN) tmp
$(JSONNET_LOCAL_OR_INSTALLED) -e "{groups: (import 'telemeter/client.libsonnet').telemeterClient.prometheusRule.spec.groups}" -J jsonnet/vendor | $(GOJSONTOYAML_BIN) > tmp/alert-rules.yaml

.PHONY: check-rules
check-rules: $(PROMTOOL_BIN) tmp/rules.yaml
check-rules: $(PROMTOOL_BIN) tmp/rules.yaml tmp/alert-rules.yaml
rm -f tmp/"$@".out
$(PROMTOOL_BIN) check rules tmp/rules.yaml | tee "tmp/[email protected]"
$(PROMTOOL_BIN) check rules tmp/rules.yaml tmp/alert-rules.yaml | tee "tmp/[email protected]"

.PHONY: test-rules
test-rules: check-rules
Expand Down
45 changes: 37 additions & 8 deletions jsonnet/telemeter/client/kubernetes.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,23 +252,52 @@ local securePort = 8443;
expr: 'max(federate_samples - federate_filtered_samples)',
},
{
alert: 'TelemeterClientSendErrors',
expr: |||
sum by (namespace) (
rate(federate_requests_failed_total{job="telemeter-client"}[15m])
) /
sum by (namespace) (
rate(federate_requests_total{job="telemeter-client"}[15m])
(
sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code!~"2.."}[15m]))
/
sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_to"}[15m]))
) > 0.2
|||,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
description: 'The telemeter client in namespace {{ $labels.namespace }} fails {{ $value | humanize }} of the requests to the telemeter service.\nCheck the logs of the telemeter-client pod with the following command:\noc logs -n openshift-monitoring deployment.apps/telemeter-client -c telemeter-client\nIf the telemeter client fails to authenticate with the telemeter service, make sure that the global pull secret is up to date, see https://docs.openshift.com/container-platform/latest/openshift_images/managing_images/using-image-pull-secrets.html#images-update-global-pull-secret_using-image-pull-secrets for more details.',
summary: 'Telemeter client fails to send metrics',
summary: 'Telemeter client is failing to send metrics.',
description: |||
The telemeter client in namespace {{ $labels.namespace }} has {{ $value | humanizePercentage }} error rate when sending metrics to the telemeter service.
4xx errors typically indicate an authentication or authorization issue — check that the global pull secret is valid and up to date:
oc get secret pull-secret -n openshift-config -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d
5xx errors indicate a server-side issue on Red Hat telemeter service and are not actionable by the cluster administrator.

@simonpasquier simonpasquier Sep 9, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤔 do we even need to alert in this case? maybe we just need an info alert for 5xx responses.

@slashpai slashpai Sep 9, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

info is good.

See https://docs.openshift.com/container-platform/latest/openshift_images/managing_images/using-image-pull-secrets.html#images-update-global-pull-secret_using-image-pull-secrets for more details.
|||,
},
alert: 'TelemeterClientFailures',
},
{
alert: 'TelemeterClientRetrieveErrors',
expr: |||
(
sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code!~"2.."}[15m]))
/
sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_from"}[15m]))
) > 0.2
|||,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
summary: 'Telemeter client is failing to retrieve metrics from Prometheus.',
description: |||
The telemeter client in namespace {{ $labels.namespace }} has {{ $value | humanizePercentage }} error rate when retrieving metrics from the in-cluster Prometheus.
4xx errors may indicate an RBAC issue or misconfigured service account token.
5xx errors may indicate that Prometheus is overloaded or unhealthy.
Check the telemeter-client logs:
oc logs -n openshift-monitoring deployment/telemeter-client -c telemeter-client
|||,
},
},
],
},
Expand Down
42 changes: 30 additions & 12 deletions manifests/client/prometheusRule.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,38 @@ spec:
rules:
- expr: max(federate_samples - federate_filtered_samples)
record: cluster:telemetry_selected_series:count
- alert: TelemeterClientFailures
- alert: TelemeterClientSendErrors
annotations:
description: |-
The telemeter client in namespace {{ $labels.namespace }} fails {{ $value | humanize }} of the requests to the telemeter service.
Check the logs of the telemeter-client pod with the following command:
oc logs -n openshift-monitoring deployment.apps/telemeter-client -c telemeter-client
If the telemeter client fails to authenticate with the telemeter service, make sure that the global pull secret is up to date, see https://docs.openshift.com/container-platform/latest/openshift_images/managing_images/using-image-pull-secrets.html#images-update-global-pull-secret_using-image-pull-secrets for more details.
summary: Telemeter client fails to send metrics
description: |
The telemeter client in namespace {{ $labels.namespace }} has {{ $value | humanizePercentage }} error rate when sending metrics to the telemeter service.
4xx errors typically indicate an authentication or authorization issue — check that the global pull secret is valid and up to date:
oc get secret pull-secret -n openshift-config -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d
5xx errors indicate a server-side issue on Red Hat telemeter service and are not actionable by the cluster administrator.
See https://docs.openshift.com/container-platform/latest/openshift_images/managing_images/using-image-pull-secrets.html#images-update-global-pull-secret_using-image-pull-secrets for more details.
summary: Telemeter client is failing to send metrics.
expr: |
sum by (namespace) (
rate(federate_requests_failed_total{job="telemeter-client"}[15m])
) /
sum by (namespace) (
rate(federate_requests_total{job="telemeter-client"}[15m])
(
sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code!~"2.."}[15m]))
/
sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_to"}[15m]))
) > 0.2
for: 1h
labels:
severity: warning
- alert: TelemeterClientRetrieveErrors
annotations:
description: |
The telemeter client in namespace {{ $labels.namespace }} has {{ $value | humanizePercentage }} error rate when retrieving metrics from the in-cluster Prometheus.
4xx errors may indicate an RBAC issue or misconfigured service account token.
5xx errors may indicate that Prometheus is overloaded or unhealthy.
Check the telemeter-client logs:
oc logs -n openshift-monitoring deployment/telemeter-client -c telemeter-client
summary: Telemeter client is failing to retrieve metrics from Prometheus.
expr: |
(
sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code!~"2.."}[15m]))
/
sum by (namespace) (rate(metricsclient_http_requests_total{job="telemeter-client",client="federate_from"}[15m]))
) > 0.2
for: 1h
labels:
Expand Down
106 changes: 106 additions & 0 deletions test/rulestests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
rule_files:
- ../tmp/rules.yaml
- ../tmp/alert-rules.yaml

evaluation_interval: 1m

Expand Down Expand Up @@ -354,3 +355,108 @@ tests:
value: 72
- labels: 'acm:managed_cluster_worker_cores:managed:sum{_id="another_hub_cluster"}'
value: 80

# TelemeterClientSendErrors fires when >20% of federate_to requests are non-2xx
- interval: 1m
input_series:
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="401",namespace="openshift-monitoring"}'
values: '0+1x120'
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="200",namespace="openshift-monitoring"}'
values: '0+1x120'
alert_rule_test:
- alertname: TelemeterClientSendErrors
eval_time: 60m
exp_alerts: []
- alertname: TelemeterClientSendErrors
eval_time: 61m
exp_alerts:
- exp_labels:
severity: warning
namespace: openshift-monitoring
exp_annotations:
summary: Telemeter client is failing to send metrics.
description: "The telemeter client in namespace openshift-monitoring has 50% error rate when sending metrics to the telemeter service.\n4xx errors typically indicate an authentication or authorization issue \u2014 check that the global pull secret is valid and up to date:\n oc get secret pull-secret -n openshift-config -o jsonpath='{.data.\\.dockerconfigjson}' | base64 -d\n5xx errors indicate a server-side issue on Red Hat telemeter service and are not actionable by the cluster administrator.\nSee https://docs.openshift.com/container-platform/latest/openshift_images/managing_images/using-image-pull-secrets.html#images-update-global-pull-secret_using-image-pull-secrets for more details.\n"

# TelemeterClientSendErrors does NOT fire when error rate < 20%
- interval: 1m
input_series:
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="503",namespace="openshift-monitoring"}'
values: '0+1x120'
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="200",namespace="openshift-monitoring"}'
values: '0+9x120'
alert_rule_test:
- alertname: TelemeterClientSendErrors
eval_time: 75m
exp_alerts: []
Comment thread
coderabbitai[bot] marked this conversation as resolved.

# TelemeterClientSendErrors does NOT fire at exactly 20% error rate (> 0.2)
- interval: 1m
input_series:
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="401",namespace="openshift-monitoring"}'
values: '0+1x120'
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="200",namespace="openshift-monitoring"}'
values: '0+4x120'
alert_rule_test:
- alertname: TelemeterClientSendErrors
eval_time: 75m
exp_alerts: []

# TelemeterClientRetrieveErrors fires when >20% of federate_from requests are non-2xx
- interval: 1m
input_series:
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="403",namespace="openshift-monitoring"}'
values: '0+1x120'
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="200",namespace="openshift-monitoring"}'
values: '0+1x120'
alert_rule_test:
- alertname: TelemeterClientRetrieveErrors
eval_time: 60m
exp_alerts: []
- alertname: TelemeterClientRetrieveErrors
eval_time: 61m
exp_alerts:
- exp_labels:
severity: warning
namespace: openshift-monitoring
exp_annotations:
summary: Telemeter client is failing to retrieve metrics from Prometheus.
description: "The telemeter client in namespace openshift-monitoring has 50% error rate when retrieving metrics from the in-cluster Prometheus.\n4xx errors may indicate an RBAC issue or misconfigured service account token.\n5xx errors may indicate that Prometheus is overloaded or unhealthy.\nCheck the telemeter-client logs:\n oc logs -n openshift-monitoring deployment/telemeter-client -c telemeter-client\n"

# TelemeterClientRetrieveErrors does NOT fire when error rate < 20%
- interval: 1m
input_series:
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="500",namespace="openshift-monitoring"}'
values: '0+1x120'
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="200",namespace="openshift-monitoring"}'
values: '0+9x120'
alert_rule_test:
- alertname: TelemeterClientRetrieveErrors
eval_time: 75m
exp_alerts: []

# TelemeterClientRetrieveErrors does NOT fire at exactly 20% error rate (> 0.2)
- interval: 1m
input_series:
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="500",namespace="openshift-monitoring"}'
values: '0+1x120'
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="200",namespace="openshift-monitoring"}'
values: '0+4x120'
alert_rule_test:
- alertname: TelemeterClientRetrieveErrors
eval_time: 75m
exp_alerts: []

# No alerts fire when all requests are successful
- interval: 1m
input_series:
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_to",status_code="200",namespace="openshift-monitoring"}'
values: '0+10x120'
- series: 'metricsclient_http_requests_total{job="telemeter-client",client="federate_from",status_code="200",namespace="openshift-monitoring"}'
values: '0+10x120'
alert_rule_test:
- alertname: TelemeterClientSendErrors
eval_time: 75m
exp_alerts: []
- alertname: TelemeterClientRetrieveErrors
eval_time: 75m
exp_alerts: []