From 7cef9f1ffc3d4aa537aed433208f553a1a550149 Mon Sep 17 00:00:00 2001 From: Sai Asish Y Date: Mon, 20 Apr 2026 20:20:04 +0000 Subject: [PATCH] controllers/crd: preserve strategy retry counter across failed-CR replace SubmitCustomResource clears the strategy retry counter on every successful Create. When the Create happens because we just deleted a failed CR (ReplaceConcurrencyPolicy), the reset turned the retry budget into an infinite loop: every replacement re-zeroed the counter and the instance group would keep submitting a new CR forever regardless of MaxRetries (#317). Track whether we replaced anything on this pass with a didReplace bool and only zero the counter on the genuine first-time submission branch. On replace we keep the accumulated count so the strategy-level retry budget is actually enforced. Signed-off-by: SAY-5 Fixes #317 Signed-off-by: Sai Asish Y --- controllers/providers/kubernetes/crd.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/controllers/providers/kubernetes/crd.go b/controllers/providers/kubernetes/crd.go index 9a1579e7..3a22c777 100644 --- a/controllers/providers/kubernetes/crd.go +++ b/controllers/providers/kubernetes/crd.go @@ -104,6 +104,11 @@ func ProcessCRDStrategy(kube dynamic.Interface, instanceGroup *v1alpha1.Instance return false, errors.Wrap(err, "failed to discover active custom resources") } + // didReplace tracks whether we deleted a failed/stale CR on this pass. + // When we replace, the retry counter must NOT be reset: the retry counter + // exists to bound how many times we replace a failing CR, and resetting + // it on every successful re-create produced an infinite loop (#317). + var didReplace bool if len(activeResources) > 0 { switch strings.ToLower(policy) { @@ -132,6 +137,7 @@ func ProcessCRDStrategy(kube dynamic.Interface, instanceGroup *v1alpha1.Instance return false, errors.Wrap(err, "failed to delete custom resource") } } + didReplace = true } if isRunning { // finally, if an active resource is still running, requeue until it's done @@ -152,7 +158,12 @@ func ProcessCRDStrategy(kube dynamic.Interface, instanceGroup *v1alpha1.Instance } } else { log.Info("submitted custom resource", "instancegroup", instanceGroupNamespacedName) - status.SetStrategyRetryCount(0) + if !didReplace { + // First-time submission for this strategy attempt: safe to clear + // the retry counter. On replace we keep the accumulated count so + // the retry budget is actually enforced. + status.SetStrategyRetryCount(0) + } } // get created resource