Skip to content

Commit 913621d

Browse files
authored
Merge pull request #1620 from gianlucam76/validate-health-retries
(opt) Add HealthCheckError and Configurable Retry Logic
2 parents b3ff6ea + 248f283 commit 913621d

8 files changed

Lines changed: 99 additions & 37 deletions

File tree

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ KUBECTL := $(TOOLS_BIN_DIR)/kubectl
7373

7474
GOVULNCHECK_VERSION := "v1.1.4"
7575
GOLANGCI_LINT_VERSION := "v2.8.0"
76-
CLUSTERCTL_VERSION := v1.12.2
76+
CLUSTERCTL_VERSION := v1.12.3
7777

7878
KUSTOMIZE_VER := v5.8.0
7979
KUSTOMIZE_BIN := kustomize

cmd/main.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ var (
8888
webhookPort int
8989
syncPeriod time.Duration
9090
conflictRetryTime time.Duration
91+
healthErrorRetryTime time.Duration
9192
version string
9293
healthAddr string
9394
profilerAddress string
@@ -287,6 +288,11 @@ func initFlags(fs *pflag.FlagSet) {
287288
fmt.Sprintf("The minimum interval at which watched ClusterProfile with conflicts are retried. Defaul: %d seconds",
288289
defaultConflictRetryTime))
289290

291+
const defaultHealthErrorRetryTime = 60
292+
fs.DurationVar(&healthErrorRetryTime, "health-error-retry-time", defaultHealthErrorRetryTime*time.Second,
293+
fmt.Sprintf("The minimum interval at which health check failures are retried. Default: %d seconds",
294+
defaultHealthErrorRetryTime))
295+
290296
// AutoDeployDependencies enables automatic deployment of prerequisite profiles.
291297
//
292298
// Profile instances can specify dependencies on other profiles using the
@@ -518,6 +524,7 @@ func getClusterSummaryReconciler(ctx context.Context, mgr manager.Manager) *cont
518524
PolicyMux: sync.Mutex{},
519525
ConcurrentReconciles: concurrentReconciles,
520526
ConflictRetryTime: conflictRetryTime,
527+
HealthErrorRetryTime: healthErrorRetryTime,
521528
Logger: ctrl.Log.WithName("clustersummaryreconciler"),
522529
}
523530
}

controllers/clustersummary_controller.go

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,9 @@ type ClusterSummaryReconciler struct {
113113
ReferenceMap map[corev1.ObjectReference]*libsveltosset.Set // key: Referenced object; value: set of all ClusterSummaries referencing the resource
114114
ClusterMap map[corev1.ObjectReference]*libsveltosset.Set // key: Sveltos/Cluster; value: set of all ClusterSummaries for that Cluster
115115

116-
ConflictRetryTime time.Duration
117-
ctrl controller.Controller
116+
ConflictRetryTime time.Duration
117+
HealthErrorRetryTime time.Duration
118+
ctrl controller.Controller
118119

119120
eventRecorder events.EventRecorder
120121

@@ -463,6 +464,17 @@ func (r *ClusterSummaryReconciler) proceedDeployingClusterSummary(ctx context.Co
463464
return reconcile.Result{Requeue: true, RequeueAfter: r.ConflictRetryTime}, nil
464465
}
465466

467+
var healthCheckError *clusterops.HealthCheckError
468+
ok = errors.As(err, &healthCheckError)
469+
if ok {
470+
logger.V(logs.LogInfo).Info("failed to deploy health check not passing",
471+
"feature", healthCheckError.FeatureID,
472+
"checkName", healthCheckError.CheckName,
473+
"reason", healthCheckError.InternalErr.Error(),
474+
"requeueAfter", r.HealthErrorRetryTime.String())
475+
return reconcile.Result{Requeue: true, RequeueAfter: r.HealthErrorRetryTime}, nil
476+
}
477+
466478
requeueAfter := normalRequeueAfter
467479
maxFailures := r.getMaxConsecutiveFailures(clusterSummaryScope)
468480

@@ -1764,6 +1776,11 @@ func (r *ClusterSummaryReconciler) processUndeployError(clusterSummaryScope *sco
17641776
return reconcile.Result{Requeue: true, RequeueAfter: deleteHandOverRequeueAfter}, nil
17651777
}
17661778

1779+
var healthCheckError *clusterops.HealthCheckError
1780+
if errors.As(undeployError, &healthCheckError) {
1781+
return reconcile.Result{Requeue: true, RequeueAfter: r.HealthErrorRetryTime}, nil
1782+
}
1783+
17671784
return reconcile.Result{Requeue: true, RequeueAfter: deleteRequeueAfter}, nil
17681785
}
17691786

controllers/clustersummary_deployer.go

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"sigs.k8s.io/controller-runtime/pkg/client"
3131

3232
configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1"
33+
"github.com/projectsveltos/addon-controller/lib/clusterops"
3334
"github.com/projectsveltos/addon-controller/pkg/scope"
3435
libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1"
3536
"github.com/projectsveltos/libsveltos/lib/clusterproxy"
@@ -170,24 +171,9 @@ func (r *ClusterSummaryReconciler) proceedDeployingFeature(ctx context.Context,
170171

171172
r.updateFeatureStatus(clusterSummaryScope, f.id, deployerStatus, currentHash, deployerError, logger)
172173
if deployerError != nil {
173-
// Check if error is a NonRetriableError type
174-
var nonRetriableError *configv1beta1.NonRetriableError
175-
if errors.As(deployerError, &nonRetriableError) {
176-
nonRetriableStatus := libsveltosv1beta1.FeatureStatusFailedNonRetriable
177-
r.updateFeatureStatus(clusterSummaryScope, f.id, &nonRetriableStatus, currentHash, deployerError, logger)
178-
return nil
179-
}
180-
var templateError *configv1beta1.TemplateInstantiationError
181-
if errors.As(deployerError, &templateError) {
182-
nonRetriableStatus := libsveltosv1beta1.FeatureStatusFailedNonRetriable
183-
r.updateFeatureStatus(clusterSummaryScope, f.id, &nonRetriableStatus, currentHash, deployerError, logger)
184-
return nil
185-
}
186-
if r.maxNumberOfConsecutiveFailureReached(clusterSummaryScope, f, logger) {
187-
nonRetriableStatus := libsveltosv1beta1.FeatureStatusFailedNonRetriable
188-
resultError := errors.New("the maximum number of consecutive errors has been reached")
189-
r.updateFeatureStatus(clusterSummaryScope, f.id, &nonRetriableStatus, currentHash, resultError, logger)
190-
return nil
174+
shouldReturn, err := r.handleDeployerError(deployerError, clusterSummaryScope, f, currentHash, logger)
175+
if shouldReturn {
176+
return err
191177
}
192178
}
193179
if *deployerStatus == libsveltosv1beta1.FeatureStatusProvisioning {
@@ -224,6 +210,38 @@ func (r *ClusterSummaryReconciler) proceedDeployingFeature(ctx context.Context,
224210
return fmt.Errorf("request is queued")
225211
}
226212

213+
func (r *ClusterSummaryReconciler) handleDeployerError(deployerError error, clusterSummaryScope *scope.ClusterSummaryScope,
214+
f feature, currentHash []byte, logger logr.Logger) (bool, error) {
215+
216+
// Check if error is a NonRetriableError type
217+
var nonRetriableError *configv1beta1.NonRetriableError
218+
if errors.As(deployerError, &nonRetriableError) {
219+
nonRetriableStatus := libsveltosv1beta1.FeatureStatusFailedNonRetriable
220+
r.updateFeatureStatus(clusterSummaryScope, f.id, &nonRetriableStatus, currentHash, deployerError, logger)
221+
return true, nil
222+
}
223+
var templateError *configv1beta1.TemplateInstantiationError
224+
if errors.As(deployerError, &templateError) {
225+
nonRetriableStatus := libsveltosv1beta1.FeatureStatusFailedNonRetriable
226+
r.updateFeatureStatus(clusterSummaryScope, f.id, &nonRetriableStatus, currentHash, deployerError, logger)
227+
return true, nil
228+
}
229+
var healthCheckError *clusterops.HealthCheckError
230+
if errors.As(deployerError, &healthCheckError) {
231+
retriableStatus := libsveltosv1beta1.FeatureStatusFailed
232+
r.updateFeatureStatus(clusterSummaryScope, f.id, &retriableStatus, currentHash, deployerError, logger)
233+
return true, healthCheckError
234+
}
235+
if r.maxNumberOfConsecutiveFailureReached(clusterSummaryScope, f, logger) {
236+
nonRetriableStatus := libsveltosv1beta1.FeatureStatusFailedNonRetriable
237+
resultError := errors.New("the maximum number of consecutive errors has been reached")
238+
r.updateFeatureStatus(clusterSummaryScope, f.id, &nonRetriableStatus, currentHash, resultError, logger)
239+
return true, nil
240+
}
241+
242+
return false, deployerError
243+
}
244+
227245
func (r *ClusterSummaryReconciler) proceedDeployingFeatureInPullMode(ctx context.Context,
228246
clusterSummaryScope *scope.ClusterSummaryScope, f feature, isConfigSame bool, currentHash []byte,
229247
logger logr.Logger) error {

controllers/delete_checks.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,16 +66,17 @@ func validateDeleteChecks(ctx context.Context, clusterSummary *configv1beta1.Clu
6666

6767
adminNamespace, adminName := getClusterSummaryAdmin(clusterSummary)
6868
cacheMgr := clustercache.GetManager()
69-
remoteRestConfig, err := cacheMgr.GetKubernetesRestConfig(ctx, getManagementClusterClient(), clusterSummary.Spec.ClusterNamespace,
70-
clusterSummary.Spec.ClusterName, adminNamespace, adminName, clusterSummary.Spec.ClusterType, logger)
69+
remoteRestConfig, err := cacheMgr.GetKubernetesRestConfig(ctx, getManagementClusterClient(),
70+
clusterSummary.Spec.ClusterNamespace, clusterSummary.Spec.ClusterName, adminNamespace,
71+
adminName, clusterSummary.Spec.ClusterType, logger)
7172
if err != nil {
7273
logger.V(logs.LogDebug).Error(err, "failed to get cluster rest.Config")
7374
return err
7475
}
7576

7677
logger.V(logs.LogDebug).Info("validate delete checks")
77-
err = clusterops.ValidateHealthPolicies(ctx, remoteRestConfig, clusterSummary.Spec.ClusterProfileSpec.PreDeleteChecks,
78-
featureID, true, logger)
78+
err = clusterops.ValidateHealthPolicies(ctx, remoteRestConfig,
79+
clusterSummary.Spec.ClusterProfileSpec.PreDeleteChecks, featureID, true, logger)
7980
if err != nil {
8081
logger.V(logs.LogDebug).Error(err, "delete check failed")
8182
return err

go.mod

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,14 @@ require (
1111
github.com/fluxcd/pkg/apis/meta v1.25.0
1212
github.com/fluxcd/pkg/http/fetch v0.22.0
1313
github.com/fluxcd/pkg/tar v0.17.0
14-
github.com/fluxcd/source-controller/api v1.7.4
14+
github.com/fluxcd/source-controller/api v1.8.0
1515
github.com/gdexlab/go-render v1.0.1
1616
github.com/go-logr/logr v1.4.3
1717
github.com/hexops/gotextdiff v1.0.3
1818
github.com/onsi/ginkgo/v2 v2.28.1
1919
github.com/onsi/gomega v1.39.1
2020
github.com/pkg/errors v0.9.1
21-
github.com/projectsveltos/libsveltos v1.5.0
21+
github.com/projectsveltos/libsveltos v1.5.1
2222
github.com/prometheus/client_golang v1.23.2
2323
github.com/robfig/cron v1.2.0
2424
github.com/spf13/pflag v1.0.10
@@ -33,7 +33,7 @@ require (
3333
k8s.io/component-base v0.35.1
3434
k8s.io/klog/v2 v2.130.1
3535
k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2
36-
sigs.k8s.io/cluster-api v1.12.2
36+
sigs.k8s.io/cluster-api v1.12.3
3737
sigs.k8s.io/controller-runtime v0.23.1
3838
sigs.k8s.io/kustomize/api v0.21.1
3939
sigs.k8s.io/kustomize/kyaml v0.21.1

go.sum

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpS
5050
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
5151
github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0=
5252
github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
53-
github.com/coredns/corefile-migration v1.0.29 h1:g4cPYMXXDDs9uLE2gFYrJaPBuUAR07eEMGyh9JBE13w=
54-
github.com/coredns/corefile-migration v1.0.29/go.mod h1:56DPqONc3njpVPsdilEnfijCwNGC3/kTJLl7i7SPavY=
53+
github.com/coredns/corefile-migration v1.0.30 h1:ljZNPGgna+4yKv81gfkvkgLEWdtz0NjBR1glaiPI140=
54+
github.com/coredns/corefile-migration v1.0.30/go.mod h1:56DPqONc3njpVPsdilEnfijCwNGC3/kTJLl7i7SPavY=
5555
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
5656
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
5757
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -104,8 +104,8 @@ github.com/fluxcd/pkg/tar v0.17.0 h1:uNxbFXy8ly8C7fJ8D7w3rjTNJFrb4Hp1aY/30XkfvxY
104104
github.com/fluxcd/pkg/tar v0.17.0/go.mod h1:b1xyIRYDD0ket4SV5u0UXYv+ZdN/O/HmIO5jZQdHQls=
105105
github.com/fluxcd/pkg/testserver v0.13.0 h1:xEpBcEYtD7bwvZ+i0ZmChxKkDo/wfQEV3xmnzVybSSg=
106106
github.com/fluxcd/pkg/testserver v0.13.0/go.mod h1:akRYv3FLQUsme15na9ihECRG6hBuqni4XEY9W8kzs8E=
107-
github.com/fluxcd/source-controller/api v1.7.4 h1:+EOVnRA9LmLxOx7J273l7IOEU39m+Slt/nQGBy69ygs=
108-
github.com/fluxcd/source-controller/api v1.7.4/go.mod h1:ruf49LEgZRBfcP+eshl2n9SX1MfHayCcViAIGnZcaDY=
107+
github.com/fluxcd/source-controller/api v1.8.0 h1:ndrYmcv6ZMcdQHFSUkOrFVDO7h16SfDBSw/DOqf/LPo=
108+
github.com/fluxcd/source-controller/api v1.8.0/go.mod h1:1O7+sMbqc1+3tPvjmtgFz+bASTl794Y9SxpebHDDSGA=
109109
github.com/foxcpp/go-mockdns v1.2.0 h1:omK3OrHRD1IWJz1FuFBCFquhXslXoF17OvBS6JPzZF0=
110110
github.com/foxcpp/go-mockdns v1.2.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk=
111111
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -279,8 +279,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
279279
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
280280
github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY=
281281
github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg=
282-
github.com/projectsveltos/libsveltos v1.5.0 h1:XUxdDMYJAujLntep9u1mgKy4ArqhsE7nZhIbZEnC0ZM=
283-
github.com/projectsveltos/libsveltos v1.5.0/go.mod h1:AM2uJ4uasH6PnK8aMeLpc1p8WW7J+uzBA1Nx2g0r5Ug=
282+
github.com/projectsveltos/libsveltos v1.5.1 h1:PDTl+JoT5uVREldz8aXITzdEj3YRo9EXkoyrB0NNeUA=
283+
github.com/projectsveltos/libsveltos v1.5.1/go.mod h1:AM2uJ4uasH6PnK8aMeLpc1p8WW7J+uzBA1Nx2g0r5Ug=
284284
github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5 h1:khnc+994UszxZYu69J+R5FKiLA/Nk1JQj0EYAkwTWz0=
285285
github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5/go.mod h1:yVL8KQFa9tmcxgwl9nwIMtKgtmIVC1zaFRSCfOwYvPY=
286286
github.com/projectsveltos/lua-utils/glua-runes v0.0.0-20251212200258-2b3cdcb7c0f5 h1:YbsebwRwTRhV8QacvEAdFqxcxHdeu7JTVtsBovbkgos=
@@ -521,8 +521,8 @@ oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc=
521521
oras.land/oras-go/v2 v2.6.0/go.mod h1:magiQDfG6H1O9APp+rOsvCPcW1GD2MM7vgnKY0Y+u1o=
522522
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM=
523523
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw=
524-
sigs.k8s.io/cluster-api v1.12.2 h1:+b+M2IygfvFZJq7bsaloNakimMEVNf81zkGR1IiuxXs=
525-
sigs.k8s.io/cluster-api v1.12.2/go.mod h1:2XuF/dmN3c/1VITb6DB44N5+Ecvsvd5KOWqrY9Q53nU=
524+
sigs.k8s.io/cluster-api v1.12.3 h1:cuOl3fWXhlXFuQcyIH4C8i3ns8rLhtcnK+x00MVdKBs=
525+
sigs.k8s.io/cluster-api v1.12.3/go.mod h1:EAiTJtf/8M5eBetPwumi6t8DJJ55Ln6Fkvh2OAa7PD4=
526526
sigs.k8s.io/controller-runtime v0.23.1 h1:TjJSM80Nf43Mg21+RCy3J70aj/W6KyvDtOlpKf+PupE=
527527
sigs.k8s.io/controller-runtime v0.23.1/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0=
528528
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=

lib/clusterops/validate_health.go

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,21 @@ type healthStatus struct {
4343
Message string `json:"message"`
4444
}
4545

46+
type HealthCheckError struct {
47+
FeatureID libsveltosv1beta1.FeatureID
48+
CheckName string
49+
InternalErr error
50+
}
51+
52+
func (e *HealthCheckError) Error() string {
53+
return fmt.Sprintf("health check '%s' for feature %s failed: %v",
54+
e.CheckName, e.FeatureID, e.InternalErr)
55+
}
56+
57+
func (e *HealthCheckError) Unwrap() error {
58+
return e.InternalErr
59+
}
60+
4661
// ValidateHealthPolicies runs all validateDeployment checks registered for the feature (Helm/Kustomize/Resources)
4762
func ValidateHealthPolicies(ctx context.Context, remoteConfig *rest.Config, validateHealths []libsveltosv1beta1.ValidateHealth,
4863
featureID libsveltosv1beta1.FeatureID, isDelete bool, logger logr.Logger) error {
@@ -61,7 +76,11 @@ func ValidateHealthPolicies(ctx context.Context, remoteConfig *rest.Config, vali
6176

6277
if err := validateHealthPolicy(ctx, remoteConfig, check, isDelete, logger); err != nil {
6378
logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to validate check: %s", err))
64-
return err
79+
return &HealthCheckError{
80+
FeatureID: featureID,
81+
CheckName: check.Name,
82+
InternalErr: err,
83+
}
6584
}
6685
}
6786

0 commit comments

Comments
 (0)