From 4844404dff8c7501ac091d33378afc665a163738 Mon Sep 17 00:00:00 2001 From: "philippe.dacosta" Date: Mon, 17 Jun 2024 18:15:50 +0200 Subject: [PATCH] feat(appset): Add a cache layer for Argo Projects to speed-up application validation Signed-off-by: Philippe Da Costa --- .../controllers/applicationset_controller.go | 24 +++++++++++++------ go.mod | 1 + go.sum | 2 ++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/applicationset/controllers/applicationset_controller.go b/applicationset/controllers/applicationset_controller.go index f13e9272bde8f..058f45b871512 100644 --- a/applicationset/controllers/applicationset_controller.go +++ b/applicationset/controllers/applicationset_controller.go @@ -57,6 +57,7 @@ import ( "github.com/argoproj/argo-cd/v2/util/argo/normalizers" "github.com/argoproj/argo-cd/v2/pkg/apis/application" + "github.com/hashicorp/golang-lru/v2/expirable" ) const ( @@ -97,6 +98,7 @@ type ApplicationSetReconciler struct { // +kubebuilder:rbac:groups=argoproj.io,resources=applicationsets/status,verbs=get;update;patch func (r *ApplicationSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + startReconcile := time.Now() logCtx := log.WithField("applicationset", req.NamespacedName) var applicationSetInfo argov1alpha1.ApplicationSet @@ -337,7 +339,7 @@ func (r *ApplicationSetReconciler) Reconcile(ctx context.Context, req ctrl.Reque requeueAfter = ReconcileRequeueOnValidationError } - logCtx.WithField("requeueAfter", requeueAfter).Info("end reconcile") + logCtx.WithField("requeueAfter", requeueAfter).Info("end reconcile in ", time.Since(startReconcile)) return ctrl.Result{ RequeueAfter: requeueAfter, @@ -446,6 +448,9 @@ func (r *ApplicationSetReconciler) setApplicationSetStatusCondition(ctx context. return nil } +// TODO add an option to choose cache expiration +var argoProjCache = expirable.NewLRU[string, bool](0, nil, time.Hour*24) + // validateGeneratedApplications uses the Argo CD validation functions to verify the correctness of the // generated applications. func (r *ApplicationSetReconciler) validateGeneratedApplications(ctx context.Context, desiredApplications []argov1alpha1.Application, applicationSetInfo argov1alpha1.ApplicationSet) (map[int]error, error) { @@ -458,13 +463,18 @@ func (r *ApplicationSetReconciler) validateGeneratedApplications(ctx context.Con errorsByIndex[i] = fmt.Errorf("ApplicationSet %s contains applications with duplicate name: %s", applicationSetInfo.Name, app.Name) continue } - _, err := r.ArgoAppClientset.ArgoprojV1alpha1().AppProjects(r.ArgoCDNamespace).Get(ctx, app.Spec.GetProject(), metav1.GetOptions{}) - if err != nil { - if apierr.IsNotFound(err) { - errorsByIndex[i] = fmt.Errorf("application references project %s which does not exist", app.Spec.Project) - continue + + if !argoProjCache.Contains(app.Spec.GetProject()) { + _, err := r.ArgoAppClientset.ArgoprojV1alpha1().AppProjects(r.ArgoCDNamespace).Get(ctx, app.Spec.GetProject(), metav1.GetOptions{}) + if err != nil { + if apierr.IsNotFound(err) { + errorsByIndex[i] = fmt.Errorf("application references project %s which does not exist", app.Spec.Project) + continue + } + return nil, err + } else { + argoProjCache.Add(app.Spec.GetProject(), true) } - return nil, err } if err := utils.ValidateDestination(ctx, &app.Spec.Destination, r.KubeClientset, r.ArgoCDNamespace); err != nil { diff --git a/go.mod b/go.mod index a68b4caed4d22..b3403bb2f1260 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/hashicorp/go-retryablehttp v0.7.7 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/imdario/mergo v0.3.16 github.com/improbable-eng/grpc-web v0.15.0 github.com/itchyny/gojq v0.12.13 diff --git a/go.sum b/go.sum index 662e5e53ee96c..4c603edadbaa1 100644 --- a/go.sum +++ b/go.sum @@ -1263,6 +1263,8 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=