From 7852eb28cad3217f0ab090afcbe9b4fbb4c1a760 Mon Sep 17 00:00:00 2001 From: Parikshith Date: Wed, 21 May 2025 14:32:03 +0530 Subject: [PATCH 1/8] e2e: Add wait for resource deletion functions This commit adds infra to wait for resources are fully deleted before tests continue, preventing conditions where subsequent tests might be affected by lingering resources. - Add WaitForResourceDelete generic function to wait for resources to be fully deleted - Implement wait funcs for different resource types: - ApplicationSet - ConfigMap - Placement - ManagedClusterSetBinding Signed-off-by: Parikshith --- e2e/util/wait.go | 113 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 e2e/util/wait.go diff --git a/e2e/util/wait.go b/e2e/util/wait.go new file mode 100644 index 000000000..40d3d1e77 --- /dev/null +++ b/e2e/util/wait.go @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: The RamenDR authors +// SPDX-License-Identifier: Apache-2.0 + +package util + +import ( + "fmt" + "reflect" + "time" + + argocdv1alpha1hack "github.com/ramendr/ramen/e2e/argocd" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + ocmv1b1 "open-cluster-management.io/api/cluster/v1beta1" + ocmv1b2 "open-cluster-management.io/api/cluster/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/ramendr/ramen/e2e/types" +) + +func WaitForApplicationSetDelete(ctx types.Context, cluster types.Cluster, name, namespace string) error { + obj := &argocdv1alpha1hack.ApplicationSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + + return waitForResourceDelete(ctx, cluster, obj) +} + +func WaitForConfigMapDelete(ctx types.Context, cluster types.Cluster, name, namespace string) error { + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + + return waitForResourceDelete(ctx, cluster, obj) +} + +func WaitForPlacementDelete(ctx types.Context, cluster types.Cluster, name, namespace string) error { + obj := &ocmv1b1.Placement{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + + return waitForResourceDelete(ctx, cluster, obj) +} + +func WaitForManagedClusterSetBindingDelete(ctx types.Context, cluster types.Cluster, name, namespace string) error { + obj := &ocmv1b2.ManagedClusterSetBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + + return waitForResourceDelete(ctx, cluster, obj) +} + +// waitForResourceDelete waits until a resource is deleted or deadline is reached +func waitForResourceDelete(ctx types.Context, cluster types.Cluster, obj client.Object) error { + log := ctx.Logger() + kind := getKind(obj) + key := k8stypes.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + } + resourceName := logName(obj) + + log.Debugf("Waiting until %s %q is deleted in cluster %q", kind, resourceName, cluster.Name) + + for { + if err := cluster.Client.Get(ctx.Context(), key, obj); err != nil { + if !k8serrors.IsNotFound(err) { + return err + } + + log.Debugf("%s %q deleted in cluster %q", kind, resourceName, cluster.Name) + + return nil + } + + if err := Sleep(ctx.Context(), time.Second); err != nil { + return fmt.Errorf("%s %q not deleted in cluster %q: %w", kind, resourceName, cluster.Name, err) + } + } +} + +// getKind extracts resource type name from the object +func getKind(obj client.Object) string { + t := reflect.TypeOf(obj) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + + return t.Name() +} + +// logName returns the resource name for logging (namespace/name or just name) +func logName(obj client.Object) string { + if obj.GetNamespace() != "" { + return obj.GetNamespace() + "/" + obj.GetName() + } + + return obj.GetName() +} From 40ce476c871f4cc7de344cc4a79fa6485d96b1b6 Mon Sep 17 00:00:00 2001 From: Parikshith Date: Wed, 21 May 2025 14:39:11 +0530 Subject: [PATCH 2/8] e2e: wait for resources to be deleted in undeploy and unprotect Before we deleted resources but we never waited for them, we could detect issue if resource never deleted like if the finalizer for the resource exists after deleted. This commit updates the undeploy and unprotect functions for the appset and dissapp respectively by adding wait for all resource deleted with a deadline which is part of the context. Signed-off-by: Parikshith --- e2e/deployers/appset.go | 12 ++++++++++++ e2e/dractions/disapp.go | 9 +++++++++ 2 files changed, 21 insertions(+) diff --git a/e2e/deployers/appset.go b/e2e/deployers/appset.go index 09e1a4ba0..4cde20d03 100644 --- a/e2e/deployers/appset.go +++ b/e2e/deployers/appset.go @@ -77,6 +77,18 @@ func (a ApplicationSet) Undeploy(ctx types.TestContext) error { return err } + if err := util.WaitForApplicationSetDelete(ctx, ctx.Env().Hub, name, managementNamespace); err != nil { + return err + } + + if err := util.WaitForConfigMapDelete(ctx, ctx.Env().Hub, name, managementNamespace); err != nil { + return err + } + + if err := util.WaitForPlacementDelete(ctx, ctx.Env().Hub, name, managementNamespace); err != nil { + return err + } + log.Info("Workload undeployed") return nil diff --git a/e2e/dractions/disapp.go b/e2e/dractions/disapp.go index 4be89663f..2df72b23c 100644 --- a/e2e/dractions/disapp.go +++ b/e2e/dractions/disapp.go @@ -110,6 +110,15 @@ func DisableProtectionDiscoveredApps(ctx types.TestContext) error { return err } + if err := util.WaitForPlacementDelete(ctx, ctx.Env().Hub, placementName, managementNamespace); err != nil { + return err + } + + if err := util.WaitForManagedClusterSetBindingDelete(ctx, ctx.Env().Hub, config.ClusterSet, + managementNamespace); err != nil { + return err + } + log.Info("Workload unprotected") return nil From 68d0e2dfbf1e7e5a2326fceea1745106f5debfb7 Mon Sep 17 00:00:00 2001 From: Parikshith Date: Wed, 21 May 2025 14:47:49 +0530 Subject: [PATCH 3/8] e2e: refactor namespace deletion to separate delete and wait Separate namespace deletion from waiting to enable parallel operations across clusters. Previously, deleting and waiting sequentially on each cluster could cause timeouts if the first cluster's deletion was slow, blocking ops on the second cluster. This refactoring allows namespace deletion to be initiated on both clusters before waiting, improving overall deletion perf and reliability. The separation provides better control over the deletion process by explicitly managing when to wait for completion. Updated undeploy functions in DiscoveredApp and Subscription deployers, as well as EnsureChannelDeleted function, to use separate delete and wait operations with shared context deadlines for consistent timeout handling across clusters. Signed-off-by: Parikshith --- e2e/deployers/disapp.go | 10 ++++++++++ e2e/deployers/subscr.go | 4 ++++ e2e/util/channel.go | 6 +++++- e2e/util/namespace.go | 23 +---------------------- e2e/util/wait.go | 10 ++++++++++ 5 files changed, 30 insertions(+), 23 deletions(-) diff --git a/e2e/deployers/disapp.go b/e2e/deployers/disapp.go index eeb36fe25..7ca015261 100644 --- a/e2e/deployers/disapp.go +++ b/e2e/deployers/disapp.go @@ -97,6 +97,16 @@ func (d DiscoveredApp) Undeploy(ctx types.TestContext) error { return err } + // wait for namespace to be deleted on both clusters + + if err := util.WaitForNamespaceDelete(ctx, ctx.Env().C1, appNamespace); err != nil { + return err + } + + if err := util.WaitForNamespaceDelete(ctx, ctx.Env().C2, appNamespace); err != nil { + return err + } + log.Info("Workload undeployed") return nil diff --git a/e2e/deployers/subscr.go b/e2e/deployers/subscr.go index a8dbe9264..3434ca8b9 100644 --- a/e2e/deployers/subscr.go +++ b/e2e/deployers/subscr.go @@ -112,6 +112,10 @@ func (s Subscription) Undeploy(ctx types.TestContext) error { return err } + if err := util.WaitForNamespaceDelete(ctx, ctx.Env().Hub, managementNamespace); err != nil { + return err + } + log.Info("Workload undeployed") return nil diff --git a/e2e/util/channel.go b/e2e/util/channel.go index dbf21f860..b997460ea 100644 --- a/e2e/util/channel.go +++ b/e2e/util/channel.go @@ -26,7 +26,11 @@ func EnsureChannelDeleted(ctx types.Context) error { return err } - return DeleteNamespace(ctx, ctx.Env().Hub, ctx.Config().Channel.Namespace) + if err := DeleteNamespace(ctx, ctx.Env().Hub, ctx.Config().Channel.Namespace); err != nil { + return err + } + + return WaitForNamespaceDelete(ctx, ctx.Env().Hub, ctx.Config().Channel.Namespace) } func createChannel(ctx types.Context) error { diff --git a/e2e/util/namespace.go b/e2e/util/namespace.go index 46a5fe15a..bc01080aa 100644 --- a/e2e/util/namespace.go +++ b/e2e/util/namespace.go @@ -4,9 +4,6 @@ package util import ( - "fmt" - "time" - corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -63,25 +60,7 @@ func DeleteNamespace(ctx types.Context, cluster types.Cluster, namespace string) return nil } - log.Debugf("Waiting until namespace %q is deleted in cluster %q", namespace, cluster.Name) - - key := k8stypes.NamespacedName{Name: namespace} - - for { - if err := cluster.Client.Get(ctx.Context(), key, ns); err != nil { - if !k8serrors.IsNotFound(err) { - return err - } - - log.Debugf("Namespace %q deleted in cluster %q", namespace, cluster.Name) - - return nil - } - - if err := Sleep(ctx.Context(), time.Second); err != nil { - return fmt.Errorf("namespace %q not deleted in cluster %q: %w", namespace, cluster.Name, err) - } - } + return nil } // Problem: currently we must manually add an annotation to application’s namespace to make volsync work. diff --git a/e2e/util/wait.go b/e2e/util/wait.go index 40d3d1e77..30b2e90de 100644 --- a/e2e/util/wait.go +++ b/e2e/util/wait.go @@ -64,6 +64,16 @@ func WaitForManagedClusterSetBindingDelete(ctx types.Context, cluster types.Clus return waitForResourceDelete(ctx, cluster, obj) } +func WaitForNamespaceDelete(ctx types.Context, cluster types.Cluster, namespace string) error { + obj := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + + return waitForResourceDelete(ctx, cluster, obj) +} + // waitForResourceDelete waits until a resource is deleted or deadline is reached func waitForResourceDelete(ctx types.Context, cluster types.Cluster, obj client.Object) error { log := ctx.Logger() From bb357b7e5529cb10f11796b325e19fcddc9f5359 Mon Sep 17 00:00:00 2001 From: Parikshith Date: Wed, 21 May 2025 15:00:32 +0530 Subject: [PATCH 4/8] e2e: use --wait=false in DeleteDiscoveredApps Replace --timeout=5m with --wait=false in the kubectl delete command so that DeleteDiscoveredApps returns immediately after initiating deletion instead of waiting for resources to be fully deleted. This change makes the function's behavior consistent with other delete operations that separately handle deletion initiation and waiting for app namespace to be deleted. Signed-off-by: Parikshith --- e2e/deployers/crud.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/deployers/crud.go b/e2e/deployers/crud.go index 869a6c318..6759f7943 100644 --- a/e2e/deployers/crud.go +++ b/e2e/deployers/crud.go @@ -431,7 +431,7 @@ func DeleteDiscoveredApps(ctx types.TestContext, cluster types.Cluster, namespac } cmd := exec.Command("kubectl", "delete", "-k", tempDir, "-n", namespace, - "--kubeconfig", cluster.Kubeconfig, "--timeout=5m", "--ignore-not-found=true") + "--kubeconfig", cluster.Kubeconfig, "--wait=false", "--ignore-not-found=true") if out, err := cmd.Output(); err != nil { if ee, ok := err.(*exec.ExitError); ok { From 0ab9a089f6da9a45b4edeb54391ba2a002affef9 Mon Sep 17 00:00:00 2001 From: Parikshith Date: Wed, 21 May 2025 14:50:04 +0530 Subject: [PATCH 5/8] e2e: add line after comment in dissapp undeploy Since we are doing common delete ops on both clusters current comment needs to updated to indicate same. Signed-off-by: Parikshith --- e2e/deployers/disapp.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/deployers/disapp.go b/e2e/deployers/disapp.go index 7ca015261..ae11c51eb 100644 --- a/e2e/deployers/disapp.go +++ b/e2e/deployers/disapp.go @@ -80,6 +80,7 @@ func (d DiscoveredApp) Undeploy(ctx types.TestContext) error { appNamespace, ctx.Workload().GetAppName(), ctx.Env().C1.Name, ctx.Env().C2.Name) // delete app on both clusters + if err := DeleteDiscoveredApps(ctx, ctx.Env().C1, appNamespace); err != nil { return err } @@ -89,6 +90,7 @@ func (d DiscoveredApp) Undeploy(ctx types.TestContext) error { } // delete namespace on both clusters + if err := util.DeleteNamespace(ctx, ctx.Env().C1, appNamespace); err != nil { return err } From 75e6f4af32bb30844a5c0d1bd9190fee86133e71 Mon Sep 17 00:00:00 2001 From: Parikshith Date: Thu, 22 May 2025 18:24:13 +0530 Subject: [PATCH 6/8] e2e: replace waitDRPCDeleted with WaitForDRPCDelete This change replace waitDRPCDeleted with WaitForDRPCDelete with the deadline similar to other resource delete waits. Signed-off-by: Parikshith --- e2e/dractions/actions.go | 3 +-- e2e/dractions/disapp.go | 2 +- e2e/dractions/retry.go | 25 ------------------------- e2e/util/wait.go | 12 ++++++++++++ 4 files changed, 14 insertions(+), 28 deletions(-) diff --git a/e2e/dractions/actions.go b/e2e/dractions/actions.go index 1f108b136..0e90f90b8 100644 --- a/e2e/dractions/actions.go +++ b/e2e/dractions/actions.go @@ -127,8 +127,7 @@ func DisableProtection(ctx types.TestContext) error { return err } - err = waitDRPCDeleted(ctx, managementNamespace, drpcName) - if err != nil { + if err := util.WaitForDRPCDelete(ctx, ctx.Env().Hub, drpcName, managementNamespace); err != nil { return err } diff --git a/e2e/dractions/disapp.go b/e2e/dractions/disapp.go index 2df72b23c..e8d1907cc 100644 --- a/e2e/dractions/disapp.go +++ b/e2e/dractions/disapp.go @@ -96,7 +96,7 @@ func DisableProtectionDiscoveredApps(ctx types.TestContext) error { return err } - if err := waitDRPCDeleted(ctx, managementNamespace, drpcName); err != nil { + if err := util.WaitForDRPCDelete(ctx, ctx.Env().Hub, drpcName, managementNamespace); err != nil { return err } diff --git a/e2e/dractions/retry.go b/e2e/dractions/retry.go index 3e599aa4f..12694c9a0 100644 --- a/e2e/dractions/retry.go +++ b/e2e/dractions/retry.go @@ -7,7 +7,6 @@ import ( "fmt" ramen "github.com/ramendr/ramen/api/v1alpha1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -103,30 +102,6 @@ func getTargetCluster( return targetCluster, nil } -func waitDRPCDeleted(ctx types.TestContext, namespace string, name string) error { - log := ctx.Logger() - hub := ctx.Env().Hub - - log.Debugf("Waiting until drpc \"%s/%s\" is deleted in cluster %q", namespace, name, hub.Name) - - for { - _, err := getDRPC(ctx, namespace, name) - if err != nil { - if k8serrors.IsNotFound(err) { - log.Debugf("drpc \"%s/%s\" is deleted in cluster %q", namespace, name, hub.Name) - - return nil - } - - log.Debugf("Failed to get drpc \"%s/%s\" in cluster %q: %s", namespace, name, hub.Name, err) - } - - if err := util.Sleep(ctx.Context(), util.RetryInterval); err != nil { - return fmt.Errorf("drpc %q is not deleted in cluster %q: %w", name, hub.Name, err) - } - } -} - // nolint:unparam func waitDRPCProgression( ctx types.TestContext, diff --git a/e2e/util/wait.go b/e2e/util/wait.go index 30b2e90de..d315b2647 100644 --- a/e2e/util/wait.go +++ b/e2e/util/wait.go @@ -8,6 +8,7 @@ import ( "reflect" "time" + ramen "github.com/ramendr/ramen/api/v1alpha1" argocdv1alpha1hack "github.com/ramendr/ramen/e2e/argocd" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -64,6 +65,17 @@ func WaitForManagedClusterSetBindingDelete(ctx types.Context, cluster types.Clus return waitForResourceDelete(ctx, cluster, obj) } +func WaitForDRPCDelete(ctx types.Context, cluster types.Cluster, name, namespace string) error { + obj := &ramen.DRPlacementControl{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + + return waitForResourceDelete(ctx, cluster, obj) +} + func WaitForNamespaceDelete(ctx types.Context, cluster types.Cluster, namespace string) error { obj := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ From faa0cee542fd45a25e4057e68a6ba00e53bf4a17 Mon Sep 17 00:00:00 2001 From: Parikshith Date: Thu, 22 May 2025 19:34:20 +0530 Subject: [PATCH 7/8] e2e: reorder DRPC delete wait in DisableProtectionDiscoveredApps Reorder wait sequence to wait for DRPC deletion after initiating placement and mcsb deletion, ensuring the order in which first all resource are deleted and then wait for them to be deleted. Signed-off-by: Parikshith --- e2e/dractions/disapp.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/dractions/disapp.go b/e2e/dractions/disapp.go index e8d1907cc..827334d35 100644 --- a/e2e/dractions/disapp.go +++ b/e2e/dractions/disapp.go @@ -96,10 +96,6 @@ func DisableProtectionDiscoveredApps(ctx types.TestContext) error { return err } - if err := util.WaitForDRPCDelete(ctx, ctx.Env().Hub, drpcName, managementNamespace); err != nil { - return err - } - // delete placement if err := deployers.DeletePlacement(ctx, placementName, managementNamespace); err != nil { return err @@ -110,6 +106,10 @@ func DisableProtectionDiscoveredApps(ctx types.TestContext) error { return err } + if err := util.WaitForDRPCDelete(ctx, ctx.Env().Hub, drpcName, managementNamespace); err != nil { + return err + } + if err := util.WaitForPlacementDelete(ctx, ctx.Env().Hub, placementName, managementNamespace); err != nil { return err } From 413d215a29c0e10bfc65f1f4d455f8bcdc95031c Mon Sep 17 00:00:00 2001 From: Parikshith Date: Thu, 22 May 2025 20:04:01 +0530 Subject: [PATCH 8/8] e2e: Rename namespace parameter to name in namespace funcs Updated function parameters from 'namespace' to 'name' in functions handling Namespace resources. Since Namespace is a cluster-scoped resource and does not belong to a namespace, so the field is correctly referred to as 'Name' in Kubernetes APIs. This change improves aligns with Kubernetes conventions. Signed-off-by: Parikshith --- e2e/util/namespace.go | 14 +++++++------- e2e/util/wait.go | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/e2e/util/namespace.go b/e2e/util/namespace.go index bc01080aa..67131329a 100644 --- a/e2e/util/namespace.go +++ b/e2e/util/namespace.go @@ -17,12 +17,12 @@ import ( // More info: https://volsync.readthedocs.io/en/stable/usage/permissionmodel.html#controlling-mover-permissions const volsyncPrivilegedMovers = "volsync.backube/privileged-movers" -func CreateNamespace(ctx types.Context, cluster types.Cluster, namespace string) error { +func CreateNamespace(ctx types.Context, cluster types.Cluster, name string) error { log := ctx.Logger() ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: namespace, + Name: name, }, } @@ -32,20 +32,20 @@ func CreateNamespace(ctx types.Context, cluster types.Cluster, namespace string) return err } - log.Debugf("Namespace %q already exist in cluster %q", namespace, cluster.Name) + log.Debugf("Namespace %q already exist in cluster %q", name, cluster.Name) } - log.Debugf("Created namespace %q in cluster %q", namespace, cluster.Name) + log.Debugf("Created namespace %q in cluster %q", name, cluster.Name) return nil } -func DeleteNamespace(ctx types.Context, cluster types.Cluster, namespace string) error { +func DeleteNamespace(ctx types.Context, cluster types.Cluster, name string) error { log := ctx.Logger() ns := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: namespace, + Name: name, }, } @@ -55,7 +55,7 @@ func DeleteNamespace(ctx types.Context, cluster types.Cluster, namespace string) return err } - log.Debugf("Namespace %q not found in cluster %q", namespace, cluster.Name) + log.Debugf("Namespace %q not found in cluster %q", name, cluster.Name) return nil } diff --git a/e2e/util/wait.go b/e2e/util/wait.go index d315b2647..0e0f1439b 100644 --- a/e2e/util/wait.go +++ b/e2e/util/wait.go @@ -76,10 +76,10 @@ func WaitForDRPCDelete(ctx types.Context, cluster types.Cluster, name, namespace return waitForResourceDelete(ctx, cluster, obj) } -func WaitForNamespaceDelete(ctx types.Context, cluster types.Cluster, namespace string) error { +func WaitForNamespaceDelete(ctx types.Context, cluster types.Cluster, name string) error { obj := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: namespace, + Name: name, }, }