fix: sentinel should not reconcile until replication cluster ready

controllers/redissentinel_controller.go

@@ -11,11 +11,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/handler"
 )
 
 // RedisSentinelReconciler reconciles a RedisSentinel object
@@ -80,22 +77,13 @@ func (r *RedisSentinelReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{}, err
 	}
 
-	reqLogger.Info("Will reconcile after 600 seconds")
-	return ctrl.Result{RequeueAfter: time.Second * 600}, nil
+	reqLogger.Info("Will reconcile after 10 seconds")
+	return ctrl.Result{RequeueAfter: time.Second * 10}, nil
 }
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *RedisSentinelReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&redisv1beta2.RedisSentinel{}).
-		Watches(&redisv1beta2.RedisReplication{}, &handler.Funcs{
-			CreateFunc: nil,
-			UpdateFunc: func(ctx context.Context, event event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) {
-				_ = event.ObjectNew.GetName()
-				_ = event.ObjectNew.GetNamespace()
-			},
-			DeleteFunc:  nil,
-			GenericFunc: nil,
-		}).
 		Complete(r)
 }

k8sutils/redis-replication.go

@@ -210,6 +210,12 @@ func IsRedisReplicationReady(ctx context.Context, logger logr.Logger, client kub
 	if sts.Status.ReadyReplicas != *sts.Spec.Replicas {
 		return false
 	}
+	if sts.Status.ObservedGeneration != sts.Generation {
+		return false
+	}
+	if sts.Status.UpdateRevision != sts.Status.CurrentRevision {
+		return false
+	}
 	// Enhanced check: When the pod is ready, it may not have been
 	// created as part of a replication cluster, so we should verify
 	// whether there is an actual master node.

k8sutils/redis-sentinel.go

@@ -316,30 +316,27 @@ func getRedisReplicationMasterIP(ctx context.Context, client kubernetes.Interfac
 	}
 
 	masterPods := GetRedisNodesByRole(ctx, client, logger, &replicationInstance, "master")
-
 	if len(masterPods) == 0 {
-		realMasterPod = ""
-		err := errors.New("no master pods found")
-		logger.Error(err, "")
-	} else if len(masterPods) == 1 {
-		realMasterPod = masterPods[0]
-	} else {
-		for _, podName := range masterPods {
-			redisClient := configureRedisReplicationClient(client, logger, &replicationInstance, podName)
-			defer redisClient.Close()
-
-			if checkAttachedSlave(ctx, redisClient, logger, podName) > 0 {
-				realMasterPod = podName
-				break
-			}
+		logger.Error(errors.New("no master pods found"), "")
+		return ""
+	}
+	for _, podName := range masterPods {
+		redisClient := configureRedisReplicationClient(client, logger, &replicationInstance, podName)
+		defer redisClient.Close()
+
+		if checkAttachedSlave(ctx, redisClient, logger, podName) > 0 {
+			realMasterPod = podName
+			break
 		}
 	}
+	if realMasterPod == "" {
+		logger.Error(errors.New("no real master pod found"), "")
+		return ""
+	}
 
 	realMasterInfo := RedisDetails{
 		PodName:   realMasterPod,
 		Namespace: replicationNamespace,
 	}
-
-	realMasterPodIP := getRedisServerIP(client, logger, realMasterInfo)
-	return realMasterPodIP
+	return getRedisServerIP(client, logger, realMasterInfo)
 }

tests/e2e-chainsaw/v1beta2/ha-failover/chainsaw-test.yaml

@@ -50,18 +50,11 @@ spec:
             timeout: 10s
             content: |
               kubectl --namespace ${NAMESPACE} delete pod redis-replication-0
-      catch:
-        - description: Redis Operator Logs
-          podLogs:
-            namespace: redis-operator-system
-            selector: control-plane=redis-operator
-            container: manager
-            tail: -1  # tail all logs
 
-    - name: Sleep for 3 minutes
+    - name: Sleep for 5 minutes
       try:
         - sleep:
-            duration: 3m
+            duration: 5m
 
     - name: Test sentinel monitoring
       try: