add test

Author: drivebyer

api/status/redis-cluster_status.go

@@ -14,6 +14,6 @@ const (
 	RedisClusterInitializing RedisClusterState = "Initializing"
 	RedisClusterBootstrap    RedisClusterState = "Bootstrap"
 	// RedisClusterReady means the RedisCluster is ready for use, we use redis-cli --cluster check 127.0.0.1:6379 to check the cluster status
-	RedisClusterReady RedisClusterState = "Ready"
-	// RedisClusterFailed       RedisClusterState = "Failed"
+	RedisClusterReady  RedisClusterState = "Ready"
+	RedisClusterFailed RedisClusterState = "Failed"
 )

pkg/controllers/rediscluster/rediscluster_controller.go

@@ -47,7 +47,7 @@ type RedisClusterReconciler struct {
 
 func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	reqLogger := r.Log.WithValues("Request.Namespace", req.Namespace, "Request.Name", req.Name)
-	reqLogger.Info("Reconciling opstree redis Cluster controller")
+	reqLogger.V(1).Info("Reconciling opstree redis Cluster controller")
 	instance := &redisv1beta2.RedisCluster{}
 
 	err := r.Client.Get(context.TODO(), req.NamespacedName, instance)
@@ -188,12 +188,18 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "Redis cluster count is not desired", "Current.Count", nc, "Desired.Count", totalReplicas)
 	}
 
-	reqLogger.Info("Number of Redis nodes match desired")
+	reqLogger.V(1).Info("Number of Redis nodes match desired")
 	unhealthyNodeCount, err := k8sutils.UnhealthyNodesInCluster(ctx, r.K8sClient, r.Log, instance)
 	if err != nil {
 		reqLogger.Error(err, "failed to determine unhealthy node count in cluster")
 	}
 	if int(totalReplicas) > 1 && unhealthyNodeCount >= int(totalReplicas)-1 {
+
+		err = k8sutils.UpdateRedisClusterStatus(instance, status.RedisClusterFailed, "RedisCluster has too many unhealthy nodes", leaderReplicas, followerReplicas, r.Dk8sClient)
+		if err != nil {
+			return intctrlutil.RequeueWithError(err, reqLogger, "")
+		}
+
 		reqLogger.Info("healthy leader count does not match desired; attempting to repair disconnected masters")
 		if err = k8sutils.RepairDisconnectedMasters(ctx, r.K8sClient, r.Log, instance); err != nil {
 			reqLogger.Error(err, "failed to repair disconnected masters")

tests/e2e-chainsaw/v1beta2/redis-cluster-restart/chainsaw-test.yaml

@@ -0,0 +1,51 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: redis-cluster
+spec:
+  steps:
+    - try:
+        - apply:
+            file: cluster.yaml
+        - assert:
+            file: ready-cluster.yaml
+
+    - name: Try saving a key With Password
+      try:
+        - script:
+            timeout: 30s
+            content: >
+              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 --
+              redis-cli -c set foo-0 bar-0
+            check:
+              (contains($stdout, 'OK')): true
+
+    - name: Restart pods at same time
+      try:
+          - script:
+              timeout: 30s
+              content: >
+                kubectl delete pod --namespace ${NAMESPACE} -l app=redis-cluster-v1beta2-leader
+
+    - name: Wait cluster to be failed
+      try:
+        - assert:
+            file: failed-cluster.yaml
+
+    - name: Wait cluster to be ready
+      try:
+        - assert:
+            file: ready-cluster.yaml
+
+    - name: Get key
+      try:
+        - script:
+            timeout: 30s
+            content: >
+              kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 --
+              redis-cli -c get foo-0
+            check:
+              (contains($stdout, 'bar-0')): true
+

tests/e2e-chainsaw/v1beta2/redis-cluster-restart/cluster.yaml

@@ -0,0 +1,52 @@
+---
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisCluster
+metadata:
+  name: redis-cluster-v1beta2
+spec:
+  clusterSize: 3
+  clusterVersion: v7
+  persistenceEnabled: true
+  podSecurityContext:
+    runAsUser: 1000
+    fsGroup: 1000
+  redisLeader:
+    replicas: 3
+  redisFollower:
+    replicas: 0
+  kubernetesConfig:
+    image: quay.io/opstree/redis:latest
+    imagePullPolicy: Always
+    resources:
+      requests:
+        cpu: 101m
+        memory: 128Mi
+      limits:
+        cpu: 101m
+        memory: 128Mi
+  redisExporter:
+    enabled: true
+    image: quay.io/opstree/redis-exporter:v1.44.0
+    imagePullPolicy: Always
+    resources:
+      requests:
+        cpu: 100m
+        memory: 128Mi
+      limits:
+        cpu: 100m
+        memory: 128Mi
+  storage:
+    volumeClaimTemplate:
+      spec:
+        # storageClassName: standard
+        accessModes: [ReadWriteOnce]
+        resources:
+          requests:
+            storage: 1Gi
+    nodeConfVolume: true
+    nodeConfVolumeClaimTemplate:
+      spec:
+        accessModes: [ReadWriteOnce]
+        resources:
+          requests:
+            storage: 1Gi

tests/e2e-chainsaw/v1beta2/redis-cluster-restart/failed-cluster.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisCluster
+metadata:
+  name: redis-cluster-v1beta2
+status:
+  state: Failed

tests/e2e-chainsaw/v1beta2/redis-cluster-restart/ready-cluster.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: RedisCluster
+metadata:
+  name: redis-cluster-v1beta2
+status:
+  state: Ready
+  reason: RedisCluster is ready