Skip to content

Commit 92ac24f

Browse files
drivebyerdrivebyer
authored
feat: rolling update sequence from leader to follower (#966)
* feat: rolling update sequence from leader to follower Signed-off-by: drivebyer <yang.wu@daocloud.io> * fix Signed-off-by: drivebyer <yang.wu@daocloud.io> * add suffix Signed-off-by: drivebyer <yang.wu@daocloud.io> * fix test Signed-off-by: drivebyer <yang.wu@daocloud.io> * fix lint Signed-off-by: drivebyer <yang.wu@daocloud.io> --------- Signed-off-by: drivebyer <yang.wu@daocloud.io>
1 parent 96a5ccf commit 92ac24f

File tree

9 files changed

+257
-16
lines changed

9 files changed

+257
-16
lines changed

‎.github/workflows/publish-image.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,6 @@ jobs:
4040
file: Dockerfile
4141
push: true
4242
tags: |
43-
${{ env.REGISTRY }}/${{ env.REPOSITORY }}:${{ env.TAG }}
44-
${{ env.REGISTRY }}/${{ env.REPOSITORY }}:latest
43+
${{ env.REGISTRY }}/${{ env.REPOSITORY }}/redis-operator:${{ env.TAG }}
44+
${{ env.REGISTRY }}/${{ env.REPOSITORY }}/redis-operator:latest
4545
platforms: linux/amd64,linux/arm64

‎controllers/rediscluster_controller.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import (
3636
// RedisClusterReconciler reconciles a RedisCluster object
3737
type RedisClusterReconciler struct {
3838
client.Client
39+
k8sutils.StatefulSet
3940
K8sClient kubernetes.Interface
4041
Dk8sClient dynamic.Interface
4142
Log logr.Logger
@@ -125,6 +126,7 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
125126
return ctrl.Result{}, err
126127
}
127128

129+
// todo: remove me after watch statefulset in controller
128130
redisLeaderInfo, err := k8sutils.GetStatefulSet(r.K8sClient, r.Log, instance.GetNamespace(), instance.GetName()+"-leader")
129131
if err != nil {
130132
if errors.IsNotFound(err) {
@@ -133,7 +135,7 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
133135
return ctrl.Result{}, err
134136
}
135137

136-
if redisLeaderInfo.Status.ReadyReplicas == leaderReplicas {
138+
if r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-leader") {
137139
// Mark the cluster status as initializing if there are no follower nodes
138140
if (instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0) ||
139141
instance.Status.ReadyFollowerReplicas != followerReplicas {
@@ -158,6 +160,7 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
158160
return ctrl.Result{}, err
159161
}
160162
}
163+
// todo: remove me after watch statefulset in controller
161164
redisFollowerInfo, err := k8sutils.GetStatefulSet(r.K8sClient, r.Log, instance.GetNamespace(), instance.GetName()+"-follower")
162165
if err != nil {
163166
if errors.IsNotFound(err) {
@@ -171,9 +174,9 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
171174
return ctrl.Result{RequeueAfter: time.Second * 60}, nil
172175
}
173176

174-
if !(redisLeaderInfo.Status.ReadyReplicas == leaderReplicas && redisFollowerInfo.Status.ReadyReplicas == followerReplicas) {
175-
reqLogger.Info("Redis leader and follower nodes are not ready yet", "Ready.Replicas", strconv.Itoa(int(redisLeaderInfo.Status.ReadyReplicas)), "Expected.Replicas", leaderReplicas)
176-
return ctrl.Result{RequeueAfter: time.Second * 60}, nil
177+
if !(r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-leader") && r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-follower")) {
178+
reqLogger.Info("Redis leader and follower nodes are not ready yet")
179+
return ctrl.Result{RequeueAfter: time.Second * 30}, nil
177180
}
178181

179182
// Mark the cluster status as bootstrapping if all the leader and follower nodes are ready

‎controllers/suite_test.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ import (
2121
"testing"
2222
"time"
2323

24-
// redisv1beta1 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta1"
2524
redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2"
25+
"github.com/OT-CONTAINER-KIT/redis-operator/k8sutils"
2626
. "github.com/onsi/ginkgo/v2"
2727
. "github.com/onsi/gomega"
2828
"github.com/onsi/gomega/gexec"
@@ -101,11 +101,13 @@ var _ = BeforeSuite(func() {
101101
}).SetupWithManager(k8sManager)
102102
Expect(err).ToNot(HaveOccurred())
103103

104+
rrLog := ctrl.Log.WithName("controllers").WithName("RedisReplication")
104105
err = (&RedisClusterReconciler{
105-
Client: k8sManager.GetClient(),
106-
K8sClient: k8sClient,
107-
Dk8sClient: dk8sClient,
108-
Scheme: k8sManager.GetScheme(),
106+
Client: k8sManager.GetClient(),
107+
K8sClient: k8sClient,
108+
Dk8sClient: dk8sClient,
109+
Scheme: k8sManager.GetScheme(),
110+
StatefulSet: k8sutils.NewStatefulSetService(k8sClient, rrLog),
109111
}).SetupWithManager(k8sManager)
110112
Expect(err).ToNot(HaveOccurred())
111113

‎k8sutils/statefulset.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,60 @@ import (
2424
"k8s.io/utils/ptr"
2525
)
2626

27+
type StatefulSet interface {
28+
IsStatefulSetReady(ctx context.Context, namespace, name string) bool
29+
}
30+
31+
type StatefulSetService struct {
32+
kubeClient kubernetes.Interface
33+
log logr.Logger
34+
}
35+
36+
func NewStatefulSetService(kubeClient kubernetes.Interface, log logr.Logger) *StatefulSetService {
37+
log = log.WithValues("service", "k8s.statefulset")
38+
return &StatefulSetService{
39+
kubeClient: kubeClient,
40+
log: log,
41+
}
42+
}
43+
44+
func (s *StatefulSetService) IsStatefulSetReady(ctx context.Context, namespace, name string) bool {
45+
var (
46+
partition = 0
47+
replicas = 1
48+
49+
logger = s.log.WithValues("namespace", namespace, "name", name)
50+
)
51+
52+
sts, err := s.kubeClient.AppsV1().StatefulSets(namespace).Get(ctx, name, metav1.GetOptions{})
53+
if err != nil {
54+
logger.Error(err, "failed to get statefulset")
55+
return false
56+
}
57+
58+
if sts.Spec.UpdateStrategy.RollingUpdate != nil && sts.Spec.UpdateStrategy.RollingUpdate.Partition != nil {
59+
partition = int(*sts.Spec.UpdateStrategy.RollingUpdate.Partition)
60+
}
61+
if sts.Spec.Replicas != nil {
62+
replicas = int(*sts.Spec.Replicas)
63+
}
64+
65+
if expectedUpdateReplicas := replicas - partition; sts.Status.UpdatedReplicas < int32(expectedUpdateReplicas) {
66+
logger.V(1).Info("StatefulSet is not ready", "Status.UpdatedReplicas", sts.Status.UpdatedReplicas, "ExpectedUpdateReplicas", expectedUpdateReplicas)
67+
return false
68+
}
69+
if partition == 0 && sts.Status.CurrentRevision != sts.Status.UpdateRevision {
70+
logger.V(1).Info("StatefulSet is not ready", "Status.CurrentRevision", sts.Status.CurrentRevision, "Status.UpdateRevision", sts.Status.UpdateRevision)
71+
return false
72+
}
73+
if sts.Status.ObservedGeneration != sts.ObjectMeta.Generation {
74+
logger.V(1).Info("StatefulSet is not ready", "Status.ObservedGeneration", sts.Status.ObservedGeneration, "ObjectMeta.Generation", sts.ObjectMeta.Generation)
75+
return false
76+
}
77+
78+
return true
79+
}
80+
2781
const (
2882
redisExporterContainer = "redis-exporter"
2983
)

‎main.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -124,12 +124,14 @@ func main() {
124124
setupLog.Error(err, "unable to create controller", "controller", "Redis")
125125
os.Exit(1)
126126
}
127+
rcLog := ctrl.Log.WithName("controllers").WithName("RedisCluster")
127128
if err = (&controllers.RedisClusterReconciler{
128-
Client: mgr.GetClient(),
129-
K8sClient: k8sclient,
130-
Dk8sClient: dk8sClient,
131-
Log: ctrl.Log.WithName("controllers").WithName("RedisCluster"),
132-
Scheme: mgr.GetScheme(),
129+
Client: mgr.GetClient(),
130+
K8sClient: k8sclient,
131+
Dk8sClient: dk8sClient,
132+
Log: rcLog,
133+
Scheme: mgr.GetScheme(),
134+
StatefulSet: k8sutils.NewStatefulSetService(k8sclient, rcLog),
133135
}).SetupWithManager(mgr); err != nil {
134136
setupLog.Error(err, "unable to create controller", "controller", "RedisCluster")
135137
os.Exit(1)

‎tests/e2e-chainsaw/v1beta2/disable-persistence/redis-cluster/chainsaw-test.yaml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
---
2+
# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
3+
apiVersion: chainsaw.kyverno.io/v1alpha1
4+
kind: Test
5+
metadata:
6+
name: redis-cluster
7+
spec:
8+
steps:
9+
- try:
10+
- apply:
11+
file: cluster.yaml
12+
- assert:
13+
file: ready-cluster.yaml
14+
15+
- name: Try saving a key
16+
try:
17+
- script:
18+
timeout: 30s
19+
content: >
20+
kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 -- redis-cli -c -p 6379 set foo-0 bar-0
21+
check:
22+
($stdout=='OK'): true
23+
- script:
24+
timeout: 30s
25+
content: >
26+
kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-1 -- redis-cli -c -p 6379 set foo-1 bar-1
27+
check:
28+
($stdout=='OK'): true
29+
- script:
30+
timeout: 30s
31+
content: >
32+
kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-2 -- redis-cli -c -p 6379 set foo-2 bar-2
33+
check:
34+
($stdout=='OK'): true
35+
- script:
36+
timeout: 30s
37+
content: >
38+
kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-0 -- redis-cli -c -p 6379 set foo-3 bar-3
39+
check:
40+
($stdout=='OK'): true
41+
- script:
42+
timeout: 30s
43+
content: >
44+
kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-1 -- redis-cli -c -p 6379 set foo-4 bar-4
45+
check:
46+
($stdout=='OK'): true
47+
- script:
48+
timeout: 30s
49+
content: >
50+
kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-2 -- redis-cli -c -p 6379 set foo-5 bar-5
51+
check:
52+
($stdout=='OK'): true
53+
54+
- name: Rolling update the cluster
55+
try:
56+
- apply:
57+
file: cluster-hscale.yaml
58+
- assert:
59+
file: ready-cluster.yaml
60+
61+
- name: Check if all keys exist
62+
try:
63+
- script:
64+
timeout: 30s
65+
content: >
66+
kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-0 -- redis-cli -c -p 6379 get foo-0
67+
check:
68+
($stdout=='bar-0'): true
69+
- script:
70+
timeout: 30s
71+
content: >
72+
kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-1 -- redis-cli -c -p 6379 get foo-1
73+
check:
74+
($stdout=='bar-1'): true
75+
- script:
76+
timeout: 30s
77+
content: >
78+
kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-leader redis-cluster-v1beta2-leader-2 -- redis-cli -c -p 6379 get foo-2
79+
check:
80+
($stdout=='bar-2'): true
81+
- script:
82+
timeout: 30s
83+
content: >
84+
kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-0 -- redis-cli -c -p 6379 get foo-3
85+
check:
86+
($stdout=='bar-3'): true
87+
- script:
88+
timeout: 30s
89+
content: >
90+
kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-1 -- redis-cli -c -p 6379 get foo-4
91+
check:
92+
($stdout=='bar-4'): true
93+
- script:
94+
timeout: 30s
95+
content: >
96+
kubectl exec --namespace ${NAMESPACE} --container redis-cluster-v1beta2-follower redis-cluster-v1beta2-follower-2 -- redis-cli -c -p 6379 get foo-5
97+
check:
98+
($stdout=='bar-5'): true

‎tests/e2e-chainsaw/v1beta2/disable-persistence/redis-cluster/cluster-hscale.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
---
2+
apiVersion: redis.redis.opstreelabs.in/v1beta2
3+
kind: RedisCluster
4+
metadata:
5+
name: redis-cluster-v1beta2
6+
spec:
7+
clusterSize: 3
8+
clusterVersion: v7
9+
persistenceEnabled: false
10+
podSecurityContext:
11+
runAsUser: 1000
12+
fsGroup: 1000
13+
kubernetesConfig:
14+
image: quay.io/opstree/redis:v7.0.12
15+
imagePullPolicy: Always
16+
resources:
17+
requests:
18+
cpu: 101m
19+
memory: 128Mi
20+
limits:
21+
cpu: 101m
22+
memory: 256Mi # Increased memory limit
23+
storage:
24+
volumeClaimTemplate:
25+
spec:
26+
accessModes: [ReadWriteOnce]
27+
resources:
28+
requests:
29+
storage: 1Gi
30+
nodeConfVolume: true
31+
nodeConfVolumeClaimTemplate:
32+
spec:
33+
accessModes: [ReadWriteOnce]
34+
resources:
35+
requests:
36+
storage: 1Gi

‎tests/e2e-chainsaw/v1beta2/disable-persistence/redis-cluster/cluster.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
---
2+
apiVersion: redis.redis.opstreelabs.in/v1beta2
3+
kind: RedisCluster
4+
metadata:
5+
name: redis-cluster-v1beta2
6+
spec:
7+
clusterSize: 3
8+
clusterVersion: v7
9+
persistenceEnabled: false
10+
podSecurityContext:
11+
runAsUser: 1000
12+
fsGroup: 1000
13+
kubernetesConfig:
14+
image: quay.io/opstree/redis:v7.0.12
15+
imagePullPolicy: Always
16+
resources:
17+
requests:
18+
cpu: 101m
19+
memory: 128Mi
20+
limits:
21+
cpu: 101m
22+
memory: 128Mi
23+
storage:
24+
volumeClaimTemplate:
25+
spec:
26+
accessModes: [ReadWriteOnce]
27+
resources:
28+
requests:
29+
storage: 1Gi
30+
nodeConfVolume: true
31+
nodeConfVolumeClaimTemplate:
32+
spec:
33+
accessModes: [ReadWriteOnce]
34+
resources:
35+
requests:
36+
storage: 1Gi

‎tests/e2e-chainsaw/v1beta2/disable-persistence/redis-cluster/ready-cluster.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
---
2+
apiVersion: redis.redis.opstreelabs.in/v1beta2
3+
kind: RedisCluster
4+
metadata:
5+
name: redis-cluster-v1beta2
6+
status:
7+
readyFollowerReplicas: 3
8+
readyLeaderReplicas: 3
9+
state: Ready
10+
reason: RedisCluster is ready

0 commit comments

Comments
 (0)