[Fix] : Fix the backup and restore script and manifest (#624)

shubham-cmyk · mattrobinsonsre · commit a3941a4e0197 · 2024-07-11T08:25:22.000+01:00
* Add : job for bacup from the redis cluster

Signed-off-by: Shubham Gupta &lt;iamshubhamgupta2001@gmail.com&gt;

* remove unwanted var rom env_vars.env

Signed-off-by: Shubham Gupta &lt;iamshubhamgupta2001@gmail.com&gt;

* fix the backup scripts

Signed-off-by: Shubham Gupta &lt;iamshubhamgupta2001@gmail.com&gt;

* docker file

Signed-off-by: Shubham Gupta &lt;iamshubhamgupta2001@gmail.com&gt;

* write manifest for easy backup and restore

Signed-off-by: Shubham Gupta &lt;iamshubhamgupta2001@gmail.com&gt;

---------

Signed-off-by: Shubham Gupta &lt;iamshubhamgupta2001@gmail.com&gt;
Signed-off-by: Matt Robinson &lt;mattrobinsonsre@gmail.com&gt;
diff --git a/example/v1beta2/backup_restore/backup/env-secrets.yaml b/example/v1beta2/backup_restore/backup/env-secrets.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: env-secrets
+  namespace: default
+type: Opaque
+stringData:
+  CLUSTER_NAME: redis-cluster
+  CLUSTER_NAMESPACE: default
+  RESTIC_PASSWORD: abc@123
+  RESTIC_CACHE_DIR: "/tmp/restic_cache"
+  REDIS_PORT: "6379"
+  REDIS_PASSWORD: ""
+  BACKUP_DESTINATION: AWS_S3
+  AWS_S3_BUCKET: shubham-redis
+  AWS_DEFAULT_REGION: ap-south-1
+  AWS_ACCESS_KEY_ID: ""
+  AWS_SECRET_ACCESS_KEY: ""
diff --git a/example/v1beta2/backup_restore/backup/job.yaml b/example/v1beta2/backup_restore/backup/job.yaml
@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: redis-operator-backup-job
+  namespace: default
+spec:
+  template:
+    metadata:
+      name: redis-operator-backup-pod
+    spec:
+      serviceAccountName: backup-service-account
+      containers:
+      - name: redis-operator-backup-container
+        image: "quay.io/opstree/redis-operator-backup:v0.15.1"
+        command: ["/bin/bash", "-c", "/backup/backup.bash"]
+        imagePullPolicy: Always
+        envFrom:
+        - secretRef:
+            name: env-secrets
+      restartPolicy: Never
diff --git a/example/v1beta2/backup_restore/backup/role-binding.yaml b/example/v1beta2/backup_restore/backup/role-binding.yaml
@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: backup-role-binding
+subjects:
+  - kind: ServiceAccount
+    name: backup-service-account
+roleRef:
+  kind: Role
+  name: modify-pods
+  apiGroup: rbac.authorization.k8s.io
diff --git a/example/v1beta2/backup_restore/backup/role.yaml b/example/v1beta2/backup_restore/backup/role.yaml
@@ -0,0 +1,8 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: modify-pods
+rules:
+  - apiGroups: ["*"]
+    resources: ["*"]
+    verbs: ["*"]
diff --git a/example/v1beta2/backup_restore/backup/service-account.yaml b/example/v1beta2/backup_restore/backup/service-account.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: backup-service-account
diff --git a/example/v1beta2/backup_restore/restore/configmap.yaml b/example/v1beta2/backup_restore/restore/configmap.yaml
diff --git a/example/v1beta2/backup_restore/restore/env-secret.yaml b/example/v1beta2/backup_restore/restore/env-secret.yaml
@@ -10,9 +10,6 @@ stringData:
   DEFAULT_FILE_PATH: /data/dump.rdb
   RESTIC_PASSWORD: abc@123
   RESTIC_CACHE_DIR: "/tmp/restic_cache"
-  DEFAULT_REDIS_HOST: redis-cluster-leader-0
-  DEFAULT_REDIS_PORT: "6379"
-  DEFAULT_REDIS_PASSWORD: ""
   BACKUP_DESTINATION: AWS_S3
   AWS_S3_BUCKET: shubham-redis
   AWS_DEFAULT_REGION: ap-south-1
diff --git a/example/v1beta2/backup_restore/restore/redis-cluster.yaml b/example/v1beta2/backup_restore/restore/redis-cluster.yaml
@@ -27,7 +27,7 @@ spec:
     #   - name: regcred
   initContainer:
     enabled: true
-    image: shubham192001/restore:latest
+    image: quay.io/opstree/redis-operator-restore:latest
     imagePullPolicy: Always
     command: ["/bin/bash", "-c", "/app/restore.bash"]
     resources:
@@ -63,21 +63,6 @@ spec:
           secretKeyRef:
             name: env-secrets
             key: RESTIC_CACHE_DIR
-      - name: DEFAULT_REDIS_HOST
-        valueFrom:
-          secretKeyRef:
-            name: env-secrets
-            key: DEFAULT_REDIS_HOST
-      - name: DEFAULT_REDIS_PORT
-        valueFrom:
-          secretKeyRef:
-            name: env-secrets
-            key: DEFAULT_REDIS_PORT
-      - name: DEFAULT_REDIS_PASSWORD
-        valueFrom:
-          secretKeyRef:
-            name: env-secrets
-            key: DEFAULT_REDIS_PASSWORD
       - name: BACKUP_DESTINATION
         valueFrom:
           secretKeyRef:
@@ -138,6 +123,7 @@ spec:
         resources:
           requests:
             storage: 1Gi
+    nodeConfVolume: true
     nodeConfVolumeClaimTemplate:
       spec:
         accessModes: ["ReadWriteOnce"]
diff --git a/example/v1beta2/redis-cluster.yaml b/example/v1beta2/redis-cluster.yaml
@@ -11,7 +11,7 @@ spec:
     runAsUser: 1000
     fsGroup: 1000
   kubernetesConfig:
-    image: quay.io/opstree/redis:v7.0.5
+    image: quay.io/opstree/redis:v7.0.11
     imagePullPolicy: IfNotPresent
     resources:
       requests:
@@ -64,6 +64,7 @@ spec:
         resources:
           requests:
             storage: 1Gi
+    nodeConfVolume: true
     nodeConfVolumeClaimTemplate:
       spec:
         accessModes: ["ReadWriteOnce"]
diff --git a/scripts/backup/Dockerfile.kubectlpod b/scripts/backup/Dockerfile.kubectlpod
@@ -15,6 +15,11 @@ RUN curl -LO "https://dl.k8s.io/$(cat stable.txt)/bin/linux/amd64/kubectl"
 RUN chmod +x kubectl
 RUN mv kubectl /usr/local/bin/
 
-COPY backup-user.bash /backup
-# COPY backup.bash /backup
+# COPY backup-user.bash /backup/backup-user.bash
+# COPY backup.bash /backup/backup.bash
+COPY backup-v0.15.1.bash /backup/backup.bash
+
+# Make the script executable
+RUN chmod +x /backup/backup.bash
+# RUN chmod +x /backup/backup-user.bash
 # COPY env_vars.env /backup
diff --git a/scripts/backup/backup-user.bash b/scripts/backup/backup-user.bash
@@ -62,10 +62,13 @@ TOTAL_LEADERS=$(kubectl get redisclusters.redis.redis.opstreelabs.in "${CLUSTER_
 MASTERS_IP=($(redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" -a "$REDIS_PASSWORD" --no-auth-warning cluster nodes | grep "master" | awk '{print $2}' | cut -d "@" -f1))
 
 check_total_leaders_from_cr() {
-  # Check if TOTAL_LEADERS is 0 or nil
   if [[ -z "$TOTAL_LEADERS" || "$TOTAL_LEADERS" == 0 ]]; then
-    echo "Error: Total number of leader pods is 0"
-    exit 1
+    TOTAL_LEADERS=$(kubectl get redisclusters.redis.redis.opstreelabs.in "${CLUSTER_NAME}" -n "${CLUSTER_NAMESPACE}" -o jsonpath='{.spec.clusterSize}')
+    # After setting, check again if it's still zero or empty
+    if [[ -z "$TOTAL_LEADERS" || "$TOTAL_LEADERS" == 0 ]]; then
+      echo "Error: Total number of leader pods is 0"
+      exit 1
+    fi
   fi
 }
 
diff --git a/scripts/backup/backup-v0.15.1.bash b/scripts/backup/backup-v0.15.1.bash
@@ -0,0 +1,91 @@
+#!/bin/bash
+
+set -e  # Exit on error
+
+case "$BACKUP_DESTINATION" in
+    "aws_s3"|"AWS_S3")
+        RESTIC_REPOSITORY="s3:s3.${AWS_DEFAULT_REGION}.amazonaws.com/${AWS_S3_BUCKET}/${CLUSTER_NAME}-${CLUSTER_NAMESPACE}"
+        ;;
+    "azure_blob"|"AZURE_BLOB")
+        RESTIC_REPOSITORY="azure:${AZURE_CONTAINER}:${CLUSTER_NAME}-${CLUSTER_NAMESPACE}"
+        ;;
+    "google_cloud"|"GOOGLE_CLOUD")
+        RESTIC_REPOSITORY="gs:${GCP_BUCKET}/${CLUSTER_NAME}-${CLUSTER_NAMESPACE}"
+        ;;
+    *)
+        echo "Invalid choice. Exiting."
+        exit 1
+        ;;
+esac
+
+
+REDIS_HOST="${CLUSTER_NAME}-leader-0.${CLUSTER_NAME}-leader-headless.${CLUSTER_NAMESPACE}.svc"
+
+# Check the Total Leader Present in Redis Cluster using cr and redis-cli
+TOTAL_LEADERS=$(kubectl get redisclusters.redis.redis.opstreelabs.in "${CLUSTER_NAME}" -n "${CLUSTER_NAMESPACE}" -o jsonpath='{.spec.redisLeader.replicas}')
+MASTERS_IP=($(redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" -a "$REDIS_PASSWORD" --no-auth-warning cluster nodes | grep "master" | awk '{print $2}' | cut -d "@" -f1))
+
+check_total_leaders_from_cr() {
+  if [[ -z "$TOTAL_LEADERS" || "$TOTAL_LEADERS" == 0 ]]; then
+    TOTAL_LEADERS=$(kubectl get redisclusters.redis.redis.opstreelabs.in "${CLUSTER_NAME}" -n "${CLUSTER_NAMESPACE}" -o jsonpath='{.spec.clusterSize}')
+    # After setting, check again if it's still zero or empty
+    if [[ -z "$TOTAL_LEADERS" || "$TOTAL_LEADERS" == 0 ]]; then
+      echo "Error: Total number of leader pods is 0"
+      exit 1
+    fi
+  fi
+}
+
+check_total_masters_from_redis() {
+  # IF the total master by the redis custom-resource and the redis-cli doesn't match throw an error
+  if [[ "${TOTAL_LEADERS}" -ne "${#MASTERS_IP[@]}" ]]; then
+    echo "Error: Total number of leaders (${TOTAL_LEADERS}) is not equal to total number of masters (${#MASTERS_IP[@]})!"
+    exit 1
+  fi
+}
+
+
+initialize_repository() {
+    # To set the password of the repo you must pass it the env Variable  RESTIC_PASSWORD
+    if ! restic -r "$RESTIC_REPOSITORY" snapshots &>/dev/null ; then
+        echo "Initializing restic repository..."
+        restic init --repo "$RESTIC_REPOSITORY"
+    else
+        echo "Restic repository already initialized."
+    fi
+}
+
+
+perform_redis_backup(){
+    # Start performing backup
+    for ((i = 0; i < TOTAL_LEADERS; i++))
+    do  
+        # Get the name of the Redis pod
+        POD="${CLUSTER_NAME}-leader-${i}"
+
+        # Get the IP address and port of the Redis instance
+        IP_PORT="${MASTERS_IP[${i}]}"
+
+        # Extract the IP address and port from the IP_PORT variable
+        IP=$(echo "$IP_PORT" | cut -d ':' -f 1)
+        PORT=$(echo "$IP_PORT" | cut -d ':' -f 2)
+
+        echo "Performing backup on Redis instance at $IP:$PORT"
+
+        # Copy the file from the Redis instance to a local file
+        redis-cli -h "$IP" -p "$PORT" -a "$REDIS_PASSWORD" --rdb "/tmp/${POD}.rdb"
+
+        # Upload the file to the selected backup destination using restic
+        restic -r "$RESTIC_REPOSITORY" backup "/tmp/${POD}.rdb" --host "${CLUSTER_NAME}_${CLUSTER_NAMESPACE}" --tag "${POD}" --tag "redis"
+
+        # Clean up the local file
+        rm "/tmp/${POD}.rdb"
+    done
+}
+
+
+check_total_leaders_from_cr
+check_total_masters_from_redis
+initialize_repository
+perform_redis_backup
+echo "Backup completed successfully."
diff --git a/scripts/backup/backup.bash b/scripts/backup/backup.bash
@@ -23,7 +23,7 @@ REDIS_HOST="${CLUSTER_NAME}-leader-0.${CLUSTER_NAME}-leader-headless.${CLUSTER_N
 
 # Check the Total Leader Present in Redis Cluster using cr and redis-cli
 TOTAL_LEADERS=$(kubectl get redisclusters.redis.redis.opstreelabs.in "${CLUSTER_NAME}" -n "${CLUSTER_NAMESPACE}" -o jsonpath='{.spec.redisLeader.replicas}')
-MASTERS_IP=$(redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" -a "$REDIS_PASSWORD" cluster nodes --no-auth-warning | grep "master" | awk '{print $2}' | cut -d "@" -f1)
+MASTERS_IP=($(redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" -a "$REDIS_PASSWORD" --no-auth-warning cluster nodes | grep "master" | awk '{print $2}' | cut -d "@" -f1))
 
 check_total_leaders_from_cr() {
   # Check if TOTAL_LEADERS is 0 or nil
@@ -33,11 +33,14 @@ check_total_leaders_from_cr() {
   fi
 }
 
-check_total_masters_from_redis() {
-  # IF the total master by the redis custom-resource and the redis-cli doesn't match throw an error
-  if [[ "${TOTAL_LEADERS}" -ne "${#MASTERS_IP[@]}" ]]; then
-    echo "Error: Total number of leaders (${TOTAL_LEADERS}) is not equal to total number of masters (${#MASTERS_IP[@]})!"
-    exit 1
+check_total_leaders_from_cr() {
+  if [[ -z "$TOTAL_LEADERS" || "$TOTAL_LEADERS" == 0 ]]; then
+    TOTAL_LEADERS=$(kubectl get redisclusters.redis.redis.opstreelabs.in "${CLUSTER_NAME}" -n "${CLUSTER_NAMESPACE}" -o jsonpath='{.spec.clusterSize}')
+    # After setting, check again if it's still zero or empty
+    if [[ -z "$TOTAL_LEADERS" || "$TOTAL_LEADERS" == 0 ]]; then
+      echo "Error: Total number of leader pods is 0"
+      exit 1
+    fi
   fi
 }
 
@@ -73,7 +76,7 @@ perform_redis_backup(){
         redis-cli -h "$IP" -p "$PORT" -a "$REDIS_PASSWORD" --rdb "/tmp/${POD}.rdb"
 
         # Upload the file to the selected backup destination using restic
-        restic -r "$RESTIC_REPOSITORY" backup "/tmp/${POD}.rdb" --host"${CLUSTER_NAME}_${CLUSTER_NAMESPACE}" --tag "${POD}" --tag "redis"
+        restic -r "$RESTIC_REPOSITORY" backup "/tmp/${POD}.rdb" --host "${CLUSTER_NAME}_${CLUSTER_NAMESPACE}" --tag "${POD}" --tag "redis"
 
         # Clean up the local file
         rm "/tmp/${POD}.rdb"
diff --git a/scripts/backup/env_vars.env b/scripts/backup/env_vars.env
@@ -7,14 +7,13 @@ export CLUSTER_NAMESPACE="default"
 # Restic
 export DEFAULT_FILE_PATH="/data/dump.rdb"
 export RESTIC_PASSWORD="abc@123"
+export RESTIC_CACHE_DIR=/tmp/restic_cache
 
 # Redis 
-export DEFAULT_REDIS_HOST="redis-cluster-leader-0"
 export DEFAULT_REDIS_PORT="6379"
 export DEFAULT_REDIS_PASSWORD=""
 
 # Backup destination
-
 export BACKUP_DESTINATION=AWS_S3
 
 # AWS
diff --git a/scripts/restore/env_vars.env b/scripts/restore/env_vars.env
@@ -7,9 +7,9 @@ export CLUSTER_NAMESPACE="default"
 # Restic
 export DEFAULT_FILE_PATH="/data/dump.rdb"
 export RESTIC_PASSWORD="abc@123"
+export RESTIC_CACHE_DIR=/tmp/restic_cache
 
 # Redis 
-export DEFAULT_REDIS_HOST="redis-cluster-leader-0"
 export DEFAULT_REDIS_PORT="6379"
 export DEFAULT_REDIS_PASSWORD=""
 
