feat(node): try to manage scale up and scale down

resources/node/files/conf/replication.conf.tpl

@@ -28,7 +28,6 @@ aof-timestamp-enabled no
 
 save ""
 
-
 replica-announce-port {{ $port }}
 replica-announce-ip {{ $fqdn }}
 

resources/node/files/conf/sentinel.conf.tpl

@@ -8,6 +8,7 @@
 {{- $fqdn := printf "%s.%s" $hostname $domain }}
 {{- $hostid := sha1sum $hostname }}
 {{- $quorum := env "VALKEY_QUORUM" }}
+#-REPLICAS:{{ $replicas }}
 
 dir "/tmp"
 port {{ $sentinel_port }}

resources/node/files/scripts/common.sh

@@ -14,6 +14,31 @@ pingSentinel() {
     return ${ret}
 }
 
+getPrimaryInfo() {
+    valkey-cli --csv -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \
+        awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }'
+    return ${?}
+}
+
+getSentinelMasterName() {
+    getPrimaryInfo | awk -F ' ' '{print $1}'
+}
+
+getFailOverStatus() {
+    # Check if the failover is finished
+    local failoverStatus
+    primaryInfo=$(getPrimaryInfo)
+    primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}')
+    currentHost=$(hostname -f)
+    if [[ "${primaryHost}" != "${currentHost}" ]]; then
+        echo "Failover finished"
+        return 0
+    else
+        echo "Failover in progress"
+        return 1
+    fi
+}
+
 waitForSentinel() {
     tout=60
     while true; do
@@ -35,16 +60,6 @@ waitForSentinel() {
     sleep 10
 }
 
-getPrimaryInfo() {
-    valkey-cli --csv -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \
-        awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }'
-    return ${?}
-}
-
-getSentinelMasterName() {
-    getPrimaryInfo | awk -F ' ' '{print $1}'
-}
-
 sentinelIsMaster() {
     # Check if the sentinel is master
     primaryInfo=$(getPrimaryInfo)
@@ -57,4 +72,10 @@ sentinelIsMaster() {
         echo "Sentinel is master"
         return 0
     fi
+}
+
+sentinelReset() {
+    # Reset the sentinel
+    valkey-cli -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel reset "${1}"
+    return $?
 }

resources/node/files/scripts/pre-stop-sentinel.sh

@@ -1,29 +1,16 @@
 #!/bin/sh
 
-vcli() {
-    valkey-cli -h 127.0.0.1 -p "${VALKEY_SENTINEL_PORT}" "sentinel" "$@"
-    return $?
-}
-
-getFailOverStatus() {
-    # Check if the failover is finished
-    local failoverStatus
-    primaryInfo=($(vcli get-primary-addr-by-name "mymaster"))
-    primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}')
-    currentHost=$(hostname -f)
-    if [[ "${primaryHost}" != "${currentHost}" ]] then
-        echo "Failover finished"
-        return 0
-    else
-        echo "Failover in progress"
-        return 1
-    fi
-}
+source /opt/scripts/common.sh
 
 tmout=120
-while getFailOverStatus; do
+while true ; do
     echo "I'm the primary pod and you are stopping me. Starting sentinel failover"
     echo "Waiting for failover to finish..."
+    fo=$(getFailOverStatus)
+    if [ $? -eq 0 ]; then
+        echo "Primary has been successfully failed over to a different pod."
+        break
+    fi
 
     sleep 1
     tmout=$((tmout - 1))
@@ -32,5 +19,14 @@ while getFailOverStatus; do
         exit 1
     fi
 done
+
 echo "Primary has been successfuly failed over to a different pod."
+
+echo "Removing myself from the sentinel configuration"
+sentinelReset $(hostname -f)
+if [ $? -ne 0 ]; then
+    echo "Failed to remove myself from the sentinel configuration"
+    exit 1
+fi
+
 exit 0

resources/node/files/scripts/pre-stop.sh

@@ -19,7 +19,7 @@ getFailOverStatus() {
     primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}')
     currentHost=$(hostname -f)
     if [[ "${primaryHost}" != "${currentHost}" ]]; then
-        echo "I'm not the primary, failover finished"
+        echo "Failover finished"
         return 0
     else
         echo "Failover in progress"

resources/node/files/scripts/start-node.sh

@@ -12,7 +12,7 @@ pingSentinel() {
 }
 
 getPrimaryInfo() {
-    valkey-cli --csv -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \
+    valkey-cli -t 15 --csv -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \
         awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }'
     return ${?}
 }

resources/node/files/scripts/startSentinel.sh

@@ -1,14 +1,12 @@
 #!/bin/sh
 
-PATH=${PATH}:/
-SENTINEL_MASTER=""
-
 # Generate sentinel.conf file for Valkey Sentinel
 source /opt/scripts/common.sh
 
-if [ -f /etc/valkey/sentinel.conf ]; then
-    echo "Sentinel configuration file already exists, starting with this."
-else
+PATH=${PATH}:/
+SENTINEL_MASTER=""
+
+updateSentinelMaster(){
     cp /etc/valkey/sentinel.conf.orig /etc/valkey/sentinel.conf
     hostname=$(hostname -f)
 
@@ -16,11 +14,25 @@ else
     if [ $? -ne 0 ]; then
         SENTINEL_MASTER=$(hostname -f)
     else
-        getPrimaryInfo
         SENTINEL_MASTER=$(getSentinelMasterName)
     fi
 
     sed -i "s/SENTINEL_MASTER/${SENTINEL_MASTER}/g" /etc/valkey/sentinel.conf
+
+}
+
+
+if [ -f /etc/valkey/sentinel.conf ]; then
+    echo "Sentinel configuration file already exists, starting with this."
+    reps=$(awk -F ':' '/REPLICAS/ {print $2}' /etc/valkey/sentinel.conf)
+    if [ "${reps}" = "${VALKEY_REPLICAS}" ]; then
+        echo "Sentinel configuration file already has the correct number of replicas (${VALKEY_REPLICAS})."
+    else
+        echo "Updating Sentinel configuration file with the correct number of replicas (${VALKEY_REPLICAS})."
+        updateSentinelMaster
+    fi
+else
+    updateSentinelMaster
 fi
 
 valkey-sentinel /etc/valkey/sentinel.conf

resources/node/kustomization.yaml

@@ -22,7 +22,7 @@ configMapGenerator:
   - VALKEY_TLS_ENABLED="no"
   - VALKEY_SENTINEL_TLS_ENABLED="no"
   - VALKEY_DATA_DIR="/data"
-  - VALKEY_LOG_LEVEL="debug"
+  - VALKEY_LOG_LEVEL="warning"
   - VALKEY_QUORUM="2"
 - name: valkey-config
   files:
@@ -33,6 +33,7 @@ configMapGenerator:
   - files/scripts/common.sh
   - files/scripts/startSentinel.sh
   - files/scripts/pre-stop.sh
+  - files/scripts/pre-stop-sentinel.sh
   - files/scripts/start-node.sh
   - files/scripts/ping-sentinel.sh
   - files/scripts/liveness-local.sh