diff --git a/resources/node/files/conf/replication.conf.tpl b/resources/node/files/conf/replication.conf.tpl index d595d6c..6eaec23 100644 --- a/resources/node/files/conf/replication.conf.tpl +++ b/resources/node/files/conf/replication.conf.tpl @@ -28,7 +28,6 @@ aof-timestamp-enabled no save "" - replica-announce-port {{ $port }} replica-announce-ip {{ $fqdn }} diff --git a/resources/node/files/conf/sentinel.conf.tpl b/resources/node/files/conf/sentinel.conf.tpl index 22dce69..76df91c 100644 --- a/resources/node/files/conf/sentinel.conf.tpl +++ b/resources/node/files/conf/sentinel.conf.tpl @@ -8,6 +8,7 @@ {{- $fqdn := printf "%s.%s" $hostname $domain }} {{- $hostid := sha1sum $hostname }} {{- $quorum := env "VALKEY_QUORUM" }} +#-REPLICAS:{{ $replicas }} dir "/tmp" port {{ $sentinel_port }} diff --git a/resources/node/files/scripts/common.sh b/resources/node/files/scripts/common.sh index 0217bdc..4d18377 100644 --- a/resources/node/files/scripts/common.sh +++ b/resources/node/files/scripts/common.sh @@ -14,6 +14,31 @@ pingSentinel() { return ${ret} } +getPrimaryInfo() { + valkey-cli --csv -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \ + awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }' + return ${?} +} + +getSentinelMasterName() { + getPrimaryInfo | awk -F ' ' '{print $1}' +} + +getFailOverStatus() { + # Check if the failover is finished + local failoverStatus + primaryInfo=$(getPrimaryInfo) + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + currentHost=$(hostname -f) + if [[ "${primaryHost}" != "${currentHost}" ]]; then + echo "Failover finished" + return 0 + else + echo "Failover in progress" + return 1 + fi +} + waitForSentinel() { tout=60 while true; do @@ -35,16 +60,6 @@ waitForSentinel() { sleep 10 } -getPrimaryInfo() { - valkey-cli --csv -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \ - awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }' - return ${?} -} - -getSentinelMasterName() { - getPrimaryInfo | awk -F ' ' '{print $1}' -} - sentinelIsMaster() { # Check if the sentinel is master primaryInfo=$(getPrimaryInfo) @@ -57,4 +72,10 @@ sentinelIsMaster() { echo "Sentinel is master" return 0 fi +} + +sentinelReset() { + # Reset the sentinel + valkey-cli -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel reset "${1}" + return $? } \ No newline at end of file diff --git a/resources/node/files/scripts/pre-stop-sentinel.sh b/resources/node/files/scripts/pre-stop-sentinel.sh index c52ac3b..d4941aa 100644 --- a/resources/node/files/scripts/pre-stop-sentinel.sh +++ b/resources/node/files/scripts/pre-stop-sentinel.sh @@ -1,29 +1,16 @@ #!/bin/sh -vcli() { - valkey-cli -h 127.0.0.1 -p "${VALKEY_SENTINEL_PORT}" "sentinel" "$@" - return $? -} - -getFailOverStatus() { - # Check if the failover is finished - local failoverStatus - primaryInfo=($(vcli get-primary-addr-by-name "mymaster")) - primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') - currentHost=$(hostname -f) - if [[ "${primaryHost}" != "${currentHost}" ]] then - echo "Failover finished" - return 0 - else - echo "Failover in progress" - return 1 - fi -} +source /opt/scripts/common.sh tmout=120 -while getFailOverStatus; do +while true ; do echo "I'm the primary pod and you are stopping me. Starting sentinel failover" echo "Waiting for failover to finish..." + fo=$(getFailOverStatus) + if [ $? -eq 0 ]; then + echo "Primary has been successfully failed over to a different pod." + break + fi sleep 1 tmout=$((tmout - 1)) @@ -32,5 +19,14 @@ while getFailOverStatus; do exit 1 fi done + echo "Primary has been successfuly failed over to a different pod." + +echo "Removing myself from the sentinel configuration" +sentinelReset $(hostname -f) +if [ $? -ne 0 ]; then + echo "Failed to remove myself from the sentinel configuration" + exit 1 +fi + exit 0 diff --git a/resources/node/files/scripts/pre-stop.sh b/resources/node/files/scripts/pre-stop.sh index f96057d..67e9b8c 100644 --- a/resources/node/files/scripts/pre-stop.sh +++ b/resources/node/files/scripts/pre-stop.sh @@ -19,7 +19,7 @@ getFailOverStatus() { primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') currentHost=$(hostname -f) if [[ "${primaryHost}" != "${currentHost}" ]]; then - echo "I'm not the primary, failover finished" + echo "Failover finished" return 0 else echo "Failover in progress" diff --git a/resources/node/files/scripts/start-node.sh b/resources/node/files/scripts/start-node.sh index c958faa..ecdedde 100644 --- a/resources/node/files/scripts/start-node.sh +++ b/resources/node/files/scripts/start-node.sh @@ -12,7 +12,7 @@ pingSentinel() { } getPrimaryInfo() { - valkey-cli --csv -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \ + valkey-cli -t 15 --csv -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \ awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }' return ${?} } diff --git a/resources/node/files/scripts/startSentinel.sh b/resources/node/files/scripts/startSentinel.sh index 8fc1194..ac6fc40 100644 --- a/resources/node/files/scripts/startSentinel.sh +++ b/resources/node/files/scripts/startSentinel.sh @@ -1,14 +1,12 @@ #!/bin/sh -PATH=${PATH}:/ -SENTINEL_MASTER="" - # Generate sentinel.conf file for Valkey Sentinel source /opt/scripts/common.sh -if [ -f /etc/valkey/sentinel.conf ]; then - echo "Sentinel configuration file already exists, starting with this." -else +PATH=${PATH}:/ +SENTINEL_MASTER="" + +updateSentinelMaster(){ cp /etc/valkey/sentinel.conf.orig /etc/valkey/sentinel.conf hostname=$(hostname -f) @@ -16,11 +14,25 @@ else if [ $? -ne 0 ]; then SENTINEL_MASTER=$(hostname -f) else - getPrimaryInfo SENTINEL_MASTER=$(getSentinelMasterName) fi sed -i "s/SENTINEL_MASTER/${SENTINEL_MASTER}/g" /etc/valkey/sentinel.conf + +} + + +if [ -f /etc/valkey/sentinel.conf ]; then + echo "Sentinel configuration file already exists, starting with this." + reps=$(awk -F ':' '/REPLICAS/ {print $2}' /etc/valkey/sentinel.conf) + if [ "${reps}" = "${VALKEY_REPLICAS}" ]; then + echo "Sentinel configuration file already has the correct number of replicas (${VALKEY_REPLICAS})." + else + echo "Updating Sentinel configuration file with the correct number of replicas (${VALKEY_REPLICAS})." + updateSentinelMaster + fi +else + updateSentinelMaster fi valkey-sentinel /etc/valkey/sentinel.conf \ No newline at end of file diff --git a/resources/node/kustomization.yaml b/resources/node/kustomization.yaml index f9b5f62..3661822 100644 --- a/resources/node/kustomization.yaml +++ b/resources/node/kustomization.yaml @@ -22,7 +22,7 @@ configMapGenerator: - VALKEY_TLS_ENABLED="no" - VALKEY_SENTINEL_TLS_ENABLED="no" - VALKEY_DATA_DIR="/data" - - VALKEY_LOG_LEVEL="debug" + - VALKEY_LOG_LEVEL="warning" - VALKEY_QUORUM="2" - name: valkey-config files: @@ -33,6 +33,7 @@ configMapGenerator: - files/scripts/common.sh - files/scripts/startSentinel.sh - files/scripts/pre-stop.sh + - files/scripts/pre-stop-sentinel.sh - files/scripts/start-node.sh - files/scripts/ping-sentinel.sh - files/scripts/liveness-local.sh