diff --git a/files/default.vcl b/files/default.vcl deleted file mode 100644 index 1cb8790..0000000 --- a/files/default.vcl +++ /dev/null @@ -1,6 +0,0 @@ -vcl 4.0; - -backend default { - .host = "127.0.0.1"; - .port = "8080"; -} \ No newline at end of file diff --git a/kustomization.yaml b/kustomization.yaml index 10dfa43..ad68128 100644 --- a/kustomization.yaml +++ b/kustomization.yaml @@ -2,13 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- resources/deployment.yaml -- resources/svc.yaml - -configMapGenerator: -- name: varnish-env - literals: - - VARNISH_SIZE=1G -- name: varnish-config - files: - - files/default.vcl +- resources/node diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile index d468f70..ad67f69 100644 --- a/misc/docker/Dockerfile +++ b/misc/docker/Dockerfile @@ -1,18 +1,22 @@ # Base image -FROM golang AS builder +FROM golang:tip-alpine3.21 AS builder # Set directory to known value WORKDIR /app # Define the version as a build argument ARG GOTEMPLATE_VERSION=3.12.0 +RUN apk add valkey-cli git + # Git clone the repo for gotemplate, checkout the desired tag, and build the executable RUN git clone https://github.com/coveooss/gotemplate.git . && \ git checkout v${GOTEMPLATE_VERSION} && \ CGO_ENABLED=0 go build -FROM busybox +FROM alpine:3.21 +# COPY --from=builder /app/gotemplate /gotemplate +COPY --from=builder /usr/bin/valkey-cli /valkey-cli ENTRYPOINT [ "/gotemplate" ] \ No newline at end of file diff --git a/resources/deployment.yaml b/resources/deployment.yaml deleted file mode 100644 index 3ae5e80..0000000 --- a/resources/deployment.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: varnish - labels: - app: varnish -spec: - replicas: 1 - selector: - matchLabels: - app: varnish - template: - metadata: - labels: - app: varnish - spec: - containers: - - name: varnish - image: reg.cadoles.com/dh/library/varnish:7.6.1-alpine - imagePullPolicy: IfNotPresent - env: - - name: VARNISH_HTTP_PORT - value: "8080" - envFrom: - - configMapRef: - name: varnish-env - ports: - - containerPort: 8080 - volumeMounts: - - mountPath: /etc/varnish/default.vcl - name: varnish-config - subPath: default.vcl - - mountPath: /var/lib/varnish/varnishd - name: varnish-lib - securityContext: - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: - add: [ "IPC_LOCK" ] - volumes: - - name: varnish-config - configMap: - name: varnish-config - items: - - key: default.vcl - path: default.vcl - - name: varnish-lib - emptyDir: - sizeLimit: 1Gi - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - runAsNonRoot: true diff --git a/resources/node/files/conf/replication.conf.tpl b/resources/node/files/conf/replication.conf.tpl new file mode 100644 index 0000000..d595d6c --- /dev/null +++ b/resources/node/files/conf/replication.conf.tpl @@ -0,0 +1,38 @@ +{{- $hostname := env "HOSTNAME" }} +{{- $service := env "VALKEY_SERVICE" }} +{{- $namespace := env "NAMESPACE" }} +{{- $port := env "VALKEY_PORT" }} +{{- $sentinel_port := env "VALKEY_SENTINEL_PORT" }} +{{- $replicas := env "VALKEY_REPLICAS" }} +{{- $domain := printf "%s.%s.svc.cluster.local" $service $namespace }} +{{- $fqdn := printf "%s.%s" $hostname $domain }} +{{- $hostid := sha1sum $hostname }} +{{- $datadir := env "VALKEY_DATA_DIR" }} + +dir {{ $datadir }} + +protected-mode no + +loglevel {{ env "VALKEY_LOG_LEVEL" }} + +appendonly yes +appendfilename "appendonly.aof" +appenddirname "appendonlydir" +appendfsync everysec +no-appendfsync-on-rewrite no +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb +aof-load-truncated yes +aof-use-rdb-preamble yes +aof-timestamp-enabled no + +save "" + + +replica-announce-port {{ $port }} +replica-announce-ip {{ $fqdn }} + +# User-supplied replica configuration: +rename-command FLUSHDB "" +rename-command FLUSHALL "" + diff --git a/resources/node/files/conf/replication.conf.tpl.full b/resources/node/files/conf/replication.conf.tpl.full new file mode 100644 index 0000000..20b18c5 --- /dev/null +++ b/resources/node/files/conf/replication.conf.tpl.full @@ -0,0 +1,119 @@ +{{- $hostname := env "HOSTNAME" }} +{{- $service := env "VALKEY_SERVICE" }} +{{- $namespace := env "NAMESPACE" }} +{{- $port := env "VALKEY_PORT" }} +{{- $sentinel_port := env "VALKEY_SENTINEL_PORT" }} +{{- $replicas := env "VALKEY_REPLICAS" }} +{{- $domain := printf "%s.%s.svc.cluster.local" $service $namespace }} +{{- $fqdn := printf "%s.%s" $hostname $domain }} +{{- $hostid := sha1sum $hostname }} +################################## INCLUDES ################################### +################################## MODULES ##################################### +################################## NETWORK ##################################### +bind * -::* +protected-mode no +port {{ $port }} +tcp-backlog 511 +timeout 0 +tcp-keepalive 300 +################################# TLS/SSL ##################################### +port {{ env "VALKEY_PORT" }} +################################### RDMA ###################################### +################################# GENERAL ##################################### +daemonize no +pidfile /opt/bitnami/valkey/tmp/valkey.pid +loglevel notice +logfile "" +databases 16 +always-show-logo no +hide-user-data-from-log yes +set-proc-title yes +proc-title-template "{title} {listen-addr} {server-mode}" +locale-collate "" +################################ SNAPSHOTTING ################################ +stop-writes-on-bgsave-error yes +rdbcompression yes +rdbchecksum yes +rdb-version-check strict +dbfilename dump.rdb +rdb-del-sync-files no +dir {{ env "VALKEY_DATA_DIR" }} +################################# REPLICATION ################################# +replica-serve-stale-data yes +replica-read-only yes +repl-diskless-sync yes +repl-diskless-sync-delay 5 +repl-diskless-sync-max-replicas 0 +repl-diskless-load disabled +dual-channel-replication-enabled no +repl-disable-tcp-nodelay no +replica-priority 100 +replica-announce-port {{ $port }} +replica-announce-ip {{ $fqdn }} +############################### KEYS TRACKING ################################# +################################## SECURITY ################################### +acllog-max-len 128 +################################### CLIENTS #################################### +############################## MEMORY MANAGEMENT ################################ +############################# LAZY FREEING #################################### +lazyfree-lazy-eviction yes +lazyfree-lazy-expire yes +lazyfree-lazy-server-del yes +replica-lazy-flush yes +lazyfree-lazy-user-del yes +lazyfree-lazy-user-flush yes +################################ THREADED I/O ################################# +############################ KERNEL OOM CONTROL ############################## +oom-score-adj no +oom-score-adj-values 0 200 800 +#################### KERNEL transparent hugepage CONTROL ###################### +disable-thp yes +############################## APPEND ONLY MODE ############################### +appendonly no +appendfilename "appendonly.aof" +appenddirname "appendonlydir" +appendfsync everysec +no-appendfsync-on-rewrite no +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb +aof-load-truncated yes +aof-use-rdb-preamble yes +aof-timestamp-enabled no +################################ SHUTDOWN ##################################### +################ NON-DETERMINISTIC LONG BLOCKING COMMANDS ##################### +################################ VALKEY CLUSTER ############################### +########################## CLUSTER DOCKER/NAT support ######################## +################################## COMMAND LOG ################################### +commandlog-execution-slower-than 10000 +commandlog-slow-execution-max-len 128 +commandlog-request-larger-than 1048576 +commandlog-large-request-max-len 128 +commandlog-reply-larger-than 1048576 +commandlog-large-reply-max-len 128 +################################ LATENCY MONITOR ############################## +latency-monitor-threshold 0 +################################ LATENCY TRACKING ############################## +############################# EVENT NOTIFICATION ############################## +notify-keyspace-events "" +############################### ADVANCED CONFIG ############################### +hash-max-listpack-entries 512 +hash-max-listpack-value 64 +list-max-listpack-size -2 +list-compress-depth 0 +set-max-intset-entries 512 +set-max-listpack-entries 128 +set-max-listpack-value 64 +zset-max-listpack-entries 128 +zset-max-listpack-value 64 +hll-sparse-max-bytes 3000 +stream-node-max-bytes 4096 +stream-node-max-entries 100 +activerehashing yes +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit replica 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 +hz 10 +aof-rewrite-incremental-fsync yes +rdb-save-incremental-fsync yes +########################### ACTIVE DEFRAGMENTATION ####################### +jemalloc-bg-thread yes diff --git a/resources/node/files/conf/sentinel.conf.tpl b/resources/node/files/conf/sentinel.conf.tpl new file mode 100644 index 0000000..22dce69 --- /dev/null +++ b/resources/node/files/conf/sentinel.conf.tpl @@ -0,0 +1,45 @@ +{{- $hostname := env "HOSTNAME" }} +{{- $service := env "VALKEY_SERVICE" }} +{{- $namespace := env "NAMESPACE" }} +{{- $port := env "VALKEY_PORT" }} +{{- $sentinel_port := env "VALKEY_SENTINEL_PORT" }} +{{- $replicas := env "VALKEY_REPLICAS" }} +{{- $domain := printf "%s.%s.svc.cluster.local" $service $namespace }} +{{- $fqdn := printf "%s.%s" $hostname $domain }} +{{- $hostid := sha1sum $hostname }} +{{- $quorum := env "VALKEY_QUORUM" }} + +dir "/tmp" +port {{ $sentinel_port }} +sentinel monitor mymaster SENTINEL_MASTER {{ $port }} {{ $quorum }} +sentinel down-after-milliseconds mymaster 60000 + +# User-supplied sentinel configuration: +# End of sentinel configuration +{{ printf "sentinel myid %s" $hostid }} + +sentinel announce-hostnames yes +sentinel resolve-hostnames yes + +sentinel announce-port {{ $sentinel_port }} +sentinel announce-ip {{ $fqdn }} + +# Generated by CONFIG REWRITE +latency-tracking-info-percentiles 50 99 99.9 +protected-mode no +# gotemplate-pause! +user default on nopass sanitize-payload ~* &* +@all +# gotemplate-resume! +sentinel config-epoch mymaster 0 +sentinel leader-epoch mymaster 0 +sentinel current-epoch 0 + +{{- range $i, $e := until ( int $replicas ) }} + {{- $ndeHostname := printf "valkey-node-%d" $i }} + {{- $ndeFQDN := printf "%s.%s" $ndeHostname $domain }} + {{- $nodeID := sha1sum $ndeHostname }} + {{- if (ne $fqdn $ndeFQDN) }} +sentinel {{ printf "known-sentinel mymaster %s %s %s" $ndeFQDN $sentinel_port $nodeID }} +sentinel {{ printf "known-replica mymaster %s %s" $ndeFQDN $port }} + {{- end}} +{{- end}} \ No newline at end of file diff --git a/resources/node/files/scripts/common.sh b/resources/node/files/scripts/common.sh new file mode 100644 index 0000000..0217bdc --- /dev/null +++ b/resources/node/files/scripts/common.sh @@ -0,0 +1,60 @@ +#!/bin/sh + + +pingSentinel() { + svc=${VALKEY_SERVICE:-"localhost"} + + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h ${svc} \ + -p $VALKEY_SENTINEL_PORT \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +waitForSentinel() { + tout=60 + while true; do + response=$(pingSentinel 5) + if [ "${response}" = "PONG" ]; then + echo "Sentinel is responding" + break + return 0 + fi + + echo "Sentinel is not responding [${response}]" + sleep 1 + tout=$((tout - 1)) + if [ "${tout}" -le 0 ]; then + echo "Sentinel ping timed out" + return 124 + fi + done + sleep 10 +} + +getPrimaryInfo() { + valkey-cli --csv -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \ + awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }' + return ${?} +} + +getSentinelMasterName() { + getPrimaryInfo | awk -F ' ' '{print $1}' +} + +sentinelIsMaster() { + # Check if the sentinel is master + primaryInfo=$(getPrimaryInfo) + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + currentHost=$(hostname -f) + if [[ "${primaryHost}" != "${currentHost}" ]]; then + echo "Sentinel is not master" + return 1 + else + echo "Sentinel is master" + return 0 + fi +} \ No newline at end of file diff --git a/resources/node/files/scripts/liveness-local.sh b/resources/node/files/scripts/liveness-local.sh new file mode 100644 index 0000000..f56965f --- /dev/null +++ b/resources/node/files/scripts/liveness-local.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +ping_valkey() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h localhost \ + -p $VALKEY_PORT \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +response=$(ping_valkey 5) +if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 +fi + +firstWord=$(echo $response | awk 'NR==1 {print $1;}') +if [ "$response" != "PONG" ] && [ "$firstWord" != "LOADING" ] && [ "$firstWord" != "MASTERDOWN" ]; then + echo "Valey is not alive [${response}]" + exit 1 +fi + +echo "$( date +'[%Y/%m/%d %H:%M:%S]') Valkey is alive" +exit 0 diff --git a/resources/node/files/scripts/ping-sentinel.sh b/resources/node/files/scripts/ping-sentinel.sh new file mode 100644 index 0000000..498627c --- /dev/null +++ b/resources/node/files/scripts/ping-sentinel.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +ping_sentinel() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h localhost \ + -p $VALKEY_SENTINEL_PORT \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +response=$(ping_sentinel 5) +if [ "${?}" -eq 124 ]; then + echo "Sentinel ping timed out" + exit 124 +fi + +if [ "${response}" != "PONG" ]; +then + echo "Sentinel is not responding" + exit 1 +fi + +echo "$( date +'[%Y/%m/%d %H:%M:%S]') Sentinel is responding" +exit 0 \ No newline at end of file diff --git a/resources/node/files/scripts/pre-stop-sentinel.sh b/resources/node/files/scripts/pre-stop-sentinel.sh new file mode 100644 index 0000000..c52ac3b --- /dev/null +++ b/resources/node/files/scripts/pre-stop-sentinel.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +vcli() { + valkey-cli -h 127.0.0.1 -p "${VALKEY_SENTINEL_PORT}" "sentinel" "$@" + return $? +} + +getFailOverStatus() { + # Check if the failover is finished + local failoverStatus + primaryInfo=($(vcli get-primary-addr-by-name "mymaster")) + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + currentHost=$(hostname -f) + if [[ "${primaryHost}" != "${currentHost}" ]] then + echo "Failover finished" + return 0 + else + echo "Failover in progress" + return 1 + fi +} + +tmout=120 +while getFailOverStatus; do + echo "I'm the primary pod and you are stopping me. Starting sentinel failover" + echo "Waiting for failover to finish..." + + sleep 1 + tmout=$((tmout - 1)) + if [ "${tmout}" -le 0 ]; then + echo "Failover timed out" + exit 1 + fi +done +echo "Primary has been successfuly failed over to a different pod." +exit 0 diff --git a/resources/node/files/scripts/pre-stop.sh b/resources/node/files/scripts/pre-stop.sh new file mode 100644 index 0000000..f96057d --- /dev/null +++ b/resources/node/files/scripts/pre-stop.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +# Run Valkey command +vcli() { + valkey-cli -h 127.0.0.1 -p "${VALKEY_PORT}" "$@" + return $? +} + +# Run Sentinel command +vcli-sentinel() { + valkey-cli -h "$VALKEY_SERVICE" -p "$VALKEY_SENTINEL_PORT" sentinel "$@" + return $? +} + +getFailOverStatus() { + # Check if the failover is finished + local failoverStatus + primaryInfo=$(vcli-sentinel get-primary-addr-by-name "mymaster") + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + currentHost=$(hostname -f) + if [[ "${primaryHost}" != "${currentHost}" ]]; then + echo "I'm not the primary, failover finished" + return 0 + else + echo "Failover in progress" + return 1 + fi +} + +getRole() { + # Get the role of the current node + VALKEY_ROLE=$(vcli role | head -1) + echo "${VALKEY_ROLE}" +} + +isPrimary() { + # Check if the current node is the primary + role=$(getRole) + if [ "${role}" = "master" ]; then + echo "I'm the master" + else + echo "I'm not the master, I'm a ${role}" + fi +} + +if isPrimary && ! getFailOverStatus; then + echo "I'm the primary and you are stopping me, pausing client connections" + #Pausing write connections to avoid data loss" + vcli CLIENT PAUSE "22000" WRITE + + echo "Starting failover" + vcli-sentinel failover "mymaster" + echo "Waiting for sentinel to complete failover for up to 120s" + tmout=120 + while true ; do + echo "I'm the primary pod and you are stopping me. Starting sentinel failover" + getFailOverStatus + if [ $? -eq 0 ]; then + echo "Primary has been successfully failed over to a different pod." + break + fi + echo "Waiting for failover to finish..." + + sleep 1 + tmout=$((tmout - 1)) + if [ "${tmout}" -le 0 ]; then + echo "Failover timed out" + exit 1 + fi + done +fi + +exit 0 \ No newline at end of file diff --git a/resources/node/files/scripts/readiness-local.sh b/resources/node/files/scripts/readiness-local.sh new file mode 100644 index 0000000..c852cb1 --- /dev/null +++ b/resources/node/files/scripts/readiness-local.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +ping_valkey() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h localhost \ + -p $VALKEY_PORT \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +response=$(ping_valkey 5) +if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 +fi + +if [ "$response" != "PONG" ]; then + echo "Valey is not ready [${response}]" + exit 1 +fi + +echo "$( date +'[%Y/%m/%d %H:%M:%S]') Valkey is ready" +exit 0 diff --git a/resources/node/files/scripts/start-node.sh b/resources/node/files/scripts/start-node.sh new file mode 100644 index 0000000..c958faa --- /dev/null +++ b/resources/node/files/scripts/start-node.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +pingSentinel() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h ${VALKEY_SERVICE} \ + -p ${VALKEY_SENTINEL_PORT} \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +getPrimaryInfo() { + valkey-cli --csv -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \ + awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }' + return ${?} +} + +waitForSentinel() { + tout=60 + while true; do + response=$(pingSentinel 5) + if [ "${response}" = "PONG" ]; then + echo "Sentinel is responding" + break + return 0 + fi + + echo "Sentinel is not responding [${response}]" + sleep 1 + tout=$((tout - 1)) + if [ "${tout}" -le 0 ]; then + echo "Sentinel ping timed out" + return 124 + fi + done + sleep 10 +} + +startValkey() { + # Start Valkey + echo "Running : [valkey-server ${@}]" + valkey-server ${@} + ret=${?} + if [ "${ret}" -ne 0 ]; then + echo "Failed to start Valkey" + exit ${ret} + fi +} + +setupPrimary=0 +primaryHost="" +primaryPort="" + + +if [[ -f /etc/valkey/sentinel.conf ]]; then + primaryHost="$(awk '/monitor/ {print $4}' /etc/valkey/sentinel.conf)" + primaryPort="$(awk '/monitor/ {print $5}' /etc/valkey/sentinel.conf)" + echo "Found previous primary ${primaryHost}:${primaryPort}" +fi + +if [[ ! -f /etc/valkey/replication.conf ]]; then + cp /etc/valkey/replication.conf.orig /etc/valkey/replication.conf +fi + +waitForSentinel +ret=${?} +if [ "${ret}" -ne 0 ]; then + exit ${ret} +fi + +primaryInfo=$(getPrimaryInfo) +if [ "${?}" -ne 0 ]; then + echo "No primary found, seting up node as primary" + startPrimary=1 +else + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + primaryPort=$(echo ${primaryInfo} | awk -F ' ' '{print $2}') + echo "Primary host is : ${primaryHost}, port: ${primaryPort}" + currentHost=$(hostname -f) + echo "Current host is : ${currentHost}" + if [ "${primaryHost}" != "${currentHost}" ]; then + echo "Not the primary, setting up as replica" + startPrimary=0 + else + echo "This is the primary" + startPrimary=1 + fi +fi + +if [ "${startPrimary}" -eq 1 ]; then + echo "Starting Valkey as primary" + cat $1 + startValkey ${@} +else + echo "Starting Valkey as replica" + startValkey ${@} "--replicaof" "${primaryHost}" "${primaryPort}" +fi \ No newline at end of file diff --git a/resources/node/files/scripts/startSentinel.sh b/resources/node/files/scripts/startSentinel.sh new file mode 100644 index 0000000..8fc1194 --- /dev/null +++ b/resources/node/files/scripts/startSentinel.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +PATH=${PATH}:/ +SENTINEL_MASTER="" + +# Generate sentinel.conf file for Valkey Sentinel +source /opt/scripts/common.sh + +if [ -f /etc/valkey/sentinel.conf ]; then + echo "Sentinel configuration file already exists, starting with this." +else + cp /etc/valkey/sentinel.conf.orig /etc/valkey/sentinel.conf + hostname=$(hostname -f) + + pingSentinel 1 + if [ $? -ne 0 ]; then + SENTINEL_MASTER=$(hostname -f) + else + getPrimaryInfo + SENTINEL_MASTER=$(getSentinelMasterName) + fi + + sed -i "s/SENTINEL_MASTER/${SENTINEL_MASTER}/g" /etc/valkey/sentinel.conf +fi + +valkey-sentinel /etc/valkey/sentinel.conf \ No newline at end of file diff --git a/resources/node/files/scripts/startnode.sh b/resources/node/files/scripts/startnode.sh new file mode 100644 index 0000000..67061fd --- /dev/null +++ b/resources/node/files/scripts/startnode.sh @@ -0,0 +1,151 @@ +#!/bin/bash + +. /opt/bitnami/scripts/libos.sh +. /opt/bitnami/scripts/liblog.sh +. /opt/bitnami/scripts/libvalidations.sh + +get_port() { + hostname="$1" + type="$2" + + port_var=$(echo "${hostname^^}_SERVICE_PORT_$type" | sed "s/-/_/g") + port=${!port_var} + + if [ -z "$port" ]; then + case $type in + "SENTINEL") + echo 26379 + ;; + "VALKEY") + echo 6379 + ;; + esac + else + echo $port + fi +} + +get_full_hostname() { + hostname="$1" + full_hostname="${hostname}.${HEADLESS_SERVICE}" + echo "${full_hostname}" +} + +VALKEYPORT=$(get_port "$HOSTNAME" "VALKEY") + +HEADLESS_SERVICE="valkey-headless.mse-dev.svc.cluster.local" + +if [ -n "$VALKEY_EXTERNAL_PRIMARY_HOST" ]; then + VALKEY_SERVICE="$VALKEY_EXTERNAL_PRIMARY_HOST" +else + VALKEY_SERVICE="valkey.mse-dev.svc.cluster.local" +fi + +SENTINEL_SERVICE_PORT=$(get_port "valkey" "SENTINEL") +validate_quorum() { + if is_boolean_yes "$VALKEY_TLS_ENABLED"; then + quorum_info_command="valkey-cli -h $VALKEY_SERVICE -p $SENTINEL_SERVICE_PORT --tls --cert ${VALKEY_TLS_CERT_FILE} --key ${VALKEY +_TLS_KEY_FILE} --cacert ${VALKEY_TLS_CA_FILE} sentinel primary mymaster" + else + quorum_info_command="valkey-cli -h $VALKEY_SERVICE -p $SENTINEL_SERVICE_PORT sentinel primary mymaster" + fi + info "about to run the command: $quorum_info_command" + eval $quorum_info_command | grep -Fq "s_down" +} + +trigger_manual_failover() { + if is_boolean_yes "$VALKEY_TLS_ENABLED"; then + failover_command="valkey-cli -h $VALKEY_SERVICE -p $SENTINEL_SERVICE_PORT --tls --cert ${VALKEY_TLS_CERT_FILE} --key ${VALKEY_TL +S_KEY_FILE} --cacert ${VALKEY_TLS_CA_FILE} sentinel failover mymaster" + else + failover_command="valkey-cli -h $VALKEY_SERVICE -p $SENTINEL_SERVICE_PORT sentinel failover mymaster" + fi + + info "about to run the command: $failover_command" + eval $failover_command +} + +get_sentinel_primary_info() { + if is_boolean_yes "$VALKEY_TLS_ENABLED"; then + sentinel_info_command="timeout 90 valkey-cli -h $VALKEY_SERVICE -p $SENTINEL_SERVICE_PORT --tls --cert ${VALKEY_TLS_CERT_FILE} - +-key ${VALKEY_TLS_KEY_FILE} --cacert ${VALKEY_TLS_CA_FILE} sentinel get-primary-addr-by-name mymaster" + else + sentinel_info_command="timeout 90 valkey-cli -h $VALKEY_SERVICE -p $SENTINEL_SERVICE_PORT sentinel get-primary-addr-by-name myma +ster" + fi + info "about to run the command: $sentinel_info_command" + retry_while "eval $sentinel_info_command" 2 5 +} + +[[ -f $VALKEY_PASSWORD_FILE ]] && export VALKEY_PASSWORD="$(< "${VALKEY_PASSWORD_FILE}")" +[[ -f $VALKEY_PRIMARY_PASSWORD_FILE ]] && export VALKEY_PRIMARY_PASSWORD="$(< "${VALKEY_PRIMARY_PASSWORD_FILE}")" + +# check if there is a primary +primary_in_persisted_conf="$(get_full_hostname "$HOSTNAME")" +primary_port_in_persisted_conf="$VALKEY_PRIMARY_PORT_NUMBER" +primary_in_sentinel="$(get_sentinel_primary_info)" +valkeyRetVal=$? + +if [[ -f /opt/bitnami/valkey-sentinel/etc/sentinel.conf ]]; then + primary_in_persisted_conf="$(awk '/monitor/ {print $4}' /opt/bitnami/valkey-sentinel/etc/sentinel.conf)" + primary_port_in_persisted_conf="$(awk '/monitor/ {print $5}' /opt/bitnami/valkey-sentinel/etc/sentinel.conf)" + info "Found previous primary ${primary_in_persisted_conf}:${primary_port_in_persisted_conf} in /opt/bitnami/valkey-sentinel/etc/sent +inel.conf" + debug "$(cat /opt/bitnami/valkey-sentinel/etc/sentinel.conf | grep monitor)" +fi + +if [[ $valkeyRetVal -ne 0 ]]; then + if [[ "$primary_in_persisted_conf" == "$(get_full_hostname "$HOSTNAME")" ]]; then + # Case 1: No active sentinel and in previous sentinel.conf we were the primary --> PRIMARY + info "Configuring the node as primary" + export VALKEY_REPLICATION_MODE="primary" + else + # Case 2: No active sentinel and in previous sentinel.conf we were not primary --> REPLICA + info "Configuring the node as replica" + export VALKEY_REPLICATION_MODE="replica" + VALKEY_PRIMARY_HOST=${primary_in_persisted_conf} + VALKEY_PRIMARY_PORT_NUMBER=${primary_port_in_persisted_conf} + fi +else + # Fetches current primary's host and port + VALKEY_SENTINEL_INFO=($(get_sentinel_primary_info)) + info "Current primary: VALKEY_SENTINEL_INFO=(${VALKEY_SENTINEL_INFO[0]},${VALKEY_SENTINEL_INFO[1]})" + VALKEY_PRIMARY_HOST=${VALKEY_SENTINEL_INFO[0]} + VALKEY_PRIMARY_PORT_NUMBER=${VALKEY_SENTINEL_INFO[1]} + + if [[ "$VALKEY_PRIMARY_HOST" == "$(get_full_hostname "$HOSTNAME")" ]]; then + # Case 3: Active sentinel and primary it is this node --> PRIMARY + info "Configuring the node as primary" + export VALKEY_REPLICATION_MODE="primary" + else + # Case 4: Active sentinel and primary is not this node --> REPLICA + info "Configuring the node as replica" + export VALKEY_REPLICATION_MODE="replica" + fi +fi + +if [[ -n "$VALKEY_EXTERNAL_PRIMARY_HOST" ]]; then + VALKEY_PRIMARY_HOST="$VALKEY_EXTERNAL_PRIMARY_HOST" + VALKEY_PRIMARY_PORT_NUMBER="${VALKEY_EXTERNAL_PRIMARY_PORT}" +fi + +if [[ -f /opt/bitnami/valkey/mounted-etc/replica.conf ]];then + cp /opt/bitnami/valkey/mounted-etc/replica.conf /opt/bitnami/valkey/etc/replica.conf +fi + +if [[ -f /opt/bitnami/valkey/mounted-etc/valkey.conf ]];then + cp /opt/bitnami/valkey/mounted-etc/valkey.conf /opt/bitnami/valkey/etc/valkey.conf +fi + +echo "" >> /opt/bitnami/valkey/etc/replica.conf +echo "replica-announce-port $VALKEYPORT" >> /opt/bitnami/valkey/etc/replica.conf +echo "replica-announce-ip $(get_full_hostname "$HOSTNAME")" >> /opt/bitnami/valkey/etc/replica.conf +ARGS=("--port" "${VALKEY_PORT}") + +if [[ "$VALKEY_REPLICATION_MODE" = "replica" ]]; then + ARGS+=("--replicaof" "${VALKEY_PRIMARY_HOST}" "${VALKEY_PRIMARY_PORT_NUMBER}") +fi +ARGS+=("--protected-mode" "no") +ARGS+=("--include" "/opt/bitnami/valkey/etc/replica.conf") +ARGS+=("--include" "/opt/bitnami/valkey/etc/valkey.conf") +exec valkey-server "${ARGS[@]}" diff --git a/resources/node/kustomization.yaml b/resources/node/kustomization.yaml new file mode 100644 index 0000000..f9b5f62 --- /dev/null +++ b/resources/node/kustomization.yaml @@ -0,0 +1,39 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- resources/sa.yaml +- resources/statefulset.yaml +- resources/svc.yaml + +replacements: +- path: ./replacements/replicas.yaml + +configMapGenerator: +- name: valkey-env + literals: + - NAMESPACE="default" + - VALKEY_ENV="base" + - VALKEY_SERVICE="valkey-headless" + - VALKEY_REPLICAS="4" + - VALKEY_PORT="6379" + - VALKEY_SENTINEL_PORT="26379" + - ALLOW_EMPTY_PASSWORD="yes" + - VALKEY_TLS_ENABLED="no" + - VALKEY_SENTINEL_TLS_ENABLED="no" + - VALKEY_DATA_DIR="/data" + - VALKEY_LOG_LEVEL="debug" + - VALKEY_QUORUM="2" +- name: valkey-config + files: + - files/conf/replication.conf.tpl + - files/conf/sentinel.conf.tpl +- name: valkey-scripts + files: + - files/scripts/common.sh + - files/scripts/startSentinel.sh + - files/scripts/pre-stop.sh + - files/scripts/start-node.sh + - files/scripts/ping-sentinel.sh + - files/scripts/liveness-local.sh + - files/scripts/readiness-local.sh diff --git a/resources/node/replacements/replicas.yaml b/resources/node/replacements/replicas.yaml new file mode 100644 index 0000000..232fa84 --- /dev/null +++ b/resources/node/replacements/replicas.yaml @@ -0,0 +1,10 @@ +- source: + kind: ConfigMap + name: valkey-env + fieldPath: data.VALKEY_REPLICAS + targets: + - select: + kind: StatefulSet + name: valkey-node + fieldPaths: + - spec.replicas diff --git a/resources/node/resources/sa.yaml b/resources/node/resources/sa.yaml new file mode 100644 index 0000000..4e0e6e6 --- /dev/null +++ b/resources/node/resources/sa.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey diff --git a/resources/node/resources/statefulset.yaml b/resources/node/resources/statefulset.yaml new file mode 100644 index 0000000..fcbae0b --- /dev/null +++ b/resources/node/resources/statefulset.yaml @@ -0,0 +1,343 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey-node +spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + podManagementPolicy: OrderedReady + # DO NOT CHANGE THIS LINE HERE, USE THE VARIABLE VALKEY_REPLICAS INSTEAD + replicas: 4 + # END OF DO NOT CHANGE THIS LINE + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey + serviceName: valkey-headless + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: valkey + app.kubernetes.io/version: 8.1.1 + helm.sh/chart: valkey-3.0.7 + spec: + shareProcessNamespace: true + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey + topologyKey: kubernetes.io/hostname + weight: 1 + automountServiceAccountToken: false + initContainers: + - name: copy-config-templates + image: reg.cadoles.com/dh/library/busybox:1.37.0-musl + command: + - /bin/cp + args: + - -R + - /templates + - /tmp/ + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + volumeMounts: + - mountPath: /templates/replication.conf.orig + name: valkey-config + subPath: replication.conf.tpl + - mountPath: /templates/sentinel.conf.orig + name: valkey-config + subPath: sentinel.conf.tpl + - mountPath: /tmp + name: tmp + - mountPath: /data + name: valkey-data + - name: generate-config + image: reg.cadoles.com/cadoles/gotemplate:0.0.6-dev + imagePullPolicy: IfNotPresent + args: + - --source + - /tmp/templates + - --target + - /etc/valkey/ + - --no-overwrite + - replication.conf.orig + - sentinel.conf.orig + envFrom: + - configMapRef: + name: valkey-env + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + volumeMounts: + - mountPath: /etc/valkey/ + name: valkey-etc + - mountPath: /tmp + name: tmp + - mountPath: /data + name: valkey-data + containers: + - name: valkey + image: reg.cadoles.com/dh/valkey/valkey:8.1.1-alpine3.21 + command: + - /opt/scripts/start-node.sh + args: + - /etc/valkey/replication.conf + imagePullPolicy: IfNotPresent + env: + - name: VALKEY_ROLE + value: "replication" + envFrom: + - configMapRef: + name: valkey-env + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - /opt/scripts/pre-stop.sh + livenessProbe: + exec: + command: + - sh + - -c + - /opt/scripts/liveness-local.sh 5 + failureThreshold: 5 + initialDelaySeconds: 20 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + ports: + - containerPort: 6379 + name: valkey + protocol: TCP + readinessProbe: + exec: + command: + - sh + - -c + - /opt/scripts/readiness-local.sh 1 + failureThreshold: 5 + initialDelaySeconds: 20 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 150m + memory: 192Mi + ephemeral-storage: 2Gi + requests: + cpu: 100m + memory: 128Mi + ephemeral-storage: 50Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + startupProbe: + exec: + command: + - sh + - -c + - /opt/scripts/liveness-local.sh 5 + failureThreshold: 22 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/valkey/ + name: valkey-etc + - mountPath: /opt/scripts + name: valkey-scripts + - mountPath: /data + name: valkey-data + - name: sentinel + image: reg.cadoles.com/dh/valkey/valkey:8.1.1-alpine3.21 + imagePullPolicy: IfNotPresent + command: + - /opt/scripts/startSentinel.sh + env: + - name: ALLOW_EMPTY_PASSWORD + value: "yes" + - name: VALKEY_SENTINEL_TLS_ENABLED + value: "no" + envFrom: + - configMapRef: + name: valkey-env + lifecycle: + preStop: + exec: + command: + - /bin/bash + - -c + - /opt/scripts/pre-stop-sentinel.sh + livenessProbe: + exec: + command: + - sh + - -c + - /opt/scripts/ping-sentinel.sh 5 + failureThreshold: 6 + initialDelaySeconds: 20 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + ports: + - containerPort: 26379 + name: valkey-sentinel + protocol: TCP + readinessProbe: + exec: + command: + - sh + - -c + - /opt/scripts/ping-sentinel.sh 5 + failureThreshold: 6 + initialDelaySeconds: 20 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 150m + ephemeral-storage: 2Gi + memory: 192Mi + requests: + cpu: 100m + ephemeral-storage: 50Mi + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + startupProbe: + exec: + command: + - sh + - -c + - /opt/scripts/ping-sentinel.sh 5 + failureThreshold: 22 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /opt/scripts + name: valkey-scripts + - mountPath: /etc/valkey/ + name: valkey-etc + dnsPolicy: ClusterFirst + enableServiceLinks: true + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + runAsUser: 1001 + runAsNonRoot: true + runAsGroup: 1001 + fsGroup: 1001 + fsGroupChangePolicy: Always + serviceAccount: valkey + serviceAccountName: valkey + terminationGracePeriodSeconds: 30 + volumes: + - name: valkey-scripts + configMap: + defaultMode: 493 + name: valkey-scripts + - name: valkey-config + configMap: + defaultMode: 420 + name: valkey-config + - emptyDir: + sizeLimit: 64Mi + medium: Memory + name: tmp + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey + name: valkey-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + volumeMode: Filesystem + status: + phase: Pending + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey + name: valkey-etc + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 32Mi + volumeMode: Filesystem + status: + phase: Pending diff --git a/resources/node/resources/svc.yaml b/resources/node/resources/svc.yaml new file mode 100644 index 0000000..6c0b102 --- /dev/null +++ b/resources/node/resources/svc.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey +spec: + ports: + - name: tcp-redis + port: 6379 + protocol: TCP + targetPort: 6379 + - name: tcp-sentinel + port: 26379 + protocol: TCP + targetPort: 26379 + selector: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey-headless +spec: + clusterIP: None + ports: + - name: tcp-redis + port: 6379 + protocol: TCP + targetPort: redis + - name: tcp-sentinel + port: 26379 + protocol: TCP + targetPort: valkey-sentinel + publishNotReadyAddresses: true + selector: + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey diff --git a/resources/svc.yaml b/resources/svc.yaml deleted file mode 100644 index 8e5047d..0000000 --- a/resources/svc.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: varnish -spec: - selector: - app: varnish - ports: - - name: varnish-http - protocol: TCP - port: 8080 - targetPort: 8080