From 429ea72ccb2296e366403be6d3f550cd49d781e9 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Wed, 21 May 2025 16:00:20 +0200 Subject: [PATCH] feat(node): more node stuff fup --- .../node/files/conf/replication.conf.tpl | 129 +++------------ .../node/files/conf/replication.conf.tpl.full | 119 ++++++++++++++ resources/node/files/conf/sentinel.conf.tpl | 6 +- .../node/files/scripts/pre-start-sentinel.sh | 0 resources/node/files/scripts/start-node.sh | 85 ++++++++++ resources/node/files/scripts/startnode.sh | 151 ++++++++++++++++++ resources/node/kustomization.yaml | 9 +- resources/node/replacements/replicas.yaml | 10 ++ resources/node/resources/statefulset.yaml | 45 ++++-- resources/node/resources/svc.yaml | 52 ++++++ 10 files changed, 479 insertions(+), 127 deletions(-) create mode 100644 resources/node/files/conf/replication.conf.tpl.full delete mode 100644 resources/node/files/scripts/pre-start-sentinel.sh create mode 100644 resources/node/files/scripts/start-node.sh create mode 100644 resources/node/files/scripts/startnode.sh create mode 100644 resources/node/replacements/replicas.yaml create mode 100644 resources/node/resources/svc.yaml diff --git a/resources/node/files/conf/replication.conf.tpl b/resources/node/files/conf/replication.conf.tpl index a2cc086..499e91d 100644 --- a/resources/node/files/conf/replication.conf.tpl +++ b/resources/node/files/conf/replication.conf.tpl @@ -1,108 +1,21 @@ -################################## INCLUDES ################################### -################################## MODULES ##################################### -################################## NETWORK ##################################### -bind 127.0.0.1 -::1 -protected-mode yes -port 6379 -tcp-backlog 511 -timeout 0 -tcp-keepalive 300 -################################# TLS/SSL ##################################### -port {{ env "VALKEY_PORT" }} -################################### RDMA ###################################### -################################# GENERAL ##################################### -daemonize yes -pidfile /opt/bitnami/valkey/tmp/valkey.pid -loglevel notice -logfile "" -databases 16 -always-show-logo no -hide-user-data-from-log yes -set-proc-title yes -proc-title-template "{title} {listen-addr} {server-mode}" -locale-collate "" -################################ SNAPSHOTTING ################################ -stop-writes-on-bgsave-error yes -rdbcompression yes -rdbchecksum yes -rdb-version-check strict -dbfilename dump.rdb -rdb-del-sync-files no -dir {{ env "VALKEY_DATA_DIR" }} -################################# REPLICATION ################################# -replica-serve-stale-data yes -replica-read-only yes -repl-diskless-sync yes -repl-diskless-sync-delay 5 -repl-diskless-sync-max-replicas 0 -repl-diskless-load disabled -dual-channel-replication-enabled no -repl-disable-tcp-nodelay no -replica-priority 100 -############################### KEYS TRACKING ################################# -################################## SECURITY ################################### -acllog-max-len 128 -################################### CLIENTS #################################### -############################## MEMORY MANAGEMENT ################################ -############################# LAZY FREEING #################################### -lazyfree-lazy-eviction yes -lazyfree-lazy-expire yes -lazyfree-lazy-server-del yes -replica-lazy-flush yes -lazyfree-lazy-user-del yes -lazyfree-lazy-user-flush yes -################################ THREADED I/O ################################# -############################ KERNEL OOM CONTROL ############################## -oom-score-adj no -oom-score-adj-values 0 200 800 -#################### KERNEL transparent hugepage CONTROL ###################### -disable-thp yes -############################## APPEND ONLY MODE ############################### -appendonly no -appendfilename "appendonly.aof" -appenddirname "appendonlydir" -appendfsync everysec -no-appendfsync-on-rewrite no -auto-aof-rewrite-percentage 100 -auto-aof-rewrite-min-size 64mb -aof-load-truncated yes -aof-use-rdb-preamble yes -aof-timestamp-enabled no -################################ SHUTDOWN ##################################### -################ NON-DETERMINISTIC LONG BLOCKING COMMANDS ##################### -################################ VALKEY CLUSTER ############################### -########################## CLUSTER DOCKER/NAT support ######################## -################################## COMMAND LOG ################################### -commandlog-execution-slower-than 10000 -commandlog-slow-execution-max-len 128 -commandlog-request-larger-than 1048576 -commandlog-large-request-max-len 128 -commandlog-reply-larger-than 1048576 -commandlog-large-reply-max-len 128 -################################ LATENCY MONITOR ############################## -latency-monitor-threshold 0 -################################ LATENCY TRACKING ############################## -############################# EVENT NOTIFICATION ############################## -notify-keyspace-events "" -############################### ADVANCED CONFIG ############################### -hash-max-listpack-entries 512 -hash-max-listpack-value 64 -list-max-listpack-size -2 -list-compress-depth 0 -set-max-intset-entries 512 -set-max-listpack-entries 128 -set-max-listpack-value 64 -zset-max-listpack-entries 128 -zset-max-listpack-value 64 -hll-sparse-max-bytes 3000 -stream-node-max-bytes 4096 -stream-node-max-entries 100 -activerehashing yes -client-output-buffer-limit normal 0 0 0 -client-output-buffer-limit replica 256mb 64mb 60 -client-output-buffer-limit pubsub 32mb 8mb 60 -hz 10 -aof-rewrite-incremental-fsync yes -rdb-save-incremental-fsync yes -########################### ACTIVE DEFRAGMENTATION ####################### -jemalloc-bg-thread yes +{{- $hostname := env "HOSTNAME" }} +{{- $service := env "VALKEY_SERVICE" }} +{{- $namespace := env "NAMESPACE" }} +{{- $port := env "VALKEY_PORT" }} +{{- $sentinel_port := env "VALKEY_SENTINEL_PORT" }} +{{- $replicas := env "VALKEY_REPLICAS" }} +{{- $domain := printf "%s.%s.svc.cluster.local" $service $namespace }} +{{- $fqdn := printf "%s.%s" $hostname $domain }} +{{- $hostid := sha1sum $hostname }} +{{- $datadir := env "VALKEY_DATA_DIR" }} +appendonly yes +save "" + +dir {{ $datadir }} + +# User-supplied replica configuration: +rename-command FLUSHDB "" +rename-command FLUSHALL "" + +replica-announce-port {{ $port }} +replica-announce-ip {{ $fqdn }} diff --git a/resources/node/files/conf/replication.conf.tpl.full b/resources/node/files/conf/replication.conf.tpl.full new file mode 100644 index 0000000..20b18c5 --- /dev/null +++ b/resources/node/files/conf/replication.conf.tpl.full @@ -0,0 +1,119 @@ +{{- $hostname := env "HOSTNAME" }} +{{- $service := env "VALKEY_SERVICE" }} +{{- $namespace := env "NAMESPACE" }} +{{- $port := env "VALKEY_PORT" }} +{{- $sentinel_port := env "VALKEY_SENTINEL_PORT" }} +{{- $replicas := env "VALKEY_REPLICAS" }} +{{- $domain := printf "%s.%s.svc.cluster.local" $service $namespace }} +{{- $fqdn := printf "%s.%s" $hostname $domain }} +{{- $hostid := sha1sum $hostname }} +################################## INCLUDES ################################### +################################## MODULES ##################################### +################################## NETWORK ##################################### +bind * -::* +protected-mode no +port {{ $port }} +tcp-backlog 511 +timeout 0 +tcp-keepalive 300 +################################# TLS/SSL ##################################### +port {{ env "VALKEY_PORT" }} +################################### RDMA ###################################### +################################# GENERAL ##################################### +daemonize no +pidfile /opt/bitnami/valkey/tmp/valkey.pid +loglevel notice +logfile "" +databases 16 +always-show-logo no +hide-user-data-from-log yes +set-proc-title yes +proc-title-template "{title} {listen-addr} {server-mode}" +locale-collate "" +################################ SNAPSHOTTING ################################ +stop-writes-on-bgsave-error yes +rdbcompression yes +rdbchecksum yes +rdb-version-check strict +dbfilename dump.rdb +rdb-del-sync-files no +dir {{ env "VALKEY_DATA_DIR" }} +################################# REPLICATION ################################# +replica-serve-stale-data yes +replica-read-only yes +repl-diskless-sync yes +repl-diskless-sync-delay 5 +repl-diskless-sync-max-replicas 0 +repl-diskless-load disabled +dual-channel-replication-enabled no +repl-disable-tcp-nodelay no +replica-priority 100 +replica-announce-port {{ $port }} +replica-announce-ip {{ $fqdn }} +############################### KEYS TRACKING ################################# +################################## SECURITY ################################### +acllog-max-len 128 +################################### CLIENTS #################################### +############################## MEMORY MANAGEMENT ################################ +############################# LAZY FREEING #################################### +lazyfree-lazy-eviction yes +lazyfree-lazy-expire yes +lazyfree-lazy-server-del yes +replica-lazy-flush yes +lazyfree-lazy-user-del yes +lazyfree-lazy-user-flush yes +################################ THREADED I/O ################################# +############################ KERNEL OOM CONTROL ############################## +oom-score-adj no +oom-score-adj-values 0 200 800 +#################### KERNEL transparent hugepage CONTROL ###################### +disable-thp yes +############################## APPEND ONLY MODE ############################### +appendonly no +appendfilename "appendonly.aof" +appenddirname "appendonlydir" +appendfsync everysec +no-appendfsync-on-rewrite no +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb +aof-load-truncated yes +aof-use-rdb-preamble yes +aof-timestamp-enabled no +################################ SHUTDOWN ##################################### +################ NON-DETERMINISTIC LONG BLOCKING COMMANDS ##################### +################################ VALKEY CLUSTER ############################### +########################## CLUSTER DOCKER/NAT support ######################## +################################## COMMAND LOG ################################### +commandlog-execution-slower-than 10000 +commandlog-slow-execution-max-len 128 +commandlog-request-larger-than 1048576 +commandlog-large-request-max-len 128 +commandlog-reply-larger-than 1048576 +commandlog-large-reply-max-len 128 +################################ LATENCY MONITOR ############################## +latency-monitor-threshold 0 +################################ LATENCY TRACKING ############################## +############################# EVENT NOTIFICATION ############################## +notify-keyspace-events "" +############################### ADVANCED CONFIG ############################### +hash-max-listpack-entries 512 +hash-max-listpack-value 64 +list-max-listpack-size -2 +list-compress-depth 0 +set-max-intset-entries 512 +set-max-listpack-entries 128 +set-max-listpack-value 64 +zset-max-listpack-entries 128 +zset-max-listpack-value 64 +hll-sparse-max-bytes 3000 +stream-node-max-bytes 4096 +stream-node-max-entries 100 +activerehashing yes +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit replica 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 +hz 10 +aof-rewrite-incremental-fsync yes +rdb-save-incremental-fsync yes +########################### ACTIVE DEFRAGMENTATION ####################### +jemalloc-bg-thread yes diff --git a/resources/node/files/conf/sentinel.conf.tpl b/resources/node/files/conf/sentinel.conf.tpl index 5a22f32..7b83f7a 100644 --- a/resources/node/files/conf/sentinel.conf.tpl +++ b/resources/node/files/conf/sentinel.conf.tpl @@ -9,7 +9,7 @@ {{- $hostid := sha1sum $hostname }} dir "/tmp" -port {{ $port }} +port {{ $sentinel_port }} sentinel monitor mymaster {{ printf "%s %s" $fqdn $port }} 2 sentinel down-after-milliseconds mymaster 60000 @@ -19,8 +19,8 @@ sentinel down-after-milliseconds mymaster 60000 sentinel announce-hostnames yes sentinel resolve-hostnames yes -sentinel announce-port {{- $sentinel_port }} -sentinel announce-ip {{- $fqdn }} +sentinel announce-port {{ $sentinel_port }} +sentinel announce-ip {{ $fqdn }} # Generated by CONFIG REWRITE latency-tracking-info-percentiles 50 99 99.9 diff --git a/resources/node/files/scripts/pre-start-sentinel.sh b/resources/node/files/scripts/pre-start-sentinel.sh deleted file mode 100644 index e69de29..0000000 diff --git a/resources/node/files/scripts/start-node.sh b/resources/node/files/scripts/start-node.sh new file mode 100644 index 0000000..b0f6d3a --- /dev/null +++ b/resources/node/files/scripts/start-node.sh @@ -0,0 +1,85 @@ +#!/bin/sh + +pingSentinel() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h ${VALKEY_SERVICE} \ + -p ${VALKEY_SENTINEL_PORT} \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +getPrimaryInfo() { + valkey-cli --csv -h ${VALKEY_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \ + awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }' + return ${?} +} + +waitFroSentinel() { + tout=60 + while true; do + response=$(pingSentinel 5) + if [ "${response}" = "PONG" ]; then + echo "Sentinel is responding" + break + return 0 + fi + + echo "Sentinel is not responding [${response}]" + sleep 1 + tout=$((tout - 1)) + if [ "${tout}" -le 0 ]; then + echo "Sentinel ping timed out" + return 124 + fi + done +} + +startValkey() { + # Start Valkey + echo "Running : [valkey-server ${@}]" + valkey-server ${@} + ret=${?} + if [ "${ret}" -ne 0 ]; then + echo "Failed to start Valkey" + exit ${ret} + fi +} + +setupPrimary=0 +primaryHost="" +primaryPort="" + +waitFroSentinel +ret=${?} +if [ "${ret}" -ne 0 ]; then + exit ${ret} +fi + +primaryInfo=$(getPrimaryInfo) +if [ "${?}" -ne 0 ]; then + echo "No primary found, seting up node as primary" + setupPrimary=1 +else + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + primaryPort=$(echo ${primaryInfo} | awk -F ' ' '{print $2}') + currentHost=$(hostname -f) + if [ "${primaryHost}" != "${currentHost}" ]; then + echo "Not the primary, setting up as replica" + setupPrimary=0 + else + echo "This is the primary" + setupPrimary=1 + fi +fi + +if [ "${setupPrimary}" -eq 1 ]; then + echo "Starting Valkey as primary" + cat $1 + startValkey ${@} +else + echo "Starting Valkey as replica" + startValkey ${@} "--replicaof" "${primaryHost}" "${primaryPort}" +fi \ No newline at end of file diff --git a/resources/node/files/scripts/startnode.sh b/resources/node/files/scripts/startnode.sh new file mode 100644 index 0000000..67061fd --- /dev/null +++ b/resources/node/files/scripts/startnode.sh @@ -0,0 +1,151 @@ +#!/bin/bash + +. /opt/bitnami/scripts/libos.sh +. /opt/bitnami/scripts/liblog.sh +. /opt/bitnami/scripts/libvalidations.sh + +get_port() { + hostname="$1" + type="$2" + + port_var=$(echo "${hostname^^}_SERVICE_PORT_$type" | sed "s/-/_/g") + port=${!port_var} + + if [ -z "$port" ]; then + case $type in + "SENTINEL") + echo 26379 + ;; + "VALKEY") + echo 6379 + ;; + esac + else + echo $port + fi +} + +get_full_hostname() { + hostname="$1" + full_hostname="${hostname}.${HEADLESS_SERVICE}" + echo "${full_hostname}" +} + +VALKEYPORT=$(get_port "$HOSTNAME" "VALKEY") + +HEADLESS_SERVICE="valkey-headless.mse-dev.svc.cluster.local" + +if [ -n "$VALKEY_EXTERNAL_PRIMARY_HOST" ]; then + VALKEY_SERVICE="$VALKEY_EXTERNAL_PRIMARY_HOST" +else + VALKEY_SERVICE="valkey.mse-dev.svc.cluster.local" +fi + +SENTINEL_SERVICE_PORT=$(get_port "valkey" "SENTINEL") +validate_quorum() { + if is_boolean_yes "$VALKEY_TLS_ENABLED"; then + quorum_info_command="valkey-cli -h $VALKEY_SERVICE -p $SENTINEL_SERVICE_PORT --tls --cert ${VALKEY_TLS_CERT_FILE} --key ${VALKEY +_TLS_KEY_FILE} --cacert ${VALKEY_TLS_CA_FILE} sentinel primary mymaster" + else + quorum_info_command="valkey-cli -h $VALKEY_SERVICE -p $SENTINEL_SERVICE_PORT sentinel primary mymaster" + fi + info "about to run the command: $quorum_info_command" + eval $quorum_info_command | grep -Fq "s_down" +} + +trigger_manual_failover() { + if is_boolean_yes "$VALKEY_TLS_ENABLED"; then + failover_command="valkey-cli -h $VALKEY_SERVICE -p $SENTINEL_SERVICE_PORT --tls --cert ${VALKEY_TLS_CERT_FILE} --key ${VALKEY_TL +S_KEY_FILE} --cacert ${VALKEY_TLS_CA_FILE} sentinel failover mymaster" + else + failover_command="valkey-cli -h $VALKEY_SERVICE -p $SENTINEL_SERVICE_PORT sentinel failover mymaster" + fi + + info "about to run the command: $failover_command" + eval $failover_command +} + +get_sentinel_primary_info() { + if is_boolean_yes "$VALKEY_TLS_ENABLED"; then + sentinel_info_command="timeout 90 valkey-cli -h $VALKEY_SERVICE -p $SENTINEL_SERVICE_PORT --tls --cert ${VALKEY_TLS_CERT_FILE} - +-key ${VALKEY_TLS_KEY_FILE} --cacert ${VALKEY_TLS_CA_FILE} sentinel get-primary-addr-by-name mymaster" + else + sentinel_info_command="timeout 90 valkey-cli -h $VALKEY_SERVICE -p $SENTINEL_SERVICE_PORT sentinel get-primary-addr-by-name myma +ster" + fi + info "about to run the command: $sentinel_info_command" + retry_while "eval $sentinel_info_command" 2 5 +} + +[[ -f $VALKEY_PASSWORD_FILE ]] && export VALKEY_PASSWORD="$(< "${VALKEY_PASSWORD_FILE}")" +[[ -f $VALKEY_PRIMARY_PASSWORD_FILE ]] && export VALKEY_PRIMARY_PASSWORD="$(< "${VALKEY_PRIMARY_PASSWORD_FILE}")" + +# check if there is a primary +primary_in_persisted_conf="$(get_full_hostname "$HOSTNAME")" +primary_port_in_persisted_conf="$VALKEY_PRIMARY_PORT_NUMBER" +primary_in_sentinel="$(get_sentinel_primary_info)" +valkeyRetVal=$? + +if [[ -f /opt/bitnami/valkey-sentinel/etc/sentinel.conf ]]; then + primary_in_persisted_conf="$(awk '/monitor/ {print $4}' /opt/bitnami/valkey-sentinel/etc/sentinel.conf)" + primary_port_in_persisted_conf="$(awk '/monitor/ {print $5}' /opt/bitnami/valkey-sentinel/etc/sentinel.conf)" + info "Found previous primary ${primary_in_persisted_conf}:${primary_port_in_persisted_conf} in /opt/bitnami/valkey-sentinel/etc/sent +inel.conf" + debug "$(cat /opt/bitnami/valkey-sentinel/etc/sentinel.conf | grep monitor)" +fi + +if [[ $valkeyRetVal -ne 0 ]]; then + if [[ "$primary_in_persisted_conf" == "$(get_full_hostname "$HOSTNAME")" ]]; then + # Case 1: No active sentinel and in previous sentinel.conf we were the primary --> PRIMARY + info "Configuring the node as primary" + export VALKEY_REPLICATION_MODE="primary" + else + # Case 2: No active sentinel and in previous sentinel.conf we were not primary --> REPLICA + info "Configuring the node as replica" + export VALKEY_REPLICATION_MODE="replica" + VALKEY_PRIMARY_HOST=${primary_in_persisted_conf} + VALKEY_PRIMARY_PORT_NUMBER=${primary_port_in_persisted_conf} + fi +else + # Fetches current primary's host and port + VALKEY_SENTINEL_INFO=($(get_sentinel_primary_info)) + info "Current primary: VALKEY_SENTINEL_INFO=(${VALKEY_SENTINEL_INFO[0]},${VALKEY_SENTINEL_INFO[1]})" + VALKEY_PRIMARY_HOST=${VALKEY_SENTINEL_INFO[0]} + VALKEY_PRIMARY_PORT_NUMBER=${VALKEY_SENTINEL_INFO[1]} + + if [[ "$VALKEY_PRIMARY_HOST" == "$(get_full_hostname "$HOSTNAME")" ]]; then + # Case 3: Active sentinel and primary it is this node --> PRIMARY + info "Configuring the node as primary" + export VALKEY_REPLICATION_MODE="primary" + else + # Case 4: Active sentinel and primary is not this node --> REPLICA + info "Configuring the node as replica" + export VALKEY_REPLICATION_MODE="replica" + fi +fi + +if [[ -n "$VALKEY_EXTERNAL_PRIMARY_HOST" ]]; then + VALKEY_PRIMARY_HOST="$VALKEY_EXTERNAL_PRIMARY_HOST" + VALKEY_PRIMARY_PORT_NUMBER="${VALKEY_EXTERNAL_PRIMARY_PORT}" +fi + +if [[ -f /opt/bitnami/valkey/mounted-etc/replica.conf ]];then + cp /opt/bitnami/valkey/mounted-etc/replica.conf /opt/bitnami/valkey/etc/replica.conf +fi + +if [[ -f /opt/bitnami/valkey/mounted-etc/valkey.conf ]];then + cp /opt/bitnami/valkey/mounted-etc/valkey.conf /opt/bitnami/valkey/etc/valkey.conf +fi + +echo "" >> /opt/bitnami/valkey/etc/replica.conf +echo "replica-announce-port $VALKEYPORT" >> /opt/bitnami/valkey/etc/replica.conf +echo "replica-announce-ip $(get_full_hostname "$HOSTNAME")" >> /opt/bitnami/valkey/etc/replica.conf +ARGS=("--port" "${VALKEY_PORT}") + +if [[ "$VALKEY_REPLICATION_MODE" = "replica" ]]; then + ARGS+=("--replicaof" "${VALKEY_PRIMARY_HOST}" "${VALKEY_PRIMARY_PORT_NUMBER}") +fi +ARGS+=("--protected-mode" "no") +ARGS+=("--include" "/opt/bitnami/valkey/etc/replica.conf") +ARGS+=("--include" "/opt/bitnami/valkey/etc/valkey.conf") +exec valkey-server "${ARGS[@]}" diff --git a/resources/node/kustomization.yaml b/resources/node/kustomization.yaml index d49648c..f8d1e7e 100644 --- a/resources/node/kustomization.yaml +++ b/resources/node/kustomization.yaml @@ -4,13 +4,18 @@ kind: Kustomization resources: - resources/sa.yaml - resources/statefulset.yaml +- resources/svc.yaml + +replacements: +- path: ./replacements/replicas.yaml configMapGenerator: - name: valkey-env literals: - NAMESPACE="default" - VALKEY_ENV="base" - - VALKEY_REPLICAS="4" + - VALKEY_SERVICE="valkey-headless" + - VALKEY_REPLICAS="3" - VALKEY_PORT="6379" - VALKEY_SENTINEL_PORT="26379" - ALLOW_EMPTY_PASSWORD="yes" @@ -24,7 +29,7 @@ configMapGenerator: - name: valkey-scripts files: - files/scripts/pre-stop.sh - - files/scripts/pre-start-sentinel.sh + - files/scripts/start-node.sh - files/scripts/ping-sentinel.sh - files/scripts/liveness-local.sh - files/scripts/readiness-local.sh diff --git a/resources/node/replacements/replicas.yaml b/resources/node/replacements/replicas.yaml new file mode 100644 index 0000000..232fa84 --- /dev/null +++ b/resources/node/replacements/replicas.yaml @@ -0,0 +1,10 @@ +- source: + kind: ConfigMap + name: valkey-env + fieldPath: data.VALKEY_REPLICAS + targets: + - select: + kind: StatefulSet + name: valkey-node + fieldPaths: + - spec.replicas diff --git a/resources/node/resources/statefulset.yaml b/resources/node/resources/statefulset.yaml index 02f6652..3abd162 100644 --- a/resources/node/resources/statefulset.yaml +++ b/resources/node/resources/statefulset.yaml @@ -14,7 +14,9 @@ spec: whenDeleted: Retain whenScaled: Retain podManagementPolicy: OrderedReady + # DO NOT CHANGE THIS LINE HERE, USE THE VARIABLE VALKEY_REPLICAS INSTEAD replicas: 4 + # END OF DO NOT CHANGE THIS LINE revisionHistoryLimit: 10 selector: matchLabels: @@ -72,6 +74,8 @@ spec: subPath: sentinel.conf.tpl - mountPath: /tmp/ name: tmp + - mountPath: /data + name: valkey-data - name: generate-config image: reg.cadoles.com/cadoles/gotemplate:0.0.5-dev imagePullPolicy: IfNotPresent @@ -98,17 +102,22 @@ spec: name: valkey-etc - mountPath: /tmp/ name: tmp + - mountPath: /data + name: valkey-data containers: - name: valkey image: reg.cadoles.com/dh/valkey/valkey:8.1.1-alpine3.21 command: - - valkey-server + - /opt/scripts/start-node.sh args: - /etc/valkey/replication.conf imagePullPolicy: IfNotPresent env: - name: VALKEY_ROLE value: "replication" + envFrom: + - configMapRef: + name: valkey-env lifecycle: preStop: exec: @@ -181,33 +190,36 @@ spec: name: valkey-etc - mountPath: /opt/scripts name: valkey-scripts + - mountPath: /data + name: valkey-data - name: sentinel image: reg.cadoles.com/dh/valkey/valkey:8.1.1-alpine3.21 imagePullPolicy: IfNotPresent command: args: - - --sentienel - /etc/valkey/sentinel.conf + - --sentinel env: - name: ALLOW_EMPTY_PASSWORD value: "yes" - name: VALKEY_SENTINEL_TLS_ENABLED value: "no" - - name: VALKEY_SENTINEL_PORT - value: "26379" - # lifecycle: - # preStop: - # exec: - # command: - # - /bin/bash - # - -c - # - /opt/scripts/pre-stop-sentinel.sh + envFrom: + - configMapRef: + name: valkey-env + lifecycle: + preStop: + exec: + command: + - /bin/bash + - -c + - /opt/scripts/pre-stop-sentinel.sh livenessProbe: exec: command: - sh - -c - - /health/ping_sentinel.sh 5 + - /opt/scripts/ping-sentinel.sh 5 failureThreshold: 6 initialDelaySeconds: 20 periodSeconds: 10 @@ -222,7 +234,7 @@ spec: command: - sh - -c - - /health/ping_sentinel.sh 1 + - /opt/scripts/ping-sentinel.sh 5 failureThreshold: 6 initialDelaySeconds: 20 periodSeconds: 5 @@ -272,6 +284,9 @@ spec: restartPolicy: Always schedulerName: default-scheduler securityContext: + runAsUser: 1001 + runAsNonRoot: true + runAsGroup: 1001 fsGroup: 1001 fsGroupChangePolicy: Always serviceAccount: valkey @@ -288,7 +303,9 @@ spec: name: valkey-config - emptyDir: {} name: valkey-etc - - emptyDir: {} + - emptyDir: + sizeLimit: 64Mi + medium: Memory name: tmp volumeClaimTemplates: - apiVersion: v1 diff --git a/resources/node/resources/svc.yaml b/resources/node/resources/svc.yaml new file mode 100644 index 0000000..6c0b102 --- /dev/null +++ b/resources/node/resources/svc.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey +spec: + ports: + - name: tcp-redis + port: 6379 + protocol: TCP + targetPort: 6379 + - name: tcp-sentinel + port: 26379 + protocol: TCP + targetPort: 26379 + selector: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey-headless +spec: + clusterIP: None + ports: + - name: tcp-redis + port: 6379 + protocol: TCP + targetPort: redis + - name: tcp-sentinel + port: 26379 + protocol: TCP + targetPort: valkey-sentinel + publishNotReadyAddresses: true + selector: + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey