From 2564eed2092333b870398fd4a8ca7dce150e78d7 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Wed, 12 Feb 2025 10:09:13 +0100 Subject: [PATCH] First commit --- .gitea/workflows/build.yaml | 54 +++ README.md | 1 + kustomization.yaml | 5 + misc/docker/Dockerfile | 22 ++ .../node/files/conf/replication.conf.tpl | 37 ++ .../node/files/conf/replication.conf.tpl.full | 119 ++++++ resources/node/files/conf/sentinel.conf.tpl | 47 +++ resources/node/files/scripts/common.sh | 81 +++++ .../node/files/scripts/liveness-local.sh | 27 ++ resources/node/files/scripts/ping-sentinel.sh | 27 ++ .../node/files/scripts/pre-stop-sentinel.sh | 32 ++ resources/node/files/scripts/pre-stop.sh | 73 ++++ .../node/files/scripts/readiness-local.sh | 26 ++ resources/node/files/scripts/start-node.sh | 99 +++++ resources/node/files/scripts/startSentinel.sh | 38 ++ resources/node/kustomization.yaml | 43 +++ resources/node/replacements/services.yaml | 20 + resources/node/replacements/statefulset.yaml | 20 + resources/node/resources/sa.yaml | 10 + resources/node/resources/statefulset.yaml | 343 ++++++++++++++++++ resources/node/resources/svc.yaml | 52 +++ 21 files changed, 1176 insertions(+) create mode 100644 .gitea/workflows/build.yaml create mode 100644 kustomization.yaml create mode 100644 misc/docker/Dockerfile create mode 100644 resources/node/files/conf/replication.conf.tpl create mode 100644 resources/node/files/conf/replication.conf.tpl.full create mode 100644 resources/node/files/conf/sentinel.conf.tpl create mode 100644 resources/node/files/scripts/common.sh create mode 100644 resources/node/files/scripts/liveness-local.sh create mode 100644 resources/node/files/scripts/ping-sentinel.sh create mode 100644 resources/node/files/scripts/pre-stop-sentinel.sh create mode 100644 resources/node/files/scripts/pre-stop.sh create mode 100644 resources/node/files/scripts/readiness-local.sh create mode 100644 resources/node/files/scripts/start-node.sh create mode 100644 resources/node/files/scripts/startSentinel.sh create mode 100644 resources/node/kustomization.yaml create mode 100644 resources/node/replacements/services.yaml create mode 100644 resources/node/replacements/statefulset.yaml create mode 100644 resources/node/resources/sa.yaml create mode 100644 resources/node/resources/statefulset.yaml create mode 100644 resources/node/resources/svc.yaml diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml new file mode 100644 index 0000000..4150577 --- /dev/null +++ b/.gitea/workflows/build.yaml @@ -0,0 +1,54 @@ +name: Build and Push Image +on: + push: + tags: + - '*' +env: + REGISTRY: reg.cadoles.com + +jobs: + build: + name: Build and push image + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Login to Docker Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta base + id: metabase + uses: docker/metadata-action@v5 + with: + images: | + reg.cadoles.com/cadoles/gotemplate + flavor: | + latest=auto + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,prefix=,suffix= + + - name: Build and push + uses: docker/build-push-action@v6 + with: + build-args: | + GOTEMPLATE_VERSION=3.12.0 + context: ./misc/docker + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.metabase.outputs.tags }} + labels: ${{ steps.metabase.outputs.labels }} diff --git a/README.md b/README.md index e097d4e..b7cc446 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # varnish-kustom +Deploy a simple varnish server with kustomize \ No newline at end of file diff --git a/kustomization.yaml b/kustomization.yaml new file mode 100644 index 0000000..ad68128 --- /dev/null +++ b/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- resources/node diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile new file mode 100644 index 0000000..ad67f69 --- /dev/null +++ b/misc/docker/Dockerfile @@ -0,0 +1,22 @@ +# Base image +FROM golang:tip-alpine3.21 AS builder + +# Set directory to known value +WORKDIR /app +# Define the version as a build argument +ARG GOTEMPLATE_VERSION=3.12.0 + +RUN apk add valkey-cli git + +# Git clone the repo for gotemplate, checkout the desired tag, and build the executable +RUN git clone https://github.com/coveooss/gotemplate.git . && \ + git checkout v${GOTEMPLATE_VERSION} && \ + CGO_ENABLED=0 go build + + +FROM alpine:3.21 +# +COPY --from=builder /app/gotemplate /gotemplate +COPY --from=builder /usr/bin/valkey-cli /valkey-cli + +ENTRYPOINT [ "/gotemplate" ] \ No newline at end of file diff --git a/resources/node/files/conf/replication.conf.tpl b/resources/node/files/conf/replication.conf.tpl new file mode 100644 index 0000000..7b50acc --- /dev/null +++ b/resources/node/files/conf/replication.conf.tpl @@ -0,0 +1,37 @@ +{{- $hostname := env "HOSTNAME" }} +{{- $service := env "VALKEY_HEADLESS_SERVICE" }} +{{- $namespace := env "NAMESPACE" }} +{{- $port := env "VALKEY_PORT" }} +{{- $sentinel_port := env "VALKEY_SENTINEL_PORT" }} +{{- $replicas := env "VALKEY_REPLICAS" }} +{{- $domain := printf "%s.%s.svc.cluster.local" $service $namespace }} +{{- $fqdn := printf "%s.%s" $hostname $domain }} +{{- $hostid := sha1sum $hostname }} +{{- $datadir := env "VALKEY_DATA_DIR" }} + +dir {{ $datadir }} + +protected-mode no + +loglevel {{ env "VALKEY_LOG_LEVEL" }} + +appendonly yes +appendfilename "appendonly.aof" +appenddirname "appendonlydir" +appendfsync everysec +no-appendfsync-on-rewrite no +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb +aof-load-truncated yes +aof-use-rdb-preamble yes +aof-timestamp-enabled no + +save "" + +replica-announce-port {{ $port }} +replica-announce-ip {{ $fqdn }} + +# User-supplied replica configuration: +rename-command FLUSHDB "" +rename-command FLUSHALL "" + diff --git a/resources/node/files/conf/replication.conf.tpl.full b/resources/node/files/conf/replication.conf.tpl.full new file mode 100644 index 0000000..f0720b4 --- /dev/null +++ b/resources/node/files/conf/replication.conf.tpl.full @@ -0,0 +1,119 @@ +{{- $hostname := env "HOSTNAME" }} +{{- $service := env "VALKEY_HEADLESS_SERVICE" }} +{{- $namespace := env "NAMESPACE" }} +{{- $port := env "VALKEY_PORT" }} +{{- $sentinel_port := env "VALKEY_SENTINEL_PORT" }} +{{- $replicas := env "VALKEY_REPLICAS" }} +{{- $domain := printf "%s.%s.svc.cluster.local" $service $namespace }} +{{- $fqdn := printf "%s.%s" $hostname $domain }} +{{- $hostid := sha1sum $hostname }} +################################## INCLUDES ################################### +################################## MODULES ##################################### +################################## NETWORK ##################################### +bind * -::* +protected-mode no +port {{ $port }} +tcp-backlog 511 +timeout 0 +tcp-keepalive 300 +################################# TLS/SSL ##################################### +port {{ env "VALKEY_PORT" }} +################################### RDMA ###################################### +################################# GENERAL ##################################### +daemonize no +pidfile /opt/bitnami/valkey/tmp/valkey.pid +loglevel notice +logfile "" +databases 16 +always-show-logo no +hide-user-data-from-log yes +set-proc-title yes +proc-title-template "{title} {listen-addr} {server-mode}" +locale-collate "" +################################ SNAPSHOTTING ################################ +stop-writes-on-bgsave-error yes +rdbcompression yes +rdbchecksum yes +rdb-version-check strict +dbfilename dump.rdb +rdb-del-sync-files no +dir {{ env "VALKEY_DATA_DIR" }} +################################# REPLICATION ################################# +replica-serve-stale-data yes +replica-read-only yes +repl-diskless-sync yes +repl-diskless-sync-delay 5 +repl-diskless-sync-max-replicas 0 +repl-diskless-load disabled +dual-channel-replication-enabled no +repl-disable-tcp-nodelay no +replica-priority 100 +replica-announce-port {{ $port }} +replica-announce-ip {{ $fqdn }} +############################### KEYS TRACKING ################################# +################################## SECURITY ################################### +acllog-max-len 128 +################################### CLIENTS #################################### +############################## MEMORY MANAGEMENT ################################ +############################# LAZY FREEING #################################### +lazyfree-lazy-eviction yes +lazyfree-lazy-expire yes +lazyfree-lazy-server-del yes +replica-lazy-flush yes +lazyfree-lazy-user-del yes +lazyfree-lazy-user-flush yes +################################ THREADED I/O ################################# +############################ KERNEL OOM CONTROL ############################## +oom-score-adj no +oom-score-adj-values 0 200 800 +#################### KERNEL transparent hugepage CONTROL ###################### +disable-thp yes +############################## APPEND ONLY MODE ############################### +appendonly no +appendfilename "appendonly.aof" +appenddirname "appendonlydir" +appendfsync everysec +no-appendfsync-on-rewrite no +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb +aof-load-truncated yes +aof-use-rdb-preamble yes +aof-timestamp-enabled no +################################ SHUTDOWN ##################################### +################ NON-DETERMINISTIC LONG BLOCKING COMMANDS ##################### +################################ VALKEY CLUSTER ############################### +########################## CLUSTER DOCKER/NAT support ######################## +################################## COMMAND LOG ################################### +commandlog-execution-slower-than 10000 +commandlog-slow-execution-max-len 128 +commandlog-request-larger-than 1048576 +commandlog-large-request-max-len 128 +commandlog-reply-larger-than 1048576 +commandlog-large-reply-max-len 128 +################################ LATENCY MONITOR ############################## +latency-monitor-threshold 0 +################################ LATENCY TRACKING ############################## +############################# EVENT NOTIFICATION ############################## +notify-keyspace-events "" +############################### ADVANCED CONFIG ############################### +hash-max-listpack-entries 512 +hash-max-listpack-value 64 +list-max-listpack-size -2 +list-compress-depth 0 +set-max-intset-entries 512 +set-max-listpack-entries 128 +set-max-listpack-value 64 +zset-max-listpack-entries 128 +zset-max-listpack-value 64 +hll-sparse-max-bytes 3000 +stream-node-max-bytes 4096 +stream-node-max-entries 100 +activerehashing yes +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit replica 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 +hz 10 +aof-rewrite-incremental-fsync yes +rdb-save-incremental-fsync yes +########################### ACTIVE DEFRAGMENTATION ####################### +jemalloc-bg-thread yes diff --git a/resources/node/files/conf/sentinel.conf.tpl b/resources/node/files/conf/sentinel.conf.tpl new file mode 100644 index 0000000..fcc6aff --- /dev/null +++ b/resources/node/files/conf/sentinel.conf.tpl @@ -0,0 +1,47 @@ +{{- $hostname := env "HOSTNAME" }} +{{- $service := env "VALKEY_HEADLESS_SERVICE" }} +{{- $namespace := env "NAMESPACE" }} +{{- $port := env "VALKEY_PORT" }} +{{- $name := env "VALKEY_NAME" }} +{{- $sentinel_port := env "VALKEY_SENTINEL_PORT" }} +{{- $replicas := env "VALKEY_REPLICAS" }} +{{- $domain := printf "%s.%s.svc.cluster.local" $service $namespace }} +{{- $fqdn := printf "%s.%s" $hostname $domain }} +{{- $hostid := sha1sum $hostname }} +{{- $quorum := env "VALKEY_QUORUM" }} +#-REPLICAS:{{ $replicas }} + +dir "/tmp" +port {{ $sentinel_port }} +sentinel monitor mymaster SENTINEL_MASTER {{ $port }} {{ $quorum }} +sentinel down-after-milliseconds mymaster 60000 + +# User-supplied sentinel configuration: +# End of sentinel configuration +{{ printf "sentinel myid %s" $hostid }} + +sentinel announce-hostnames yes +sentinel resolve-hostnames yes + +sentinel announce-port {{ $sentinel_port }} +sentinel announce-ip {{ $fqdn }} + +# Generated by CONFIG REWRITE +latency-tracking-info-percentiles 50 99 99.9 +protected-mode no +# gotemplate-pause! +user default on nopass sanitize-payload ~* &* +@all +# gotemplate-resume! +sentinel config-epoch mymaster 0 +sentinel leader-epoch mymaster 0 +sentinel current-epoch 0 + +{{- range $i, $e := until ( int $replicas ) }} + {{- $ndeHostname := printf "%s-%d" $name $i }} + {{- $ndeFQDN := printf "%s.%s" $ndeHostname $domain }} + {{- $nodeID := sha1sum $ndeHostname }} + {{- if (ne $fqdn $ndeFQDN) }} +sentinel {{ printf "known-sentinel mymaster %s %s %s" $ndeFQDN $sentinel_port $nodeID }} +sentinel {{ printf "known-replica mymaster %s %s" $ndeFQDN $port }} + {{- end}} +{{- end}} \ No newline at end of file diff --git a/resources/node/files/scripts/common.sh b/resources/node/files/scripts/common.sh new file mode 100644 index 0000000..391b885 --- /dev/null +++ b/resources/node/files/scripts/common.sh @@ -0,0 +1,81 @@ +#!/bin/sh + + +pingSentinel() { + svc=${VALKEY_HEADLESS_SERVICE:-"localhost"} + + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h ${svc} \ + -p $VALKEY_SENTINEL_PORT \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +getPrimaryInfo() { + valkey-cli --csv -h ${VALKEY_HEADLESS_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \ + awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }' + return ${?} +} + +getSentinelMasterName() { + getPrimaryInfo | awk -F ' ' '{print $1}' +} + +getFailOverStatus() { + # Check if the failover is finished + local failoverStatus + primaryInfo=$(getPrimaryInfo) + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + currentHost=$(hostname -f) + if [[ "${primaryHost}" != "${currentHost}" ]]; then + echo "Failover finished" + return 0 + else + echo "Failover in progress" + return 1 + fi +} + +waitForSentinel() { + tout=60 + while true; do + response=$(pingSentinel 5) + if [ "${response}" = "PONG" ]; then + echo "Sentinel is responding" + break + return 0 + fi + + echo "Sentinel is not responding [${response}]" + sleep 1 + tout=$((tout - 1)) + if [ "${tout}" -le 0 ]; then + echo "Sentinel ping timed out" + return 124 + fi + done + sleep 10 +} + +sentinelIsMaster() { + # Check if the sentinel is master + primaryInfo=$(getPrimaryInfo) + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + currentHost=$(hostname -f) + if [[ "${primaryHost}" != "${currentHost}" ]]; then + echo "Sentinel is not master" + return 1 + else + echo "Sentinel is master" + return 0 + fi +} + +sentinelReset() { + # Reset the sentinel + valkey-cli -h ${VALKEY_HEADLESS_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel reset "${1}" + return $? +} \ No newline at end of file diff --git a/resources/node/files/scripts/liveness-local.sh b/resources/node/files/scripts/liveness-local.sh new file mode 100644 index 0000000..f56965f --- /dev/null +++ b/resources/node/files/scripts/liveness-local.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +ping_valkey() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h localhost \ + -p $VALKEY_PORT \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +response=$(ping_valkey 5) +if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 +fi + +firstWord=$(echo $response | awk 'NR==1 {print $1;}') +if [ "$response" != "PONG" ] && [ "$firstWord" != "LOADING" ] && [ "$firstWord" != "MASTERDOWN" ]; then + echo "Valey is not alive [${response}]" + exit 1 +fi + +echo "$( date +'[%Y/%m/%d %H:%M:%S]') Valkey is alive" +exit 0 diff --git a/resources/node/files/scripts/ping-sentinel.sh b/resources/node/files/scripts/ping-sentinel.sh new file mode 100644 index 0000000..498627c --- /dev/null +++ b/resources/node/files/scripts/ping-sentinel.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +ping_sentinel() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h localhost \ + -p $VALKEY_SENTINEL_PORT \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +response=$(ping_sentinel 5) +if [ "${?}" -eq 124 ]; then + echo "Sentinel ping timed out" + exit 124 +fi + +if [ "${response}" != "PONG" ]; +then + echo "Sentinel is not responding" + exit 1 +fi + +echo "$( date +'[%Y/%m/%d %H:%M:%S]') Sentinel is responding" +exit 0 \ No newline at end of file diff --git a/resources/node/files/scripts/pre-stop-sentinel.sh b/resources/node/files/scripts/pre-stop-sentinel.sh new file mode 100644 index 0000000..d4941aa --- /dev/null +++ b/resources/node/files/scripts/pre-stop-sentinel.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +source /opt/scripts/common.sh + +tmout=120 +while true ; do + echo "I'm the primary pod and you are stopping me. Starting sentinel failover" + echo "Waiting for failover to finish..." + fo=$(getFailOverStatus) + if [ $? -eq 0 ]; then + echo "Primary has been successfully failed over to a different pod." + break + fi + + sleep 1 + tmout=$((tmout - 1)) + if [ "${tmout}" -le 0 ]; then + echo "Failover timed out" + exit 1 + fi +done + +echo "Primary has been successfuly failed over to a different pod." + +echo "Removing myself from the sentinel configuration" +sentinelReset $(hostname -f) +if [ $? -ne 0 ]; then + echo "Failed to remove myself from the sentinel configuration" + exit 1 +fi + +exit 0 diff --git a/resources/node/files/scripts/pre-stop.sh b/resources/node/files/scripts/pre-stop.sh new file mode 100644 index 0000000..2bbc799 --- /dev/null +++ b/resources/node/files/scripts/pre-stop.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +# Run Valkey command +vcli() { + valkey-cli -h 127.0.0.1 -p "${VALKEY_PORT}" "$@" + return $? +} + +# Run Sentinel command +vcli-sentinel() { + valkey-cli -h "${VALKEY_HEADLESS_SERVICE}" -p "${VALKEY_SENTINEL_PORT}" sentinel "$@" + return $? +} + +getFailOverStatus() { + # Check if the failover is finished + local failoverStatus + primaryInfo=$(vcli-sentinel get-primary-addr-by-name "mymaster") + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + currentHost=$(hostname -f) + if [[ "${primaryHost}" != "${currentHost}" ]]; then + echo "Failover finished" + return 0 + else + echo "Failover in progress" + return 1 + fi +} + +getRole() { + # Get the role of the current node + VALKEY_ROLE=$(vcli role | head -1) + echo "${VALKEY_ROLE}" +} + +isPrimary() { + # Check if the current node is the primary + role=$(getRole) + if [ "${role}" = "master" ]; then + echo "I'm the master" + else + echo "I'm not the master, I'm a ${role}" + fi +} + +if isPrimary && ! getFailOverStatus; then + echo "I'm the primary and you are stopping me, pausing client connections" + #Pausing write connections to avoid data loss" + vcli CLIENT PAUSE "22000" WRITE + + echo "Starting failover" + vcli-sentinel failover "mymaster" + echo "Waiting for sentinel to complete failover for up to 120s" + tmout=120 + while true ; do + echo "I'm the primary pod and you are stopping me. Starting sentinel failover" + getFailOverStatus + if [ $? -eq 0 ]; then + echo "Primary has been successfully failed over to a different pod." + break + fi + echo "Waiting for failover to finish..." + + sleep 1 + tmout=$((tmout - 1)) + if [ "${tmout}" -le 0 ]; then + echo "Failover timed out" + exit 1 + fi + done +fi + +exit 0 \ No newline at end of file diff --git a/resources/node/files/scripts/readiness-local.sh b/resources/node/files/scripts/readiness-local.sh new file mode 100644 index 0000000..c852cb1 --- /dev/null +++ b/resources/node/files/scripts/readiness-local.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +ping_valkey() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h localhost \ + -p $VALKEY_PORT \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +response=$(ping_valkey 5) +if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 +fi + +if [ "$response" != "PONG" ]; then + echo "Valey is not ready [${response}]" + exit 1 +fi + +echo "$( date +'[%Y/%m/%d %H:%M:%S]') Valkey is ready" +exit 0 diff --git a/resources/node/files/scripts/start-node.sh b/resources/node/files/scripts/start-node.sh new file mode 100644 index 0000000..10dc695 --- /dev/null +++ b/resources/node/files/scripts/start-node.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +pingSentinel() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h ${VALKEY_HEADLESS_SERVICE} \ + -p ${VALKEY_SENTINEL_PORT} \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +getPrimaryInfo() { + valkey-cli -t 15 --csv -h ${VALKEY_HEADLESS_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \ + awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }' + return ${?} +} + +waitForSentinel() { + tout=60 + while true; do + response=$(pingSentinel 5) + if [ "${response}" = "PONG" ]; then + echo "Sentinel is responding" + break + return 0 + fi + + echo "Sentinel is not responding [${response}]" + sleep 1 + tout=$((tout - 1)) + if [ "${tout}" -le 0 ]; then + echo "Sentinel ping timed out" + return 124 + fi + done + sleep 10 +} + +startValkey() { + # Start Valkey + echo "Running : [valkey-server ${@}]" + valkey-server ${@} + ret=${?} + if [ "${ret}" -ne 0 ]; then + echo "Failed to start Valkey" + exit ${ret} + fi +} + +setupPrimary=0 +primaryHost="" +primaryPort="" + + +#if [[ -f /etc/valkey/sentinel.conf ]]; then +# primaryHost="$(awk '/monitor/ {print $4}' /etc/valkey/sentinel.conf)" +# primaryPort="$(awk '/monitor/ {print $5}' /etc/valkey/sentinel.conf)" +# echo "Found previous primary ${primaryHost}:${primaryPort}" +#fi + +if [[ ! -f /etc/valkey/replication.conf ]]; then + cp /etc/valkey/replication.conf.orig /etc/valkey/replication.conf +fi + +waitForSentinel +ret=${?} +if [ "${ret}" -ne 0 ]; then + exit ${ret} +fi + +primaryInfo=$(getPrimaryInfo) +if [ "${?}" -ne 0 ]; then + echo "No primary found, seting up node as primary" + startPrimary=1 +else + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + primaryPort=$(echo ${primaryInfo} | awk -F ' ' '{print $2}') + echo "Primary host is : ${primaryHost}, port: ${primaryPort}" + currentHost=$(hostname -f) + echo "Current host is : ${currentHost}" + if [ "${primaryHost}" != "${currentHost}" ]; then + echo "Not the primary, setting up as replica" + startPrimary=0 + else + echo "This is the primary" + startPrimary=1 + fi +fi + +if [ "${startPrimary}" -eq 1 ]; then + echo "Starting Valkey as primary" + cat $1 + startValkey ${@} +else + echo "Starting Valkey as replica" + startValkey ${@} "--replicaof" "${primaryHost}" "${primaryPort}" +fi \ No newline at end of file diff --git a/resources/node/files/scripts/startSentinel.sh b/resources/node/files/scripts/startSentinel.sh new file mode 100644 index 0000000..ac6fc40 --- /dev/null +++ b/resources/node/files/scripts/startSentinel.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +# Generate sentinel.conf file for Valkey Sentinel +source /opt/scripts/common.sh + +PATH=${PATH}:/ +SENTINEL_MASTER="" + +updateSentinelMaster(){ + cp /etc/valkey/sentinel.conf.orig /etc/valkey/sentinel.conf + hostname=$(hostname -f) + + pingSentinel 1 + if [ $? -ne 0 ]; then + SENTINEL_MASTER=$(hostname -f) + else + SENTINEL_MASTER=$(getSentinelMasterName) + fi + + sed -i "s/SENTINEL_MASTER/${SENTINEL_MASTER}/g" /etc/valkey/sentinel.conf + +} + + +if [ -f /etc/valkey/sentinel.conf ]; then + echo "Sentinel configuration file already exists, starting with this." + reps=$(awk -F ':' '/REPLICAS/ {print $2}' /etc/valkey/sentinel.conf) + if [ "${reps}" = "${VALKEY_REPLICAS}" ]; then + echo "Sentinel configuration file already has the correct number of replicas (${VALKEY_REPLICAS})." + else + echo "Updating Sentinel configuration file with the correct number of replicas (${VALKEY_REPLICAS})." + updateSentinelMaster + fi +else + updateSentinelMaster +fi + +valkey-sentinel /etc/valkey/sentinel.conf \ No newline at end of file diff --git a/resources/node/kustomization.yaml b/resources/node/kustomization.yaml new file mode 100644 index 0000000..bec987a --- /dev/null +++ b/resources/node/kustomization.yaml @@ -0,0 +1,43 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- resources/sa.yaml +- resources/statefulset.yaml +- resources/svc.yaml + +configMapGenerator: +- name: valkey-env + literals: + - NAMESPACE="default" + - VALKEY_ENV="base" + - VALKEY_NAME="valkey-node" + - VALKEY_SERVICE="valkey-node" + - VALKEY_HEADLESS_SERVICE="valkey-node-headless" + - VALKEY_REPLICAS="4" + - VALKEY_PORT="6379" + - VALKEY_SENTINEL_PORT="26379" + - ALLOW_EMPTY_PASSWORD="yes" + - VALKEY_TLS_ENABLED="no" + - VALKEY_SENTINEL_TLS_ENABLED="no" + - VALKEY_DATA_DIR="/data" + - VALKEY_LOG_LEVEL="warning" + - VALKEY_QUORUM="2" +- name: valkey-config + files: + - files/conf/replication.conf.tpl + - files/conf/sentinel.conf.tpl +- name: valkey-scripts + files: + - files/scripts/common.sh + - files/scripts/startSentinel.sh + - files/scripts/pre-stop.sh + - files/scripts/pre-stop-sentinel.sh + - files/scripts/start-node.sh + - files/scripts/ping-sentinel.sh + - files/scripts/liveness-local.sh + - files/scripts/readiness-local.sh + +#replacements: +#- path: ./replacements/services.yaml +#- path: ./replacements/statefulset.yaml diff --git a/resources/node/replacements/services.yaml b/resources/node/replacements/services.yaml new file mode 100644 index 0000000..d325149 --- /dev/null +++ b/resources/node/replacements/services.yaml @@ -0,0 +1,20 @@ +- source: + kind: ConfigMap + name: valkey-env + fieldPath: data.VALKEY_HEADLESS_SERVICE + targets: + - select: + kind: Service + name: valkey-headless + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: valkey-env + fieldPath: data.VALKEY_SERVICE + targets: + - select: + kind: Service + name: valkey + fieldPaths: + - metadata.name diff --git a/resources/node/replacements/statefulset.yaml b/resources/node/replacements/statefulset.yaml new file mode 100644 index 0000000..4f82603 --- /dev/null +++ b/resources/node/replacements/statefulset.yaml @@ -0,0 +1,20 @@ +- source: + kind: ConfigMap + name: valkey-env + fieldPath: data.VALKEY_REPLICAS + targets: + - select: + kind: StatefulSet + name: valkey-node + fieldPaths: + - spec.replicas +- source: + kind: ConfigMap + name: valkey-env + fieldPath: data.VALKEY_HEADLESS_SERVICE + targets: + - select: + kind: StatefulSet + name: valkey-node + fieldPaths: + - spec.serviceName diff --git a/resources/node/resources/sa.yaml b/resources/node/resources/sa.yaml new file mode 100644 index 0000000..4e0e6e6 --- /dev/null +++ b/resources/node/resources/sa.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey diff --git a/resources/node/resources/statefulset.yaml b/resources/node/resources/statefulset.yaml new file mode 100644 index 0000000..fcbae0b --- /dev/null +++ b/resources/node/resources/statefulset.yaml @@ -0,0 +1,343 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey-node +spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + podManagementPolicy: OrderedReady + # DO NOT CHANGE THIS LINE HERE, USE THE VARIABLE VALKEY_REPLICAS INSTEAD + replicas: 4 + # END OF DO NOT CHANGE THIS LINE + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey + serviceName: valkey-headless + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: valkey + app.kubernetes.io/version: 8.1.1 + helm.sh/chart: valkey-3.0.7 + spec: + shareProcessNamespace: true + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey + topologyKey: kubernetes.io/hostname + weight: 1 + automountServiceAccountToken: false + initContainers: + - name: copy-config-templates + image: reg.cadoles.com/dh/library/busybox:1.37.0-musl + command: + - /bin/cp + args: + - -R + - /templates + - /tmp/ + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + volumeMounts: + - mountPath: /templates/replication.conf.orig + name: valkey-config + subPath: replication.conf.tpl + - mountPath: /templates/sentinel.conf.orig + name: valkey-config + subPath: sentinel.conf.tpl + - mountPath: /tmp + name: tmp + - mountPath: /data + name: valkey-data + - name: generate-config + image: reg.cadoles.com/cadoles/gotemplate:0.0.6-dev + imagePullPolicy: IfNotPresent + args: + - --source + - /tmp/templates + - --target + - /etc/valkey/ + - --no-overwrite + - replication.conf.orig + - sentinel.conf.orig + envFrom: + - configMapRef: + name: valkey-env + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + volumeMounts: + - mountPath: /etc/valkey/ + name: valkey-etc + - mountPath: /tmp + name: tmp + - mountPath: /data + name: valkey-data + containers: + - name: valkey + image: reg.cadoles.com/dh/valkey/valkey:8.1.1-alpine3.21 + command: + - /opt/scripts/start-node.sh + args: + - /etc/valkey/replication.conf + imagePullPolicy: IfNotPresent + env: + - name: VALKEY_ROLE + value: "replication" + envFrom: + - configMapRef: + name: valkey-env + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - /opt/scripts/pre-stop.sh + livenessProbe: + exec: + command: + - sh + - -c + - /opt/scripts/liveness-local.sh 5 + failureThreshold: 5 + initialDelaySeconds: 20 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + ports: + - containerPort: 6379 + name: valkey + protocol: TCP + readinessProbe: + exec: + command: + - sh + - -c + - /opt/scripts/readiness-local.sh 1 + failureThreshold: 5 + initialDelaySeconds: 20 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 150m + memory: 192Mi + ephemeral-storage: 2Gi + requests: + cpu: 100m + memory: 128Mi + ephemeral-storage: 50Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + startupProbe: + exec: + command: + - sh + - -c + - /opt/scripts/liveness-local.sh 5 + failureThreshold: 22 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/valkey/ + name: valkey-etc + - mountPath: /opt/scripts + name: valkey-scripts + - mountPath: /data + name: valkey-data + - name: sentinel + image: reg.cadoles.com/dh/valkey/valkey:8.1.1-alpine3.21 + imagePullPolicy: IfNotPresent + command: + - /opt/scripts/startSentinel.sh + env: + - name: ALLOW_EMPTY_PASSWORD + value: "yes" + - name: VALKEY_SENTINEL_TLS_ENABLED + value: "no" + envFrom: + - configMapRef: + name: valkey-env + lifecycle: + preStop: + exec: + command: + - /bin/bash + - -c + - /opt/scripts/pre-stop-sentinel.sh + livenessProbe: + exec: + command: + - sh + - -c + - /opt/scripts/ping-sentinel.sh 5 + failureThreshold: 6 + initialDelaySeconds: 20 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + ports: + - containerPort: 26379 + name: valkey-sentinel + protocol: TCP + readinessProbe: + exec: + command: + - sh + - -c + - /opt/scripts/ping-sentinel.sh 5 + failureThreshold: 6 + initialDelaySeconds: 20 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 150m + ephemeral-storage: 2Gi + memory: 192Mi + requests: + cpu: 100m + ephemeral-storage: 50Mi + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + startupProbe: + exec: + command: + - sh + - -c + - /opt/scripts/ping-sentinel.sh 5 + failureThreshold: 22 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /opt/scripts + name: valkey-scripts + - mountPath: /etc/valkey/ + name: valkey-etc + dnsPolicy: ClusterFirst + enableServiceLinks: true + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + runAsUser: 1001 + runAsNonRoot: true + runAsGroup: 1001 + fsGroup: 1001 + fsGroupChangePolicy: Always + serviceAccount: valkey + serviceAccountName: valkey + terminationGracePeriodSeconds: 30 + volumes: + - name: valkey-scripts + configMap: + defaultMode: 493 + name: valkey-scripts + - name: valkey-config + configMap: + defaultMode: 420 + name: valkey-config + - emptyDir: + sizeLimit: 64Mi + medium: Memory + name: tmp + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey + name: valkey-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + volumeMode: Filesystem + status: + phase: Pending + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey + name: valkey-etc + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 32Mi + volumeMode: Filesystem + status: + phase: Pending diff --git a/resources/node/resources/svc.yaml b/resources/node/resources/svc.yaml new file mode 100644 index 0000000..6c0b102 --- /dev/null +++ b/resources/node/resources/svc.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey +spec: + ports: + - name: tcp-redis + port: 6379 + protocol: TCP + targetPort: 6379 + - name: tcp-sentinel + port: 26379 + protocol: TCP + targetPort: 26379 + selector: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey-headless +spec: + clusterIP: None + ports: + - name: tcp-redis + port: 6379 + protocol: TCP + targetPort: redis + - name: tcp-sentinel + port: 26379 + protocol: TCP + targetPort: valkey-sentinel + publishNotReadyAddresses: true + selector: + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey