diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml new file mode 100644 index 0000000..4150577 --- /dev/null +++ b/.gitea/workflows/build.yaml @@ -0,0 +1,54 @@ +name: Build and Push Image +on: + push: + tags: + - '*' +env: + REGISTRY: reg.cadoles.com + +jobs: + build: + name: Build and push image + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Login to Docker Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta base + id: metabase + uses: docker/metadata-action@v5 + with: + images: | + reg.cadoles.com/cadoles/gotemplate + flavor: | + latest=auto + tags: | + type=schedule + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,prefix=,suffix= + + - name: Build and push + uses: docker/build-push-action@v6 + with: + build-args: | + GOTEMPLATE_VERSION=3.12.0 + context: ./misc/docker + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.metabase.outputs.tags }} + labels: ${{ steps.metabase.outputs.labels }} diff --git a/README.md b/README.md index e097d4e..b7cc446 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # varnish-kustom +Deploy a simple varnish server with kustomize \ No newline at end of file diff --git a/components/node/files/conf/replication.conf.tpl b/components/node/files/conf/replication.conf.tpl new file mode 100644 index 0000000..7b50acc --- /dev/null +++ b/components/node/files/conf/replication.conf.tpl @@ -0,0 +1,37 @@ +{{- $hostname := env "HOSTNAME" }} +{{- $service := env "VALKEY_HEADLESS_SERVICE" }} +{{- $namespace := env "NAMESPACE" }} +{{- $port := env "VALKEY_PORT" }} +{{- $sentinel_port := env "VALKEY_SENTINEL_PORT" }} +{{- $replicas := env "VALKEY_REPLICAS" }} +{{- $domain := printf "%s.%s.svc.cluster.local" $service $namespace }} +{{- $fqdn := printf "%s.%s" $hostname $domain }} +{{- $hostid := sha1sum $hostname }} +{{- $datadir := env "VALKEY_DATA_DIR" }} + +dir {{ $datadir }} + +protected-mode no + +loglevel {{ env "VALKEY_LOG_LEVEL" }} + +appendonly yes +appendfilename "appendonly.aof" +appenddirname "appendonlydir" +appendfsync everysec +no-appendfsync-on-rewrite no +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb +aof-load-truncated yes +aof-use-rdb-preamble yes +aof-timestamp-enabled no + +save "" + +replica-announce-port {{ $port }} +replica-announce-ip {{ $fqdn }} + +# User-supplied replica configuration: +rename-command FLUSHDB "" +rename-command FLUSHALL "" + diff --git a/components/node/files/conf/replication.conf.tpl.full b/components/node/files/conf/replication.conf.tpl.full new file mode 100644 index 0000000..f0720b4 --- /dev/null +++ b/components/node/files/conf/replication.conf.tpl.full @@ -0,0 +1,119 @@ +{{- $hostname := env "HOSTNAME" }} +{{- $service := env "VALKEY_HEADLESS_SERVICE" }} +{{- $namespace := env "NAMESPACE" }} +{{- $port := env "VALKEY_PORT" }} +{{- $sentinel_port := env "VALKEY_SENTINEL_PORT" }} +{{- $replicas := env "VALKEY_REPLICAS" }} +{{- $domain := printf "%s.%s.svc.cluster.local" $service $namespace }} +{{- $fqdn := printf "%s.%s" $hostname $domain }} +{{- $hostid := sha1sum $hostname }} +################################## INCLUDES ################################### +################################## MODULES ##################################### +################################## NETWORK ##################################### +bind * -::* +protected-mode no +port {{ $port }} +tcp-backlog 511 +timeout 0 +tcp-keepalive 300 +################################# TLS/SSL ##################################### +port {{ env "VALKEY_PORT" }} +################################### RDMA ###################################### +################################# GENERAL ##################################### +daemonize no +pidfile /opt/bitnami/valkey/tmp/valkey.pid +loglevel notice +logfile "" +databases 16 +always-show-logo no +hide-user-data-from-log yes +set-proc-title yes +proc-title-template "{title} {listen-addr} {server-mode}" +locale-collate "" +################################ SNAPSHOTTING ################################ +stop-writes-on-bgsave-error yes +rdbcompression yes +rdbchecksum yes +rdb-version-check strict +dbfilename dump.rdb +rdb-del-sync-files no +dir {{ env "VALKEY_DATA_DIR" }} +################################# REPLICATION ################################# +replica-serve-stale-data yes +replica-read-only yes +repl-diskless-sync yes +repl-diskless-sync-delay 5 +repl-diskless-sync-max-replicas 0 +repl-diskless-load disabled +dual-channel-replication-enabled no +repl-disable-tcp-nodelay no +replica-priority 100 +replica-announce-port {{ $port }} +replica-announce-ip {{ $fqdn }} +############################### KEYS TRACKING ################################# +################################## SECURITY ################################### +acllog-max-len 128 +################################### CLIENTS #################################### +############################## MEMORY MANAGEMENT ################################ +############################# LAZY FREEING #################################### +lazyfree-lazy-eviction yes +lazyfree-lazy-expire yes +lazyfree-lazy-server-del yes +replica-lazy-flush yes +lazyfree-lazy-user-del yes +lazyfree-lazy-user-flush yes +################################ THREADED I/O ################################# +############################ KERNEL OOM CONTROL ############################## +oom-score-adj no +oom-score-adj-values 0 200 800 +#################### KERNEL transparent hugepage CONTROL ###################### +disable-thp yes +############################## APPEND ONLY MODE ############################### +appendonly no +appendfilename "appendonly.aof" +appenddirname "appendonlydir" +appendfsync everysec +no-appendfsync-on-rewrite no +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb +aof-load-truncated yes +aof-use-rdb-preamble yes +aof-timestamp-enabled no +################################ SHUTDOWN ##################################### +################ NON-DETERMINISTIC LONG BLOCKING COMMANDS ##################### +################################ VALKEY CLUSTER ############################### +########################## CLUSTER DOCKER/NAT support ######################## +################################## COMMAND LOG ################################### +commandlog-execution-slower-than 10000 +commandlog-slow-execution-max-len 128 +commandlog-request-larger-than 1048576 +commandlog-large-request-max-len 128 +commandlog-reply-larger-than 1048576 +commandlog-large-reply-max-len 128 +################################ LATENCY MONITOR ############################## +latency-monitor-threshold 0 +################################ LATENCY TRACKING ############################## +############################# EVENT NOTIFICATION ############################## +notify-keyspace-events "" +############################### ADVANCED CONFIG ############################### +hash-max-listpack-entries 512 +hash-max-listpack-value 64 +list-max-listpack-size -2 +list-compress-depth 0 +set-max-intset-entries 512 +set-max-listpack-entries 128 +set-max-listpack-value 64 +zset-max-listpack-entries 128 +zset-max-listpack-value 64 +hll-sparse-max-bytes 3000 +stream-node-max-bytes 4096 +stream-node-max-entries 100 +activerehashing yes +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit replica 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 +hz 10 +aof-rewrite-incremental-fsync yes +rdb-save-incremental-fsync yes +########################### ACTIVE DEFRAGMENTATION ####################### +jemalloc-bg-thread yes diff --git a/components/node/files/conf/sentinel.conf.tpl b/components/node/files/conf/sentinel.conf.tpl new file mode 100644 index 0000000..113114e --- /dev/null +++ b/components/node/files/conf/sentinel.conf.tpl @@ -0,0 +1,49 @@ +{{- $hostname := env "HOSTNAME" }} +{{- $service := env "VALKEY_HEADLESS_SERVICE" }} +{{- $namespace := env "NAMESPACE" }} +{{- $port := env "VALKEY_PORT" }} +{{- $name := env "VALKEY_NAME" }} +{{- $sentinel_port := env "VALKEY_SENTINEL_PORT" }} +{{- $replicas := env "VALKEY_REPLICAS" }} +{{- $domain := printf "%s.%s.svc.cluster.local" $service $namespace }} +{{- $fqdn := printf "%s.%s" $hostname $domain }} +{{- $hostid := sha1sum $hostname }} +{{- $quorum := env "VALKEY_QUORUM" }} +{{- $master_name := env "VALKEY_MASTER_NAME" }} +#-REPLICAS:{{ $replicas }} + +dir "/tmp" +port {{ $sentinel_port }} +sentinel monitor {{ $master_name }} SENTINEL_MASTER {{ $port }} {{ $quorum }} +sentinel down-after-milliseconds {{ $master_name }} 60000 + +# User-supplied sentinel configuration: +# End of sentinel configuration +{{ printf "sentinel myid %s" $hostid }} + +sentinel announce-hostnames yes +sentinel resolve-hostnames yes + +sentinel announce-port {{ $sentinel_port }} +sentinel announce-ip {{ $fqdn }} + +# Generated by CONFIG REWRITE +latency-tracking-info-percentiles 50 99 99.9 +protected-mode no +# gotemplate-pause! +user default on nopass sanitize-payload ~* &* +@all +# gotemplate-resume! +sentinel config-epoch {{ $master_name }} 0 +sentinel leader-epoch {{ $master_name }} 0 +sentinel current-epoch 0 + +{{- range $i, $e := until ( int $replicas ) }} + {{- $masterName := printf "%s" $master_name}} + {{- $ndeHostname := printf "%s-%d" $name $i }} + {{- $ndeFQDN := printf "%s.%s" $ndeHostname $domain }} + {{- $nodeID := sha1sum $ndeHostname }} + {{- if (ne $fqdn $ndeFQDN) }} +sentinel {{ printf "known-sentinel %s %s %s %s" $masterName $ndeFQDN $sentinel_port $nodeID }} +sentinel {{ printf "known-replica %s %s %s" $masterName $ndeFQDN $port }} + {{- end}} +{{- end}} \ No newline at end of file diff --git a/components/node/files/scripts/common.sh b/components/node/files/scripts/common.sh new file mode 100644 index 0000000..feb0923 --- /dev/null +++ b/components/node/files/scripts/common.sh @@ -0,0 +1,103 @@ +#!/bin/sh + + +pingSentinel() { + svc=${VALKEY_HEADLESS_SERVICE:-"localhost"} + + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h ${svc} \ + -p $VALKEY_SENTINEL_PORT \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +getPrimaryInfo() { + masterName=${VALKEY_MASTER_NAME:-"mymaster"} + it="${1:-0}" + + info=$(valkey-cli --csv -h ${VALKEY_HEADLESS_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "${masterName}"| \ + awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }') + + if [ -z "${info}" ]; then + echo "Failed to get primary info for master '${masterName}'" + it=$((it + 1)) + getPrimaryInfo ${it} + if [ ${it} -ge 10 ]; then + echo "Failed to get primary info after 5 attempts" + return 1 + fi + fi + if [[ "${info}" =~ /^NULL/ ]]; then + it=$((it + 1)) + getPrimaryInfo ${it} + if [ ${it} -ge 10 ]; then + echo "Failed to get primary info after 5 attempts" + return 1 + fi + fi + echo "${info}" + return ${?} +} + +getSentinelMasterName() { + getPrimaryInfo | awk -F ' ' '{print $1}' +} + +getFailOverStatus() { + # Check if the failover is finished + local failoverStatus + primaryInfo=$(getPrimaryInfo) + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + currentHost=$(hostname -f) + if [[ "${primaryHost}" != "${currentHost}" ]]; then + echo "Failover finished" + return 0 + else + echo "Failover in progress" + return 1 + fi +} + +waitForSentinel() { + tout=60 + while true; do + response=$(pingSentinel 5) + if [ "${response}" = "PONG" ]; then + echo "Sentinel is responding" + break + return 0 + fi + + echo "Sentinel is not responding [${response}]" + sleep 1 + tout=$((tout - 1)) + if [ "${tout}" -le 0 ]; then + echo "Sentinel ping timed out" + return 124 + fi + done + sleep 10 +} + +sentinelIsMaster() { + # Check if the sentinel is master + primaryInfo=$(getPrimaryInfo) + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + currentHost=$(hostname -f) + if [[ "${primaryHost}" != "${currentHost}" ]]; then + echo "Sentinel is not master" + return 1 + else + echo "Sentinel is master" + return 0 + fi +} + +sentinelReset() { + # Reset the sentinel + valkey-cli -h ${VALKEY_HEADLESS_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel reset "${1}" + return $? +} \ No newline at end of file diff --git a/components/node/files/scripts/liveness-local.sh b/components/node/files/scripts/liveness-local.sh new file mode 100644 index 0000000..f56965f --- /dev/null +++ b/components/node/files/scripts/liveness-local.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +ping_valkey() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h localhost \ + -p $VALKEY_PORT \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +response=$(ping_valkey 5) +if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 +fi + +firstWord=$(echo $response | awk 'NR==1 {print $1;}') +if [ "$response" != "PONG" ] && [ "$firstWord" != "LOADING" ] && [ "$firstWord" != "MASTERDOWN" ]; then + echo "Valey is not alive [${response}]" + exit 1 +fi + +echo "$( date +'[%Y/%m/%d %H:%M:%S]') Valkey is alive" +exit 0 diff --git a/components/node/files/scripts/ping-sentinel.sh b/components/node/files/scripts/ping-sentinel.sh new file mode 100644 index 0000000..498627c --- /dev/null +++ b/components/node/files/scripts/ping-sentinel.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +ping_sentinel() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h localhost \ + -p $VALKEY_SENTINEL_PORT \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +response=$(ping_sentinel 5) +if [ "${?}" -eq 124 ]; then + echo "Sentinel ping timed out" + exit 124 +fi + +if [ "${response}" != "PONG" ]; +then + echo "Sentinel is not responding" + exit 1 +fi + +echo "$( date +'[%Y/%m/%d %H:%M:%S]') Sentinel is responding" +exit 0 \ No newline at end of file diff --git a/components/node/files/scripts/pre-stop-sentinel.sh b/components/node/files/scripts/pre-stop-sentinel.sh new file mode 100644 index 0000000..d4941aa --- /dev/null +++ b/components/node/files/scripts/pre-stop-sentinel.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +source /opt/scripts/common.sh + +tmout=120 +while true ; do + echo "I'm the primary pod and you are stopping me. Starting sentinel failover" + echo "Waiting for failover to finish..." + fo=$(getFailOverStatus) + if [ $? -eq 0 ]; then + echo "Primary has been successfully failed over to a different pod." + break + fi + + sleep 1 + tmout=$((tmout - 1)) + if [ "${tmout}" -le 0 ]; then + echo "Failover timed out" + exit 1 + fi +done + +echo "Primary has been successfuly failed over to a different pod." + +echo "Removing myself from the sentinel configuration" +sentinelReset $(hostname -f) +if [ $? -ne 0 ]; then + echo "Failed to remove myself from the sentinel configuration" + exit 1 +fi + +exit 0 diff --git a/components/node/files/scripts/pre-stop.sh b/components/node/files/scripts/pre-stop.sh new file mode 100644 index 0000000..58e373e --- /dev/null +++ b/components/node/files/scripts/pre-stop.sh @@ -0,0 +1,75 @@ +#!/bin/sh + +# Run Valkey command +vcli() { + valkey-cli -h 127.0.0.1 -p "${VALKEY_PORT}" "$@" + return $? +} + +# Run Sentinel command +vcli-sentinel() { + valkey-cli -h "${VALKEY_HEADLESS_SERVICE}" -p "${VALKEY_SENTINEL_PORT}" sentinel "$@" + return $? +} + +getFailOverStatus() { + # Check if the failover is finished + local failoverStatus + sentinelMasterName="${VALKEY_MASTER_NAME:-"mymaster"}" + primaryInfo=$(vcli-sentinel get-primary-addr-by-name "${sentinelMasterName}") + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + currentHost=$(hostname -f) + if [[ "${primaryHost}" != "${currentHost}" ]]; then + echo "Failover finished" + return 0 + else + echo "Failover in progress" + return 1 + fi +} + +getRole() { + # Get the role of the current node + VALKEY_ROLE=$(vcli role | head -1) + echo "${VALKEY_ROLE}" +} + +isPrimary() { + # Check if the current node is the primary + role=$(getRole) + if [ "${role}" = "master" ]; then + echo "I'm the master" + else + echo "I'm not the master, I'm a ${role}" + fi +} + +if isPrimary && ! getFailOverStatus; then + sentinelMasterName="${VALKEY_MASTER_NAME}" + echo "I'm the primary and you are stopping me, pausing client connections" + #Pausing write connections to avoid data loss" + vcli CLIENT PAUSE "22000" WRITE + + echo "Starting failover" + vcli-sentinel failover "${sentinelMasterName}" + echo "Waiting for sentinel to complete failover for up to 120s" + tmout=120 + while true ; do + echo "I'm the primary pod and you are stopping me. Starting sentinel failover" + getFailOverStatus + if [ $? -eq 0 ]; then + echo "Primary has been successfully failed over to a different pod." + break + fi + echo "Waiting for failover to finish..." + + sleep 1 + tmout=$((tmout - 1)) + if [ "${tmout}" -le 0 ]; then + echo "Failover timed out" + exit 1 + fi + done +fi + +exit 0 \ No newline at end of file diff --git a/components/node/files/scripts/readiness-local.sh b/components/node/files/scripts/readiness-local.sh new file mode 100644 index 0000000..c852cb1 --- /dev/null +++ b/components/node/files/scripts/readiness-local.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +ping_valkey() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h localhost \ + -p $VALKEY_PORT \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +response=$(ping_valkey 5) +if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 +fi + +if [ "$response" != "PONG" ]; then + echo "Valey is not ready [${response}]" + exit 1 +fi + +echo "$( date +'[%Y/%m/%d %H:%M:%S]') Valkey is ready" +exit 0 diff --git a/components/node/files/scripts/start-node.sh b/components/node/files/scripts/start-node.sh new file mode 100644 index 0000000..57d3837 --- /dev/null +++ b/components/node/files/scripts/start-node.sh @@ -0,0 +1,105 @@ +#!/bin/sh + +pingSentinel() { + resp=$(timeout -s 15 $1 \ + valkey-cli \ + -h ${VALKEY_HEADLESS_SERVICE} \ + -p ${VALKEY_SENTINEL_PORT} \ + ping) + ret=${?} + echo $resp + return ${ret} +} + +getPrimaryInfo() { + sentinelMasterName="${VALKEY_MASTER_NAME:-"mymaster"}" + valkey-cli -t 15 --csv -h ${VALKEY_HEADLESS_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "${sentinelMasterName}"| \ + awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }' + return ${?} +} + +waitForSentinel() { + tout=60 + while true; do + response=$(pingSentinel 5) + if [ "${response}" = "PONG" ]; then + echo "Sentinel is responding" + break + return 0 + fi + + echo "Sentinel is not responding [${response}]" + sleep 1 + tout=$((tout - 1)) + if [ "${tout}" -le 0 ]; then + echo "Sentinel ping timed out" + return 124 + fi + done + sleep 10 +} + +startValkey() { + # Start Valkey + echo "Running : [valkey-server ${@}]" + valkey-server ${@} + ret=${?} + if [ "${ret}" -ne 0 ]; then + echo "Failed to start Valkey" + exit ${ret} + fi +} + +setupPrimary=0 +primaryHost="" +primaryPort="" + + +#if [[ -f /etc/valkey/sentinel.conf ]]; then +# primaryHost="$(awk '/monitor/ {print $4}' /etc/valkey/sentinel.conf)" +# primaryPort="$(awk '/monitor/ {print $5}' /etc/valkey/sentinel.conf)" +# echo "Found previous primary ${primaryHost}:${primaryPort}" +#fi + +if [[ ! -f /etc/valkey/replication.conf ]]; then + cp /etc/valkey/replication.conf.orig /etc/valkey/replication.conf +fi + +waitForSentinel +ret=${?} +if [ "${ret}" -ne 0 ]; then + exit ${ret} +fi + +primaryInfo=$(getPrimaryInfo) +if [ "${?}" -ne 0 ]; then + echo "No primary found, seting up node as primary" + startPrimary=1 +else + primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}') + primaryPort=$(echo ${primaryInfo} | awk -F ' ' '{print $2}') + echo "Primary host is : ${primaryHost}, port: ${primaryPort}" + currentHost=$(hostname -f) + echo "Current host is : ${currentHost}" + if [ "${primaryHost}" = "NULL" ]; then + echo "Not primary yet, starting as primary" + startPrimary=1 + else + if [ "${primaryHost}" != "${currentHost}" ]; then + echo "Not the primary, setting up as replica" + startPrimary=0 + else + echo "This is the primary" + startPrimary=1 + fi + fi +fi + +if [ "${startPrimary}" -eq 1 ]; then + echo "Starting Valkey as primary" + cat $1 + startValkey ${@} +else + echo "Starting Valkey as replica" + startValkey ${@} "--replicaof" "${primaryHost}" "${primaryPort}" +fi \ No newline at end of file diff --git a/components/node/files/scripts/startSentinel.sh b/components/node/files/scripts/startSentinel.sh new file mode 100644 index 0000000..ac6fc40 --- /dev/null +++ b/components/node/files/scripts/startSentinel.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +# Generate sentinel.conf file for Valkey Sentinel +source /opt/scripts/common.sh + +PATH=${PATH}:/ +SENTINEL_MASTER="" + +updateSentinelMaster(){ + cp /etc/valkey/sentinel.conf.orig /etc/valkey/sentinel.conf + hostname=$(hostname -f) + + pingSentinel 1 + if [ $? -ne 0 ]; then + SENTINEL_MASTER=$(hostname -f) + else + SENTINEL_MASTER=$(getSentinelMasterName) + fi + + sed -i "s/SENTINEL_MASTER/${SENTINEL_MASTER}/g" /etc/valkey/sentinel.conf + +} + + +if [ -f /etc/valkey/sentinel.conf ]; then + echo "Sentinel configuration file already exists, starting with this." + reps=$(awk -F ':' '/REPLICAS/ {print $2}' /etc/valkey/sentinel.conf) + if [ "${reps}" = "${VALKEY_REPLICAS}" ]; then + echo "Sentinel configuration file already has the correct number of replicas (${VALKEY_REPLICAS})." + else + echo "Updating Sentinel configuration file with the correct number of replicas (${VALKEY_REPLICAS})." + updateSentinelMaster + fi +else + updateSentinelMaster +fi + +valkey-sentinel /etc/valkey/sentinel.conf \ No newline at end of file diff --git a/components/node/kustomization.yaml b/components/node/kustomization.yaml new file mode 100644 index 0000000..3314971 --- /dev/null +++ b/components/node/kustomization.yaml @@ -0,0 +1,44 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: +- resources/sa.yaml +- resources/statefulset.yaml +- resources/svc.yaml + +configMapGenerator: +#- name: valkey-env +# literals: +# - NAMESPACE="default" +# - VALKEY_ENV="base" +# - VALKEY_NAME="valkey-node" +# - VALKEY_SERVICE="valkey-node" +# - VALKEY_HEADLESS_SERVICE="valkey-node-headless" +# - VALKEY_MASTER_NAME="mymaster" +# - VALKEY_REPLICAS="4" +# - VALKEY_PORT="6379" +# - VALKEY_SENTINEL_PORT="26379" +# - ALLOW_EMPTY_PASSWORD="yes" +# - VALKEY_TLS_ENABLED="no" +# - VALKEY_SENTINEL_TLS_ENABLED="no" +# - VALKEY_DATA_DIR="/data" +# - VALKEY_LOG_LEVEL="warning" +# - VALKEY_QUORUM="2" +- name: valkey-config + files: + - files/conf/replication.conf.tpl + - files/conf/sentinel.conf.tpl +- name: valkey-scripts + files: + - files/scripts/common.sh + - files/scripts/startSentinel.sh + - files/scripts/pre-stop.sh + - files/scripts/pre-stop-sentinel.sh + - files/scripts/start-node.sh + - files/scripts/ping-sentinel.sh + - files/scripts/liveness-local.sh + - files/scripts/readiness-local.sh + +replacements: +- path: replacements/service.yaml +- path: replacements/statefulset.yaml diff --git a/components/node/replacements/service.yaml b/components/node/replacements/service.yaml new file mode 100644 index 0000000..7387c1e --- /dev/null +++ b/components/node/replacements/service.yaml @@ -0,0 +1,17 @@ +- source: + kind: ConfigMap + name: valkey-env + fieldPath: data.VALKEY_SERVICE + targets: + - select: + kind: Service + name: valkey-headless + fieldPaths: + - metadata.labels.app + - spec.selector.app + - select: + kind: Service + name: valkey + fieldPaths: + - metadata.labels.app + - spec.selector.app diff --git a/components/node/replacements/statefulset.yaml b/components/node/replacements/statefulset.yaml new file mode 100644 index 0000000..9274624 --- /dev/null +++ b/components/node/replacements/statefulset.yaml @@ -0,0 +1,12 @@ +- source: + kind: ConfigMap + name: valkey-env + fieldPath: data.VALKEY_SERVICE + targets: + - select: + kind: StatefulSet + name: valkey-node + fieldPaths: + - metadata.labels.app + - spec.selector.matchLabels.app + - spec.template.metadata.labels.app diff --git a/components/node/resources/sa.yaml b/components/node/resources/sa.yaml new file mode 100644 index 0000000..4e0e6e6 --- /dev/null +++ b/components/node/resources/sa.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey diff --git a/components/node/resources/statefulset.yaml b/components/node/resources/statefulset.yaml new file mode 100644 index 0000000..281b64b --- /dev/null +++ b/components/node/resources/statefulset.yaml @@ -0,0 +1,340 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: CHANGE_ME + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.3 + name: valkey-node +spec: + persistentVolumeClaimRetentionPolicy: + whenDeleted: Retain + whenScaled: Retain + podManagementPolicy: OrderedReady + # DO NOT CHANGE THIS LINE HERE, USE THE VARIABLE VALKEY_REPLICAS INSTEAD + replicas: 4 + # END OF DO NOT CHANGE THIS LINE + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey + app: CHANGE_ME + serviceName: valkey-headless + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: CHANGE_ME + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: valkey + app.kubernetes.io/version: 8.1.3 + helm.sh/chart: valkey-3.0.7 + spec: + shareProcessNamespace: true + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey + topologyKey: kubernetes.io/hostname + weight: 1 + automountServiceAccountToken: false + initContainers: + - name: copy-config-templates + image: reg.cadoles.com/dh/library/busybox:1.37.0-musl + command: + - /bin/cp + args: + - -R + - /templates + - /tmp/ + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + volumeMounts: + - mountPath: /templates/replication.conf.orig + name: valkey-config + subPath: replication.conf.tpl + - mountPath: /templates/sentinel.conf.orig + name: valkey-config + subPath: sentinel.conf.tpl + - mountPath: /tmp + name: tmp + - mountPath: /data + name: valkey-data + - name: generate-config + image: reg.cadoles.com/cadoles/gotemplate:0.0.6-dev + imagePullPolicy: IfNotPresent + args: + - --source + - /tmp/templates + - --target + - /etc/valkey/ + - --no-overwrite + - replication.conf.orig + - sentinel.conf.orig + envFrom: + - configMapRef: + name: valkey-env + resources: + limits: + cpu: 100m + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + volumeMounts: + - mountPath: /etc/valkey/ + name: valkey-etc + - mountPath: /tmp + name: tmp + - mountPath: /data + name: valkey-data + containers: + - name: valkey + image: reg.cadoles.com/dh/valkey/valkey:8.1.3-alpine3.22 + command: + - /opt/scripts/start-node.sh + args: + - /etc/valkey/replication.conf + imagePullPolicy: IfNotPresent + env: + - name: VALKEY_ROLE + value: "replication" + envFrom: + - configMapRef: + name: valkey-env + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - /opt/scripts/pre-stop.sh + livenessProbe: + exec: + command: + - sh + - -c + - /opt/scripts/liveness-local.sh 5 + failureThreshold: 5 + initialDelaySeconds: 20 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + ports: + - containerPort: 6379 + name: valkey + protocol: TCP + readinessProbe: + exec: + command: + - sh + - -c + - /opt/scripts/readiness-local.sh 1 + failureThreshold: 5 + initialDelaySeconds: 20 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 150m + memory: 192Mi + ephemeral-storage: 2Gi + requests: + cpu: 100m + memory: 128Mi + ephemeral-storage: 50Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + startupProbe: + exec: + command: + - sh + - -c + - /opt/scripts/liveness-local.sh 5 + failureThreshold: 22 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/valkey/ + name: valkey-etc + - mountPath: /opt/scripts + name: valkey-scripts + - mountPath: /data + name: valkey-data + - name: sentinel + image: reg.cadoles.com/dh/valkey/valkey:8.1.3-alpine3.22 + imagePullPolicy: IfNotPresent + command: + - /opt/scripts/startSentinel.sh + env: + - name: ALLOW_EMPTY_PASSWORD + value: "yes" + - name: VALKEY_SENTINEL_TLS_ENABLED + value: "no" + envFrom: + - configMapRef: + name: valkey-env + lifecycle: + preStop: + exec: + command: + - /bin/bash + - -c + - /opt/scripts/pre-stop-sentinel.sh + livenessProbe: + exec: + command: + - sh + - -c + - /opt/scripts/ping-sentinel.sh 5 + failureThreshold: 6 + initialDelaySeconds: 20 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + ports: + - containerPort: 26379 + name: valkey-sentinel + protocol: TCP + readinessProbe: + exec: + command: + - sh + - -c + - /opt/scripts/ping-sentinel.sh 5 + failureThreshold: 6 + initialDelaySeconds: 20 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 150m + ephemeral-storage: 2Gi + memory: 192Mi + requests: + cpu: 100m + ephemeral-storage: 50Mi + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seLinuxOptions: {} + seccompProfile: + type: RuntimeDefault + startupProbe: + exec: + command: + - sh + - -c + - /opt/scripts/ping-sentinel.sh 5 + failureThreshold: 22 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /opt/scripts + name: valkey-scripts + - mountPath: /etc/valkey/ + name: valkey-etc + dnsPolicy: ClusterFirst + enableServiceLinks: true + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + runAsUser: 1001 + runAsNonRoot: true + runAsGroup: 1001 + fsGroup: 1001 + fsGroupChangePolicy: Always + #serviceAccount: valkey + #serviceAccountName: valkey + terminationGracePeriodSeconds: 30 + volumes: + - name: valkey-scripts + configMap: + defaultMode: 493 + name: valkey-scripts + - name: valkey-config + configMap: + defaultMode: 420 + name: valkey-config + - emptyDir: + sizeLimit: 64Mi + medium: Memory + name: tmp + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey + name: valkey-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey + name: valkey-etc + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 32Mi diff --git a/components/node/resources/svc.yaml b/components/node/resources/svc.yaml new file mode 100644 index 0000000..53ec321 --- /dev/null +++ b/components/node/resources/svc.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: CHANGE_ME + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey +spec: + ports: + - name: tcp-redis + port: 6379 + protocol: TCP + targetPort: 6379 + - name: tcp-sentinel + port: 26379 + protocol: TCP + targetPort: 26379 + selector: + app: CHANGE_ME + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: CHANGE_ME + app.kubernetes.io/component: node + app.kubernetes.io/instance: valkey + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: valkey-headless + app.kubernetes.io/part-of: valkey + app.kubernetes.io/version: 8.1.1 + name: valkey-headless +spec: + clusterIP: None + ports: + - name: tcp-redis + port: 6379 + protocol: TCP + targetPort: redis + - name: tcp-sentinel + port: 26379 + protocol: TCP + targetPort: valkey-sentinel + publishNotReadyAddresses: true + selector: + app: CHANGE_ME + app.kubernetes.io/instance: valkey + app.kubernetes.io/name: valkey diff --git a/kustomization.yaml b/kustomization.yaml new file mode 100644 index 0000000..35e1765 --- /dev/null +++ b/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: +- resources/node diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile new file mode 100644 index 0000000..c880761 --- /dev/null +++ b/misc/docker/Dockerfile @@ -0,0 +1,22 @@ +# Base image +FROM golang:tip-alpine3.22 AS builder + +# Set directory to known value +WORKDIR /app +# Define the version as a build argument +ARG GOTEMPLATE_VERSION=3.12.0 + +RUN apk add valkey-cli git + +# Git clone the repo for gotemplate, checkout the desired tag, and build the executable +RUN git clone https://github.com/coveooss/gotemplate.git . && \ + git checkout v${GOTEMPLATE_VERSION} && \ + CGO_ENABLED=0 go build + + +FROM alpine:3.22 +# +COPY --from=builder /app/gotemplate /gotemplate +COPY --from=builder /usr/bin/valkey-cli /valkey-cli + +ENTRYPOINT [ "/gotemplate" ] \ No newline at end of file diff --git a/misc/tests/kustomization.yaml b/misc/tests/kustomization.yaml new file mode 100644 index 0000000..abfb210 --- /dev/null +++ b/misc/tests/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- resources/namespace.yaml +- resources/vlone +- resources/vltwo diff --git a/misc/tests/resources/namespace.yaml b/misc/tests/resources/namespace.yaml new file mode 100644 index 0000000..80cda30 --- /dev/null +++ b/misc/tests/resources/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: vltest diff --git a/misc/tests/resources/vlone/kustomization.yaml b/misc/tests/resources/vlone/kustomization.yaml new file mode 100644 index 0000000..d54490e --- /dev/null +++ b/misc/tests/resources/vlone/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namePrefix: vlone- + +components: +- https://forge.cadoles.com/CadolesKube/valkey-kustom//components/node + +configMapGenerator: +- name: valkey-env + literals: + - NAMESPACE="vltest" + - VALKEY_ENV="vltest" + - VALKEY_NAME="vlone-valkey-node" + - VALKEY_SERVICE="vlone-valkey" + - VALKEY_HEADLESS_SERVICE="vlone-valkey-headless" + - VALKEY_MASTER_NAME="mymaster" + - VALKEY_REPLICAS="6" + - VALKEY_PORT="6379" + - VALKEY_SENTINEL_PORT="26379" + - ALLOW_EMPTY_PASSWORD="yes" + - VALKEY_TLS_ENABLED="no" + - VALKEY_SENTINEL_TLS_ENABLED="no" + - VALKEY_DATA_DIR="/data" + - VALKEY_LOG_LEVEL="warning" + - VALKEY_QUORUM="2" diff --git a/misc/tests/resources/vlone/replacements/services.yaml b/misc/tests/resources/vlone/replacements/services.yaml new file mode 100644 index 0000000..62f4229 --- /dev/null +++ b/misc/tests/resources/vlone/replacements/services.yaml @@ -0,0 +1,20 @@ +- source: + kind: ConfigMap + name: vlone + fieldPath: data.VALKEY_HEADLESS_SERVICE + targets: + - select: + kind: Service + name: valkey-headless + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: vlone + fieldPath: data.VALKEY_SERVICE + targets: + - select: + kind: Service + name: valkey + fieldPaths: + - metadata.name diff --git a/misc/tests/resources/vlone/replacements/statefulset.yaml b/misc/tests/resources/vlone/replacements/statefulset.yaml new file mode 100644 index 0000000..edc9412 --- /dev/null +++ b/misc/tests/resources/vlone/replacements/statefulset.yaml @@ -0,0 +1,31 @@ +- source: + kind: ConfigMap + name: vlone + fieldPath: data.VALKEY_NAME + targets: + - select: + kind: StatefulSet + name: valkey-node + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: vlone + fieldPath: data.VALKEY_ENV_NAME + targets: + - select: + kind: StatefulSet + name: valkey-node + fieldPaths: + - spec.template.spec.containers[*].envFrom[0].configMapRef.name + - spec.template.spec.initContainers[*].envFrom[0].configMapRef.name +- source: + kind: ConfigMap + name: vlone + fieldPath: data.VALKEY_CONF_NAME + targets: + - select: + kind: StatefulSet + name: valkey-node + fieldPaths: + - spec.template.spec.initContainers[*].volumeMounts[valkey-config].name diff --git a/misc/tests/resources/vltwo/kustomization.yaml b/misc/tests/resources/vltwo/kustomization.yaml new file mode 100644 index 0000000..23f645e --- /dev/null +++ b/misc/tests/resources/vltwo/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namePrefix: vltwo- + +components: +- https://forge.cadoles.com/CadolesKube/valkey-kustom//components/node + +configMapGenerator: +- name: valkey-env + literals: + - NAMESPACE="vltest" + - VALKEY_ENV="vltest" + - VALKEY_NAME="vltwo-valkey-node" + - VALKEY_SERVICE="vltwo-valkey" + - VALKEY_HEADLESS_SERVICE="vltwo-valkey-headless" + - VALKEY_MASTER_NAME="vltwomaster" + - VALKEY_REPLICAS="4" + - VALKEY_PORT="6379" + - VALKEY_SENTINEL_PORT="26379" + - ALLOW_EMPTY_PASSWORD="yes" + - VALKEY_TLS_ENABLED="no" + - VALKEY_SENTINEL_TLS_ENABLED="no" + - VALKEY_DATA_DIR="/data" + - VALKEY_LOG_LEVEL="warning" + - VALKEY_QUORUM="2" diff --git a/misc/tests/resources/vltwo/replacements/services.yaml b/misc/tests/resources/vltwo/replacements/services.yaml new file mode 100644 index 0000000..3e2137e --- /dev/null +++ b/misc/tests/resources/vltwo/replacements/services.yaml @@ -0,0 +1,20 @@ +- source: + kind: ConfigMap + name: vltwo + fieldPath: data.VALKEY_HEADLESS_SERVICE + targets: + - select: + kind: Service + name: valkey-headless + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: vltwo + fieldPath: data.VALKEY_SERVICE + targets: + - select: + kind: Service + name: valkey + fieldPaths: + - metadata.name diff --git a/misc/tests/resources/vltwo/replacements/statefulset.yaml b/misc/tests/resources/vltwo/replacements/statefulset.yaml new file mode 100644 index 0000000..fb5ee88 --- /dev/null +++ b/misc/tests/resources/vltwo/replacements/statefulset.yaml @@ -0,0 +1,31 @@ +- source: + kind: ConfigMap + name: vltwo + fieldPath: data.VALKEY_NAME + targets: + - select: + kind: StatefulSet + name: valkey-node + fieldPaths: + - metadata.name +- source: + kind: ConfigMap + name: vltwo + fieldPath: data.VALKEY_ENV_NAME + targets: + - select: + kind: StatefulSet + name: valkey-node + fieldPaths: + - spec.template.spec.containers[*].envFrom[0].configMapRef.name + - spec.template.spec.initContainers[*].envFrom[0].configMapRef.name +- source: + kind: ConfigMap + name: vltwo + fieldPath: data.VALKEY_CONF_NAME + targets: + - select: + kind: StatefulSet + name: valkey-node + fieldPaths: + - spec.template.spec.initContainers[*].volumeMounts[valkey-config].name