CadolesKube/valkey-kustom
7
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge pull request '"refactor valkey deployment to combine sentinel and replication in single pod component"' (#1) from f-tests into develop

Browse Source 
Reviewed-on: #1
This commit is contained in:
pcaseiro
2025-07-24 16:18:57 +02:00
parent fef4f1d899 4530518b94
commit fd94cae2b7
29 changed files with 283 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
resources/node/files/conf/replication.conf.tpl → components/node/files/conf/replication.conf.tpl
View File

0
resources/node/files/conf/replication.conf.tpl.full → components/node/files/conf/replication.conf.tpl.full
View File

14
resources/node/files/conf/sentinel.conf.tpl → components/node/files/conf/sentinel.conf.tpl
View File

@ -9,12 +9,13 @@
{{- $fqdn := printf "%s.%s" $hostname $domain }}
{{- $hostid := sha1sum $hostname }}
{{- $quorum := env "VALKEY_QUORUM" }}
{{- $master_name := env "VALKEY_MASTER_NAME" }}
#-REPLICAS:{{ $replicas }}
dir "/tmp"
port {{ $sentinel_port }}
sentinel monitor mymaster SENTINEL_MASTER {{ $port }} {{ $quorum }}
sentinel down-after-milliseconds mymaster 60000
sentinel monitor {{ $master_name }} SENTINEL_MASTER {{ $port }} {{ $quorum }}
sentinel down-after-milliseconds {{ $master_name }} 60000
# User-supplied sentinel configuration:
# End of sentinel configuration
@ -32,16 +33,17 @@ protected-mode no
# gotemplate-pause!
user default on nopass sanitize-payload ~* &* +@all
# gotemplate-resume!
sentinel config-epoch mymaster 0
sentinel leader-epoch mymaster 0
sentinel config-epoch {{ $master_name }} 0
sentinel leader-epoch {{ $master_name }} 0
sentinel current-epoch 0
{{- range $i, $e := until ( int $replicas ) }}
{{- $masterName := printf "%s" $master_name}}
{{- $ndeHostname := printf "%s-%d" $name $i }}
{{- $ndeFQDN := printf "%s.%s" $ndeHostname $domain }}
{{- $nodeID := sha1sum $ndeHostname }}
{{- if (ne $fqdn $ndeFQDN) }}
sentinel {{ printf "known-sentinel mymaster %s %s %s" $ndeFQDN $sentinel_port $nodeID }}
sentinel {{ printf "known-replica mymaster %s %s" $ndeFQDN $port }}
sentinel {{ printf "known-sentinel %s %s %s %s" $masterName $ndeFQDN $sentinel_port $nodeID }}
sentinel {{ printf "known-replica %s %s %s" $masterName $ndeFQDN $port }}
{{- end}}
{{- end}}

26
resources/node/files/scripts/common.sh → components/node/files/scripts/common.sh
View File

@ -15,8 +15,30 @@ pingSentinel() {
}
getPrimaryInfo() {
valkey-cli --csv -h ${VALKEY_HEADLESS_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \
awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }'
masterName=${VALKEY_MASTER_NAME:-"mymaster"}
it="${1:-0}"
info=$(valkey-cli --csv -h ${VALKEY_HEADLESS_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "${masterName}"| \
awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }')
if [ -z "${info}" ]; then
echo "Failed to get primary info for master '${masterName}'"
it=$((it + 1))
getPrimaryInfo ${it}
if [ ${it} -ge 10 ]; then
echo "Failed to get primary info after 5 attempts"
return 1
fi
fi
if [[ "${info}" =~ /^NULL/ ]]; then
it=$((it + 1))
getPrimaryInfo ${it}
if [ ${it} -ge 10 ]; then
echo "Failed to get primary info after 5 attempts"
return 1
fi
fi
echo "${info}"
return ${?}
}

0
resources/node/files/scripts/liveness-local.sh → components/node/files/scripts/liveness-local.sh
View File

0
resources/node/files/scripts/ping-sentinel.sh → components/node/files/scripts/ping-sentinel.sh
View File

0
resources/node/files/scripts/pre-stop-sentinel.sh → components/node/files/scripts/pre-stop-sentinel.sh
View File

6
resources/node/files/scripts/pre-stop.sh → components/node/files/scripts/pre-stop.sh
View File

@ -15,7 +15,8 @@ vcli-sentinel() {
getFailOverStatus() {
# Check if the failover is finished
local failoverStatus
primaryInfo=$(vcli-sentinel get-primary-addr-by-name "mymaster")
sentinelMasterName="${VALKEY_MASTER_NAME:-"mymaster"}"
primaryInfo=$(vcli-sentinel get-primary-addr-by-name "${sentinelMasterName}")
primaryHost=$(echo ${primaryInfo} | awk -F ' ' '{print $1}')
currentHost=$(hostname -f)
if [[ "${primaryHost}" != "${currentHost}" ]]; then
@ -44,12 +45,13 @@ isPrimary() {
}
if isPrimary && ! getFailOverStatus; then
sentinelMasterName="${VALKEY_MASTER_NAME}"
echo "I'm the primary and you are stopping me, pausing client connections"
#Pausing write connections to avoid data loss"
vcli CLIENT PAUSE "22000" WRITE
echo "Starting failover"
vcli-sentinel failover "mymaster"
vcli-sentinel failover "${sentinelMasterName}"
echo "Waiting for sentinel to complete failover for up to 120s"
tmout=120
while true ; do

0
resources/node/files/scripts/readiness-local.sh → components/node/files/scripts/readiness-local.sh
View File

18
resources/node/files/scripts/start-node.sh → components/node/files/scripts/start-node.sh
View File

@ -12,7 +12,8 @@ pingSentinel() {
}
getPrimaryInfo() {
valkey-cli -t 15 --csv -h ${VALKEY_HEADLESS_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "mymaster"| \
sentinelMasterName="${VALKEY_MASTER_NAME:-"mymaster"}"
valkey-cli -t 15 --csv -h ${VALKEY_HEADLESS_SERVICE} -p ${VALKEY_SENTINEL_PORT} sentinel get-primary-addr-by-name "${sentinelMasterName}"| \
awk -F ',' '{ gsub(/"/,"",$0); print $1 " " $2 }'
return ${?}
}
@ -80,12 +81,17 @@ else
echo "Primary host is : ${primaryHost}, port: ${primaryPort}"
currentHost=$(hostname -f)
echo "Current host is : ${currentHost}"
if [ "${primaryHost}" != "${currentHost}" ]; then
echo "Not the primary, setting up as replica"
startPrimary=0
else
echo "This is the primary"
if [ "${primaryHost}" = "NULL" ]; then
echo "Not primary yet, starting as primary"
startPrimary=1
else
if [ "${primaryHost}" != "${currentHost}" ]; then
echo "Not the primary, setting up as replica"
startPrimary=0
else
echo "This is the primary"
startPrimary=1
fi
fi
fi

0
resources/node/files/scripts/startSentinel.sh → components/node/files/scripts/startSentinel.sh
View File

44
components/node/kustomization.yaml Normal file
View File

@ -0,0 +1,44 @@
apiVersion: kustomize.config.k8s.io/v1alpha1
kind: Component
resources:
- resources/sa.yaml
- resources/statefulset.yaml
- resources/svc.yaml
configMapGenerator:
#- name: valkey-env
# literals:
# - NAMESPACE="default"
# - VALKEY_ENV="base"
# - VALKEY_NAME="valkey-node"
# - VALKEY_SERVICE="valkey-node"
# - VALKEY_HEADLESS_SERVICE="valkey-node-headless"
# - VALKEY_MASTER_NAME="mymaster"
# - VALKEY_REPLICAS="4"
# - VALKEY_PORT="6379"
# - VALKEY_SENTINEL_PORT="26379"
# - ALLOW_EMPTY_PASSWORD="yes"
# - VALKEY_TLS_ENABLED="no"
# - VALKEY_SENTINEL_TLS_ENABLED="no"
# - VALKEY_DATA_DIR="/data"
# - VALKEY_LOG_LEVEL="warning"
# - VALKEY_QUORUM="2"
- name: valkey-config
files:
- files/conf/replication.conf.tpl
- files/conf/sentinel.conf.tpl
- name: valkey-scripts
files:
- files/scripts/common.sh
- files/scripts/startSentinel.sh
- files/scripts/pre-stop.sh
- files/scripts/pre-stop-sentinel.sh
- files/scripts/start-node.sh
- files/scripts/ping-sentinel.sh
- files/scripts/liveness-local.sh
- files/scripts/readiness-local.sh
replacements:
- path: replacements/service.yaml
- path: replacements/statefulset.yaml

17
components/node/replacements/service.yaml Normal file
View File

@ -0,0 +1,17 @@
- source:
kind: ConfigMap
name: valkey-env
fieldPath: data.VALKEY_SERVICE
targets:
- select:
kind: Service
name: valkey-headless
fieldPaths:
- metadata.labels.app
- spec.selector.app
- select:
kind: Service
name: valkey
fieldPaths:
- metadata.labels.app
- spec.selector.app

12
components/node/replacements/statefulset.yaml Normal file
View File

@ -0,0 +1,12 @@
- source:
kind: ConfigMap
name: valkey-env
fieldPath: data.VALKEY_SERVICE
targets:
- select:
kind: StatefulSet
name: valkey-node
fieldPaths:
- metadata.labels.app
- spec.selector.matchLabels.app
- spec.template.metadata.labels.app

0
resources/node/resources/sa.yaml → components/node/resources/sa.yaml
View File

21
resources/node/resources/statefulset.yaml → components/node/resources/statefulset.yaml
View File

@ -2,12 +2,13 @@ apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
app: CHANGE_ME
app.kubernetes.io/component: node
app.kubernetes.io/instance: valkey
app.kubernetes.io/managed-by: kustomize
app.kubernetes.io/name: valkey
app.kubernetes.io/part-of: valkey
app.kubernetes.io/version: 8.1.1
app.kubernetes.io/version: 8.1.3
name: valkey-node
spec:
persistentVolumeClaimRetentionPolicy:
@ -23,17 +24,19 @@ spec:
app.kubernetes.io/component: node
app.kubernetes.io/instance: valkey
app.kubernetes.io/name: valkey
app: CHANGE_ME
serviceName: valkey-headless
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app: CHANGE_ME
app.kubernetes.io/component: node
app.kubernetes.io/instance: valkey
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: valkey
app.kubernetes.io/version: 8.1.1
app.kubernetes.io/version: 8.1.3
helm.sh/chart: valkey-3.0.7
spec:
shareProcessNamespace: true
@ -107,7 +110,7 @@ spec:
name: valkey-data
containers:
- name: valkey
image: reg.cadoles.com/dh/valkey/valkey:8.1.1-alpine3.21
image: reg.cadoles.com/dh/valkey/valkey:8.1.3-alpine3.22
command:
- /opt/scripts/start-node.sh
args:
@ -194,7 +197,7 @@ spec:
- mountPath: /data
name: valkey-data
- name: sentinel
image: reg.cadoles.com/dh/valkey/valkey:8.1.1-alpine3.21
image: reg.cadoles.com/dh/valkey/valkey:8.1.3-alpine3.22
imagePullPolicy: IfNotPresent
command:
- /opt/scripts/startSentinel.sh
@ -288,8 +291,8 @@ spec:
runAsGroup: 1001
fsGroup: 1001
fsGroupChangePolicy: Always
serviceAccount: valkey
serviceAccountName: valkey
#serviceAccount: valkey
#serviceAccountName: valkey
terminationGracePeriodSeconds: 30
volumes:
- name: valkey-scripts
@ -320,9 +323,6 @@ spec:
resources:
requests:
storage: 8Gi
volumeMode: Filesystem
status:
phase: Pending
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
@ -338,6 +338,3 @@ spec:
resources:
requests:
storage: 32Mi
volumeMode: Filesystem
status:
phase: Pending

6
resources/node/resources/svc.yaml → components/node/resources/svc.yaml
View File

@ -2,6 +2,7 @@ apiVersion: v1
kind: Service
metadata:
labels:
app: CHANGE_ME
app.kubernetes.io/component: node
app.kubernetes.io/instance: valkey
app.kubernetes.io/managed-by: kustomize
@ -20,6 +21,7 @@ spec:
protocol: TCP
targetPort: 26379
selector:
app: CHANGE_ME
app.kubernetes.io/component: node
app.kubernetes.io/instance: valkey
app.kubernetes.io/name: valkey
@ -28,10 +30,11 @@ apiVersion: v1
kind: Service
metadata:
labels:
app: CHANGE_ME
app.kubernetes.io/component: node
app.kubernetes.io/instance: valkey
app.kubernetes.io/managed-by: kustomize
app.kubernetes.io/name: valkey
app.kubernetes.io/name: valkey-headless
app.kubernetes.io/part-of: valkey
app.kubernetes.io/version: 8.1.1
name: valkey-headless
@ -48,5 +51,6 @@ spec:
targetPort: valkey-sentinel
publishNotReadyAddresses: true
selector:
app: CHANGE_ME
app.kubernetes.io/instance: valkey
app.kubernetes.io/name: valkey

2
kustomization.yaml
View File

@ -1,5 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
components:
- resources/node

4
misc/docker/Dockerfile
View File

@ -1,5 +1,5 @@
# Base image
FROM golang:tip-alpine3.21 AS builder
FROM golang:tip-alpine3.22 AS builder
# Set directory to known value
WORKDIR /app
@ -14,7 +14,7 @@ RUN git clone https://github.com/coveooss/gotemplate.git . && \
CGO_ENABLED=0 go build
FROM alpine:3.21
FROM alpine:3.22
#
COPY --from=builder /app/gotemplate /gotemplate
COPY --from=builder /usr/bin/valkey-cli /valkey-cli

7
misc/tests/kustomization.yaml Normal file
View File

@ -0,0 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- resources/namespace.yaml
- resources/vlone
- resources/vltwo

4
misc/tests/resources/namespace.yaml Normal file
View File

@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: vltest

25
misc/tests/resources/vlone/kustomization.yaml Normal file
View File

@ -0,0 +1,25 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namePrefix: vlone-
components:
- https://forge.cadoles.com/CadolesKube/valkey-kustom//components/node
configMapGenerator:
- name: valkey-env
literals:
- NAMESPACE="vltest"
- VALKEY_ENV="vltest"
- VALKEY_NAME="vlone-valkey-node"
- VALKEY_SERVICE="vlone-valkey"
- VALKEY_HEADLESS_SERVICE="vlone-valkey-headless"
- VALKEY_MASTER_NAME="mymaster"
- VALKEY_REPLICAS="6"
- VALKEY_PORT="6379"
- VALKEY_SENTINEL_PORT="26379"
- ALLOW_EMPTY_PASSWORD="yes"
- VALKEY_TLS_ENABLED="no"
- VALKEY_SENTINEL_TLS_ENABLED="no"
- VALKEY_DATA_DIR="/data"
- VALKEY_LOG_LEVEL="warning"
- VALKEY_QUORUM="2"

4
resources/node/replacements/services.yaml → misc/tests/resources/vlone/replacements/services.yaml
View File

@ -1,6 +1,6 @@
- source:
kind: ConfigMap
name: valkey-env
name: vlone
fieldPath: data.VALKEY_HEADLESS_SERVICE
targets:
- select:
@ -10,7 +10,7 @@
- metadata.name
- source:
kind: ConfigMap
name: valkey-env
name: vlone
fieldPath: data.VALKEY_SERVICE
targets:
- select:

31
misc/tests/resources/vlone/replacements/statefulset.yaml Normal file
View File

@ -0,0 +1,31 @@
- source:
kind: ConfigMap
name: vlone
fieldPath: data.VALKEY_NAME
targets:
- select:
kind: StatefulSet
name: valkey-node
fieldPaths:
- metadata.name
- source:
kind: ConfigMap
name: vlone
fieldPath: data.VALKEY_ENV_NAME
targets:
- select:
kind: StatefulSet
name: valkey-node
fieldPaths:
- spec.template.spec.containers[*].envFrom[0].configMapRef.name
- spec.template.spec.initContainers[*].envFrom[0].configMapRef.name
- source:
kind: ConfigMap
name: vlone
fieldPath: data.VALKEY_CONF_NAME
targets:
- select:
kind: StatefulSet
name: valkey-node
fieldPaths:
- spec.template.spec.initContainers[*].volumeMounts[valkey-config].name

25
misc/tests/resources/vltwo/kustomization.yaml Normal file
View File

@ -0,0 +1,25 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namePrefix: vltwo-
components:
- https://forge.cadoles.com/CadolesKube/valkey-kustom//components/node
configMapGenerator:
- name: valkey-env
literals:
- NAMESPACE="vltest"
- VALKEY_ENV="vltest"
- VALKEY_NAME="vltwo-valkey-node"
- VALKEY_SERVICE="vltwo-valkey"
- VALKEY_HEADLESS_SERVICE="vltwo-valkey-headless"
- VALKEY_MASTER_NAME="vltwomaster"
- VALKEY_REPLICAS="4"
- VALKEY_PORT="6379"
- VALKEY_SENTINEL_PORT="26379"
- ALLOW_EMPTY_PASSWORD="yes"
- VALKEY_TLS_ENABLED="no"
- VALKEY_SENTINEL_TLS_ENABLED="no"
- VALKEY_DATA_DIR="/data"
- VALKEY_LOG_LEVEL="warning"
- VALKEY_QUORUM="2"

20
misc/tests/resources/vltwo/replacements/services.yaml Normal file
View File

@ -0,0 +1,20 @@
- source:
kind: ConfigMap
name: vltwo
fieldPath: data.VALKEY_HEADLESS_SERVICE
targets:
- select:
kind: Service
name: valkey-headless
fieldPaths:
- metadata.name
- source:
kind: ConfigMap
name: vltwo
fieldPath: data.VALKEY_SERVICE
targets:
- select:
kind: Service
name: valkey
fieldPaths:
- metadata.name

31
misc/tests/resources/vltwo/replacements/statefulset.yaml Normal file
View File

@ -0,0 +1,31 @@
- source:
kind: ConfigMap
name: vltwo
fieldPath: data.VALKEY_NAME
targets:
- select:
kind: StatefulSet
name: valkey-node
fieldPaths:
- metadata.name
- source:
kind: ConfigMap
name: vltwo
fieldPath: data.VALKEY_ENV_NAME
targets:
- select:
kind: StatefulSet
name: valkey-node
fieldPaths:
- spec.template.spec.containers[*].envFrom[0].configMapRef.name
- spec.template.spec.initContainers[*].envFrom[0].configMapRef.name
- source:
kind: ConfigMap
name: vltwo
fieldPath: data.VALKEY_CONF_NAME
targets:
- select:
kind: StatefulSet
name: valkey-node
fieldPaths:
- spec.template.spec.initContainers[*].volumeMounts[valkey-config].name

43
resources/node/kustomization.yaml
View File

@ -1,43 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- resources/sa.yaml
- resources/statefulset.yaml
- resources/svc.yaml
configMapGenerator:
- name: valkey-env
literals:
- NAMESPACE="default"
- VALKEY_ENV="base"
- VALKEY_NAME="valkey-node"
- VALKEY_SERVICE="valkey-node"
- VALKEY_HEADLESS_SERVICE="valkey-node-headless"
- VALKEY_REPLICAS="4"
- VALKEY_PORT="6379"
- VALKEY_SENTINEL_PORT="26379"
- ALLOW_EMPTY_PASSWORD="yes"
- VALKEY_TLS_ENABLED="no"
- VALKEY_SENTINEL_TLS_ENABLED="no"
- VALKEY_DATA_DIR="/data"
- VALKEY_LOG_LEVEL="warning"
- VALKEY_QUORUM="2"
- name: valkey-config
files:
- files/conf/replication.conf.tpl
- files/conf/sentinel.conf.tpl
- name: valkey-scripts
files:
- files/scripts/common.sh
- files/scripts/startSentinel.sh
- files/scripts/pre-stop.sh
- files/scripts/pre-stop-sentinel.sh
- files/scripts/start-node.sh
- files/scripts/ping-sentinel.sh
- files/scripts/liveness-local.sh
- files/scripts/readiness-local.sh
#replacements:
#- path: ./replacements/services.yaml
#- path: ./replacements/statefulset.yaml

20
resources/node/replacements/statefulset.yaml
View File

@ -1,20 +0,0 @@
- source:
kind: ConfigMap
name: valkey-env
fieldPath: data.VALKEY_REPLICAS
targets:
- select:
kind: StatefulSet
name: valkey-node
fieldPaths:
- spec.replicas
- source:
kind: ConfigMap
name: valkey-env
fieldPath: data.VALKEY_HEADLESS_SERVICE
targets:
- select:
kind: StatefulSet
name: valkey-node
fieldPaths:
- spec.serviceName