Merge pull request 'feat(hydra-dispatcher): update image ref' (#70 ) from f/update_hydra_dispatcher into unstable

Reviewed-on: #70
feat(hydra-dispatcher): update image ref
2025-03-18 16:13:32 +01:00 · 2025-03-18 15:12:14 +01:00 · 2025-03-11 15:46:07 +01:00 · 2025-03-11 15:42:32 +01:00 · 2025-03-11 15:35:46 +01:00 · 2025-03-07 15:31:59 +01:00
19 changed files with 268 additions and 30 deletions
--- a/components/hydra-cleaner/files/hydra-cleaner.sh
+++ b/components/hydra-cleaner/files/hydra-cleaner.sh
@ -0,0 +1,116 @@
+#!/bin/sh
+
+set -e
+set -o nounset
+
+# 4 tables to empty, at least
+# oidc, code, flow, authentication_session
+
+# \d hydra_oauth2_flow
+#Referenced by:
+#    TABLE "hydra_oauth2_access" CONSTRAINT "hydra_oauth2_access_challenge_id_fk" FOREIGN KEY (challenge_id) REFERENCES hydra_oauth2_flow(consent_challenge_id) ON DELETE CASCADE
+#    TABLE "hydra_oauth2_code" CONSTRAINT "hydra_oauth2_code_challenge_id_fk" FOREIGN KEY (challenge_id) REFERENCES hydra_oauth2_flow(consent_challenge_id) ON DELETE CASCADE
+#    TABLE "hydra_oauth2_oidc" CONSTRAINT "hydra_oauth2_oidc_challenge_id_fk" FOREIGN KEY (challenge_id) REFERENCES hydra_oauth2_flow(consent_challenge_id) ON DELETE CASCADE
+#    TABLE "hydra_oauth2_pkce" CONSTRAINT "hydra_oauth2_pkce_challenge_id_fk" FOREIGN KEY (challenge_id) REFERENCES hydra_oauth2_flow(consent_challenge_id) ON DELETE CASCADE
+#    TABLE "hydra_oauth2_refresh" CONSTRAINT "hydra_oauth2_refresh_challenge_id_fk" FOREIGN KEY (challenge_id) REFERENCES hydra_oauth2_flow(consent_challenge_id) ON DELETE CASCADE
+
+# -> delete "cascade" on table "flow" cleans access, code, oidc, pkce and refresh tables.
+
+
+DSN="${DSN:-postgresql://${HYDRA_DATABASE_USER}:${HYDRA_DATABASE_PASSWORD}@${HYDRA_DATABASE_SERVICE_NAME}:${HYDRA_DATABASE_SERVICE_PORT:-5432}/hydra?sslmode=disable}"
+RETENTION_HOURS="${RETENTION_HOURS:-48}"
+BATCH_SIZE="${BATCH_SIZE:-50}"
+LIMIT="${LIMIT:-1000}"
+BEFORE_DATE="$(date +'%Y-%m-%d %H:%M:%S' --date=@$(($(date +%s) - RETENTION_HOURS * 3600)))"
+
+
+log() {
+    echo "$(date +'%d-%m-%y %H:%M:%S%z')| $1"
+}
+
+perror() {
+    log "Something went wrong, exiting."
+    trap - EXIT
+    exit 1
+}
+
+trap perror EXIT
+
+if ! [[ ${RETENTION_HOURS} =~ '^[0-9]+$' ]]; then
+    log "Error: variable RETENTION_HOURS is not a positive integer."
+    perror
+fi
+
+if ! [[ ${LIMIT} =~ '^[0-9]+$' ]]; then
+    log "Error: variable LIMIT is not a positive integer."
+    perror
+fi
+
+if ! [[ ${BATCH_SIZE} =~ '^[0-9]+$' ]]; then
+    log "Error: variable BATCH_SIZE is not a positive integer."
+    perror
+fi
+
+log "Starting hydra cleaner"
+
+log "Removing up to ${LIMIT} elements before ${BEFORE_DATE} by batch of ${BATCH_SIZE}"
+
+log "Beginning estimated size:"
+psql "${DSN}" <<EOF
+select
+  table_name, reltuples as estimate,
+  pg_size_pretty(pg_total_relation_size(quote_ident(table_name))),
+  pg_total_relation_size(quote_ident(table_name))
+from information_schema.tables left join pg_class on information_schema.tables.table_name=pg_class.relname
+where table_schema = 'public'
+order by 4 desc;
+EOF
+
+
+REMAINING_ELMTS="${LIMIT}"
+while [ "${REMAINING_ELMTS}" -gt 0 ]; do
+    OUTPUT=$(psql "${DSN}" <<EOF
+DELETE
+FROM hydra_oauth2_flow
+WHERE login_challenge = ANY (
+  array(
+    SELECT login_challenge
+    FROM hydra_oauth2_flow
+    WHERE requested_at < '${BEFORE_DATE}'
+    LIMIT ${BATCH_SIZE}
+  )
+);
+EOF
+    )
+
+    log "${OUTPUT}"
+
+    if ! [[ "${OUTPUT}" =~ '^DELETE ' ]] ; then
+	log "Output doesn't seems OK..."
+	break
+    fi
+    OUTPUT_NB=$(echo "${OUTPUT}" | cut -d' ' -f 2)
+
+    if [ "${OUTPUT_NB}" -lt "${BATCH_SIZE}" ]; then
+	break
+    fi
+
+    REMAINING_ELMTS=$((REMAINING_ELMTS - BATCH_SIZE))
+    if [ "${REMAINING_ELMTS}" -lt "${BATCH_SIZE}" ]; then
+	BATCH_SIZE="${REMAINING_ELMTS}"
+    fi
+done
+
+
+log "Final estimated size:"
+psql "${DSN}" <<EOF
+select
+  table_name, reltuples as estimate,
+  pg_size_pretty(pg_total_relation_size(quote_ident(table_name))),
+  pg_total_relation_size(quote_ident(table_name))
+from information_schema.tables left join pg_class on information_schema.tables.table_name=pg_class.relname
+where table_schema = 'public'
+order by 4 desc;
+EOF
+
+trap - EXIT
--- a/components/hydra-cleaner/kustomization.yaml
+++ b/components/hydra-cleaner/kustomization.yaml
@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+resources:
+- ./resources/hydra-cleaner-cronjob.yaml
+
+configMapGenerator:
+- name: hydra-cleaner-env
+  behavior: create
+  literals:
+  - RETENTION_HOURS="48"
+  - BATCH_SIZE="100"
+  - LIMIT="1000"
+- name: hydra-cleaner-script
+  behavior: create
+  files:
+  - ./files/hydra-cleaner.sh
--- a/components/hydra-cleaner/resources/hydra-cleaner-cronjob.yaml
+++ b/components/hydra-cleaner/resources/hydra-cleaner-cronjob.yaml
@ -0,0 +1,54 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: hydra-cleaner
+  labels:
+    app.kubernetes.io/name: hydra-cleaner
+spec:
+  concurrencyPolicy: Forbid
+  schedule: "30 */1 * * *"
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: hydra-cleaner
+        spec:
+          restartPolicy: OnFailure
+          serviceAccountName: hydra-sa
+          containers:
+          - name: hydra-cleaner
+            image: reg.cadoles.com/proxy_cache/alpine/psql:17.4
+            envFrom:
+            - configMapRef:
+                name: hydra-env
+            - configMapRef:
+                name: hydra-cleaner-env
+            imagePullPolicy: IfNotPresent
+            command: ["/hydra-cleaner.sh"]
+            env:
+            - name: HYDRA_DATABASE_USER
+              valueFrom:
+                secretKeyRef:
+                  name: hydra-postgres-app
+                  key: username
+            - name: HYDRA_DATABASE_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: hydra-postgres-app
+                  key: password
+            - name: HYDRA_DATABASE_SERVICE_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: hydra-postgres-app
+                  key: host
+            args: []
+            volumeMounts:
+            - name: hydra-cleaner-script
+              mountPath: "/hydra-cleaner.sh"
+              subPath: "hydra-cleaner.sh"
+          volumes:
+          - name: hydra-cleaner-script
+            configMap:
+              name: hydra-cleaner-script
+              defaultMode: 0544
--- a/components/hydra-cnpg-database/patches/hydra-deployment.yaml
+++ b/components/hydra-cnpg-database/patches/hydra-deployment.yaml
@ -26,4 +26,4 @@
  path: "/spec/template/spec/containers/0/env/-"
  value:
    name: DSN
-    value: "postgres://$(HYDRA_DATABASE_USER):$(HYDRA_DATABASE_PASSWORD)@$(HYDRA_DATABASE_SERVICE_NAME):5432/hydra?sslmode=disable&max_conns=$(HYDRA_DATABASE_MAX_CONN)"
+    value: "postgres://$(HYDRA_DATABASE_USER):$(HYDRA_DATABASE_PASSWORD)@$(HYDRA_DATABASE_SERVICE_NAME):5432/hydra?sslmode=disable&max_conns=$(HYDRA_DATABASE_MAX_CONN)&max_idle_conns=$(HYDRA_DATABASE_MAX_IDLE_CONNS)&max_conn_lifetime=$(HYDRA_DATABASE_MAX_CONN_LIFETIME)&max_conn_idle_time=$(HYDRA_DATABASE_MAX_CONN_IDLE_TIME)&connect_timeout=$(HYDRA_DATABASE_CONNECT_TIMEOUT)"
--- a/components/hydra-saml/files/hydra/saml.yaml
+++ b/components/hydra-saml/files/hydra/saml.yaml
@ -20,11 +20,3 @@ hydra:
        eduPersonAffiliation:
          rules:
          - "property_exists(consent.session.id_token, 'eduPersonAffiliation') ? consent.session.id_token.eduPersonAffiliation : null"
-  firewall:
-    rules:
-      email:
-        required: false
-      uid:
-        required: false
-      eduPersonAffiliation:
-        required: false
--- a/components/hydra-sql/resources/hydra-sql-deployment.yaml
+++ b/components/hydra-sql/resources/hydra-sql-deployment.yaml
@ -21,7 +21,7 @@ spec:
    spec:
      containers:
      - name: hydra-sql-fpm
-        image: reg.cadoles.com/cadoles/hydra-sql-base:2024.11.6-develop.1113.075be9b
+        image: reg.cadoles.com/cadoles/hydra-sql-base:2025.3.7-develop.1415.7239d84
        imagePullPolicy: IfNotPresent
        args: ["/usr/sbin/php-fpm81", "-F", "-e"]
        readinessProbe:
@ -68,7 +68,7 @@ spec:
          subPath: 03_base.ini

      - name: hydra-sql-caddy
-        image: reg.cadoles.com/cadoles/hydra-sql-base:2024.11.6-develop.1113.075be9b
+        image: reg.cadoles.com/cadoles/hydra-sql-base:2025.3.7-develop.1415.7239d84
        imagePullPolicy: IfNotPresent
        args: ["/usr/sbin/caddy", "run", "--adapter", "caddyfile", "--config", "/etc/caddy/Caddyfile"]
        readinessProbe:
--- a/components/oidc-test/resources/deployment.yaml
+++ b/components/oidc-test/resources/deployment.yaml
@ -17,7 +17,7 @@ spec:
        app.kubernetes.io/name: oidc-test
    spec:
      containers:
-        - image: reg.cadoles.com/cadoles/oidc-test:2023.12.6-stable.1502.ebfd504
+        - image: reg.cadoles.com/cadoles/oidc-test:2025.3.11-stable.1428.6545cb3
          name: oidc-test
          ports:
            - containerPort: 8080
--- a/components/redis/configurations/redis-conf.yaml
+++ b/components/redis/configurations/redis-conf.yaml
@ -0,0 +1,5 @@
+nameReference:
+  - kind: ConfigMap
+    fieldSpecs:
+      - kind: Redis
+        path: spec/redisConfig/additionalRedisConfig
--- a/components/redis/files/redis-additional.conf
+++ b/components/redis/files/redis-additional.conf
@ -0,0 +1,3 @@
+maxmemory-policy allkeys-lru
+maxmemory 1536mb
+tcp-keepalive 90
--- a/components/redis/kustomization.yaml
+++ b/components/redis/kustomization.yaml
@ -1,9 +1,17 @@
 apiVersion: kustomize.config.k8s.io/v1alpha1
 kind: Component

+configurations:
+  - ./configurations/redis-conf.yaml
+
 resources:
  - ./resources/redis-sso.yaml

+configMapGenerator:
+- name: redis-sso-extra-conf
+  files:
+    - ./files/redis-additional.conf
+
 patches:
  - path: ./patches/hydra-apps.yaml
    target:
--- a/components/redis/resources/redis-sso.yaml
+++ b/components/redis/resources/redis-sso.yaml
@ -6,6 +6,15 @@ spec:
  kubernetesConfig:
    image: reg.cadoles.com/quay/opstree/redis:v7.0.15
    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1024Mi
+      limits:
+        cpu: 2000m
+        memory: 2048Mi
+  redisConfig:
+    additionalRedisConfig: redis-sso-extra-conf
  storage:
    volumeClaimTemplate:
      spec:
@ -16,4 +25,3 @@ spec:
            storage: 1Gi
  securityContext:
    runAsUser: 1000
-    fsGroup: 1000
--- a/examples/authenticated-app/files/hydra-dispatcher-apps.yaml
+++ b/examples/authenticated-app/files/hydra-dispatcher-apps.yaml
@ -25,17 +25,6 @@ hydra:
        email:
          rules:
            - "property_exists(consent.session.id_token, 'email') ? consent.session.id_token.email : null"
-  firewall:
-    additional_properties: true
-    rules:
-      siret:
-        required: false
-      email:
-        required: false
-      given_name:
-        required: false
-      family_name:
-        required: false
  webhook:
    enabled: false
  webhook_post_login:
--- a/examples/authenticated-app/kustomization.yaml
+++ b/examples/authenticated-app/kustomization.yaml
@ -14,6 +14,7 @@ components:
  - ../../components/hydra-ldap
  - ../../components/oidc-test
  - ../../components/redis
+  - ../../components/hydra-cleaner

 patchesJson6902:
  - target:
@ -51,6 +52,16 @@ patchesJson6902:
      kind: OAuth2Client
      name: oidc-test-oauth2-client
    path: patches/oidc-test-oauth2-client.yaml
+  - target:
+      version: v1
+      kind: ConfigMap
+      name: hydra-cleaner-env
+    path: patches/hydra-cleaner-env.yaml
+  - target:
+      version: v1
+      kind: CronJob
+      name: hydra-cleaner
+    path: patches/hydra-cleaner.yaml

 configMapGenerator:
  - name: hydra-dispatcher-apps
--- a/examples/authenticated-app/patches/hydra-cleaner-env.yaml
+++ b/examples/authenticated-app/patches/hydra-cleaner-env.yaml
@ -0,0 +1,9 @@
+- op: replace
+  path: "/data/RETENTION_HOURS"
+  value: "1" # 1 HOUR
+- op: replace
+  path: "/data/BATCH_SIZE"
+  value: "100"
+- op: replace
+  path: "/data/LIMIT"
+  value: "1000"
--- a/examples/authenticated-app/patches/hydra-cleaner.yaml
+++ b/examples/authenticated-app/patches/hydra-cleaner.yaml
@ -0,0 +1,3 @@
+- op: replace
+  path: "/spec/schedule"
+  value: "* * * * *"
--- a/resources/hydra-dispatcher/files/hydra/default.yaml
+++ b/resources/hydra-dispatcher/files/hydra/default.yaml
@ -12,8 +12,5 @@ hydra:
    api_url: "%env(string:HYDRA_DISPATCHER_WEBHOOK_API_URL)%"
    api_key: "%env(string:HYDRA_DISPATCHER_WEBHOOK_API_KEY)%"
    api_method: "%env(string:HYDRA_DISPATCHER_WEBHOOK_API_METHOD)%"
-  firewall:
-    additional_properties: "%env(bool:HYDRA_DISPATCHER_FIREWALL_ADDITIONAL_PROPERTIES)%"
-    rules: {}
  webhook_post_login:
    enabled: false
--- a/resources/hydra-dispatcher/resources/hydra-dispatcher-deployment.yaml
+++ b/resources/hydra-dispatcher/resources/hydra-dispatcher-deployment.yaml
@ -19,7 +19,7 @@ spec:
    spec:
      containers:
        - name: hydra-dispatcher-php-fpm
-          image: reg.cadoles.com/cadoles/hydra-dispatcher-base:2024.9.24-develop.1122.f88a5eb
+          image: reg.cadoles.com/cadoles/hydra-dispatcher-base:2025.3.18-develop.1401.4646fbb
          args: ["/usr/sbin/php-fpm81", "-F", "-e"]
          readinessProbe:
            exec:
@ -61,7 +61,7 @@ spec:
            runAsGroup: 1000
            runAsUser: 1000
        - name: hydra-dispatcher-caddy
-          image: reg.cadoles.com/cadoles/hydra-dispatcher-base:2024.9.24-develop.1122.f88a5eb
+          image: reg.cadoles.com/cadoles/hydra-dispatcher-base:2025.3.18-develop.1401.4646fbb
          imagePullPolicy: IfNotPresent
          args:
            [
--- a/resources/hydra/kustomization.yaml
+++ b/resources/hydra/kustomization.yaml
@ -31,6 +31,11 @@ configMapGenerator:
      - URLS_LOGOUT=http://hydra-logout-app/logout
      - HYDRA_SERVE_ALL_ARGS=--dev
      - HYDRA_DATABASE_MAX_CONN="10"
+      - HYDRA_DATABASE_MAX_IDLE_CONNS="5"
+      - HYDRA_DATABASE_MAX_CONN_LIFETIME="0"  # Unlimited. ms, s, m, h
+      - HYDRA_DATABASE_MAX_CONN_IDLE_TIME="0"  # Unlimited. ms, s, m, h
+      - HYDRA_DATABASE_CONNECT_TIMEOUT="0"  # Unlimited
+      - SERVE_ADMIN_REQUEST_LOG_DISABLE_FOR_HEALTH="true"
      - LOG_LEVEL=info

 replacements:
--- a/resources/hydra/resources/hydra-deployment.yaml
+++ b/resources/hydra/resources/hydra-deployment.yaml
@ -46,10 +46,31 @@ spec:
                - wget
                - --spider
                - -q
-                - http://127.0.0.1:4444/.well-known/openid-configuration
+                - http://127.0.0.1:4445/health/alive
            failureThreshold: 6
            periodSeconds: 10
            timeoutSeconds: 10
+          readinessProbe:
+            exec:
+              command:
+                - wget
+                - --spider
+                - -q
+                - http://127.0.0.1:4445/health/ready
+            failureThreshold: 6
+            periodSeconds: 10
+            timeoutSeconds: 10
+          startupProbe:
+            exec:
+              command:
+                - wget
+                - --spider
+                - -q
+                - http://127.0.0.1:4445/health/ready
+            failureThreshold: 60
+            successThreshold: 1
+            periodSeconds: 1
+            timeoutSeconds: 1
          ports:
            - containerPort: 4444
              name: hydra-public
Author	SHA1	Message	Date
Gauthier Dupont	c1d9ca62d4	Merge pull request 'feat(hydra-dispatcher): update image ref' (#70 ) from f/update_hydra_dispatcher into unstable Reviewed-on: #70	2025-03-18 16:13:32 +01:00
Laurent Gourvénec	09c91e7cae	feat(hydra-dispatcher): update image ref Breaking change: section "firewall" is not allowed anymore	2025-03-18 15:12:14 +01:00
wpetit	3db15dfc8a	Merge pull request 'feat(oidc-test): update image ref' (#68 ) from feat/update_oidc-test_20250311 into unstable Reviewed-on: #68	2025-03-11 15:46:07 +01:00
Laurent Gourvénec	77e167b17c	feat(oidc-test): update image ref	2025-03-11 15:42:32 +01:00
Laurent Gourvenec	d09b644b5f	Merge pull request 'feat(hydra-cnpg): configure DSN with more options' (#66 ) from f/hydra-cnpg_dsn_options into unstable Reviewed-on: #66 Reviewed-by: wpetit <wpetit@cadoles.com> Reviewed-by: pcaseiro <pcaseiro@cadoles.com>	2025-03-11 15:35:46 +01:00
Laurent Gourvénec	5e5670dcdf	feat(hydra-cnpg): configure DSN with more options	2025-03-07 15:31:59 +01:00
pcaseiro	172d9def39	Merge pull request 'hydra-sql: Update ref for lower mail fix' (#65 ) from fix-hydra-sql-lower into unstable Reviewed-on: #65 Reviewed-by: pcaseiro <pcaseiro@cadoles.com>	2025-03-07 14:48:31 +01:00
Gauthier DUPONT	e4b67e0812	fix(hydra-sql): Update ref for lower mail fix	2025-03-07 14:26:20 +01:00
pcaseiro	a26b8aafe1	Merge pull request 'fix(hydra): use same liveness URL as ory's helm' (#63 ) from fix/hydra_liveness_probe_url into unstable Reviewed-on: #63 Reviewed-by: wpetit <wpetit@cadoles.com> Reviewed-by: pcaseiro <pcaseiro@cadoles.com>	2025-03-07 13:52:29 +01:00
Laurent Gourvénec	06235bccad	feat(hydra): disable logs about health requests by default	2025-03-07 12:29:46 +01:00
Laurent Gourvénec	19039c5e1c	feat(hydra): adding readiness and startup probes	2025-03-07 11:50:26 +01:00
Laurent Gourvénec	9e02d7badb	fix(hydra): use same liveness URL as ory's helm	2025-03-07 11:15:43 +01:00
wpetit	87a056be2c	Merge pull request 'feat(hydra-cleaner): add component' (#61 ) from f/hydra_cleaner into unstable Reviewed-on: #61 Reviewed-by: wpetit <wpetit@cadoles.com>	2025-03-06 11:55:17 +01:00
William Petit	fedf44a062	feat(hydra-cleaner): configurable dsn and hydra port	2025-03-06 10:12:18 +01:00
William Petit	b0506995e5	feat: integrate hydra-cleaner in example app	2025-03-06 10:12:18 +01:00
Laurent Gourvénec	7a09045e82	feat(hydra-cleaner): add component	2025-02-27 16:20:34 +01:00
wpetit	f300b91316	Merge pull request 'feat(redis): manage limits' (#60 ) from feat/redis_limits into unstable Reviewed-on: #60	2025-02-21 14:43:22 +01:00
Laurent Gourvénec	30ba1f4d5a	feat(redis): manage limits	2025-02-21 14:34:06 +01:00
wpetit	d9bdbccfe4	Merge pull request 'fix(redis): fsGroup in security context is not applied' (#59 ) from fix/redis_fsgroup into unstable Reviewed-on: #59	2025-02-19 14:55:24 +01:00
Laurent Gourvénec	2d329501c0	fix(redis): fsGroup in security context is not applied The fsGroup is not applied to configuration of the pod and it's disturbing flux.	2025-02-19 14:43:25 +01:00