feat(init): adding init image for auto creation of connections

base/kustomization.yaml

@@ -36,27 +36,25 @@ helmCharts:
       value: "$(AIRFLOW_DATABASE_SERVICE_NAME)-rw"
     - name: "DB_SERVICE_PORT"
       value: "5432"
-
     webserver:
       defaultUser:
         username: admin
         password: NotSoSecret
+      extraInitContainers:
+      - name: airflow-create-connections
+        image: reg.cadoles.com/cadoles/airflow-init:latest
+        env:
+        - name: "DB_SERVICE_HOST"
+          value: "$(AIRFLOW_DATABASE_SERVICE_NAME)-rw"
+        - name: "DB_SERVICE_PORT"
+          value: "5432"
+        envFrom:
+        - configMapRef:
+            name: airflow-connections
+        - secretRef:
+            name: airflow-postgres-connection
     workers:
       replicas: 3
     dags:
       gitSync:
         enabled: false
-    extraEnvFrom: |
-      - configMapRef:
-          name: '{{ .Release.Name }}-connections'
-    scheduler:
-      extraInitContainers:
-      - name: airflow-create-connections
-        image: reg.cadoles.com/cadoles/airflow:latest
-        args:
-        - bash
-        - -c
-        - |-
-          exec \
-          ./scripts/create-connections.sh
-        - --

images/airflow-init/Dockerfile

@@ -0,0 +1,11 @@
+FROM reg.cadoles.com/proxy_cache/apache/airflow:2.5.3-python3.10
+
+USER root
+
+COPY --chown=airflow:root ./scripts/ ${AIRFLOW_HOME}/scripts/
+RUN chmod +x ./scripts/*
+
+USER airflow
+
+ENTRYPOINT [ "/usr/bin/dumb-init", "--" ]
+CMD [ "./scripts/create-connections.sh" ]

images/airflow-init/scripts/create-connections.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Simple script to provision AIRFLOW_CONNECTIONS !
+
+export SQLALCHEMY_SILENCE_UBER_WARNING=1
+#export AIRFLOW_CONN_AIRFLOW_DB="${connection}"
+export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${connection}"
+export AIRFLOW__CORE__SQL_ALCHEMY_CONN="${connection}"
+
+echo "====================================="
+conns=$(compgen -v  -X '!*AIRFLOW_CONN_*')
+
+if [ -z "${conns}" ];then
+   echo " Nothing to do"
+   echo "====================================="
+else
+   for conn in ${conns}
+   do
+      name="${conn#"AIRFLOW_CONN_"}"
+      value=$(eval "echo -e ${!conn}")
+      echo "Creating ${name}: ${value}"
+      ex=$(airflow connections add "${name}" --conn-uri ${value} 2>&1)
+      if [ "${?}" -ne 0 ]; then
+         echo "${conn}: Bad connection definition"
+         echo "= Error =========================="
+         echo "${ex}"
+         echo "= End error======================="
+      else
+         echo "= Ok ================================"
+      fi
+   done
+fi

skaffold.yaml

@@ -36,5 +36,10 @@ build:
     kaniko:
       dockerfile: Dockerfile
       cache: {}
+  - image: reg.cadoles.com/cadoles/airflow-init
+    context: images/airflow-init
+    kaniko:
+      dockerfile: Dockerfile
+      cache: {}
 deploy:
   statusCheckDeadlineSeconds: 600