feat(init): adding init image for auto creation of connections

.gitignore

@@ -0,0 +1 @@
+charts/

README.md

@@ -3,3 +3,22 @@
 Airflow kustomization.
 
 For now this repo uses helm chart with kustomize
+
+## How to use
+
+### What do you need ?
+
+* kubernetes cluster (k3d, kind, the real thing)
+* kustomize command (on your machine)
+* kubectl command (on your machine)
+
+### Step by step
+
+* Clone this repository
+* run this command : 
+  - `kubectl kustomize --enable-helm ./airflow-kustom | kubectl apply -f -`
+* to revert run this command :
+  - `kubectl kustomize --enable-helm ./airflow-kustom | kubectl delete -f -`
+* Then you can "port-forward" any local port of your machine above 1024 to 8080 port of the service called airflow-webserver
+  - `kubectl port-forward svc/airflow-webserver 9090:8080`
+* You are looking for the default user and password ... read the code it's their

base/components/airflow-cnpg-database/configurations/cnpg-cluster.yaml

@@ -0,0 +1,8 @@
+---
+nameReference:
+- kind: Secret
+  fieldSpecs:
+  - path: spec/superuserSecret/name
+    kind: Cluster
+  - path: spec/bootstrap/initdb/secret/name
+    kind: Cluster

base/components/airflow-cnpg-database/kustomization.yaml

@@ -0,0 +1,36 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+configurations:
+- ./configurations/cnpg-cluster.yaml
+
+resources:
+- ./resources/airflow-cnpg-cluster.yaml
+
+secretgenerator:
+- name: airflow-postgres-admin
+  type: secret
+  literals:
+  - username=postgres
+  - password=notsosecret
+- name: airflow-postgres-user
+  type: Secret
+  literals:
+  - username=airflow
+  - password=NotSoSecret
+- name: airflow-postgres-connection
+  type: Secret
+  literals:
+  - connection=postgresql://airflow:NotSoSecret@${DB_SERVICE_HOST}:${DB_SERVICE_PORT}/airflow
+
+vars:
+- name: AIRFLOW_DATABASE_SERVICE_NAME
+  objref:
+    name: airflow-postgres
+    kind: Cluster
+    apiVersion: postgresql.cnpg.io/v1
+  fieldref:
+    fieldpath: metadata.name

base/components/airflow-cnpg-database/resources/airflow-cnpg-cluster.yaml

@@ -0,0 +1,17 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: airflow-postgres
+spec:
+  instances: 3
+  primaryUpdateStrategy: unsupervised
+  superuserSecret:
+    name: airflow-postgres-admin
+  bootstrap:
+    initdb:
+      database: airflow
+      owner: airflow
+      secret:
+        name: airflow-postgres-user
+  storage:
+    size: 2Gi

base/kustomization.yaml

@@ -0,0 +1,60 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+components:
+- ./components/airflow-cnpg-database
+
+secretGenerator:
+- name: airflow-webserver-secret-key
+  type: secret
+  literals:
+  - webserver-secret-key=c94b62cffbf4dd1c42747fc65007054432f10c185c5e6160
+
+configMapGenerator:
+- name: 'airflow-connections'
+  literals:
+  - AIRFLOW_CONN_TEST="test://test.do.not.use"
+
+helmCharts:
+- name: airflow
+  repo: https://airflow.apache.org
+  version: 1.9.0
+  releaseName: airflow
+  valuesInline:
+    postgresql:
+      enabled: false
+    pgbouncer:
+      enabled: true
+    data:
+      metadataSecretName: airflow-postgres-connection
+    webserverSecretKeySecretName: airflow-webserver-secret-key
+    env:
+    - name: "DB_SERVICE_HOST"
+      value: "$(AIRFLOW_DATABASE_SERVICE_NAME)-rw"
+    - name: "DB_SERVICE_PORT"
+      value: "5432"
+    webserver:
+      defaultUser:
+        username: admin
+        password: NotSoSecret
+      extraInitContainers:
+      - name: airflow-create-connections
+        image: reg.cadoles.com/cadoles/airflow-init:latest
+        env:
+        - name: "DB_SERVICE_HOST"
+          value: "$(AIRFLOW_DATABASE_SERVICE_NAME)-rw"
+        - name: "DB_SERVICE_PORT"
+          value: "5432"
+        envFrom:
+        - configMapRef:
+            name: airflow-connections
+        - secretRef:
+            name: airflow-postgres-connection
+    workers:
+      replicas: 3
+    dags:
+      gitSync:
+        enabled: false

base/resources/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+    name: airflow

base/secrets/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!.gitkeep

images/airflow-init/Dockerfile

@@ -0,0 +1,11 @@
+FROM reg.cadoles.com/proxy_cache/apache/airflow:2.5.3-python3.10
+
+USER root
+
+COPY --chown=airflow:root ./scripts/ ${AIRFLOW_HOME}/scripts/
+RUN chmod +x ./scripts/*
+
+USER airflow
+
+ENTRYPOINT [ "/usr/bin/dumb-init", "--" ]
+CMD [ "./scripts/create-connections.sh" ]

images/airflow-init/scripts/create-connections.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Simple script to provision AIRFLOW_CONNECTIONS !
+
+export SQLALCHEMY_SILENCE_UBER_WARNING=1
+#export AIRFLOW_CONN_AIRFLOW_DB="${connection}"
+export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${connection}"
+export AIRFLOW__CORE__SQL_ALCHEMY_CONN="${connection}"
+
+echo "====================================="
+conns=$(compgen -v  -X '!*AIRFLOW_CONN_*')
+
+if [ -z "${conns}" ];then
+   echo " Nothing to do"
+   echo "====================================="
+else
+   for conn in ${conns}
+   do
+      name="${conn#"AIRFLOW_CONN_"}"
+      value=$(eval "echo -e ${!conn}")
+      echo "Creating ${name}: ${value}"
+      ex=$(airflow connections add "${name}" --conn-uri ${value} 2>&1)
+      if [ "${?}" -ne 0 ]; then
+         echo "${conn}: Bad connection definition"
+         echo "= Error =========================="
+         echo "${ex}"
+         echo "= End error======================="
+      else
+         echo "= Ok ================================"
+      fi
+   done
+fi

images/airflow/Dockerfile

@@ -0,0 +1,9 @@
+FROM apache/airflow:2.5.3-python3.10
+
+USER root
+
+COPY --chown=airflow:root ./dags/ ${AIRFLOW_HOME}/dags/
+COPY --chown=airflow:root ./scripts/ ${AIRFLOW_HOME}/scripts/
+RUN chmod +x ./scripts/*
+
+USER airflow

images/airflow/dags/dag_http.py

@@ -0,0 +1,42 @@
+import json
+from airflow.utils.dates import days_ago
+from airflow import DAG
+from airflow.providers.http.operators.http import SimpleHttpOperator
+# Sensors
+from airflow.providers.http.sensors.http import HttpSensor
+from airflow.models.param import Param
+
+default_dag_args = {
+   'start_date': days_ago(2)
+}
+
+with DAG(
+    dag_id='mse_cmd_over_http',
+    default_args=default_dag_args,
+    schedule=None,
+    params={
+        "cmdName": Param("", type="string"),
+        "format": Param("", type="string"),
+        "env": Param("prod", type="string"),
+    }
+) as dag:
+    is_api_available = HttpSensor(
+        task_id='is_api_available',
+        http_conn_id='mse_api',
+        endpoint='/api/v1/cmds'
+    )
+    task = SimpleHttpOperator(
+        task_id='mse_cmd',
+        method="GET",
+        http_conn_id='mse_api',
+        endpoint='/api/v1/cmds',
+        data={
+            "cmdName": "{{ dag_run.conf.get('cmdName') }}",
+            "format": "{{ dag_run.conf.get('format') }}",
+            "env": "{{ dag_run.conf.get('env') }}"
+        },
+        headers={"Content-Type": "application/json"},
+        dag=dag
+    )
+
+is_api_available >> task

images/airflow/scripts/create-connections.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Simple script to provision AIRFLOW_CONNECTIONS !
+
+export SQLALCHEMY_SILENCE_UBER_WARNING=1
+
+conns=$(compgen -v  -X '!*AIRFLOW_CONN_*')
+
+for conn in ${conns}
+do
+   echo "====================================="
+   name="${conn#"AIRFLOW_CONN_"}"
+   value=$(eval "echo -e ${!conn}")
+   echo "Creating ${name}: ${value}"
+   ex=$(airflow connections add "${name}" --conn-uri ${value} 2>&1)
+   if [ "${?}" -ne 0 ]; then
+    echo "${conn}: Bad connection definition"
+    echo "= Error =========================="
+    echo "${ex}"
+    echo "= End error======================="
+   else
+    echo "= Ok ================================"
+   fi
+done

overlays/dev/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: airflow-dev
+
+resources:
+- ../../base
+- resources/namespace.yaml
+
+namePrefix: dev-

overlays/dev/resources/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: airflow-dev

skaffold.yaml

@@ -0,0 +1,45 @@
+apiVersion: skaffold/v3
+kind: Config
+
+metadata:
+  name: mse
+
+manifests:
+  kustomize:
+    paths:
+    - base
+
+profiles:
+- name: dev
+  manifests:
+    kustomize:
+      buildArgs:
+      - "--enable-helm"
+      paths:
+      - overlays/dev
+  activation:
+  - command: dev
+
+build:
+  cluster:
+    dockerConfig:
+      path: base/secrets/dockerconfig/.dockerconfigjson
+    randomDockerConfigSecret: true
+    randomPullSecret: true
+
+  tagPolicy:
+    sha256: {}
+
+  artifacts:
+  - image: reg.cadoles.com/cadoles/airflow
+    context: images/airflow
+    kaniko:
+      dockerfile: Dockerfile
+      cache: {}
+  - image: reg.cadoles.com/cadoles/airflow-init
+    context: images/airflow-init
+    kaniko:
+      dockerfile: Dockerfile
+      cache: {}
+deploy:
+  statusCheckDeadlineSeconds: 600