.gitignore

@@ -1 +0,0 @@
-charts/

README.md

@@ -1,24 +1,5 @@
 # airflow-kustom
 
-Airflow kustomization.
+Airflow kustomization.
-
+
-For now this repo uses helm chart with kustomize
+For now this repo uses helm chart with kustomize
-
-## How to use
-
-### What do you need ?
-
-* kubernetes cluster (k3d, kind, the real thing)
-* kustomize command (on your machine)
-* kubectl command (on your machine)
-
-### Step by step
-
-* Clone this repository
-* run this command : 
-  - `kubectl kustomize --enable-helm ./airflow-kustom | kubectl apply -f -`
-* to revert run this command :
-  - `kubectl kustomize --enable-helm ./airflow-kustom | kubectl delete -f -`
-* Then you can "port-forward" any local port of your machine above 1024 to 8080 port of the service called airflow-webserver
-  - `kubectl port-forward svc/airflow-webserver 9090:8080`
-* You are looking for the default user and password ... read the code it's their

base/components/airflow-cnpg-database/configurations/cnpg-cluster.yaml

@@ -1,8 +0,0 @@
----
-nameReference:
-- kind: Secret
-  fieldSpecs:
-  - path: spec/superuserSecret/name
-    kind: Cluster
-  - path: spec/bootstrap/initdb/secret/name
-    kind: Cluster

base/components/airflow-cnpg-database/kustomization.yaml

@@ -1,36 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1alpha1
-kind: Component
-
-generatorOptions:
-  disableNameSuffixHash: true
-
-configurations:
-- ./configurations/cnpg-cluster.yaml
-
-resources:
-- ./resources/airflow-cnpg-cluster.yaml
-
-secretgenerator:
-- name: airflow-postgres-admin
-  type: secret
-  literals:
-  - username=postgres
-  - password=notsosecret
-- name: airflow-postgres-user
-  type: Secret
-  literals:
-  - username=airflow
-  - password=NotSoSecret
-- name: airflow-postgres-connection
-  type: Secret
-  literals:
-  - connection=postgresql://airflow:NotSoSecret@${DB_SERVICE_HOST}:${DB_SERVICE_PORT}/airflow
-
-vars:
-- name: AIRFLOW_DATABASE_SERVICE_NAME
-  objref:
-    name: airflow-postgres
-    kind: Cluster
-    apiVersion: postgresql.cnpg.io/v1
-  fieldref:
-    fieldpath: metadata.name

base/components/airflow-cnpg-database/resources/airflow-cnpg-cluster.yaml

@@ -1,17 +0,0 @@
-apiVersion: postgresql.cnpg.io/v1
-kind: Cluster
-metadata:
-  name: airflow-postgres
-spec:
-  instances: 3
-  primaryUpdateStrategy: unsupervised
-  superuserSecret:
-    name: airflow-postgres-admin
-  bootstrap:
-    initdb:
-      database: airflow
-      owner: airflow
-      secret:
-        name: airflow-postgres-user
-  storage:
-    size: 2Gi

base/kustomization.yaml

@@ -1,60 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-generatorOptions:
-  disableNameSuffixHash: true
-
-components:
-- ./components/airflow-cnpg-database
-
-secretGenerator:
-- name: airflow-webserver-secret-key
-  type: secret
-  literals:
-  - webserver-secret-key=c94b62cffbf4dd1c42747fc65007054432f10c185c5e6160
-
-configMapGenerator:
-- name: 'airflow-connections'
-  literals:
-  - AIRFLOW_CONN_TEST="test://test.do.not.use"
-
-helmCharts:
-- name: airflow
-  repo: https://airflow.apache.org
-  version: 1.9.0
-  releaseName: airflow
-  valuesInline:
-    postgresql:
-      enabled: false
-    pgbouncer:
-      enabled: true
-    data:
-      metadataSecretName: airflow-postgres-connection
-    webserverSecretKeySecretName: airflow-webserver-secret-key
-    env:
-    - name: "DB_SERVICE_HOST"
-      value: "$(AIRFLOW_DATABASE_SERVICE_NAME)-rw"
-    - name: "DB_SERVICE_PORT"
-      value: "5432"
-    webserver:
-      defaultUser:
-        username: admin
-        password: NotSoSecret
-      extraInitContainers:
-      - name: airflow-create-connections
-        image: reg.cadoles.com/cadoles/airflow-init:latest
-        env:
-        - name: "DB_SERVICE_HOST"
-          value: "$(AIRFLOW_DATABASE_SERVICE_NAME)-rw"
-        - name: "DB_SERVICE_PORT"
-          value: "5432"
-        envFrom:
-        - configMapRef:
-            name: airflow-connections
-        - secretRef:
-            name: airflow-postgres-connection
-    workers:
-      replicas: 3
-    dags:
-      gitSync:
-        enabled: false

base/resources/namespace.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-    name: airflow

base/secrets/.gitignore

@@ -1,3 +0,0 @@
-*
-!.gitignore
-!.gitkeep

images/airflow-init/Dockerfile

@@ -1,11 +0,0 @@
-FROM reg.cadoles.com/proxy_cache/apache/airflow:2.5.3-python3.10
-
-USER root
-
-COPY --chown=airflow:root ./scripts/ ${AIRFLOW_HOME}/scripts/
-RUN chmod +x ./scripts/*
-
-USER airflow
-
-ENTRYPOINT [ "/usr/bin/dumb-init", "--" ]
-CMD [ "./scripts/create-connections.sh" ]

images/airflow-init/scripts/create-connections.sh

@@ -1,32 +0,0 @@
-#!/bin/bash
-
-# Simple script to provision AIRFLOW_CONNECTIONS !
-
-export SQLALCHEMY_SILENCE_UBER_WARNING=1
-#export AIRFLOW_CONN_AIRFLOW_DB="${connection}"
-export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${connection}"
-export AIRFLOW__CORE__SQL_ALCHEMY_CONN="${connection}"
-
-echo "====================================="
-conns=$(compgen -v  -X '!*AIRFLOW_CONN_*')
-
-if [ -z "${conns}" ];then
-   echo " Nothing to do"
-   echo "====================================="
-else
-   for conn in ${conns}
-   do
-      name="${conn#"AIRFLOW_CONN_"}"
-      value=$(eval "echo -e ${!conn}")
-      echo "Creating ${name}: ${value}"
-      ex=$(airflow connections add "${name}" --conn-uri ${value} 2>&1)
-      if [ "${?}" -ne 0 ]; then
-         echo "${conn}: Bad connection definition"
-         echo "= Error =========================="
-         echo "${ex}"
-         echo "= End error======================="
-      else
-         echo "= Ok ================================"
-      fi
-   done
-fi

images/airflow/Dockerfile

@@ -1,9 +0,0 @@
-FROM apache/airflow:2.5.3-python3.10
-
-USER root
-
-COPY --chown=airflow:root ./dags/ ${AIRFLOW_HOME}/dags/
-COPY --chown=airflow:root ./scripts/ ${AIRFLOW_HOME}/scripts/
-RUN chmod +x ./scripts/*
-
-USER airflow

images/airflow/dags/dag_http.py

@@ -1,42 +0,0 @@
-import json
-from airflow.utils.dates import days_ago
-from airflow import DAG
-from airflow.providers.http.operators.http import SimpleHttpOperator
-# Sensors
-from airflow.providers.http.sensors.http import HttpSensor
-from airflow.models.param import Param
-
-default_dag_args = {
-   'start_date': days_ago(2)
-}
-
-with DAG(
-    dag_id='mse_cmd_over_http',
-    default_args=default_dag_args,
-    schedule=None,
-    params={
-        "cmdName": Param("", type="string"),
-        "format": Param("", type="string"),
-        "env": Param("prod", type="string"),
-    }
-) as dag:
-    is_api_available = HttpSensor(
-        task_id='is_api_available',
-        http_conn_id='mse_api',
-        endpoint='/api/v1/cmds'
-    )
-    task = SimpleHttpOperator(
-        task_id='mse_cmd',
-        method="GET",
-        http_conn_id='mse_api',
-        endpoint='/api/v1/cmds',
-        data={
-            "cmdName": "{{ dag_run.conf.get('cmdName') }}",
-            "format": "{{ dag_run.conf.get('format') }}",
-            "env": "{{ dag_run.conf.get('env') }}"
-        },
-        headers={"Content-Type": "application/json"},
-        dag=dag
-    )
-
-is_api_available >> task

images/airflow/scripts/create-connections.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-# Simple script to provision AIRFLOW_CONNECTIONS !
-
-export SQLALCHEMY_SILENCE_UBER_WARNING=1
-
-conns=$(compgen -v  -X '!*AIRFLOW_CONN_*')
-
-for conn in ${conns}
-do
-   echo "====================================="
-   name="${conn#"AIRFLOW_CONN_"}"
-   value=$(eval "echo -e ${!conn}")
-   echo "Creating ${name}: ${value}"
-   ex=$(airflow connections add "${name}" --conn-uri ${value} 2>&1)
-   if [ "${?}" -ne 0 ]; then
-    echo "${conn}: Bad connection definition"
-    echo "= Error =========================="
-    echo "${ex}"
-    echo "= End error======================="
-   else
-    echo "= Ok ================================"
-   fi
-done

overlays/dev/kustomization.yaml

@@ -1,9 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: airflow-dev
-
-resources:
-- ../../base
-- resources/namespace.yaml
-
-namePrefix: dev-

overlays/dev/resources/namespace.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: airflow-dev

skaffold.yaml

@@ -1,45 +0,0 @@
-apiVersion: skaffold/v3
-kind: Config
-
-metadata:
-  name: mse
-
-manifests:
-  kustomize:
-    paths:
-    - base
-
-profiles:
-- name: dev
-  manifests:
-    kustomize:
-      buildArgs:
-      - "--enable-helm"
-      paths:
-      - overlays/dev
-  activation:
-  - command: dev
-
-build:
-  cluster:
-    dockerConfig:
-      path: base/secrets/dockerconfig/.dockerconfigjson
-    randomDockerConfigSecret: true
-    randomPullSecret: true
-
-  tagPolicy:
-    sha256: {}
-
-  artifacts:
-  - image: reg.cadoles.com/cadoles/airflow
-    context: images/airflow
-    kaniko:
-      dockerfile: Dockerfile
-      cache: {}
-  - image: reg.cadoles.com/cadoles/airflow-init
-    context: images/airflow-init
-    kaniko:
-      dockerfile: Dockerfile
-      cache: {}
-deploy:
-  statusCheckDeadlineSeconds: 600