feat(init): adding init image for auto creation of connections

This commit is contained in:
Philippe Caseiro 2023-05-10 16:20:58 +02:00
parent d84b111049
commit e21444bf46
4 changed files with 61 additions and 15 deletions

View File

@ -36,27 +36,25 @@ helmCharts:
value: "$(AIRFLOW_DATABASE_SERVICE_NAME)-rw" value: "$(AIRFLOW_DATABASE_SERVICE_NAME)-rw"
- name: "DB_SERVICE_PORT" - name: "DB_SERVICE_PORT"
value: "5432" value: "5432"
webserver: webserver:
defaultUser: defaultUser:
username: admin username: admin
password: NotSoSecret password: NotSoSecret
extraInitContainers:
- name: airflow-create-connections
image: reg.cadoles.com/cadoles/airflow-init:latest
env:
- name: "DB_SERVICE_HOST"
value: "$(AIRFLOW_DATABASE_SERVICE_NAME)-rw"
- name: "DB_SERVICE_PORT"
value: "5432"
envFrom:
- configMapRef:
name: airflow-connections
- secretRef:
name: airflow-postgres-connection
workers: workers:
replicas: 3 replicas: 3
dags: dags:
gitSync: gitSync:
enabled: false enabled: false
extraEnvFrom: |
- configMapRef:
name: '{{ .Release.Name }}-connections'
scheduler:
extraInitContainers:
- name: airflow-create-connections
image: reg.cadoles.com/cadoles/airflow:latest
args:
- bash
- -c
- |-
exec \
./scripts/create-connections.sh
- --

View File

@ -0,0 +1,11 @@
FROM reg.cadoles.com/proxy_cache/apache/airflow:2.5.3-python3.10
USER root
COPY --chown=airflow:root ./scripts/ ${AIRFLOW_HOME}/scripts/
RUN chmod +x ./scripts/*
USER airflow
ENTRYPOINT [ "/usr/bin/dumb-init", "--" ]
CMD [ "./scripts/create-connections.sh" ]

View File

@ -0,0 +1,32 @@
#!/bin/bash
# Simple script to provision AIRFLOW_CONNECTIONS !
export SQLALCHEMY_SILENCE_UBER_WARNING=1
#export AIRFLOW_CONN_AIRFLOW_DB="${connection}"
export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${connection}"
export AIRFLOW__CORE__SQL_ALCHEMY_CONN="${connection}"
echo "====================================="
conns=$(compgen -v -X '!*AIRFLOW_CONN_*')
if [ -z "${conns}" ];then
echo " Nothing to do"
echo "====================================="
else
for conn in ${conns}
do
name="${conn#"AIRFLOW_CONN_"}"
value=$(eval "echo -e ${!conn}")
echo "Creating ${name}: ${value}"
ex=$(airflow connections add "${name}" --conn-uri ${value} 2>&1)
if [ "${?}" -ne 0 ]; then
echo "${conn}: Bad connection definition"
echo "= Error =========================="
echo "${ex}"
echo "= End error======================="
else
echo "= Ok ================================"
fi
done
fi

View File

@ -36,5 +36,10 @@ build:
kaniko: kaniko:
dockerfile: Dockerfile dockerfile: Dockerfile
cache: {} cache: {}
- image: reg.cadoles.com/cadoles/airflow-init
context: images/airflow-init
kaniko:
dockerfile: Dockerfile
cache: {}
deploy: deploy:
statusCheckDeadlineSeconds: 600 statusCheckDeadlineSeconds: 600