feat(init): adding init image for auto creation of connections

This commit is contained in:
Philippe Caseiro 2023-05-10 16:20:58 +02:00
parent d84b111049
commit 47bc2207e6
4 changed files with 46 additions and 1 deletions

View File

@ -52,7 +52,7 @@ helmCharts:
scheduler: scheduler:
extraInitContainers: extraInitContainers:
- name: airflow-create-connections - name: airflow-create-connections
image: reg.cadoles.com/cadoles/airflow:latest image: reg.cadoles.com/cadoles/airflow-init:latest
args: args:
- bash - bash
- -c - -c

View File

@ -0,0 +1,11 @@
FROM reg.cadoles.com/proxy_cache/apache/airflow:slim-2.5.3-python3.10
USER root
COPY --chown=airflow:root ./scripts/ ${AIRFLOW_HOME}/scripts/
RUN chmod +x ./scripts/*
USER airflow
ENTRYPOINT [ "/usr/bin/dumb-init", "--" ]
CMD [ "./scripts/create-connections.sh" ]

View File

@ -0,0 +1,29 @@
#!/bin/bash
# Simple script to provision AIRFLOW_CONNECTIONS !
export SQLALCHEMY_SILENCE_UBER_WARNING=1
echo "====================================="
conns=$(compgen -v -X '!*AIRFLOW_CONN_*')
if [ -z "${conns}" ];then
echo " Nothing to do"
echo "====================================="
else
for conn in ${conns}
do
name="${conn#"AIRFLOW_CONN_"}"
value=$(eval "echo -e ${!conn}")
echo "Creating ${name}: ${value}"
ex=$(airflow connections add "${name}" --conn-uri ${value} 2>&1)
if [ "${?}" -ne 0 ]; then
echo "${conn}: Bad connection definition"
echo "= Error =========================="
echo "${ex}"
echo "= End error======================="
else
echo "= Ok ================================"
fi
done
fi

View File

@ -36,5 +36,10 @@ build:
kaniko: kaniko:
dockerfile: Dockerfile dockerfile: Dockerfile
cache: {} cache: {}
- image: reg.cadoles.com/cadoles/airflow-init
context: images/airflow-init
kaniko:
dockerfile: Dockerfile
cache: {}
deploy: deploy:
statusCheckDeadlineSeconds: 600 statusCheckDeadlineSeconds: 600