diff --git a/components/airflow-cnpg-database/configurations/cnpg-cluster.yaml b/base/components/airflow-cnpg-database/configurations/cnpg-cluster.yaml similarity index 100% rename from components/airflow-cnpg-database/configurations/cnpg-cluster.yaml rename to base/components/airflow-cnpg-database/configurations/cnpg-cluster.yaml diff --git a/components/airflow-cnpg-database/kustomization.yaml b/base/components/airflow-cnpg-database/kustomization.yaml similarity index 100% rename from components/airflow-cnpg-database/kustomization.yaml rename to base/components/airflow-cnpg-database/kustomization.yaml diff --git a/components/airflow-cnpg-database/resources/airflow-cnpg-cluster.yaml b/base/components/airflow-cnpg-database/resources/airflow-cnpg-cluster.yaml similarity index 100% rename from components/airflow-cnpg-database/resources/airflow-cnpg-cluster.yaml rename to base/components/airflow-cnpg-database/resources/airflow-cnpg-cluster.yaml diff --git a/kustomization.yaml b/base/kustomization.yaml similarity index 66% rename from kustomization.yaml rename to base/kustomization.yaml index 1aacdee..182fbca 100644 --- a/kustomization.yaml +++ b/base/kustomization.yaml @@ -13,6 +13,11 @@ secretGenerator: literals: - webserver-secret-key=c94b62cffbf4dd1c42747fc65007054432f10c185c5e6160 +configMapGenerator: +- name: 'airflow-connections' + literals: + - AIRFLOW_CONN_TEST="test://test.do.not.use" + helmCharts: - name: airflow repo: https://airflow.apache.org @@ -31,6 +36,7 @@ helmCharts: value: "$(AIRFLOW_DATABASE_SERVICE_NAME)-rw" - name: "DB_SERVICE_PORT" value: "5432" + webserver: defaultUser: username: admin @@ -40,3 +46,17 @@ helmCharts: dags: gitSync: enabled: false + extraEnvFrom: | + - configMapRef: + name: '{{ .Release.Name }}-connections' + scheduler: + extraInitContainers: + - name: airflow-create-connections + image: reg.cadoles.com/cadoles/airflow:latest + args: + - bash + - -c + - |- + exec \ + ./scripts/create-connections.sh + - -- diff --git a/resources/namespace.yaml b/base/resources/namespace.yaml similarity index 100% rename from resources/namespace.yaml rename to base/resources/namespace.yaml diff --git a/base/secrets/.gitignore b/base/secrets/.gitignore new file mode 100644 index 0000000..0318876 --- /dev/null +++ b/base/secrets/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!.gitkeep \ No newline at end of file diff --git a/base/secrets/.gitkeep b/base/secrets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/images/airflow/Dockerfile b/images/airflow/Dockerfile new file mode 100644 index 0000000..38ccf7e --- /dev/null +++ b/images/airflow/Dockerfile @@ -0,0 +1,9 @@ +FROM apache/airflow:2.5.3-python3.10 + +USER root + +COPY --chown=airflow:root ./dags/ ${AIRFLOW_HOME}/dags/ +COPY --chown=airflow:root ./scripts/ ${AIRFLOW_HOME}/scripts/ +RUN chmod +x ./scripts/* + +USER airflow diff --git a/images/airflow/dags/dag_http.py b/images/airflow/dags/dag_http.py new file mode 100644 index 0000000..a1986a5 --- /dev/null +++ b/images/airflow/dags/dag_http.py @@ -0,0 +1,42 @@ +import json +from airflow.utils.dates import days_ago +from airflow import DAG +from airflow.providers.http.operators.http import SimpleHttpOperator +# Sensors +from airflow.providers.http.sensors.http import HttpSensor +from airflow.models.param import Param + +default_dag_args = { + 'start_date': days_ago(2) +} + +with DAG( + dag_id='mse_cmd_over_http', + default_args=default_dag_args, + schedule=None, + params={ + "cmdName": Param("", type="string"), + "format": Param("", type="string"), + "env": Param("prod", type="string"), + } +) as dag: + is_api_available = HttpSensor( + task_id='is_api_available', + http_conn_id='mse_api', + endpoint='/api/v1/cmds' + ) + task = SimpleHttpOperator( + task_id='mse_cmd', + method="GET", + http_conn_id='mse_api', + endpoint='/api/v1/cmds', + data={ + "cmdName": "{{ dag_run.conf.get('cmdName') }}", + "format": "{{ dag_run.conf.get('format') }}", + "env": "{{ dag_run.conf.get('env') }}" + }, + headers={"Content-Type": "application/json"}, + dag=dag + ) + +is_api_available >> task \ No newline at end of file diff --git a/images/airflow/scripts/create-connections.sh b/images/airflow/scripts/create-connections.sh new file mode 100644 index 0000000..e55b083 --- /dev/null +++ b/images/airflow/scripts/create-connections.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Simple script to provision AIRFLOW_CONNECTIONS ! + +export SQLALCHEMY_SILENCE_UBER_WARNING=1 + +conns=$(compgen -v -X '!*AIRFLOW_CONN_*') + +for conn in ${conns} +do + echo "=====================================" + name="${conn#"AIRFLOW_CONN_"}" + value=$(eval "echo -e ${!conn}") + echo "Creating ${name}: ${value}" + ex=$(airflow connections add "${name}" --conn-uri ${value} 2>&1) + if [ "${?}" -ne 0 ]; then + echo "${conn}: Bad connection definition" + echo "= Error ==========================" + echo "${ex}" + echo "= End error=======================" + else + echo "= Ok ================================" + fi +done \ No newline at end of file diff --git a/overlays/dev/kustomization.yaml b/overlays/dev/kustomization.yaml new file mode 100644 index 0000000..8319a02 --- /dev/null +++ b/overlays/dev/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: airflow-dev + +resources: +- ../../base +- resources/namespace.yaml + +namePrefix: dev- diff --git a/overlays/dev/resources/namespace.yaml b/overlays/dev/resources/namespace.yaml new file mode 100644 index 0000000..3fc0d92 --- /dev/null +++ b/overlays/dev/resources/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: airflow-dev diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 0000000..b846c63 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,40 @@ +apiVersion: skaffold/v3 +kind: Config + +metadata: + name: mse + +manifests: + kustomize: + paths: + - base + +profiles: +- name: dev + manifests: + kustomize: + buildArgs: + - "--enable-helm" + paths: + - overlays/dev + activation: + - command: dev + +build: + cluster: + dockerConfig: + path: base/secrets/dockerconfig/.dockerconfigjson + randomDockerConfigSecret: true + randomPullSecret: true + + tagPolicy: + sha256: {} + + artifacts: + - image: reg.cadoles.com/cadoles/airflow + context: images/airflow + kaniko: + dockerfile: Dockerfile + cache: {} +deploy: + statusCheckDeadlineSeconds: 600