From b634d58f02251a00c8e5b8fde88912dfde5f4126 Mon Sep 17 00:00:00 2001 From: Gaetan Semet Date: Tue, 24 Oct 2017 13:30:08 +0200 Subject: [PATCH] Add Helm chart for Airflow - add EditorConfig files - airflow.all.yaml is splited in different chart templates - Propagate server config from env variable if defined - statup script can receive some config by ENV variable (name of rabbitmq/postgres server), at least everything derives from the same source - username and password can inherit from env - fix probable buggy fernet key definition, missing enum34 with python 2 installation - change format of template in airflow.cfg (airflow.cfg.in) - use configmap to set default environment variables - parameterize number of workers - support url prefix for flower and web ui allow ingress in endpoint such as /airflow and /flower - use python 3 in docker - optional git sync to mirror a git repo for DAGs --- .editorconfig | 19 +++ .gitignore | 1 + Dockerfile.template | 32 ++--- Makefile | 28 ++++- README.md | 35 +++++- airflow/Chart.yaml | 4 + airflow/templates/configmaps.yaml | 28 +++++ airflow/templates/deployments.yaml | 155 +++++++++++++++++++++++++ airflow/templates/ingresses.yaml | 35 ++++++ airflow/templates/services.yaml | 67 +++++++++++ airflow/values.yaml | 49 ++++++++ config.yaml | 8 ++ config/{airflow.cfg => airflow.cfg.in} | 8 +- reset-af.sh | 3 + script/entrypoint.sh | 38 ++++-- 15 files changed, 476 insertions(+), 34 deletions(-) create mode 100644 .editorconfig create mode 100644 airflow/Chart.yaml create mode 100644 airflow/templates/configmaps.yaml create mode 100644 airflow/templates/deployments.yaml create mode 100644 airflow/templates/ingresses.yaml create mode 100644 airflow/templates/services.yaml create mode 100644 airflow/values.yaml create mode 100644 config.yaml rename config/{airflow.cfg => airflow.cfg.in} (96%) create mode 100755 reset-af.sh diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..617c1c1 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,19 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.md] +trim_trailing_whitespace = false + +[*.yaml] +indent_size = 2 + +[Makefile] +indent_style = tab +indent_size = 4 diff --git a/.gitignore b/.gitignore index bdc5af0..70db2cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *~ build +rootfs diff --git a/Dockerfile.template b/Dockerfile.template index f4da7d9..8b29a5d 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -21,12 +21,12 @@ ENV LANG en_US.UTF-8 ENV LC_ALL en_US.UTF-8 ENV LC_CTYPE en_US.UTF-8 ENV LC_MESSAGES en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 RUN set -ex \ && buildDeps=' \ - python-pip \ - python-dev \ + python3-pip \ + python3-dev \ libkrb5-dev \ libsasl2-dev \ libxml2-dev \ @@ -46,20 +46,24 @@ RUN set -ex \ curl \ netcat \ locales \ - && apt-get install -yqq -t jessie-backports python-requests libpq-dev \ + && apt-get install -yqq -t jessie-backports libpq-dev git \ && sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \ && locale-gen \ && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \ && useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \ - && pip uninstall setuptools \ - && pip install setuptools==33.1.1 \ - && pip install pytz==2015.7 \ - && pip install cryptography \ - && pip install pyOpenSSL \ - && pip install ndg-httpsclient \ - && pip install pyasn1 \ - && pip install psycopg2 \ - && pip install airflow[celery,postgresql,hive]==$AIRFLOW_VERSION \ + && pip3 install --upgrade pip enum34 'setuptools!=36.0.0' \ + && pip3 install pytz==2015.7 \ + && pip3 install cryptography \ + && pip3 install requests \ + && pip3 install pyOpenSSL \ + && pip3 install ndg-httpsclient \ + && pip3 install pyasn1 \ + && pip3 install psycopg2 \ + && pip3 install airflow[celery,postgresql,hive] \ + && pip3 install click \ + && git clone https://github.com/jlowin/git-sync.git \ + && chmod +x git-sync/git-sync.py \ + && cp git-sync/git-sync.py /usr/bin \ && apt-get remove --purge -yqq $buildDeps libpq-dev \ && apt-get clean \ && rm -rf \ @@ -75,7 +79,7 @@ ENV KUBECTL_VERSION %%KUBECTL_VERSION%% RUN curl -L -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && chmod +x /usr/local/bin/kubectl COPY script/entrypoint.sh ${AIRFLOW_HOME}/entrypoint.sh -COPY config/airflow.cfg ${AIRFLOW_HOME}/airflow.cfg +COPY config/airflow.cfg.in ${AIRFLOW_HOME}/airflow.cfg.in RUN chown -R airflow: ${AIRFLOW_HOME} \ && chmod +x ${AIRFLOW_HOME}/entrypoint.sh diff --git a/Makefile b/Makefile index 58465fc..5f19f2b 100644 --- a/Makefile +++ b/Makefile @@ -10,18 +10,42 @@ ALIAS ?= $(REPOSITORY):$(AIRFLOW_VERSION)-$(KUBECTL_VERSION) BUILD_ROOT ?= build/$(TAG) DOCKERFILE ?= $(BUILD_ROOT)/Dockerfile ROOTFS ?= $(BUILD_ROOT)/rootfs -AIRFLOW_CONF ?= $(BUILD_ROOT)/config/airflow.cfg +AIRFLOW_CONF ?= $(BUILD_ROOT)/config/airflow.cfg.in ENTRYPOINT_SH ?= $(BUILD_ROOT)/script/entrypoint.sh DOCKER_CACHE ?= docker-cache SAVED_IMAGE ?= $(DOCKER_CACHE)/image-$(AIRFLOW_VERSION)-$(KUBECTL_VERSION).tar NAMESPACE ?= airflow-dev +HELM_APPLICATION_NAME ?= airflow +HELM_CONFIG ?= config.yaml +CHART_LOCATION ?= ./airflow .PHONY: build clean clean: rm -Rf build +helm-install: + helm repo update + helm install $(CHART_LOCATION) \ + --version=v0.1.0 \ + --name=$(HELM_APPLICATION_NAME) \ + --namespace=$(NAMESPACE) \ + --debug \ + -f $(HELM_CONFIG) + +helm-upgrade: + helm upgrade -f $(HELM_CONFIG) \ + --debug \ + $(HELM_APPLICATION_NAME) \ + $(CHART_LOCATION) + +helm-ls: + helm ls --all $(HELM_APPLICATION_NAME) + +helm-uninstall: + helm del --purge $(HELM_APPLICATION_NAME) + build: $(DOCKERFILE) $(ROOTFS) $(AIRFLOW_CONF) $(ENTRYPOINT_SH) cd $(BUILD_ROOT) && docker build -t $(IMAGE) . && docker tag $(IMAGE) $(ALIAS) @@ -37,7 +61,7 @@ $(ROOTFS): $(BUILD_ROOT) $(AIRFLOW_CONF): $(BUILD_ROOT) mkdir -p $(shell dirname $(AIRFLOW_CONF)) - cp config/airflow.cfg $(AIRFLOW_CONF) + cp config/airflow.cfg.in $(AIRFLOW_CONF) $(ENTRYPOINT_SH): $(BUILD_ROOT) mkdir -p $(shell dirname $(ENTRYPOINT_SH)) diff --git a/README.md b/README.md index f7835a7..eae4f38 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,8 @@ This is useful when you'd want: This repository contains: * **Dockerfile(.template)** of [airflow](https://github.com/apache/incubator-airflow) for [Docker](https://www.docker.com/) images published to the public [Docker Hub Registry](https://registry.hub.docker.com/). -* **airflow.all.yaml** for creating Kubernetes services and deployments to run Airflow on Kubernetes +* **airflow.all.yaml** for manual creating Kubernetes services and deployments to run Airflow on Kubernetes +* **Helm Chart** for deployments using Helm ## Informations @@ -23,17 +24,41 @@ This repository contains: * Based on Debian Jessie official Image [debian:jessie](https://registry.hub.docker.com/_/debian/) and uses the official [Postgres](https://hub.docker.com/_/postgres/) as backend and [RabbitMQ](https://hub.docker.com/_/rabbitmq/) as queue * Following the Airflow release from [Python Package Index](https://pypi.python.org/pypi/airflow) -## Installation +## Manual Installation Create all the deployments and services for Airflow: - kubectl create -f airflow.all.yaml + kubectl create -f airflow.all.yaml -## Build +## Helm Deployment + +Ensure your helm installation is done, you may need to have `TILLER_NAMESPACE` set a an environment +variable. + +Deploy to Kubernetes using: + + make helm-install NAMESPACE=airflow + +Remove from the clusting using: + + make helm-uninstall + +### Helm configuration customization + +Helm allow to overload the configuration to adapt to your environment. You probably want to specify +your own ingress configuration for instance. + +Note: + + Do NOT use characters such as " (double quote), ' (simple quote), / (slash) or \ (backslash) + in your passwords + + +## Build Docker image `git clone` this repository and then just run: - make build + make build ## Usage diff --git a/airflow/Chart.yaml b/airflow/Chart.yaml new file mode 100644 index 0000000..78d5951 --- /dev/null +++ b/airflow/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: Airflow installation +name: airflow +version: v0.1.0 diff --git a/airflow/templates/configmaps.yaml b/airflow/templates/configmaps.yaml new file mode 100644 index 0000000..ec4db7a --- /dev/null +++ b/airflow/templates/configmaps.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: "{{ .Values.airflow.prefix -}}airflow-cfg" +data: + AIRFLOW_HOME: "/usr/local/airflow" + {{- if .Values.airflow.fernet_key }} + FERNET_KEY: "{{- .Values.airflow.fernet_key -}}" + {{- end }} + RABBITMQ_HOST: "{{- .Values.airflow.prefix -}}{{- .Values.db.rabbitmq.basename -}}" + {{- if .Values.db.rabbitmq.user }} + RABBITMQ_CREDS: "{{- .Values.db.rabbitmq.user -}}:{{- .Values.db.rabbitmq.password -}}" + {{- end }} + POSTGRES_HOST: "{{- .Values.airflow.prefix -}}{{- .Values.db.postgres.basename -}}" + {{- if .Values.db.postgres.user }} + POSTGRES_CREDS: "{{- .Values.db.postgres.user -}}:{{- .Values.db.postgres.password -}}" + {{- end }} + {{- if .Values.airflow.init_retry_loop }} + TRY_LOOP: "{{ .Values.airflow.init_retry_loop -}}" + {{- end }} + FLOWER_URL_PREFIX: "{{ .Values.flower.url_prefix -}}" + AIRFLOW_URL_PREFIX: "{{ .Values.airflow.url_prefix -}}" + {{ if .Values.dags.git_sync_enabled }} + GIT_SYNC_REPO: "{{ .Values.dags.git_repo }}" + GIT_SYNC_BRANCH: "{{ .Values.dags.git_branch }}" + GIT_SYNC_WAIT: "{{ .Values.dags.poll_interval_sec }}" + GIT_SYNC_DEBUG: "{{ .Values.dags.git_sync_debug }}" + {{ end }} diff --git a/airflow/templates/deployments.yaml b/airflow/templates/deployments.yaml new file mode 100644 index 0000000..c04ee8e --- /dev/null +++ b/airflow/templates/deployments.yaml @@ -0,0 +1,155 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ .Values.airflow.prefix -}}postgres +spec: + replicas: 1 + template: + metadata: + labels: + app: airflow + tier: db + spec: + containers: + - name: {{ .Values.airflow.prefix -}}postgres + image: postgres + ports: + - name: {{ .Values.airflow.prefix -}}postgres + containerPort: 5432 + env: + - name: POSTGRES_USER + value: "{{ .Values.db.postgres.user -}}" + - name: POSTGRES_PASSWORD + value: "{{ .Values.db.postgres.password -}}" + - name: POSTGRES_DB + value: "{{ .Values.db.postgres.database -}}" +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ .Values.airflow.prefix -}}rabbitmq +spec: + replicas: 1 + template: + metadata: + labels: + app: airflow + tier: rabbitmq + spec: + restartPolicy: Always + containers: + - name: {{ .Values.airflow.prefix -}}rabbitmq + image: rabbitmq:3-management + ports: + - name: {{ .Values.airflow.prefix -}}management + containerPort: 15672 + - name: {{ .Values.airflow.prefix -}}node + containerPort: 5672 + env: + - name: RABBITMQ_DEFAULT_USER + value: "{{ .Values.db.rabbitmq.user -}}" + - name: RABBITMQ_DEFAULT_PASS + value: "{{ .Values.db.rabbitmq.password -}}" + - name: RABBITMQ_DEFAULT_VHOST + value: "airflow" +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ .Values.airflow.prefix -}}web +spec: + replicas: 1 + template: + metadata: + labels: + app: airflow + tier: web + spec: + restartPolicy: Always + containers: + - name: {{ .Values.airflow.prefix -}}web + image: {{ .Values.airflow.image }} + imagePullPolicy: {{ .Values.airflow.image_pull_policy}} + envFrom: + - configMapRef: + name: {{ .Values.airflow.prefix }}airflow-cfg + ports: + - name: {{ .Values.airflow.prefix -}}web + containerPort: 8080 + args: ["webserver"] +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ .Values.airflow.prefix -}}flower +spec: + replicas: 1 + template: + metadata: + labels: + app: airflow + tier: flower + spec: + restartPolicy: Always + containers: + - name: {{ .Values.airflow.prefix -}}flower + image: {{ .Values.airflow.image }} + imagePullPolicy: {{ .Values.airflow.image_pull_policy}} + env: + - name: FLOWER_PORT + value: "5555" + envFrom: + - configMapRef: + name: {{ .Values.airflow.prefix }}airflow-cfg + ports: + - name: {{ .Values.airflow.prefix -}}flower + containerPort: 5555 + args: ["flower"] +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ .Values.airflow.prefix -}}scheduler +spec: + replicas: 1 + template: + metadata: + labels: + app: airflow + tier: scheduler + spec: + restartPolicy: Always + containers: + - name: {{ .Values.airflow.prefix -}}scheduler + image: {{ .Values.airflow.image }} + imagePullPolicy: {{ .Values.airflow.image_pull_policy}} + # volumes: + # - /localpath/to/dags:/usr/local/airflow/dags + envFrom: + - configMapRef: + name: {{ .Values.airflow.prefix }}airflow-cfg + args: ["scheduler", "-n", "5"] +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ .Values.airflow.prefix -}}worker +spec: + replicas: {{ .Values.celery.num_workers }} + template: + metadata: + labels: + app: airflow + tier: worker + spec: + restartPolicy: Always + containers: + - name: {{ .Values.airflow.prefix -}}worker + image: {{ .Values.airflow.image }} + imagePullPolicy: {{ .Values.airflow.image_pull_policy}} + # volumes: + # - /localpath/to/dags:/usr/local/airflow/dags + envFrom: + - configMapRef: + name: {{ .Values.airflow.prefix }}airflow-cfg + args: ["worker"] diff --git a/airflow/templates/ingresses.yaml b/airflow/templates/ingresses.yaml new file mode 100644 index 0000000..cb50c02 --- /dev/null +++ b/airflow/templates/ingresses.yaml @@ -0,0 +1,35 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ .Values.airflow.prefix -}}airflow + annotations: + {{ range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + rules: + - http: + paths: + - path: {{ .Values.ingress.path.web }} + backend: + serviceName: {{ .Values.airflow.prefix -}}web + servicePort: {{ .Values.airflow.prefix -}}pweb + host: {{ .Values.ingress.host }} +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ .Values.airflow.prefix -}}airflow-flower + annotations: + {{ range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + rules: + - http: + paths: + - path: {{ .Values.ingress.path.flower }} + backend: + serviceName: {{ .Values.airflow.prefix -}}flower + servicePort: {{ .Values.airflow.prefix -}}pflower + host: {{ .Values.ingress.host }} diff --git a/airflow/templates/services.yaml b/airflow/templates/services.yaml new file mode 100644 index 0000000..a5514f2 --- /dev/null +++ b/airflow/templates/services.yaml @@ -0,0 +1,67 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.airflow.prefix -}}postgres +spec: + type: ClusterIP + selector: + app: airflow + tier: db + ports: + - name: {{ .Values.airflow.prefix -}}postgres + protocol: TCP + port: 5432 + targetPort: {{ .Values.airflow.prefix -}}postgres +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.airflow.prefix -}}rabbitmq +spec: + type: ClusterIP + selector: + app: airflow + tier: rabbitmq + ports: + - name: {{ .Values.airflow.prefix -}}node + protocol: TCP + port: 5672 + targetPort: {{ .Values.airflow.prefix -}}node + - name: {{ .Values.airflow.prefix -}}management + protocol: TCP + port: 15672 + targetPort: {{ .Values.airflow.prefix -}}management +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.airflow.prefix -}}web +spec: + type: NodePort + selector: + app: airflow + tier: web + ports: + - name: {{ .Values.airflow.prefix -}}web + protocol: TCP + port: 8080 + name: {{ .Values.airflow.prefix -}}pweb + targetPort: {{ .Values.airflow.prefix -}}web + nodePort: 32080 +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.airflow.prefix -}}flower +spec: + type: NodePort + selector: + app: airflow + tier: flower + ports: + - name: {{ .Values.airflow.prefix -}}flower + protocol: TCP + port: 5555 + name: {{ .Values.airflow.prefix -}}pflower + targetPort: {{ .Values.airflow.prefix -}}flower + nodePort: 32081 diff --git a/airflow/values.yaml b/airflow/values.yaml new file mode 100644 index 0000000..63a5996 --- /dev/null +++ b/airflow/values.yaml @@ -0,0 +1,49 @@ +airflow: + # common settings and setting for the webserver + # For prefix, use only lower case alphanumeric characters, '-' or '.', and must start and end + # with an alphanumeric character + prefix: "" + fernet_key: "" + service: + type: ClusterIP + baseUrl: "/airflow" + init_retry_loop: + image: mumoshu/kube-airflow:1.8.0.0-1.6.1 + # image_pull_policy: Always or IfNotPresent + image_pull_policy: Always + +celery: + num_workers: 1 + +ingress: + annotations: + host: "" + path: + web: /airflow + flower: /flower + +db: + rabbitmq: + # use_embedded == true means final hostname will be prefix + hostname + use_embedded: true + user: airflow + password: airflow + hostname: rabbitmq + database: airflow + postgres: + # use_embedded == true means final hostname will be prefix + hostname + use_embedded: true + user: airflow + password: airflow + hostname: postgres + database: airflow + +flower: + url_prefix: /flower + +dags: + git_sync_enabled: false + git_repo: + git_branch: master + poll_interval_sec: 60 + git_sync_debug: false diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..89fc49f --- /dev/null +++ b/config.yaml @@ -0,0 +1,8 @@ +airflow: + # Duplicate this file and put your customization here + # Generate fernet_key with: + # python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)" + # fernet_key: ABCDABCDABCDABCDABCDABCDABCDABCDABCDABCD + # Note: + # do NOT use characters such as " (double quote), ' (simple quote), / (slash) or \ (backslash) + # in your passwords diff --git a/config/airflow.cfg b/config/airflow.cfg.in similarity index 96% rename from config/airflow.cfg rename to config/airflow.cfg.in index 2fcbd54..405c916 100644 --- a/config/airflow.cfg +++ b/config/airflow.cfg.in @@ -27,7 +27,7 @@ executor = CeleryExecutor # The SqlAlchemy connection string to the metadata database. # SqlAlchemy supports many different database engine, more information # their website -sql_alchemy_conn = postgresql+psycopg2://airflow:airflow@postgres/airflow +sql_alchemy_conn = postgresql+psycopg2://{{ POSTGRES_CREDS }}@{{ POSTGRES_HOST }}/airflow # The SqlAlchemy pool size is the maximum number of database connections # in the pool. @@ -61,7 +61,7 @@ load_examples = True plugins_folder = /usr/local/airflow/plugins # Secret key to save connection passwords in the db -fernet_key = $FERNET_KEY +fernet_key = {{ FERNET_KEY }} # Whether to disable pickling dags donot_pickle = False @@ -141,10 +141,10 @@ worker_log_server_port = 8793 # The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally # a sqlalchemy database. Refer to the Celery documentation for more # information. -broker_url = amqp://airflow:airflow@rabbitmq:5672/airflow +broker_url = amqp://{{ RABBITMQ_CREDS }}@{{ RABBITMQ_HOST }}:5672/airflow # Another key Celery setting -celery_result_backend = amqp://airflow:airflow@rabbitmq:5672/airflow +celery_result_backend = amqp://{{ RABBITMQ_CREDS }}@{{ RABBITMQ_HOST }}:5672/airflow # Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start # it `airflow flower`. This defines the port that Celery Flower runs on diff --git a/reset-af.sh b/reset-af.sh new file mode 100755 index 0000000..0e49f67 --- /dev/null +++ b/reset-af.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +kubectl delete -f airflow.all.yaml --namespace airflow && kubectl create -f airflow.all.yaml --namespace airflow diff --git a/script/entrypoint.sh b/script/entrypoint.sh index 19d75ab..9e035a6 100644 --- a/script/entrypoint.sh +++ b/script/entrypoint.sh @@ -1,20 +1,35 @@ #!/usr/bin/env bash CMD="airflow" -TRY_LOOP="10" -POSTGRES_HOST="postgres" -POSTGRES_PORT="5432" -RABBITMQ_HOST="rabbitmq" -RABBITMQ_CREDS="airflow:airflow" -FERNET_KEY=$(python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print FERNET_KEY") +TRY_LOOP="${TRY_LOOP:-10}" +POSTGRES_HOST="${POSTGRES_HOST:-postgres}" +POSTGRES_PORT=5432 +POSTGRES_CREDS="${POSTGRES_CREDS:-airflow:airflow}" +RABBITMQ_HOST="${RABBITMQ_HOST:-rabbitmq}" +RABBITMQ_CREDS="${RABBITMQ_CREDS:-airflow:airflow}" +RABBITMQ_MANAGEMENT_PORT=15672 +FLOWER_URL_PREFIX="${FLOWER_URL_PREFIX:-/}" + +if [ -z $FERNET_KEY ]; then + FERNET_KEY=$(python3 -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY))" +fi + +echo "Postgres host: $POSTGRES_HOST" +echo "RabbitMQ host: $RABBITMQ_HOST" +echo # Generate Fernet key -sed -i "s/{FERNET_KEY}/${FERNET_KEY}/" $AIRFLOW_HOME/airflow.cfg +cp -f $AIRFLOW_HOME/airflow.cfg.in $AIRFLOW_HOME/airflow.cfg +sed -i "s/{{ FERNET_KEY }}/${FERNET_KEY}/" $AIRFLOW_HOME/airflow.cfg +sed -i "s/{{ POSTGRES_HOST }}/${POSTGRES_HOST}/" $AIRFLOW_HOME/airflow.cfg +sed -i "s/{{ POSTGRES_CREDS }}/${RABBITMQ_CREDS}/" $AIRFLOW_HOME/airflow.cfg +sed -i "s/{{ RABBITMQ_HOST }}/${RABBITMQ_HOST}/" $AIRFLOW_HOME/airflow.cfg +sed -i "s/{{ RABBITMQ_CREDS }}/${RABBITMQ_CREDS}/" $AIRFLOW_HOME/airflow.cfg # wait for rabbitmq if [ "$1" = "webserver" ] || [ "$1" = "worker" ] || [ "$1" = "scheduler" ] || [ "$1" = "flower" ] ; then j=0 - while ! curl -sI -u $RABBITMQ_CREDS http://$RABBITMQ_HOST:15672/api/whoami |grep '200 OK'; do + while ! curl -sI -u $RABBITMQ_CREDS http://$RABBITMQ_HOST:$RABBITMQ_MANAGEMENT_PORT/api/whoami |grep '200 OK'; do j=`expr $j + 1` if [ $j -ge $TRY_LOOP ]; then echo "$(date) - $RABBITMQ_HOST still not reachable, giving up" @@ -44,4 +59,9 @@ if [ "$1" = "webserver" ] || [ "$1" = "worker" ] || [ "$1" = "scheduler" ] ; the sleep 5 fi -exec $CMD "$@" +if [ -z $GIT_SYNC_REPO ]; then + echo "Executing background task git-sync on repo $GIT_SYNC_REPO" + exec git_sync --dest $AIRFLOW_HOME/dags & +fi + +$CMD --flower_conf $AIRFLOW_HOME/flowerconfig.py "$@"