Merge pull request #16 from gsemet/helm_chart

Add Helm chart for Airflow (fix #7)
mumoshu · Apr 23, 2018 · 0570ed1 · 0570ed1
2 parents 60d4f47 + 74059f2
commit 0570ed1
Show file tree

Hide file tree

Showing 24 changed files with 1,263 additions and 115 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -0,0 +1,19 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.yaml]
+indent_size = 2
+
+[Makefile]
+indent_style = tab
+indent_size = 4
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,3 @@
 *~
 build
+rootfs
diff --git a/Dockerfile.template b/Dockerfile.template
@@ -4,84 +4,94 @@
 # BUILD: docker build --rm -t mumoshu/kube-airflow
 # SOURCE: https://github.com/mumoshu/kube-airflow
 
-FROM debian:jessie
+FROM    debian:stretch
 MAINTAINER Yusuke KUOKA <ykuoka@gmail.com>
 
 # Never prompts the user for choices on installation/configuration of packages
-ENV DEBIAN_FRONTEND noninteractive
-ENV TERM linux
+ENV     DEBIAN_FRONTEND noninteractive
+ENV     TERM linux
 
 # Airflow
-ARG AIRFLOW_VERSION=%%AIRFLOW_VERSION%%
-ENV AIRFLOW_HOME /usr/local/airflow
+ARG     AIRFLOW_VERSION=%%AIRFLOW_VERSION%%
+ENV     AIRFLOW_HOME /usr/local/airflow
+ENV     EMBEDDED_DAGS_LOCATION=%%EMBEDDED_DAGS_LOCATION%%
+ENV     REQUIREMENTS_TXT_LOCATION=%%REQUIREMENTS_TXT_LOCATION%%
 
 # Define en_US.
-ENV LANGUAGE en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LC_ALL en_US.UTF-8
-ENV LC_CTYPE en_US.UTF-8
-ENV LC_MESSAGES en_US.UTF-8
-ENV LC_ALL  en_US.UTF-8
+ENV     LANGUAGE en_US.UTF-8
+ENV     LANG en_US.UTF-8
+ENV     LC_ALL en_US.UTF-8
+ENV     LC_CTYPE en_US.UTF-8
+ENV     LC_MESSAGES en_US.UTF-8
+ENV     LC_ALL en_US.UTF-8
 
-RUN set -ex \
-    && buildDeps=' \
-        python-pip \
-        python-dev \
-        libkrb5-dev \
-        libsasl2-dev \
-        libxml2-dev \
-        libssl-dev \
-        libffi-dev \
-        build-essential \
-        libblas-dev \
-        liblapack-dev \
-        libxslt1-dev \
-        zlib1g-dev \
-    ' \
-    && echo "deb http://http.debian.net/debian jessie-backports main" >/etc/apt/sources.list.d/backports.list \
-    && apt-get update -yqq \
-    && apt-get install -yqq --no-install-recommends \
-        $buildDeps \
-        apt-utils \
-        curl \
-        netcat \
-        locales \
-    && apt-get install -yqq -t jessie-backports python-requests libpq-dev \
-    && sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \
-    && locale-gen \
-    && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \
-    && useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
-    && pip uninstall setuptools \
-    && pip install setuptools==33.1.1 \
-    && pip install pytz==2015.7 \
-    && pip install cryptography \
-    && pip install pyOpenSSL \
-    && pip install ndg-httpsclient \
-    && pip install pyasn1 \
-    && pip install psycopg2 \
-    && pip install airflow[celery,postgresql,hive]==$AIRFLOW_VERSION \
-    && apt-get remove --purge -yqq $buildDeps libpq-dev \
-    && apt-get clean \
-    && rm -rf \
-        /var/lib/apt/lists/* \
-        /tmp/* \
-        /var/tmp/* \
-        /usr/share/man \
-        /usr/share/doc \
-        /usr/share/doc-base
+WORKDIR /requirements
+# Only copy needed files
+COPY    requirements/airflow.txt /requirements/airflow.txt
+COPY    ${REQUIREMENTS_TXT_LOCATION} /requirements/dags.txt
 
-ENV KUBECTL_VERSION %%KUBECTL_VERSION%%
 
-RUN curl -L -o /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && chmod +x /usr/local/bin/kubectl
+RUN         set -ex \
+        &&  buildDeps=' \
+                build-essential \
+                libblas-dev \
+                libffi-dev \
+                libkrb5-dev \
+                liblapack-dev \
+                libpq-dev \
+                libsasl2-dev \
+                libssl-dev \
+                libxml2-dev \
+                libxslt1-dev \
+                python3-dev \
+                python3-pip \
+                zlib1g-dev \
+            ' \
+        &&  apt-get update -yqq \
+        &&  apt-get install -yqq --no-install-recommends \
+                $buildDeps \
+                apt-utils \
+                curl \
+                git \
+                locales \
+                netcat \
+        &&      sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \
+        &&  locale-gen \
+        &&  update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \
+        &&  useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
+        &&  pip3 install --upgrade pip 'setuptools!=36.0.0' \
+        &&  if [ ! -e /usr/bin/pip ]; then ln -s /usr/bin/pip3 /usr/bin/pip ; fi \
+        &&  if [ ! -e /usr/bin/python ]; then ln -sf /usr/bin/python3 /usr/bin/python; fi \
+        &&  pip3 install -r /requirements/airflow.txt \
+        &&  pip3 install -r /requirements/dags.txt \
+        &&  apt-get remove --purge -yqq $buildDeps libpq-dev \
+        &&  apt-get clean \
+        &&  rm -rf \
+                /var/lib/apt/lists/* \
+                /tmp/* \
+                /var/tmp/* \
+                /usr/share/man \
+                /usr/share/doc \
+                /usr/share/doc-base
 
-COPY script/entrypoint.sh ${AIRFLOW_HOME}/entrypoint.sh
-COPY config/airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
+ENV         KUBECTL_VERSION %%KUBECTL_VERSION%%
 
-RUN chown -R airflow: ${AIRFLOW_HOME} \
-    && chmod +x ${AIRFLOW_HOME}/entrypoint.sh
+RUN         curl -L -o /usr/local/bin/kubectl \
+                https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
+        &&  chmod +x /usr/local/bin/kubectl
 
-EXPOSE 8080 5555 8793
+COPY        script/entrypoint.sh ${AIRFLOW_HOME}/entrypoint.sh
+COPY        config/airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
+COPY        script/git-sync ${AIRFLOW_HOME}/git-sync
+COPY        ${EMBEDDED_DAGS_LOCATION} ${AIRFLOW_HOME}/dags
+COPY        script/git-sync ${AIRFLOW_HOME}/git-sync
 
-USER airflow
-WORKDIR ${AIRFLOW_HOME}
-ENTRYPOINT ["./entrypoint.sh"]
+RUN         chown -R airflow: ${AIRFLOW_HOME} \
+        &&  chmod +x ${AIRFLOW_HOME}/entrypoint.sh \
+        &&  chmod +x ${AIRFLOW_HOME}/git-sync
+
+EXPOSE  8080 5555 8793
+
+USER        airflow
+WORKDIR     ${AIRFLOW_HOME}
+ENTRYPOINT  ["./entrypoint.sh"]
diff --git a/Makefile b/Makefile
@@ -12,24 +12,59 @@ DOCKERFILE ?= $(BUILD_ROOT)/Dockerfile
 ROOTFS ?= $(BUILD_ROOT)/rootfs
 AIRFLOW_CONF ?= $(BUILD_ROOT)/config/airflow.cfg
 ENTRYPOINT_SH ?= $(BUILD_ROOT)/script/entrypoint.sh
+GIT_SYNC ?= $(BUILD_ROOT)/script/git-sync
+DAGS ?= $(BUILD_ROOT)/dags
+AIRFLOW_REQUIREMENTS ?= $(BUILD_ROOT)/requirements/airflow.txt
+DAGS_REQUIREMENTS ?= $(BUILD_ROOT)/requirements/dags.txt
 DOCKER_CACHE ?= docker-cache
 SAVED_IMAGE ?= $(DOCKER_CACHE)/image-$(AIRFLOW_VERSION)-$(KUBECTL_VERSION).tar
 
 NAMESPACE ?= airflow-dev
+HELM_APPLICATION_NAME ?= airflow
+HELM_VALUES ?= airflow/values.yaml
+CHART_LOCATION ?= ./airflow
+EMBEDDED_DAGS_LOCATION ?= "./dags"
+REQUIREMENTS_TXT_LOCATION ?= "requirements/dags.txt"
 
 .PHONY: build clean
 
 clean:
 	rm -Rf build
 
-build: $(DOCKERFILE) $(ROOTFS) $(AIRFLOW_CONF) $(ENTRYPOINT_SH)
+helm-install:
+	helm upgrade -f $(HELM_VALUES) \
+	             --install \
+	             --debug \
+	             $(HELM_APPLICATION_NAME) \
+	             $(CHART_LOCATION)
+
+helm-check:
+	helm install --dry_run \
+	            $(CHART_LOCATION) \
+	             --version=v0.1.0 \
+	             --name=$(HELM_APPLICATION_NAME) \
+	             --namespace=$(NAMESPACE) \
+	             --debug \
+	             -f $(HELM_VALUES)
+
+helm-ls:
+	helm ls --all $(HELM_APPLICATION_NAME)
+
+helm-uninstall:
+	helm del --purge $(HELM_APPLICATION_NAME)
+
+build: clean $(DOCKERFILE) $(ROOTFS) $(DAGS) $(AIRFLOW_CONF) $(ENTRYPOINT_SH) $(GIT_SYNC) $(AIRFLOW_REQUIREMENTS) $(DAGS_REQUIREMENTS)
 	cd $(BUILD_ROOT) && docker build -t $(IMAGE) . && docker tag $(IMAGE) $(ALIAS)
 
 publish:
 	docker push $(IMAGE) && docker push $(ALIAS)
 
 $(DOCKERFILE): $(BUILD_ROOT)
-	sed -e 's/%%KUBECTL_VERSION%%/'"$(KUBECTL_VERSION)"'/g;' -e 's/%%AIRFLOW_VERSION%%/'"$(AIRFLOW_VERSION)"'/g;' Dockerfile.template > $(DOCKERFILE)
+	sed -e 's/%%KUBECTL_VERSION%%/'"$(KUBECTL_VERSION)"'/g;' \
+	    -e 's/%%AIRFLOW_VERSION%%/'"$(AIRFLOW_VERSION)"'/g;' \
+	    -e 's#%%EMBEDDED_DAGS_LOCATION%%#'"$(EMBEDDED_DAGS_LOCATION)"'#g;' \
+	    -e 's#%%REQUIREMENTS_TXT_LOCATION%%#'"$(REQUIREMENTS_TXT_LOCATION)"'#g;' \
+	    Dockerfile.template > $(DOCKERFILE)
 
 $(ROOTFS): $(BUILD_ROOT)
 	mkdir -p rootfs
@@ -43,6 +78,22 @@ $(ENTRYPOINT_SH): $(BUILD_ROOT)
 	mkdir -p $(shell dirname $(ENTRYPOINT_SH))
 	cp script/entrypoint.sh $(ENTRYPOINT_SH)
 
+$(GIT_SYNC): $(BUILD_ROOT)
+	mkdir -p $(shell dirname $(GIT_SYNC))
+	cp script/git-sync $(GIT_SYNC)
+
+$(AIRFLOW_REQUIREMENTS): $(BUILD_ROOT)
+	mkdir -p $(shell dirname $(AIRFLOW_REQUIREMENTS))
+	cp requirements/airflow.txt $(AIRFLOW_REQUIREMENTS)
+
+$(DAGS_REQUIREMENTS): $(BUILD_ROOT)
+	mkdir -p $(shell dirname $(DAGS_REQUIREMENTS))
+	cp $(REQUIREMENTS_TXT_LOCATION) $(DAGS_REQUIREMENTS)
+
+$(DAGS): $(BUILD_ROOT)
+	mkdir -p $(shell dirname $(DAGS))
+	cp -R $(EMBEDDED_DAGS_LOCATION) $(DAGS)
+
 $(BUILD_ROOT):
 	mkdir -p $(BUILD_ROOT)