add playbook

README.md

@@ -1,2 +1,13 @@
-# mlflow-ansible
-Ansible playbook for mlflow
+# Ansible Role: MLflow
+Ansible playbook for [MLflow](https://www.mlflow.org/)
+
+Creates docker container with MLflow. 
+
+PostgreSQL used as backend-store. Minio used for artifact storage.
+
+Minio and PostgreSQL are also installed in separate docker containers
+
+# Run MLflow on localhost example
+`ansible-playbook main.yml`
+
+MLflow will be at http://localhost:5000

docker/mlflow/Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.7.0
+RUN mkdir /mlflow/
+RUN pip install mlflow psycopg2 boto3
+
+CMD mlflow server \
+--backend-store-uri postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DOCKER_HOST}:${POSTGRES_PORT}/${DB_NAME} \
+--host 0.0.0.0 \
+--port ${MLFLOW_PORT} \
+--default-artifact-root s3://${MLFLOW_S3_BUCKET}

main.yml

@@ -0,0 +1,4 @@
+- hosts: 127.0.0.1
+  connection: local
+  roles:
+    - mlflow

roles/mlflow/tasks/docker.yml

@@ -0,0 +1,41 @@
+- name: Ensure dependencies are installed
+  apt:
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - gnupg2
+    state: present
+
+- name: Add Docker apt key
+  apt_key:
+    url: "https://download.docker.com/linux/{{ ansible_distribution | lower }}/gpg"
+    id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
+    state: present
+  register: add_repository_key
+  ignore_errors: true
+
+- name: Add Docker repository
+  apt_repository:
+    repo: "deb [arch=amd64] https://download.docker.com/linux/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} stable"
+    state: present
+    update_cache: true
+
+- name: Install Docker
+  package:
+    name: docker-ce
+    state: present
+
+- name: Ensure Docker is started and enabled at boot
+  service:
+    name: docker
+    state: started
+    enabled: true
+
+- name: Ensure handlers are notified now to avoid firewall conflicts
+  meta: flush_handlers
+
+- name: pip install docker
+  pip:
+    name: docker
+    version: 4.4.1
+    state: present

roles/mlflow/tasks/main.yml

@@ -0,0 +1,6 @@
+- include_vars: vars.yml
+
+- include_tasks: docker.yml
+- include_tasks: postgres.yml
+- include_tasks: minio.yml
+- include_tasks: mlflow.yml

roles/mlflow/tasks/minio.yml

@@ -0,0 +1,24 @@
+- name: run minio node
+  docker_container:
+    image: minio/minio:edge
+    name: minio
+    state: started
+    restart_policy: always
+    env:
+      MINIO_ACCESS_KEY: "{{ minio_access_key }}"
+      MINIO_SECRET_KEY: "{{ minio_secret_key }}"
+    volumes:
+      - /var/minio_data:/minio_data
+    ports: "{{ s3_port }}:{{ s3_port }}"
+    command: server /minio_data
+
+- name: create bucket for mlflow
+  docker_container:
+    image: minio/mc
+    name: minio_mc
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc config host add myminio http://{{ docker_host }}:{{ s3_port }} {{ minio_access_key }} {{ minio_secret_key }};
+      /usr/bin/mc mb myminio/{{ mlflow_bucket }};
+      exit 0;
+      "

roles/mlflow/tasks/mlflow.yml

@@ -0,0 +1,27 @@
+- name: build mlflow image
+  docker_image:
+    name: mlflow
+    build:
+      path: "{{ playbook_dir }}/docker/mlflow"
+      pull: no
+    state: present
+    source: build
+
+- name: run mlflow server
+  docker_container:
+    env:
+      DOCKER_HOST: "{{ docker_host }}"
+      POSTGRES_PORT: "{{ postgres_external_port|quote }}"
+      POSTGRES_USER: "{{ postgres_user }}"
+      POSTGRES_PASSWORD: "{{ postgres_password }}"
+      DB_NAME: "{{ mlflow_db_name }}"
+      MLFLOW_PORT: "{{ mlflow_port|quote }}"
+      MLFLOW_S3_ENDPOINT_URL: "http://{{ docker_host }}:{{ s3_port }}"
+      AWS_ACCESS_KEY_ID: "{{ minio_access_key }}"
+      AWS_SECRET_ACCESS_KEY: "{{ minio_secret_key }}"
+      MLFLOW_S3_BUCKET: "{{ mlflow_bucket }}"
+    name: mlflow_server
+    image: mlflow
+    state: started
+    restart_policy: always
+    ports: "{{ mlflow_port }}:{{ mlflow_port }}"

roles/mlflow/tasks/postgres.yml

@@ -0,0 +1,30 @@
+- name: check/create postgresql container
+  docker_container:
+    name: postgres
+    image: >-
+      postgres:11.4-alpine
+    state: started
+    restart_policy: always
+    volumes:
+      - /var/lib/postgresql/data:/postgres_data
+    ports: "{{ postgres_external_port }}:5432"
+
+- name : wait for postgresql starts
+  wait_for:
+    host: "{{ docker_host }}"
+    port: "{{ postgres_external_port }}"
+
+- name: Create a new database
+  postgresql_db:
+    name: "{{ mlflow_db_name }}"
+    login_host: "{{ docker_host }}"
+    port: "{{ postgres_external_port }}"
+
+- name: Create mlflow user, and grant access to database
+  postgresql_user:
+    login_host: "{{ docker_host }}"
+    port: "{{ postgres_external_port }}"
+    db: "{{ mlflow_db_name }}"
+    name: "{{ postgres_user }}"
+    password: "{{ postgres_password }}"
+    priv: "ALL"

roles/mlflow/tasks/vars.yml

@@ -0,0 +1,13 @@
+docker_host: 172.17.0.1
+
+postgres_external_port: 5433
+postgres_user: mlflow_user
+postgres_password: mlflow_password
+mlflow_db_name: mlflow
+
+s3_port: 9000
+minio_access_key: minioadmin
+minio_secret_key: minioadmin
+mlflow_bucket: mlflow/data
+
+mlflow_port: 5000