Migrate from pipenv to poetry

.circleci/config.yml

@@ -1,94 +1,234 @@
-# Python CircleCI 2.0 configuration file
-#
-# Check https://circleci.com/docs/2.0/language-python/ for more details
-#
-version: 2
+# Note that the following stanza uses CircleCI 2.1 to make use of a Reusable Executor
+# This allows defining a docker image to reuse across jobs.
+# visit https://circleci.com/docs/2.0/reusing-config/#authoring-reusable-executors to learn more.
+# NixOS stuff taken from
+# https://github.com/PrivateStorageio/PaymentServer/blob/master/.circleci/config.yml
 
-jobs:
+version: 2.1
+
+orbs:
+  check_untracked_changes: niteo/check-untracked-changes@1
+
+workflows:
+  version: 2
+
+  workflow:
+    jobs:
+      - Backend Deps
+      - Backend Lint:
+          requires:
+            - Backend Deps
+      - Backend Test:
+          requires:
+            - Backend Deps
+      - Postman Tests:
+          requires:
+            - Backend Deps
+
+executors:
+  conduit:
+    working_directory: /app
+    docker:
+      - image: nixorg/nix:circleci
+      - image: circleci/postgres:11.2-alpine
+      - image: redis
 
-  build:
-    working_directory: ~/repo
     environment:
       PGHOST: localhost
-      PIPENV_VENV_IN_PROJECT: 1
-      PIPENV_IGNORE_VIRTUALENVS: 1
       POSTGRES_USER: postgres
       POSTGRES_DB: postgres
       POSTGRES_PASSWORD: ""
-    docker:
-      - image: circleci/python:3.8.0-node
-      - image: circleci/postgres:11.2-alpine
+      NIX_BUILD_SHELL: "bash"
 
+jobs:
+
+  #################################
+  #       ---- Backend ----       #
+  # Jobs for the Pyramid backend. #
+  #################################
+
+  Backend Deps:
+    executor: conduit
+    working_directory: /app
     steps:
+
       - checkout
 
-      # Download and cache dependencies
-      - restore_cache:
-          keys:
-            - v3-dependencies-{{ checksum "Pipfile.lock" }}
-            # fallback to using the latest cache if no exact match is found
-            - v3-dependencies-
+      - run:
+          name: Build nix shell
+          command: |
+            nix-shell --pure --run "python --version"
+
+      - save_cache:
+          key: nix-deps-v1-{{ checksum "shell.nix" }}
+          paths:
+            - /nix
+
+      - persist_to_workspace:
+          root: /app
+          paths:
+            - "*"
 
       - run:
-          name: install dependencies
+          name: Touch .installed
+          command: touch .installed
+
+      - run:
+          name: Install dependencies
           command: |
-            sudo pip install pipenv
-            pipenv install --dev --deploy
-            touch .installed
+            nix-shell --pure --run "poetry install"
 
       - save_cache:
+          key: backend-deps-v3-{{ checksum "poetry.lock" }}
           paths:
-            - ./.venv
-          key: v3-dependencies-{{ checksum "Pipfile.lock" }}
+            - .venv
+            - src/conduit.egg-info
+
+  Backend Lint:
+    executor: conduit
+    working_directory: /app
+    steps:
+
+      - attach_workspace:
+          at: /app
+
+      - restore_cache:
+          name: "Restoring Cache - Nix"
+          keys:
+            - nix-deps-v1-{{ checksum "shell.nix" }}
+
+      - restore_cache:
+          name: "Restoring Cache - Python"
+          keys:
+            - backend-deps-v3-{{ checksum "poetry.lock" }}
+
+      - run:
+          name: Touch .installed
+          command: touch .installed
 
       - run:
-          name: run linters
+          name: Run Linters
           command: |
-            make lint
-            pipenv run black src/conduit --check
+            nix-shell --pure --run "poetry run pre-commit install -f --hook-type pre-push"
+            nix-shell --pure --run "make lint"
 
       - run:
           name: run mypy
           command: |
-            make types
+            nix-shell --pure --run "make types"
+
+
+  Backend Test:
+    executor: conduit
+    working_directory: /app
+    steps:
+
+      - attach_workspace:
+          at: /app
+
+      - restore_cache:
+          name: "Restoring Cache - Nix"
+          # TODO: fail build if caches are empty at this point
+          keys:
+            - nix-deps-v1-{{ checksum "shell.nix" }}
+
+      - restore_cache:
+          name: "Restoring Cache - Python"
+          keys:
+            - backend-deps-v3-{{ checksum "poetry.lock" }}
 
       - run:
-          name: Waiting for Postgres to be ready
+          name: Touch .installed
+          command: touch .installed
+
+      - run:
+          name: Wait for Postgres to be ready
           command: |
             for i in `seq 1 10`;
             do
-              nc -z localhost 5432 && echo Success && exit 0
+              nix-shell --pure --run "nc -z localhost 5432" && echo Success && exit 0
               echo -n .
               sleep 1
             done
             echo Failed waiting for Postgres && exit 1
 
       - run:
           name: Create testing db
+          # TODO: add --pure? but then PGHOST is missing
+          command: |
+            nix-shell --run ".docker/devdb.sh && .docker/testdb.sh"
+
+      - run:
+          name: Test development.ini
           command: |
-            sudo apt update
-            sudo apt install postgresql-client
-            .docker/devdb.sh
-            .docker/testdb.sh
+            nix-shell --run ".docker/devdb.sh"
+            nix-shell --pure --run "poetry run pshell etc/development.ini SKIP_CHECK_DB_MIGRATED=1 < .circleci/exit.sh"
 
       - run:
-          name: run tests
+          name: Checking consistency between alembic revision and conduit models
           command: |
-            make unit
+            nix-shell --pure --run "make devdb"
+            # As of now, the word count in an empty migration script is: 84
+            EXPECTED_WORD_COUNT=84
+            PRESENT_WORD_COUNT=$(cat $(nix-shell --pure --run "poetry run alembic -c etc/alembic.ini -x ini=etc/development.ini revision --autogenerate -m 'Test Consistency'" | cut -d' ' -f2) | wc -w)
+            if [ $EXPECTED_WORD_COUNT != $PRESENT_WORD_COUNT ]; then
+              echo "You probably forgot to generate and commit DB migration...!"
+              false
+            fi
 
       - run:
-          name: run dev app
+          name: Test production.ini
           command: |
-            pipenv run pshell etc/development.ini SKIP_CHECK_DB_MIGRATED=1 < .circleci/exit.sh
+            nix-shell --pure --run "JWT_SECRET=secret DATABASE_URL=postgresql://conduit_dev@localhost/conduit_dev poetry run pshell etc/production.ini SKIP_CHECK_DB_MIGRATED=1 < .circleci/exit.sh"
+
+      - run:
+          name: Run unit tests
+          command: |
+            nix-shell --pure --run "make unit"
+
+
+  ######################################
+  #      ---- Postman Tests ----       #
+  #       API tests with Postman       #
+  ######################################
+
+  Postman Tests:
+    executor: conduit
+    steps:
+
+      - attach_workspace:
+          at: /app
+
+      - restore_cache:
+          name: "Restoring Cache - Nix"
+          keys:
+            - nix-deps-v1-{{ checksum "shell.nix" }}
+
+      - restore_cache:
+          name: "Restoring Cache - Python"
+          keys:
+            - backend-deps-v3-{{ checksum "poetry.lock" }}
+
+      - run:
+          name: Touch .installed
+          command: touch .installed
 
       - run:
-          name: run prod app
+          name: Wait for Postgres to be ready
           command: |
-            DATABASE_URL="postgresql://conduit_dev@localhost/conduit_dev" pipenv run pshell etc/production.ini SKIP_CHECK_DB_MIGRATED=1 < .circleci/exit.sh
+            for i in `seq 1 10`;
+            do
+              nix-shell --pure --run "nc -z localhost 5432" && echo Success && exit 0
+              echo -n .
+              sleep 1
+            done
+            echo Failed waiting for Postgres && exit 1
 
       - run:
-          name: run Postman tests
+          name: Run the tests
+          # TODO: add --pure? but then PGHOST is missing
           command: |
-            make devdb
-            pipenv run pserve etc/development.ini &
-            make postman-tests
+            nix-shell --run ".docker/devdb.sh"
+            nix-shell --run "make devdb"
+            nix-shell --run "poetry run pserve etc/development.ini &"
+            nix-shell --run "make postman-tests"

.dockerignore

@@ -0,0 +1,8 @@
+.venv/
+.git/
+node_modules/
+.mypy_cache/
+.pytest_cache/
+typecov/
+htmlcov/
+htmltypecov/

.envrc

@@ -0,0 +1,2 @@
+# Use shell.nix to build the development environment
+use nix

.gitattributes

@@ -1 +1 @@
-Pipfile.lock linguist-generated
+poetry.lock linguist-generated

.heroku/release.sh

@@ -1,7 +1,9 @@
-#!/usr/bin/env bash
-
 set -e
 
+# See https://github.com/niteoweb/pyramid-realworld-example-app/issues/86
+echo "Installing required extensions"
+psql $DATABASE_URL -c "CREATE EXTENSION IF NOT EXISTS pgcrypto;"
+
 echo "Running database migrations"
 alembic -c etc/alembic.ini -x ini=etc/production.ini upgrade head || echo "Database migrations failed!"
 echo "Done"

.pre-commit-config.yaml

@@ -2,48 +2,48 @@
     hooks:
     -   id: isort
         name: isort
-        entry: pipenv run isort -rc --atomic src/conduit
+        entry: poetry run isort -rc --atomic src/conduit
         language: system
         types: [python]
 
     -   id: flake8
         name: Flake8
-        entry: pipenv run flake8
+        entry: poetry run flake8
         language: system
         types: [python]
 
     -   id: black
         name: black
-        entry: pipenv run black
+        entry: poetry run black
         language: system
         types: [python]
 
     -   id: trailing-whitespace
         name: Trim Trailing Space
-        entry: pipenv run trailing-whitespace-fixer
+        entry: poetry run trailing-whitespace-fixer
         language: system
         types: [non-executable, file, text]
         exclude_types: [svg]
 
     -   id: end-of-file-fixer
         name: Fix End of Files
         description: Ensures that a file is either empty, or ends with one newline.
-        entry: pipenv run end-of-file-fixer
+        entry: poetry run end-of-file-fixer
         language: system
         types: [non-executable, file, text]
         exclude_types: [svg]
 
     -   id: check-merge-conflict
         name: Check for merge conflicts
         description: Check for files that contain merge conflict strings.
-        entry: pipenv run check-merge-conflict
+        entry: poetry run check-merge-conflict
         language: system
         stages: [push]
 
     -   id: codespell
         name: Check Spelling
         description: Checks for common misspellings in text files.
-        entry: pipenv run codespell --ignore-words .aspell.en.pws
+        entry: poetry run codespell --ignore-words .aspell.en.pws
         language: system
         types: [non-executable, file, text]
         exclude_types: [svg]
@@ -53,7 +53,7 @@
     -   id: debug-statements
         name: Check Debug Statements Absent (Python)
         description: Checks that debug statements (pdb, ipdb, pudb) are not imported on commit.
-        entry: pipenv run debug-statement-hook
+        entry: poetry run debug-statement-hook
         language: system
         types: [python]
         stages: [push]
@@ -69,7 +69,7 @@
     -   id: bandit
         name: Check for common security issues
         args: []
-        entry: pipenv run bandit -r src/conduit/ --skip B608 -x *tests*,src/conduit/scripts/populate.py
+        entry: poetry run bandit -r src/conduit/ --skip B608 -x *tests*,src/conduit/scripts/populate.py
         language: system
         pass_filenames: false
         stages: [push]