♻️ Migrate from RQ to Celery for async jobs

.docker/rabbitmq/Dockerfile

@@ -0,0 +1,9 @@
+FROM rabbitmq:3.9-management
+
+ENV RABBITMQ_PID_FILE /var/lib/rabbitmq/mnesia/rabbitmq
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+EXPOSE 15672
+ENTRYPOINT ["/entrypoint.sh"]

.docker/rabbitmq/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -eo pipefail
+
+# Create Okuna user for RabbitMQ
+( rabbitmqctl wait --timeout 60 /var/lib/rabbitmq/mnesia/rabbitmq; \
+rabbitmqctl add_user $RABBITMQ_USERNAME $RABBITMQ_PASSWORD 2>/dev/null; \
+rabbitmqctl add_vhost okuna; \
+rabbitmqctl set_permissions -p okuna $RABBITMQ_USERNAME ".*" ".*" ".*"; \
+echo "Initial okuna user created successfully. You may authenticate now.") &
+
+rabbitmq-server $@

.docker/scheduler/entrypoint.sh

@@ -5,6 +5,8 @@ cd /opt/okuna-api
 
 /wait-for-it.sh $RDS_HOSTNAME:$RDS_PORT -t 60
 
+/wait-for-it.sh $RABBITMQ_HOST:$RABBITMQ_PORT -t 60
+
 /wait-for-it.sh $REDIS_HOST:$REDIS_PORT -t 60
 
 # install pip env deps, run migrations, collect media, start the server

.docker/scheduler/supervisord.conf

@@ -9,8 +9,8 @@ serverurl = unix:
 username=admin
 password=revproxy
 
-[program: rqschedulerdefault]
-command = python manage.py rqscheduler --queue=default
+[program: celerybeat]
+command = celery -A openbook beat -l info
 loglevel = info                ; (log level;default info; others: debug, warn, trace)
 numprocs = 1
 directory = /opt/okuna-api
@@ -22,31 +22,3 @@ exitcodes = 0, 2                 ; 'expected' exit codes for process (default 0,
 killasgroup = true              ; SIGKILL the UNIX process group (def false)
 stopasgroup = true
 stopsignal = QUIT
-
-[program: rqschedulerhigh]
-command = python manage.py rqscheduler --queue=high
-loglevel = info                ; (log level;default info; others: debug, warn, trace)
-numprocs = 1
-directory = /opt/okuna-api
-autostart = true
-autorestart = unexpected
-startsecs = 2                   ; number of secs prog must stay running (def. 1)
-startretries = 3                ; max # of serial start failures (default 3)
-exitcodes = 0, 2                 ; 'expected' exit codes for process (default 0, 2)
-killasgroup = true              ; SIGKILL the UNIX process group (def false)
-stopasgroup = true
-stopsignal = QUIT
-
-[program: rqschedulerlow]
-command = python manage.py rqscheduler --queue=low
-loglevel = info                ; (log level;default info; others: debug, warn, trace)
-numprocs = 1
-directory = /opt/okuna-api
-autostart = true
-autorestart = unexpected
-startsecs = 1                   ; number of secs prog must stay running (def. 1)
-startretries = 3                ; max # of serial start failures (default 3)
-exitcodes = 0, 2                 ; 'expected' exit codes for process (default 0, 2)
-killasgroup = true              ; SIGKILL the UNIX process group (def false)
-stopasgroup = true
-stopsignal = QUIT

.docker/worker/entrypoint.sh

@@ -5,6 +5,8 @@ cd /opt/okuna-api
 
 /wait-for-it.sh $RDS_HOSTNAME:$RDS_PORT -t 60
 
+/wait-for-it.sh $RABBITMQ_HOST:$RABBITMQ_PORT -t 60
+
 /wait-for-it.sh $REDIS_HOST:$REDIS_PORT -t 60
 
 # install pip env deps, run migrations, collect media, start the server

.docker/worker/supervisord.conf

@@ -9,8 +9,8 @@ serverurl = unix:
 username=admin
 password=revproxy
 
-[program: rqworkerdefault]
-command = python manage.py rqworker default
+[program: celerydefault]
+command = celery -A openbook worker -E -l info -Q default_priority -n okuna_default
 loglevel = info                ; (log level;default info; others: debug, warn, trace)
 numprocs = 1
 directory = /opt/okuna-api
@@ -23,8 +23,8 @@ killasgroup = true              ; SIGKILL the UNIX process group (def false)
 stopasgroup = true
 stopsignal = QUIT
 
-[program: rqworkerhigh]
-command = python manage.py rqworker high
+[program: celeryhigh]
+command = celery -A openbook worker -E -l info -Q high_priority -n okuna_high
 loglevel = info                ; (log level;default info; others: debug, warn, trace)
 numprocs = 1
 directory = /opt/okuna-api
@@ -37,8 +37,8 @@ killasgroup = true              ; SIGKILL the UNIX process group (def false)
 stopasgroup = true
 stopsignal = QUIT
 
-[program: rqworkerlow]
-command = python manage.py rqworker low
+[program: celerylow]
+command = celery -A openbook worker -E -l info -Q low_priority -n okuna_low
 loglevel = info                ; (log level;default info; others: debug, warn, trace)
 numprocs = 1
 directory = /opt/okuna-api
@@ -49,4 +49,4 @@ startretries = 3                ; max # of serial start failures (default 3)
 exitcodes = 0, 2                 ; 'expected' exit codes for process (default 0, 2)
 killasgroup = true              ; SIGKILL the UNIX process group (def false)
 stopasgroup = true
-stopsignal = QUIT
+stopsignal = QUIT

.gitignore

@@ -10,6 +10,8 @@ coverage.xml
 src/
 static/
 docker-compose.override.yml
+*.db
+celerybeat-schedule
 
 # Elastic Beanstalk Files
 .elasticbeanstalk/*

Pipfile

@@ -8,40 +8,38 @@ polib = "*"
 pytz = "*"
 
 [packages]
-django = "==2.2.16"
+Django = "==2.2.16"
 python-dotenv = "*"
 djangorestframework = "*"
 sentry-sdk = "==0.10.2"
 mysqlclient = "*"
-pillow = "*"
+Pillow = "*"
 django-nose = "*"
 nose = "*"
 pinocchio = "*"
 coverage = "*"
 nose-exclude = "*"
 mixer = "*"
-faker = "*"
-"boto3" = "*"
+Faker = "*"
+boto3 = "*"
 django-storages = "==1.8"
 django-amazon-ses = "*"
 django-media-fixtures = "==0.0.3"
 django-modeltranslation = "*"
 bandit = "==1.5.1"
-pyyaml = ">=4.2b1"
+PyYAML = ">=4.2b1"
 python-magic = "*"
 safety = "*"
-pyjwt = "*"
+PyJWT = "*"
 django-imagekit = "*"
 rest-framework-generic-relations = "*"
 onesignal-sdk = "==1.1.0"
-django-replicated = "*"
+django_replicated = "*"
 django-cacheops = "*"
 django-redis = "*"
 django-extensions = "*"
 langdetect = "*"
-django-rq = "*"
 redis = "*"
-django-rq-scheduler = "*"
 ffmpy = "*"
 django-positions = "*"
 django-ordered-model = "*"
@@ -56,6 +54,7 @@ halo = "*"
 watchdog = "*"
 spectra = "*"
 url-normalize = "*"
+celery = "*"
 
 [pipenv]
 allow_prereleases = false