- Bump Dockerfile to version 3.4

fvicent · Nov 18, 2020 · 032328c · 032328c
1 parent ff45d99
commit 032328c
Show file tree

Hide file tree

Showing 18 changed files with 209 additions and 130 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -105,9 +105,8 @@ jobs:
                   docker exec db4geonode createdb -U postgres -T postgres test_geonode_data
                   docker exec db4geonode psql -U postgres -d test_geonode -c 'CREATE EXTENSION IF NOT EXISTS postgis;'
                   docker exec db4geonode psql -U postgres -d test_geonode_data -c 'CREATE EXTENSION IF NOT EXISTS postgis;'
-                  docker exec -it django4geonode bash -c 'echo "Running <<parameters.test_suite>>"'
-                  docker exec -it django4geonode bash -c 'coverage run --branch --source=geonode manage.py test -v 3 --keepdb <<parameters.test_suite>>'
-                  docker exec -it django4geonode bash -c 'flake8 geonode'
+                  docker-compose -f docker-compose.yml run --rm django bash -c 'coverage run --branch --source=geonode manage.py test -v 3 --keepdb <<parameters.test_suite>>'
+                  docker-compose -f docker-compose.yml run --rm django bash -c 'flake8 geonode'
                 working_directory: ./
 
 workflows:

diff --git a/.env b/.env
@@ -1,12 +1,14 @@
 COMPOSE_PROJECT_NAME=geonode
-SET_DOCKER_ENV=production
-BACKUPS_VOLUME_DRIVER=local
-
 DOCKER_HOST_IP=
 DOCKER_ENV=production
 # See https://github.com/geosolutions-it/geonode-generic/issues/28
 # to see why we force API version to 1.24
 DOCKER_API_VERSION="1.24"
+BACKUPS_VOLUME_DRIVER=local
+
+C_FORCE_ROOT=1
+
+DEBUG=False
 
 DJANGO_SETTINGS_MODULE=geonode.settings
 GEONODE_INSTANCE_NAME=geonode
@@ -112,7 +114,7 @@ MONITORING_HOST_NAME=geonode
 MODIFY_TOPICCATEGORY=True
 AVATAR_GRAVATAR_SSL=True
 AVATAR_DEFAULT_URL=/geonode/img/avatar.png
-AVATAR_PROVIDERS='avatar.providers.PrimaryAvatarProvider','avatar.providers.GravatarAvatarProvider','avatar.providers.DefaultAvatarProvider'
+
 EXIF_ENABLED=True
 CREATE_LAYER=True
 FAVORITE_ENABLED=True

diff --git a/Dockerfile b/Dockerfile
@@ -1,27 +1,69 @@
-FROM geonode/geonode:latest
+FROM python:3.8.3-buster
 LABEL GeoNode development team
 
-COPY tasks.py /usr/src/app/
-COPY uwsgi.ini /usr/src/app/
-COPY entrypoint.sh /usr/src/app/
-COPY requirements.txt /usr/src/app/
+RUN mkdir -p /usr/src/geonode
 
+# Enable postgresql-client-11.2
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list
+RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+
+# This section is borrowed from the official Django image but adds GDAL and others
+RUN apt-get update && apt-get install -y \
+    libpq-dev python-dev libxml2-dev \
+    libxml2 libxslt1-dev zlib1g-dev libjpeg-dev \
+    libmemcached-dev libldap2-dev libsasl2-dev libffi-dev
+
+RUN apt-get update && apt-get install -y \
+    gcc zip gettext geoip-bin cron \
+    postgresql-client-11 \
+    sqlite3 spatialite-bin libsqlite3-mod-spatialite \
+    python3-gdal python3-psycopg2 python3-ldap \
+    python3-pip python3-pil python3-lxml python3-pylibmc \
+    python3-dev libgdal-dev \
+    uwsgi uwsgi-plugin-python3 \
+    --no-install-recommends && rm -rf /var/lib/apt/lists/*
+
+# add bower and grunt command
+COPY . /usr/src/geonode/
+WORKDIR /usr/src/geonode
+
+COPY monitoring-cron /etc/cron.d/monitoring-cron
+RUN chmod 0644 /etc/cron.d/monitoring-cron
+RUN crontab /etc/cron.d/monitoring-cron
+RUN touch /var/log/cron.log
+RUN service cron start
+
+COPY wait-for-databases.sh /usr/bin/wait-for-databases
+RUN chmod +x /usr/bin/wait-for-databases
+RUN chmod +x /usr/src/geonode/tasks.py \
+    && chmod +x /usr/src/geonode/entrypoint.sh
+
+COPY celery.sh /usr/bin/celery-commands
+RUN chmod +x /usr/bin/celery-commands
+
+# Prepraing dependencies
+RUN apt-get update && apt-get install -y devscripts build-essential debhelper pkg-kde-tools sharutils
+# RUN git clone https://salsa.debian.org/debian-gis-team/proj.git /tmp/proj
+# RUN cd /tmp/proj && debuild -i -us -uc -b && dpkg -i ../*.deb
+
+# Install pip packages
 RUN pip install pip --upgrade
-RUN pip install -r requirements.txt --upgrade
+RUN pip install --upgrade --no-cache-dir --use-feature=2020-resolver --src /usr/src -r requirements.txt \
+    && pip install pygdal==$(gdal-config --version).* \
+    && pip install flower==0.9.4
 
-WORKDIR /usr/src/geonode/
-RUN git fetch --all --prune && \
-    git reset --hard origin/master && \
-    git pull
-RUN pip install -e . --upgrade
+RUN pip install --upgrade --use-feature=2020-resolver -e .
 
 # Activate "memcached"
-RUN apt install -y cron memcached
+RUN apt install memcached
 RUN pip install pylibmc \
     && pip install sherlock
 
-RUN python manage.py makemigrations --settings=geonode.settings
-RUN python manage.py migrate --settings=geonode.settings
+# Install "geonode-contribs" apps
+RUN cd /usr/src; git clone https://github.com/GeoNode/geonode-contribs.git -b master
+# Install logstash and centralized dashboard dependencies
+RUN cd /usr/src/geonode-contribs/geonode-logstash; pip install --upgrade --use-feature=2020-resolver -e . \
+    cd /usr/src/geonode-contribs/ldap; pip install --upgrade --use-feature=2020-resolver -e .
 
 # Export ports
 EXPOSE 8000

diff --git a/celery.sh b/celery.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+nohup celery -A geonode.celery_app:app beat -l DEBUG -f /var/log/celery.log &>/dev/null &
+nohup celery -A geonode.celery_app:app worker -B -E --statedb=worker.state -s celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log &>/dev/null &
+nohup celery -A geonode.celery_app:app flower --auto_refresh=True --debug=False --broker=${BROKER_URL} --basic_auth=${ADMIN_USERNAME}:${ADMIN_PASSWORD} --address=0.0.0.0 --port=5555 &>/dev/null &
diff --git a/docker-compose-geoserver-server.yml b/docker-compose-geoserver-server.yml
@@ -26,7 +26,7 @@ services:
     volumes:
       - geoserver-data-dir:/geoserver_data/data
     env_file:
-      - ./scripts/docker/env/${SET_DOCKER_ENV}/geoserver.env
+      - ./scripts/docker/env/${DOCKER_ENV}/geoserver.env
     ports:
       - "${GEOSERVER_SERVER_PORT}:8080"
     network_mode: "bridge"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -9,8 +9,10 @@ x-common-django:
   build:
     context: ./
     dockerfile: Dockerfile
+  env_file:
+    - .env
   volumes:
-    - '.:/usr/src/app'
+    # - '.:/usr/src/geonode'
     - statics:/mnt/volumes/statics
     - geoserver-data-dir:/geoserver_data/data
     - backup-restore:/backup_restore
@@ -28,21 +30,19 @@ services:
       timeout: 10s
       retries: 1
       start_period: 60s
+    environment:
+      - IS_CELERY=False
     entrypoint: ["/usr/src/geonode/entrypoint.sh"]
-    command: "uwsgi --ini /usr/src/app/uwsgi.ini"
-    env_file:
-      - .env
-      - ./scripts/docker/env/${SET_DOCKER_ENV}/django.env
+    command: "uwsgi --ini /usr/src/geonode/uwsgi.ini"
 
   # Celery worker that executes celery tasks created by Django.
   celery:
     << : *default-common-django
     container_name: celery4${COMPOSE_PROJECT_NAME}
+    environment:
+      - IS_CELERY=True
     entrypoint: ["/usr/src/geonode/entrypoint.sh"]
     command: "celery -A geonode.celery_app:app worker -B -E --statedb=/mnt/volumes/statics/worker.state -s /mnt/volumes/statics/celerybeat-schedule --loglevel=INFO --concurrency=10 -n worker1@%h -f /var/log/celery.log"
-    env_file:
-      - .env
-      - ./scripts/docker/env/${SET_DOCKER_ENV}/celery.env
 
   # Nginx is serving django static and media files and proxies to django and geonode
   geonode:
@@ -90,7 +90,7 @@ services:
       retries: 1
       start_period: 60s
     env_file:
-      - ./scripts/docker/env/${SET_DOCKER_ENV}/geoserver.env
+      - ./scripts/docker/env/${DOCKER_ENV}/geoserver.env
     volumes:
       - statics:/mnt/volumes/statics
       - geoserver-data-dir:/geoserver_data/data
@@ -112,7 +112,7 @@ services:
     image: geonode/postgis:latest
     container_name: db4${COMPOSE_PROJECT_NAME}
     env_file:
-      - ./scripts/docker/env/${SET_DOCKER_ENV}/db.env
+      - ./scripts/docker/env/${DOCKER_ENV}/db.env
     volumes:
       - dbdata:/var/lib/postgresql/data
       - dbbackups:/pg_backups

diff --git a/entrypoint.sh b/entrypoint.sh
@@ -1,11 +1,18 @@
 #!/bin/bash
+
+# Exit script in case of error
 set -e
 
 # Start cron && memcached services
 service cron restart
 service memcached restart
 
-/usr/local/bin/invoke update >> /usr/src/app/invoke.log
+echo $"\n\n\n"
+echo "-----------------------------------------------------"
+echo "STARTING DJANGO ENTRYPOINT $(date)"
+echo "-----------------------------------------------------"
+
+/usr/local/bin/invoke update > /usr/src/geonode/invoke.log 2>&1
 
 source $HOME/.bashrc
 source $HOME/.override_env
@@ -23,11 +30,11 @@ echo MONITORING_HOST_NAME=$MONITORING_HOST_NAME
 echo MONITORING_SERVICE_NAME=$MONITORING_SERVICE_NAME
 echo MONITORING_DATA_TTL=$MONITORING_DATA_TTL
 
-/usr/local/bin/invoke waitfordbs >> /usr/src/app/invoke.log
+/usr/local/bin/invoke waitfordbs > /usr/src/geonode/invoke.log 2>&1
 echo "waitfordbs task done"
 
 echo "running migrations"
-/usr/local/bin/invoke migrations >> /usr/src/app/invoke.log
+/usr/local/bin/invoke migrations > /usr/src/geonode/invoke.log 2>&1
 echo "migrations task done"
 
 cmd="$@"
@@ -37,63 +44,66 @@ echo DOCKER_ENV=$DOCKER_ENV
 if [ -z ${DOCKER_ENV} ] || [ ${DOCKER_ENV} = "development" ]
 then
 
-    /usr/local/bin/invoke prepare
+    /usr/local/bin/invoke prepare > /usr/src/geonode/invoke.log 2>&1
     echo "prepare task done"
-    /usr/local/bin/invoke fixtures
+    /usr/local/bin/invoke fixtures > /usr/src/geonode/invoke.log 2>&1
     echo "fixture task done"
 
     if [ ${IS_CELERY} = "true" ] || [ ${IS_CELERY} = "True" ]
     then
 
-        cmd=$cmd
         echo "Executing Celery server $cmd for Development"
 
     else
 
         echo "install requirements for development"
-        /usr/local/bin/invoke devrequirements
+        /usr/local/bin/invoke devrequirements > /usr/src/geonode/invoke.log 2>&1
         echo "refresh static data"
-        /usr/local/bin/invoke statics
+        /usr/local/bin/invoke statics > /usr/src/geonode/invoke.log 2>&1
         echo "static data refreshed"
-        cmd=$cmd
+
         echo "Executing standard Django server $cmd for Development"
 
     fi
 
 else
     if [ ${IS_CELERY} = "true" ]  || [ ${IS_CELERY} = "True" ]
     then
-        cmd=$CELERY_CMD
         echo "Executing Celery server $cmd for Production"
     else
 
-        /usr/local/bin/invoke prepare >> /usr/src/app/invoke.log
+        /usr/local/bin/invoke prepare > /usr/src/geonode/invoke.log 2>&1
         echo "prepare task done"
 
         if [ ! -e "/mnt/volumes/statics/geonode_init.lock" ]; then
-            /usr/local/bin/invoke updategeoip >> /usr/src/app/invoke.log
+            /usr/local/bin/invoke updategeoip > /usr/src/geonode/invoke.log 2>&1
             echo "updategeoip task done"
-            /usr/local/bin/invoke fixtures >> /usr/src/app/invoke.log
+            /usr/local/bin/invoke fixtures > /usr/src/geonode/invoke.log 2>&1
             echo "fixture task done"
-            /usr/local/bin/invoke monitoringfixture >> /usr/src/app/invoke.log
+            /usr/local/bin/invoke monitoringfixture > /usr/src/geonode/invoke.log 2>&1
             echo "monitoringfixture task done"
-            /usr/local/bin/invoke initialized >> /usr/src/app/invoke.log
+            /usr/local/bin/invoke initialized > /usr/src/geonode/invoke.log 2>&1
             echo "initialized"
         fi
 
         echo "refresh static data"
-        /usr/local/bin/invoke statics >> /usr/src/app/invoke.log
+        /usr/local/bin/invoke statics > /usr/src/geonode/invoke.log 2>&1
         echo "static data refreshed"
-        /usr/local/bin/invoke waitforgeoserver >> /usr/src/app/invoke.log
+        /usr/local/bin/invoke waitforgeoserver > /usr/src/geonode/invoke.log 2>&1
         echo "waitforgeoserver task done"
-        /usr/local/bin/invoke geoserverfixture >> /usr/src/app/invoke.log
+        /usr/local/bin/invoke geoserverfixture > /usr/src/geonode/invoke.log 2>&1
         echo "geoserverfixture task done"
-        /usr/local/bin/invoke updateadmin >> /usr/src/app/invoke.log
+        /usr/local/bin/invoke updateadmin > /usr/src/geonode/invoke.log 2>&1
         echo "updateadmin task done"
 
-        cmd=$UWSGI_CMD
         echo "Executing UWSGI server $cmd for Production"
     fi
 fi
+
+echo "-----------------------------------------------------"
+echo "FINISHED DJANGO ENTRYPOINT --------------------------"
+echo "-----------------------------------------------------"
+
+# Run the CMD 
 echo "got command $cmd"
 exec $cmd