updates to stable version

.gitignore

@@ -1,3 +1 @@
-storage/
-
 **/*.pem

certbot/cli.ini

@@ -18,4 +18,4 @@ max-log-backups = 10
 # Uncomment to use the webroot authenticator. Replace webroot-path with the
 # path to the public_html / webroot folder being served by your web server.
 authenticator = webroot
-webroot-path = /tmp/certbotacme
+webroot-path = /tmp/certbot-acme

certbot/real-certs.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+rm -rf "/etc/letsencrypt/live/$1"
+
+comm="certbot certonly --webroot"
+comm="$comm --agree-tos -m $EMAIL"
+comm="$comm --cert-name $1"
+comm="$comm -w /tmp/certbot-acme"
+
+for domain in "$@"
+do
+    comm="$comm -d $domain"
+done
+
+eval $comm

db/init.d/01-init.sh

@@ -3,14 +3,24 @@
 # exit script if any command exits with a non-zero status
 set -e
 
+# we use `--dbname "$POSTGRES_DB"` to prevent blockings
+
 # create the `datachile` user
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
     CREATE USER datachile WITH PASSWORD 'please remember to put a password here';
 EOSQL
 
-# insert the backup `datachile_dump.custom` file
+# insert the backup `dump.custom` file
+# we cannot use a *.sql file here directly, or postgres will use it
 # see example to get the command to get this file from another production server
-pg_restore -v --username "$POSTGRES_USER" -C -d postgres /docker-entrypoint-initdb.d/datachile_dump.custom
+if [ -f /docker-entrypoint-initdb.d/dump.sql.custom ]; then
+    echo "Using a dump.sql.custom file"
+    psql -v --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < /docker-entrypoint-initdb.d/dump.sql.custom
+elif [ -f /docker-entrypoint-initdb.d/dump.custom ]; then
+    pg_restore -v -C --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" /docker-entrypoint-initdb.d/dump.custom
+else
+   echo "No dump file present."
+fi
 
 # grant privileges on the `datachile` database to the `datachile` user
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL

db/init.d/README.md

@@ -0,0 +1,16 @@
+The 01-init.sh script was made to be used with a dump obtained with this command:
+
+```bash
+pg_dump [connection params] --create --clean --no-acl --no-owner --file=/full/path/datachile_dump.custom datachile
+```
+
+The `--file` parameter is the full path to where the data will be saved.
+The output file must be placed in this folder before running `docker-compose`.
+
+Alternatively, if there are no conflicts between usernames and permissions, you can export the whole database in a custom compressed format, made by postgres, using this command:
+
+```bash
+pg_dump [connection params] --format=c --file=/full/path/datachile_dump.custom datachile
+```
+
+This command ignores all the other flags and creates an exact copy of the database. It's useful to have a backup of the production database too.

docker-compose.yml

@@ -10,15 +10,14 @@ networks:
   default:
 
 services:
-  # certbot in docker is managed by the system's crontab, use these parameters
   certbot:
     image: certbot/certbot:latest
     volumes:
       - hddletsencrypt:/etc/letsencrypt
-      - hddacme:/tmp/certbotacme
+      - hddacme:/tmp/certbot-acme
       - ./certbot/cli.ini:/etc/letsencrypt/cli.ini
-      - ./certbot/first-run.sh:/first-run.sh
-      - ./certbot/make-certs.sh:/make-certs.sh
+      - ./certbot/fake-certs.sh:/fake-certs.sh
+      - ./certbot/real-certs.sh:/real-certs.sh
     environment:
       - EMAIL=datachile@datawheel.us
 
@@ -27,7 +26,7 @@ services:
     volumes:
       - hddrestui:/app/src/build
     environment:
-      - REACT_APP_API_URL=https://chilecube.datawheel.us
+      - REACT_APP_API_URL=https://chilecube.prod.datachile.io
     command: yarn run build
 
   db:
@@ -36,9 +35,11 @@ services:
       - ./db/postgresql.conf:/etc/postgresql.conf
       - ./db/init.d:/docker-entrypoint-initdb.d
       - /datastore/postgres:/var/lib/postgresql/data
-    expose:
-      - "5432"
-    restart: always
+    environment:
+      - POSTGRES_PASSWORD="please set a password"
+    ports:
+      - "5432:5432"
+    restart: unless-stopped
     command: postgres -c config_file=/etc/postgresql.conf
 
   mondrian:
@@ -50,19 +51,12 @@ services:
       - ./mondrian/schema.xml:/mondrian-rest/schema.xml:ro
     environment:
       - MONDRIAN_REST_CONF=/mondrian-rest/config.yaml
-    # networks:
-    #   default:
-    #     # the alias must be the same as the url.origin for the mondrian server
-    #     aliases:
-    #       - chilecube.datawheel.us
     expose:
       - "9292"
-    restart: always
-    # logging:
-    #   driver: gcplogs
+    restart: unless-stopped
 
   canon:
-    build: https://github.com/datachile/datachile.git#deploy-prod
+    build: https://github.com/datachile/datachile.git#staging
     depends_on:
       - mondrian
     volumes:
@@ -71,14 +65,17 @@ services:
       - NODE_ENV=production
       - APP_HOME=/datachile
       - ROOT=/datachile
-      - CANON_CONST_API=https://chilecube.datawheel.us
-      # - CANON_GOOGLE_ANALYTICS=UA-109603240-1
+      - CANON_API=https://chilecube.prod.datachile.io
+      - CANON_CONST_API=https://chilecube.prod.datachile.io
+      - CANON_GOOGLE_ANALYTICS=UA-109603240-1
       - CANON_LANGUAGE_DEFAULT=es
       - CANON_LANGUAGES=es,en
+      - CANON_LOGICLAYER_CUBE=http://mondrian:9292/
+      - CANON_LOGICLAYER_LOGGING=true
       - CANON_PORT=4444
     expose:
       - "4444"
-    restart: always
+    restart: unless-stopped
 
   nginx:
     image: nginx:latest
@@ -87,15 +84,18 @@ services:
       - mondrian
       - restui
     volumes:
-      - hddacme:/tmp/certbotacme
+      - hddacme:/tmp/certbot-acme
       - hddcanon:/app/canon-static:ro
       - hddletsencrypt:/etc/letsencrypt
       - hddrestui:/app/restui
+      - /datastore/cache-canon:/ncache/canon
+      - /datastore/cache-mondrian:/ncache/mondrian
       - ./nginx/hosts:/etc/nginx/conf.d
       - ./nginx/snippets:/etc/nginx/snippets
-      - ./cache/canon:/ncache/canon
-      - ./cache/mondrian:/ncache/mondrian
+      - ./nginx/ssl:/etc/nginx/ssl
     ports:
       - "80:80"
       - "443:443"
-    restart: always
+    restart: unless-stopped
+    logging:
+      driver: gcplogs

init.sh

@@ -1,17 +1,37 @@
 # ==============================================================================
-# Initialize database
+# MAKE PERMANENT STORAGE FOLDERS
+sudo mkdir /datastore
+sudo mkdir /datastore/dumps
+sudo mkdir /datastore/postgres
+sudo mkdir /datastore/cache-mondrian
+sudo mkdir /datastore/cache-canon
+
+# ==============================================================================
+# INITIALIZE DATABASE
 # This will run the scripts inside db/init.d/ and restore the data and user.
 docker-compose run --rm db
 
 # ==============================================================================
-# Create fake certs
-# The nginx server won't run without these files,
-# so we have to make a few in the meantime.
+# CREATE FAKE CERTIFICATES
+# The nginx server won't run without the certificates, so we have to make a few
+# in the meantime. Input the root domain only.
 docker-compose run --rm --entrypoint sh \
-    certbot /make-certs.sh datachile.io es.datachile.io en.datachile.io static.datachile.io chilecube.datawheel.us
+    certbot /fake-certs.sh prod.datachile.io
 
 # ==============================================================================
+# CREATE CONTAINERS
+# Let's make the containers this time.
 docker-compose up -d
 
+# ==============================================================================
+# GENERATE THE ACTUAL CERTIFICATES
+# Time to run certbot.
+# Input all the domains that will be handled; the first one must be 
+# the root domain.
 docker-compose run --rm --entrypoint sh \
-    certbot /first-run.sh datachile.io es.datachile.io en.datachile.io static.datachile.io chilecube.datawheel.us
+    certbot /real-certs.sh prod.datachile.io \
+        www.prod.datachile.io \
+        es.prod.datachile.io \
+        en.prod.datachile.io \
+        chilecube.prod.datachile.io \
+        static.prod.datachile.io

nginx/hosts/01-cache.conf

@@ -5,3 +5,7 @@ proxy_cache_path /ncache/canon
 proxy_cache_path /ncache/mondrian
                  levels=1:2 keys_zone=mondriancache:100m
                  max_size=10g inactive=72h use_temp_path=off;
+
+log_format cached_log  '$remote_addr - $upstream_cache_status [$time_local] '
+                       '"$request" $status $body_bytes_sent '
+                       '"$http_referer" "$http_user_agent"';

nginx/hosts/01-logs.conf

@@ -0,0 +1,2 @@
+access_log  off;
+error_log   off;

nginx/hosts/50-chilecube.conf

@@ -1,24 +1,37 @@
 server {
-    server_name  chilecube.datawheel.us;
+    server_name  chilecube.prod.datachile.io;
     listen       443 ssl http2;
 
     include snippets/ssl.conf;
-    include snippets/ssl-chilecube.datawheel.us.conf;
+    include ssl/prod.datachile.io;
 
     include snippets/acme.conf;
 
     location /ui {
-        # sub_filter '<head>' '<head><script>window.__API_ENDPOINT__ = "https://chilecube.datawheel.us"; window.__SITE_TITLE__ = "DataChile"; </script>';
-        # sub_filter_once on;
         root /app/restui;
+        try_files $uri $uri/index.html;
     }
 
     location / {
+        access_log /var/log/nginx/access.log cached_log;
+
+        # https://www.nginx.com/blog/nginx-caching-guide/
         proxy_cache mondriancache;
+        proxy_cache_background_update on;
+        proxy_cache_lock       on;
+        proxy_cache_min_uses   2;
+        proxy_cache_revalidate on;
+        proxy_cache_use_stale  error timeout updating
+                               http_500 http_502 http_503 http_504;
+        proxy_cache_valid      200 302  60m;
+        proxy_cache_valid      404      1m;
 
         proxy_pass http://mondrian:9292;
-
-        proxy_set_header Host $host;
+
+        proxy_set_header Host            $host;
+        proxy_set_header Proxy           "";
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+        add_header         X-Proxy-Cache   $upstream_cache_status;
     }
 }

nginx/hosts/50-datachile.conf

@@ -4,28 +4,25 @@ map $http_accept_language $lang {
 }
 
 server {
-    server_name  datachile.io www.datachile.io;
+    server_name  prod.datachile.io www.prod.datachile.io;
     listen       443 ssl http2;
 
     include snippets/ssl.conf;
-    include snippets/ssl-datachile.io.conf;
+    include ssl/prod.datachile.io;
 
     include snippets/acme.conf;
 
     location / {
-        return 301 $scheme://$lang.datachile.io$request_uri;
+        return 302 https://$lang.prod.datachile.io$request_uri;
     }
 }
 
 server {
-    server_name  en.datachile.io es.datachile.io;
+    server_name  en.prod.datachile.io es.prod.datachile.io;
     listen       443 ssl http2;
 
     include snippets/ssl.conf;
-    include snippets/ssl-datachile.io.conf;
-
-    access_log  off;
-    error_log   off;
+    include ssl/prod.datachile.io;
 
     include snippets/acme.conf;
     include snippets/global.conf;
@@ -47,21 +44,26 @@ server {
     }
 
     location / {
-        access_log  off;
-        error_log   off;
+        access_log /var/log/nginx/access.log cached_log;
 
         # https://www.nginx.com/blog/nginx-caching-guide/
-        proxy_cache canoncache;
-        proxy_cache_revalidate on;
-        proxy_cache_min_uses 2;
-        proxy_cache_use_stale error timeout updating http_500 http_502
-                              http_503 http_504;
+        proxy_cache            canoncache;
         proxy_cache_background_update on;
-        proxy_cache_lock on;
+        proxy_cache_key        "$host$request_uri";
+        proxy_cache_lock       on;
+        proxy_cache_min_uses   2;
+        proxy_cache_revalidate on;
+        proxy_cache_use_stale  error timeout updating
+                               http_500 http_502 http_503 http_504;
+        proxy_cache_valid      200 302  60m;
+        proxy_cache_valid      404      1m;
 
         proxy_pass http://canon:4444;
 
-        proxy_set_header Host $host;
+        proxy_set_header Host            $host;
+        proxy_set_header Proxy           "";
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+        add_header         X-Proxy-Cache   $upstream_cache_status;
     }
 }

nginx/hosts/50-static.datachile.conf

@@ -1,9 +1,9 @@
 server {
-    server_name  static.datachile.io;
+    server_name  static.prod.datachile.io;
     listen       443 ssl http2;
 
     include snippets/ssl.conf;
-    include snippets/ssl-datachile.io.conf;
+    include ssl/prod.datachile.io;
 
     root   /app/canon-static;