From 1c7632bb0e4940513d0f535fc396ea5783ad3c2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Abarz=C3=BAa?= Date: Mon, 25 Mar 2019 16:10:01 -0300 Subject: [PATCH] updates to stable version --- .gitignore | 2 - cache/.gitignore | 3 -- certbot/cli.ini | 2 +- certbot/{make-certs.sh => fake-certs.sh} | 0 certbot/first-run.sh | 17 ------- certbot/real-certs.sh | 15 ++++++ db/init.d/01-init.sh | 14 +++++- db/init.d/README.md | 16 +++++++ db/init.d/datachile_dump.custom.example | 8 ---- docker-compose.yml | 48 +++++++++---------- init.sh | 32 ++++++++++--- nginx/hosts/01-cache.conf | 4 ++ nginx/hosts/01-logs.conf | 2 + nginx/hosts/50-chilecube.conf | 25 +++++++--- nginx/hosts/50-datachile.conf | 36 +++++++------- nginx/hosts/50-static.datachile.conf | 4 +- nginx/snippets/acme.conf | 2 +- .../snippets/ssl-chilecube.datawheel.us.conf | 2 - nginx/snippets/ssl-datachile.io.conf | 2 - nginx/ssl/prod.datachile.io | 2 + 20 files changed, 143 insertions(+), 93 deletions(-) delete mode 100644 cache/.gitignore rename certbot/{make-certs.sh => fake-certs.sh} (100%) delete mode 100644 certbot/first-run.sh create mode 100644 certbot/real-certs.sh create mode 100644 db/init.d/README.md delete mode 100644 db/init.d/datachile_dump.custom.example create mode 100644 nginx/hosts/01-logs.conf delete mode 100644 nginx/snippets/ssl-chilecube.datawheel.us.conf delete mode 100644 nginx/snippets/ssl-datachile.io.conf create mode 100644 nginx/ssl/prod.datachile.io diff --git a/.gitignore b/.gitignore index b8fed55..737c7a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1 @@ -storage/ - **/*.pem diff --git a/cache/.gitignore b/cache/.gitignore deleted file mode 100644 index 62a41ec..0000000 --- a/cache/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# This file keeps the folder available in git. -# The folder stores the cached elements by nginx's reverse proxy. -# Do not delete this file. diff --git a/certbot/cli.ini b/certbot/cli.ini index f4e0d47..8df4a7a 100644 --- a/certbot/cli.ini +++ b/certbot/cli.ini @@ -18,4 +18,4 @@ max-log-backups = 10 # Uncomment to use the webroot authenticator. Replace webroot-path with the # path to the public_html / webroot folder being served by your web server. authenticator = webroot -webroot-path = /tmp/certbotacme +webroot-path = /tmp/certbot-acme diff --git a/certbot/make-certs.sh b/certbot/fake-certs.sh similarity index 100% rename from certbot/make-certs.sh rename to certbot/fake-certs.sh diff --git a/certbot/first-run.sh b/certbot/first-run.sh deleted file mode 100644 index a54f8a7..0000000 --- a/certbot/first-run.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -rm -rf "/etc/letsencrypt/$1" - -comm="certbot certonly --webroot " -comm+="--dry-run " -comm+="--agree-tos " -comm+="-m $EMAIL " -comm+="--cert-name $1 " -comm+="-w /tmp/certbotacme " - -for domain in "$@" -do - comm+="-d $domain " -done - -eval $comm \ No newline at end of file diff --git a/certbot/real-certs.sh b/certbot/real-certs.sh new file mode 100644 index 0000000..d8b3e26 --- /dev/null +++ b/certbot/real-certs.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +rm -rf "/etc/letsencrypt/live/$1" + +comm="certbot certonly --webroot" +comm="$comm --agree-tos -m $EMAIL" +comm="$comm --cert-name $1" +comm="$comm -w /tmp/certbot-acme" + +for domain in "$@" +do + comm="$comm -d $domain" +done + +eval $comm diff --git a/db/init.d/01-init.sh b/db/init.d/01-init.sh index fa0aa7f..c10541e 100644 --- a/db/init.d/01-init.sh +++ b/db/init.d/01-init.sh @@ -3,14 +3,24 @@ # exit script if any command exits with a non-zero status set -e +# we use `--dbname "$POSTGRES_DB"` to prevent blockings + # create the `datachile` user psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL CREATE USER datachile WITH PASSWORD 'please remember to put a password here'; EOSQL -# insert the backup `datachile_dump.custom` file +# insert the backup `dump.custom` file +# we cannot use a *.sql file here directly, or postgres will use it # see example to get the command to get this file from another production server -pg_restore -v --username "$POSTGRES_USER" -C -d postgres /docker-entrypoint-initdb.d/datachile_dump.custom +if [ -f /docker-entrypoint-initdb.d/dump.sql.custom ]; then + echo "Using a dump.sql.custom file" + psql -v --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < /docker-entrypoint-initdb.d/dump.sql.custom +elif [ -f /docker-entrypoint-initdb.d/dump.custom ]; then + pg_restore -v -C --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" /docker-entrypoint-initdb.d/dump.custom +else + echo "No dump file present." +fi # grant privileges on the `datachile` database to the `datachile` user psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL diff --git a/db/init.d/README.md b/db/init.d/README.md new file mode 100644 index 0000000..d0fe333 --- /dev/null +++ b/db/init.d/README.md @@ -0,0 +1,16 @@ +The 01-init.sh script was made to be used with a dump obtained with this command: + +```bash +pg_dump [connection params] --create --clean --no-acl --no-owner --file=/full/path/datachile_dump.custom datachile +``` + +The `--file` parameter is the full path to where the data will be saved. +The output file must be placed in this folder before running `docker-compose`. + +Alternatively, if there are no conflicts between usernames and permissions, you can export the whole database in a custom compressed format, made by postgres, using this command: + +```bash +pg_dump [connection params] --format=c --file=/full/path/datachile_dump.custom datachile +``` + +This command ignores all the other flags and creates an exact copy of the database. It's useful to have a backup of the production database too. diff --git a/db/init.d/datachile_dump.custom.example b/db/init.d/datachile_dump.custom.example deleted file mode 100644 index 498d7c0..0000000 --- a/db/init.d/datachile_dump.custom.example +++ /dev/null @@ -1,8 +0,0 @@ -The 01-init.sh script was made to be used with a dump obtained with this command: - -```bash -pg_dump [connection params] --create --clean --format=c --no-acl --no-owner --file=/full/path/datachile_dump.custom datachile -``` - -The `--file` parameter is the full path to where the data will be saved. -The output file must be placed in this folder before running `docker-compose`. \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 76f72cc..297f19e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,15 +10,14 @@ networks: default: services: - # certbot in docker is managed by the system's crontab, use these parameters certbot: image: certbot/certbot:latest volumes: - hddletsencrypt:/etc/letsencrypt - - hddacme:/tmp/certbotacme + - hddacme:/tmp/certbot-acme - ./certbot/cli.ini:/etc/letsencrypt/cli.ini - - ./certbot/first-run.sh:/first-run.sh - - ./certbot/make-certs.sh:/make-certs.sh + - ./certbot/fake-certs.sh:/fake-certs.sh + - ./certbot/real-certs.sh:/real-certs.sh environment: - EMAIL=datachile@datawheel.us @@ -27,7 +26,7 @@ services: volumes: - hddrestui:/app/src/build environment: - - REACT_APP_API_URL=https://chilecube.datawheel.us + - REACT_APP_API_URL=https://chilecube.prod.datachile.io command: yarn run build db: @@ -36,9 +35,11 @@ services: - ./db/postgresql.conf:/etc/postgresql.conf - ./db/init.d:/docker-entrypoint-initdb.d - /datastore/postgres:/var/lib/postgresql/data - expose: - - "5432" - restart: always + environment: + - POSTGRES_PASSWORD="please set a password" + ports: + - "5432:5432" + restart: unless-stopped command: postgres -c config_file=/etc/postgresql.conf mondrian: @@ -50,19 +51,12 @@ services: - ./mondrian/schema.xml:/mondrian-rest/schema.xml:ro environment: - MONDRIAN_REST_CONF=/mondrian-rest/config.yaml - # networks: - # default: - # # the alias must be the same as the url.origin for the mondrian server - # aliases: - # - chilecube.datawheel.us expose: - "9292" - restart: always - # logging: - # driver: gcplogs + restart: unless-stopped canon: - build: https://github.com/datachile/datachile.git#deploy-prod + build: https://github.com/datachile/datachile.git#staging depends_on: - mondrian volumes: @@ -71,14 +65,17 @@ services: - NODE_ENV=production - APP_HOME=/datachile - ROOT=/datachile - - CANON_CONST_API=https://chilecube.datawheel.us - # - CANON_GOOGLE_ANALYTICS=UA-109603240-1 + - CANON_API=https://chilecube.prod.datachile.io + - CANON_CONST_API=https://chilecube.prod.datachile.io + - CANON_GOOGLE_ANALYTICS=UA-109603240-1 - CANON_LANGUAGE_DEFAULT=es - CANON_LANGUAGES=es,en + - CANON_LOGICLAYER_CUBE=http://mondrian:9292/ + - CANON_LOGICLAYER_LOGGING=true - CANON_PORT=4444 expose: - "4444" - restart: always + restart: unless-stopped nginx: image: nginx:latest @@ -87,15 +84,18 @@ services: - mondrian - restui volumes: - - hddacme:/tmp/certbotacme + - hddacme:/tmp/certbot-acme - hddcanon:/app/canon-static:ro - hddletsencrypt:/etc/letsencrypt - hddrestui:/app/restui + - /datastore/cache-canon:/ncache/canon + - /datastore/cache-mondrian:/ncache/mondrian - ./nginx/hosts:/etc/nginx/conf.d - ./nginx/snippets:/etc/nginx/snippets - - ./cache/canon:/ncache/canon - - ./cache/mondrian:/ncache/mondrian + - ./nginx/ssl:/etc/nginx/ssl ports: - "80:80" - "443:443" - restart: always + restart: unless-stopped + logging: + driver: gcplogs diff --git a/init.sh b/init.sh index 7ab0416..7e93bb6 100644 --- a/init.sh +++ b/init.sh @@ -1,17 +1,37 @@ # ============================================================================== -# Initialize database +# MAKE PERMANENT STORAGE FOLDERS +sudo mkdir /datastore +sudo mkdir /datastore/dumps +sudo mkdir /datastore/postgres +sudo mkdir /datastore/cache-mondrian +sudo mkdir /datastore/cache-canon + +# ============================================================================== +# INITIALIZE DATABASE # This will run the scripts inside db/init.d/ and restore the data and user. docker-compose run --rm db # ============================================================================== -# Create fake certs -# The nginx server won't run without these files, -# so we have to make a few in the meantime. +# CREATE FAKE CERTIFICATES +# The nginx server won't run without the certificates, so we have to make a few +# in the meantime. Input the root domain only. docker-compose run --rm --entrypoint sh \ - certbot /make-certs.sh datachile.io es.datachile.io en.datachile.io static.datachile.io chilecube.datawheel.us + certbot /fake-certs.sh prod.datachile.io # ============================================================================== +# CREATE CONTAINERS +# Let's make the containers this time. docker-compose up -d +# ============================================================================== +# GENERATE THE ACTUAL CERTIFICATES +# Time to run certbot. +# Input all the domains that will be handled; the first one must be +# the root domain. docker-compose run --rm --entrypoint sh \ - certbot /first-run.sh datachile.io es.datachile.io en.datachile.io static.datachile.io chilecube.datawheel.us + certbot /real-certs.sh prod.datachile.io \ + www.prod.datachile.io \ + es.prod.datachile.io \ + en.prod.datachile.io \ + chilecube.prod.datachile.io \ + static.prod.datachile.io diff --git a/nginx/hosts/01-cache.conf b/nginx/hosts/01-cache.conf index b1ed9f6..96f9404 100644 --- a/nginx/hosts/01-cache.conf +++ b/nginx/hosts/01-cache.conf @@ -5,3 +5,7 @@ proxy_cache_path /ncache/canon proxy_cache_path /ncache/mondrian levels=1:2 keys_zone=mondriancache:100m max_size=10g inactive=72h use_temp_path=off; + +log_format cached_log '$remote_addr - $upstream_cache_status [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"'; diff --git a/nginx/hosts/01-logs.conf b/nginx/hosts/01-logs.conf new file mode 100644 index 0000000..f62ce61 --- /dev/null +++ b/nginx/hosts/01-logs.conf @@ -0,0 +1,2 @@ +access_log off; +error_log off; diff --git a/nginx/hosts/50-chilecube.conf b/nginx/hosts/50-chilecube.conf index 709ecca..8282321 100644 --- a/nginx/hosts/50-chilecube.conf +++ b/nginx/hosts/50-chilecube.conf @@ -1,24 +1,37 @@ server { - server_name chilecube.datawheel.us; + server_name chilecube.prod.datachile.io; listen 443 ssl http2; include snippets/ssl.conf; - include snippets/ssl-chilecube.datawheel.us.conf; + include ssl/prod.datachile.io; include snippets/acme.conf; location /ui { - # sub_filter '' ''; - # sub_filter_once on; root /app/restui; + try_files $uri $uri/index.html; } location / { + access_log /var/log/nginx/access.log cached_log; + + # https://www.nginx.com/blog/nginx-caching-guide/ proxy_cache mondriancache; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_min_uses 2; + proxy_cache_revalidate on; + proxy_cache_use_stale error timeout updating + http_500 http_502 http_503 http_504; + proxy_cache_valid 200 302 60m; + proxy_cache_valid 404 1m; proxy_pass http://mondrian:9292; - - proxy_set_header Host $host; + + proxy_set_header Host $host; + proxy_set_header Proxy ""; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + add_header X-Proxy-Cache $upstream_cache_status; } } diff --git a/nginx/hosts/50-datachile.conf b/nginx/hosts/50-datachile.conf index 7b0588b..24206a5 100644 --- a/nginx/hosts/50-datachile.conf +++ b/nginx/hosts/50-datachile.conf @@ -4,28 +4,25 @@ map $http_accept_language $lang { } server { - server_name datachile.io www.datachile.io; + server_name prod.datachile.io www.prod.datachile.io; listen 443 ssl http2; include snippets/ssl.conf; - include snippets/ssl-datachile.io.conf; + include ssl/prod.datachile.io; include snippets/acme.conf; location / { - return 301 $scheme://$lang.datachile.io$request_uri; + return 302 https://$lang.prod.datachile.io$request_uri; } } server { - server_name en.datachile.io es.datachile.io; + server_name en.prod.datachile.io es.prod.datachile.io; listen 443 ssl http2; include snippets/ssl.conf; - include snippets/ssl-datachile.io.conf; - - access_log off; - error_log off; + include ssl/prod.datachile.io; include snippets/acme.conf; include snippets/global.conf; @@ -47,21 +44,26 @@ server { } location / { - access_log off; - error_log off; + access_log /var/log/nginx/access.log cached_log; # https://www.nginx.com/blog/nginx-caching-guide/ - proxy_cache canoncache; - proxy_cache_revalidate on; - proxy_cache_min_uses 2; - proxy_cache_use_stale error timeout updating http_500 http_502 - http_503 http_504; + proxy_cache canoncache; proxy_cache_background_update on; - proxy_cache_lock on; + proxy_cache_key "$host$request_uri"; + proxy_cache_lock on; + proxy_cache_min_uses 2; + proxy_cache_revalidate on; + proxy_cache_use_stale error timeout updating + http_500 http_502 http_503 http_504; + proxy_cache_valid 200 302 60m; + proxy_cache_valid 404 1m; proxy_pass http://canon:4444; - proxy_set_header Host $host; + proxy_set_header Host $host; + proxy_set_header Proxy ""; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + add_header X-Proxy-Cache $upstream_cache_status; } } diff --git a/nginx/hosts/50-static.datachile.conf b/nginx/hosts/50-static.datachile.conf index 28b8bd8..08499c5 100644 --- a/nginx/hosts/50-static.datachile.conf +++ b/nginx/hosts/50-static.datachile.conf @@ -1,9 +1,9 @@ server { - server_name static.datachile.io; + server_name static.prod.datachile.io; listen 443 ssl http2; include snippets/ssl.conf; - include snippets/ssl-datachile.io.conf; + include ssl/prod.datachile.io; root /app/canon-static; diff --git a/nginx/snippets/acme.conf b/nginx/snippets/acme.conf index af8e8d9..a0b47b0 100644 --- a/nginx/snippets/acme.conf +++ b/nginx/snippets/acme.conf @@ -33,7 +33,7 @@ location ^~ /.well-known/acme-challenge/ { # there to "webroot". # Do NOT use alias, use root! Target directory is located here: # /var/www/common/letsencrypt/.well-known/acme-challenge/ - root /tmp/certbotacme; + root /tmp/certbot-acme; } # Hide /acme-challenge subdirectory and return 404 on all requests. diff --git a/nginx/snippets/ssl-chilecube.datawheel.us.conf b/nginx/snippets/ssl-chilecube.datawheel.us.conf deleted file mode 100644 index 3b3cd0c..0000000 --- a/nginx/snippets/ssl-chilecube.datawheel.us.conf +++ /dev/null @@ -1,2 +0,0 @@ -ssl_certificate /etc/letsencrypt/live/chilecube.datawheel.us/fullchain.pem; -ssl_certificate_key /etc/letsencrypt/live/chilecube.datawheel.us/privkey.pem; diff --git a/nginx/snippets/ssl-datachile.io.conf b/nginx/snippets/ssl-datachile.io.conf deleted file mode 100644 index 1c3b6ff..0000000 --- a/nginx/snippets/ssl-datachile.io.conf +++ /dev/null @@ -1,2 +0,0 @@ -ssl_certificate /etc/letsencrypt/live/datachile.io/fullchain.pem; -ssl_certificate_key /etc/letsencrypt/live/datachile.io/privkey.pem; diff --git a/nginx/ssl/prod.datachile.io b/nginx/ssl/prod.datachile.io new file mode 100644 index 0000000..c95dcc3 --- /dev/null +++ b/nginx/ssl/prod.datachile.io @@ -0,0 +1,2 @@ +ssl_certificate /etc/letsencrypt/live/prod.datachile.io/fullchain.pem; +ssl_certificate_key /etc/letsencrypt/live/prod.datachile.io/privkey.pem;