Upgrade to Node 14 (#637)

* Upgrade node to v14
Update webapp babel (via manual lockfile deletion & reinstall) due to babel/babel#11216
Update webapp node-sass for Node 14 support
Update webapp element-ui due to ElemeFE/element#19389
Update graphql bcrypt to hopefully fix rebuild-on-node-version-change behavior
Update graphql knex/pg to fix knex/knex#3836
Update graphql winston to fix winstonjs/winston#1797
Update graphql ts-node-dev to fix wclr/ts-node-dev#143
Pin version of graphql graphql-binding because I remember newer versions not being compatible

Remove hacks needed for node 8 (asyncIterator polyfill, incorrect date format in ElapsedTime.spec.ts)
Updated FilterPanel snapshot. It seems the reason it was reordered was because Array.prototype.sort started using a stable sorting algorithm in Node 11, and `sortFilterKeys` in store/mutations.js sorts against the same value for most items.

* Re-enable supervisorctl autorestart

* Debug logging about node versions

* Ansible fixes

* Fix use of nvm in Ansible/Supervisor

* Update CircleCI image for Node 14

* Update graphql lockfile

* Update suggested fix for missing DB schema during provision

* Re-disable sm-cluster-autostart-daemon autorestart

* Make "sm_activate_nvm" variable to simplify nvm use

* Fix after merge

.circleci/config.yml

@@ -5,7 +5,7 @@ jobs:
 
   test-graphql:
     docker:
-      - image: metaspace2020/sm-webapp:0.10
+      - image: metaspace2020/sm-webapp:0.11
 
       - image: postgres:9.5-alpine
         environment:
@@ -48,7 +48,7 @@ jobs:
 
   generate-graphql-schema:
     docker:
-      - image: metaspace2020/sm-webapp:0.10
+      - image: metaspace2020/sm-webapp:0.11
 
       - image: postgres:9.5-alpine
         environment:
@@ -93,7 +93,7 @@ jobs:
 
   test-webapp:
     docker:
-      - image: metaspace2020/sm-webapp:0.10
+      - image: metaspace2020/sm-webapp:0.11
 
     working_directory: ~/metaspace/metaspace/webapp
     steps:
@@ -139,7 +139,7 @@ jobs:
 
   test-webapp-e2e:
     docker:
-      - image: metaspace2020/sm-webapp:0.10
+      - image: metaspace2020/sm-webapp:0.11
 
       - image: postgres:9.5-alpine
         environment:

ansible/aws/templates/all/vars.yml.template

@@ -136,7 +136,7 @@ mol_db_imports:
 #    name: "ECMDB-2018-12"
 #    version: "2018-12-13"
 
-nodejs_version: 8.*
+nodejs_version: 14.5
 
 sm_graphql_hostname: "{{ web_host }}"  # specify manually in production
 sm_graphql_home: "{{ metaspace_home }}/metaspace/graphql"
@@ -197,6 +197,7 @@ sm_webapp_features:
   advanced_ds_config: false
 
 sm_activate_venv: source {{ miniconda_prefix }}/bin/activate {{ miniconda_env.name }}
+sm_activate_nvm: source {{ ansible_env.HOME }}/.nvm/nvm.sh && nvm use
 
 spark_key_file: ~/.ssh/ubuntu_spark
 spark_version: 2.3.3-bin-hadoop2.7

ansible/roles/base/tasks/main.yml

@@ -1,12 +1,23 @@
 ---
+# NOTE: Intially the provisioned server doesn't have Python, causing Ansible to fail with an "SSH Error" for most tasks.
+# Only use "raw" tasks before the "Symlink python to python3" step
 
 - name: Run apt-get update
   become: yes
   raw: apt-get update
 
+# Include Python 2 here, even though it's not used, to prevent its later installation from overwriting the Python 3 symlink
 - name: Install python-simplejson
   become: yes
-  raw: apt-get install -y python3-simplejson
+  raw: apt-get install -y python-simplejson python3-simplejson
+
+- name: Symlink python to python3
+  become: yes
+  raw: "rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python"
+
+- name: Symlink pip to pip3
+  become: yes
+  raw: "rm -f /usr/bin/pip && ln -s /usr/bin/pip3 /usr/bin/pip"
 
 - name: Reconfigure buggy ubuntu locales
   become: yes

ansible/roles/elasticsearch/tasks/main.yml

@@ -9,12 +9,12 @@
 - name: Download ElasticSearch version={{ elasticsearch_version }}
   get_url: url=https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/{{ elasticsearch_version }}/elasticsearch-{{ elasticsearch_version }}.deb
            dest=/tmp/elasticsearch-{{ elasticsearch_version }}.deb
-  when: not "'{{ elasticsearch_version }}'.startswith('5.')"
+  when: not elasticsearch_version.startswith('5.')
 
 - name: Download ElasticSearch version={{ elasticsearch_version }}
   get_url: url=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{{ elasticsearch_version }}.deb
            dest=/tmp/elasticsearch-{{ elasticsearch_version }}.deb
-  when: "'{{ elasticsearch_version }}'.startswith('5.')"
+  when: elasticsearch_version.startswith('5.')
 
 - name: Install ElasticSearch
   become: yes

ansible/roles/nodejs/defaults/main.yml

@@ -1 +1,2 @@
-nodejs_version: 8.*
+nodejs_version: 14.5
+sm_activate_nvm: source {{ ansible_env.HOME }}/.nvm/nvm.sh && nvm use

ansible/roles/nodejs/tasks/main.yml

@@ -10,37 +10,23 @@
   become: yes
   apt: name=software-properties-common state=present
 
-- name: Download Node.js installation setup script
-  get_url:
-    url: https://deb.nodesource.com/setup_{{ nodejs_version | replace('*', 'x') }}
-    dest: /tmp/nodejs_setup.sh
-    mode: 0744
-    force: yes
-
-- name: Run NodeJS setup script
-  become: yes
-  shell: /tmp/nodejs_setup.sh
+- name: Install nvm
+  shell: >
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
+  args:
+    creates: "{{ ansible_env.HOME }}/.nvm/nvm.sh"
+
+- name: Ensure nvm installs yarn for every version
+  lineinfile:
+    path: "{{ ansible_env.HOME }}/.nvm/default-packages"
+    create: yes
+    line: yarn
+
+# This controls the default version only.
+# Individual projects can install other versions to match their own requirements.
+# NOTE: To actually use NVM in Ansible, you need to set the executable as /bin/bash and prefix the command
+# with "{{ sm_activate_nvm }} &&". Ansible doesn't support anything tidier: https://github.com/ansible/ansible/issues/4854
+- name: Install Node.js
+  shell: . ~/.nvm/nvm.sh && nvm install {{ nodejs_version }} && nvm alias default {{ nodejs_version }}
   args:
     executable: /bin/bash
-
-- name: Install NodeJS
-  become: yes
-  apt: name=nodejs state=latest
-
-- name: Add Yarn apt key
-  become: yes
-  apt_key:
-    url: https://dl.yarnpkg.com/debian/pubkey.gpg
-
-- name: Add Yarn repository
-  become: yes
-  apt_repository:
-    repo: deb https://dl.yarnpkg.com/debian/ stable main
-    state: present
-
-- name: Install Yarn apt package
-  become: yes
-  apt:
-    name: yarn
-    state: present
-    update_cache: yes

ansible/roles/rabbitmq/tasks/main.yml

@@ -2,7 +2,7 @@
 
 - name: Ensure python-software-properties is installed
   become: yes
-  apt: pkg=python-software-properties state=present
+  apt: pkg=python3-software-properties state=present
 
 - name: Add rabbitmq official apt repository
   become: yes

ansible/roles/sm_annotate_daemon/templates/sm-annotate-daemon.supervisor.j2

@@ -9,5 +9,4 @@ redirect_stderr = true
 stdout_logfile = {{ sm_home }}/logs/{{ sm_annotate_daemon_app_name }}.log
 numprocs = 1
 numprocs_start = 8000
-autorestart = false
 startsecs = 10

ansible/roles/sm_api/templates/sm-api.supervisor.j2

@@ -9,5 +9,4 @@ redirect_stderr = true
 stdout_logfile = {{ sm_home }}/logs/{{ sm_api_app_name }}.log
 numprocs = 1
 numprocs_start = 8000
-autorestart = false
 startsecs = 10

ansible/roles/sm_graphql/tasks/main.yml

@@ -1,38 +1,26 @@
 ---
 
-# TODO: Move into sm_web tasks after production has been updated, as this is a provisioning step
-- name: Create a database schema
-  become: yes
-  become_user: postgres
-  postgresql_schema:
-    database: sm
-    owner: sm
-    name: graphql
-
-# TODO: Move into sm_web tasks after production has been updated, as this is a provisioning step
-- name: Import uuid-ossp extension into sm postgres database
-  become: yes
-  become_user: postgres
-  postgresql_ext: db=sm name=uuid-ossp
+- name: Install sm-graphql's version of Node.js (if needed)
+  # This only installs a new version if there isn't a matching existing version
+  shell: "{{ sm_activate_nvm }} || nvm install"
+  args:
+    chdir: "{{ sm_graphql_home }}"
+    executable: /bin/bash
 
 - name: Install sm-graphql dependencies
-  shell: yarn install
+  shell: "{{ sm_activate_nvm }} && yarn install"
   args:
     chdir: "{{ sm_graphql_home }}"
     executable: /bin/bash
 
-- name: Re-compile and re-install nodejs binaries
-  command: npm rebuild --update-binary bcrypt
-  when: sm_graphql_update_binary | bool
-
 - name: Dereference metadata schema
-  shell: yarn run deref-schema
+  shell: "{{ sm_activate_nvm }} && yarn run deref-schema"
   args:
     chdir: "{{ sm_graphql_home }}"
     executable: /bin/bash
 
 - name: Generate binding.ts
-  shell: yarn run gen-binding
+  shell: "{{ sm_activate_nvm }} && yarn run gen-binding"
   args:
     chdir: "{{ sm_graphql_home }}"
     executable: /bin/bash

ansible/roles/sm_graphql/templates/sm-graphql.supervisor.j2

@@ -4,11 +4,10 @@
 process_name = {{ sm_graphql_app_name }}
 environment = NODE_ENV="production"
 directory = {{ sm_graphql_home }}
-command = yarn run start
+command = bash -c '. ~/.nvm/nvm.sh && nvm use && exec yarn run start'
 redirect_stderr = true
 stdout_logfile = {{ sm_graphql_home }}/logs/{{ sm_graphql_app_name }}.log
 numprocs = 1
 numprocs_start = 8000
-autorestart = false
 startsecs = 10
 stopasgroup = true

ansible/roles/sm_update_daemon/templates/sm-update-daemon.supervisor.j2

@@ -9,5 +9,4 @@ redirect_stderr = true
 stdout_logfile = {{ sm_home }}/logs/{{ sm_update_daemon_app_name }}.log
 numprocs = 1
 numprocs_start = 8000
-autorestart = false
 startsecs = 10

ansible/roles/sm_web/tasks/main.yml

@@ -41,6 +41,19 @@
   become_user: postgres
   postgresql_db: name=sm owner=sm encoding=UTF8 template=template0
 
+- name: Create a database schema
+  become: yes
+  become_user: postgres
+  postgresql_schema:
+    database: sm
+    owner: sm
+    name: graphql
+
+- name: Import uuid-ossp extension into sm postgres database
+  become: yes
+  become_user: postgres
+  postgresql_ext: db=sm name=uuid-ossp
+
 - name: Create /opt/data/metaspace directory
   file: dest=/opt/data/metaspace state=directory owner=ubuntu group=ubuntu mode=0700
   become: yes
@@ -52,6 +65,12 @@
     executable: /bin/bash
   when: sm_create_es_index
 
+
+# TODO: The DB schema for this doesn't exist yet, so this will fail.
+# WORKAROUND: Skip these steps now, run the sm-graphql TypeORM migrations, then re-provision this step
+# Suggested permanent fix: Run `engine/scripts/db_schema.sql` here. However, this will also require updating the
+# `graphql/bin/dump_db_schema.js` script to create & populate the TypeORM migrations table, so that TypeORM
+# doesn't try to run the migrations later, and conflict with `db_schema.sql`-created tables.
 - import_tasks: import_molecular_databases.yml
 
 - import_tasks: import_molecule_images.yml

ansible/roles/sm_web_app/tasks/main.yml

@@ -16,14 +16,21 @@
   file: path={{ sm_webapp_upload_destination }} state=directory mode=0755 owner=ubuntu group=ubuntu
   when: sm_webapp_upload_destination != 's3'
 
+- name: Install sm-webapp's version of Node.js (if needed)
+  # This only installs a new version if there isn't a matching existing version
+  shell: "{{ sm_activate_nvm }} || nvm install"
+  args:
+    chdir: "{{ sm_webapp_home }}"
+    executable: /bin/bash
+
 - name: Install sm-web-app dependencies
-  shell: yarn install
+  shell: "{{ sm_activate_nvm }} && yarn install"
   args:
     chdir: "{{ sm_webapp_home }}"
     executable: /bin/bash
 
 - name: Run build app
-  shell: nice yarn run build
+  shell: "{{ sm_activate_nvm }} && nice yarn run build"
   args:
     chdir: "{{ sm_webapp_home }}"
     executable: /bin/bash

ansible/roles/supervisor/tasks/main.yml

@@ -2,7 +2,7 @@
 
 - name: Install pip
   become: yes
-  apt: name=python-pip state=present
+  apt: name=python3-pip state=present
 
 - name: Install supervisor into system python site
   become: yes

docker/sm-graphql/Dockerfile

@@ -1,14 +1,15 @@
-FROM node:8.16.2
+FROM node:14.5.0-alpine3.12
 
-RUN apt-get update && apt-get install -y netcat && apt-get clean
+RUN apk add --no-cache git netcat-openbsd
 RUN yarn global add nodemon
 
 # Prefetch dependencies before getting the full source code so Docker's image cache
 # reduces the amount that needs to be downloaded to rebuild an image after a code change
-WORKDIR /opt/pkgcache
-RUN curl https://raw.githubusercontent.com/metaspace2020/metaspace/master/metaspace/graphql/package.json -o package.json
-RUN curl https://raw.githubusercontent.com/metaspace2020/metaspace/master/metaspace/graphql/yarn.lock -o yarn.lock
-RUN yarn install
+# Disabled because it's only useful for self-contained docker containers, which are no longer supported
+# WORKDIR /opt/pkgcache
+# RUN curl https://raw.githubusercontent.com/metaspace2020/metaspace/master/metaspace/graphql/package.json -o package.json
+# RUN curl https://raw.githubusercontent.com/metaspace2020/metaspace/master/metaspace/graphql/yarn.lock -o yarn.lock
+# RUN yarn install
 
 # The non-dev copy is not used anymore, so don't install it
 # ARG METASPACE_REPO=https://github.com/metaspace2020/metaspace.git

docker/sm-graphql/docker-entrypoint.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 wait_for() {
   if ! $1; then

docker/sm-webapp/Dockerfile

@@ -1,14 +1,15 @@
-FROM node:8.16.2
+FROM node:14.5.0-alpine3.12
 
-RUN apt-get update && apt-get install -y netcat && apt-get clean
+RUN apk add --no-cache git netcat-openbsd
 RUN yarn global add nodemon
 
 # Prefetch dependencies before getting the full source code so Docker's image cache
 # reduces the amount that needs to be downloaded to rebuild an image after a code change
-WORKDIR /opt/pkgcache
-RUN curl https://raw.githubusercontent.com/metaspace2020/metaspace/master/metaspace/webapp/package.json -o package.json
-RUN curl https://raw.githubusercontent.com/metaspace2020/metaspace/master/metaspace/webapp/yarn.lock -o yarn.lock
-RUN yarn install
+# Disabled because it's only useful for self-contained docker containers, which are no longer supported
+# WORKDIR /opt/pkgcache
+# RUN curl https://raw.githubusercontent.com/metaspace2020/metaspace/master/metaspace/webapp/package.json -o package.json
+# RUN curl https://raw.githubusercontent.com/metaspace2020/metaspace/master/metaspace/webapp/yarn.lock -o yarn.lock
+# RUN yarn install
 
 # The non-dev copy is not used anymore, so don't install it
 # ARG METASPACE_REPO=https://github.com/metaspace2020/metaspace.git

docker/sm-webapp/docker-entrypoint.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 wait_for() {
   if ! $1; then

metaspace/graphql/.nvmrc

@@ -0,0 +1 @@
+14.5