From c0c4ed37443118d89ca43aa4b1de8fc3aeb7260f Mon Sep 17 00:00:00 2001 From: Steven Sheehy <17552371+steven-sheehy@users.noreply.github.com> Date: Thu, 3 Feb 2022 16:35:00 -0600 Subject: [PATCH] Add auto restore functionality to Rosetta unified image (#3253) * Add auto restore functionality to Rosetta unified image * Add documentation for running unified image * Change Rosetta default start date to epoch * Fix documentation bug with wrong units for checkpoint_timeout Signed-off-by: Steven Sheehy Signed-off-by: Matheus DallRosa --- docs/database.md | 13 ++- docs/rosetta/README.md | 110 ++++++++++++++++++ hedera-mirror-rosetta/build/Dockerfile | 2 +- .../build/postgresql-restore.conf | 8 ++ .../build/run_supervisord.sh | 61 +++++++++- hedera-mirror-rosetta/build/supervisord.conf | 3 +- 6 files changed, 183 insertions(+), 14 deletions(-) create mode 100644 hedera-mirror-rosetta/build/postgresql-restore.conf diff --git a/docs/database.md b/docs/database.md index 0cb1f8d1426..d828bbd641e 100644 --- a/docs/database.md +++ b/docs/database.md @@ -52,7 +52,7 @@ required database objects including the `mirror_node` database, the roles, the s The following configuration needs to be applied to the database instance to improve the write speed. ``` -checkpoint_timeout = 30m +checkpoint_timeout = 30min maintenance_work_mem = 2GB max_parallel_maintenance_workers = 4 max_wal_size = 512GB @@ -60,9 +60,10 @@ temp_file_limit = 2147483647kB ``` Note: - - Not all flags are available in managed database services. For example, `max_parallel_maintenance_workers` is not - available in Google Cloud SQL. - - Once the data is restored, revert the values back for normal operation. + +- Not all flags are available in managed database services. For example, `max_parallel_maintenance_workers` is not + available in Google Cloud SQL. +- Once the data is restored, revert the values back for normal operation. ### Restore @@ -82,5 +83,5 @@ pg_restore -h $NEW_POSTGRESQL_DB_IP -U mirror_node \ ``` Note: `-j` works the same way as for `pg_dump`. The single transaction mode can't be used together with the parallel -mode. As a result, if the command is interrupted, the database will have partial data, and it needs to be restored -using the saved snapshot before retry. +mode. As a result, if the command is interrupted, the database will have partial data, and it needs to be restored using +the saved snapshot before retry. diff --git a/docs/rosetta/README.md b/docs/rosetta/README.md index 438d97754e6..ea45b463b99 100644 --- a/docs/rosetta/README.md +++ b/docs/rosetta/README.md @@ -37,6 +37,116 @@ client. These are structures coming out of the box with rosetta-sdk-go. These handle the raw requests, marshaling/unmarshaling the data, and triggering the business logic services. +## Getting Started + +### Hedera Managed Endpoints + +Hedera runs a number of Rosetta API endpoints that can be used in lieu of running your own Rosetta server: + +Mainnet: https://mainnet-public.mirrornode.hedera.com/rosetta + +Testnet: https://testnet.mirrornode.hedera.com/rosetta + +Previewnet: https://previewnet.mirrornode.hedera.com/rosetta + +In order to run construction API tests, you'll need a Hedera testnet or previewnet account. You can follow the +[account creation guide](https://help.hedera.com/hc/en-us/articles/360000664678-How-do-I-create-an-account-on-the-Hedera-testnet-) +to get an account. Note you can also create a previewnet account at the [Hedera Portal](https://portal.hedera.com) by +selecting 'Previewnet' from the dropdown menu. + +### Running Locally + +The recommended way to run Rosetta locally is to use the all-in-one docker image. Below are the steps to build the image +for a Hedera mirror node release. Please replace the example release `v0.49.1` with the latest release from +our [releases page](https://github.com/hashgraph/hedera-mirror-node/releases). + +1. Download the [Dockerfile](/hedera-mirror-rosetta/build/Dockerfile). + +2. Run `docker build --build-arg GIT_REF=v0.49.1 -t hedera-mirror-rosetta:0.49.1 .` + +Configure and run the server in online mode: + +1. Follow the [guide](https://docs.hedera.com/guides/mirrornet/run-your-own-beta-mirror-node) to configure requester + pays for Hedera testnet. + +2. Set the desired configuration for both the [Importer](/docs/configuration.md#importer) + and [Rosetta API](/docs/configuration.md#rosetta-api) in a new `application.yml` file to be mounted to the container. + Alternatively, every property can be also be set via corresponding environment variables that can be passed to the + container. For example, the YAML property `hedera.mirror.importer.startDate` can be set + as `-e HEDERA_MIRROR_IMPORTER_STARTDATE=1970-01-01T01:01:00Z`. + +3. Set `hedera.mirror.importer.startDate` to 15 minutes before UTC now and zero the seconds, for example, if UTC now is + `2021-12-06T15:25:20Z`, startDate should set to `2021-12-06T15:10:00Z`. Setting `startDate` properly can make the + importer get the genesis account balance file faster. + +4. To automatically restore the embedded PostgreSQL database to a particular database snapshot, pass a URL to a backup + file via `-e RESTORE=https://example.com/db.tar`. The database dump should be in the format specified in + the [backup](/docs/database.md#backup) section of the database upgrade documentation. The container database should + be empty otherwise the restore process will be skipped. + +5. Run the server from the all-in-one docker image with the appropriate `NETWORK` specified: + +```shell +docker run -d -e MODE=online -e NETWORK=testnet \ +-v ./application.yml:/app/importer/application.yml \ +-p 5432:5432 -p 5700:5700 hedera-mirror-rosetta:0.49.1 +``` + +The server should be reachable at http://localhost:5700. Note the server can also run in offline mode by +passing `-e MODE=offline`. + +Before running any tests, we need to make sure the server has ingested the genesis balance file and the genesis block. +This can be done using the shell +script [wait-for-mirror-node.sh](/hedera-mirror-rosetta/scripts/wait-for-mirror-node.sh). The script will report that +mirror node syncing has started when the genesis information is available. + +In order to run the rosetta-cli `check:data` command, run the +[script](/hedera-mirror-rosetta/scripts/validation/get-genesis-balance.sh) with the associated +[configuration file](/hedera-mirror-rosetta/scripts/validation/testnet/validation.json) to get the genesis account +balance file. Once the `get-genesis-balance.sh testnet` command is executed, it'll write the file +to `testnet/data_genesis_balances.json`. Note the script uses PostgreSQL's command line client psql to query the +database for genesis account balance information, so please install psql beforehand. + +In order to run the rosetta-cli `check:construction` command with the DSL spec in `testnet`/`testnet.ros`, you need two +testnet accounts with the private keys and set `prefunded_accounts` in `testnet/validation.json` as follows: + +```json +{ + "construction": { + "prefunded_accounts": [ + { + "privkey": "key1", + "account_identifier": { + "address": "0.0.xxx" + }, + "curve_type": "edwards25519", + "currency": { + "symbol": "HBAR", + "decimals": 8, + "metadata": { + "issuer": "Hedera" + } + } + }, + { + "privkey": "key2", + "account_identifier": { + "address": "0.0.yyy" + }, + "curve_type": "edwards25519", + "currency": { + "symbol": "HBAR", + "decimals": 8, + "metadata": { + "issuer": "Hedera" + } + } + } + ] + } +} +``` + ## Acceptance Tests The Rosetta API uses [Postman](https://www.postman.com) tests to verify proper operation. The diff --git a/hedera-mirror-rosetta/build/Dockerfile b/hedera-mirror-rosetta/build/Dockerfile index e9ba763aff7..11074d5a3f7 100644 --- a/hedera-mirror-rosetta/build/Dockerfile +++ b/hedera-mirror-rosetta/build/Dockerfile @@ -29,7 +29,7 @@ FROM ubuntu:20.04 as runner ARG DEBIAN_FRONTEND=noninteractive ENV PG_VERSION=13 RUN apt-get update \ - && apt-get install -y ca-certificates gnupg lsb-release \ + && apt-get install -y ca-certificates curl gnupg lsb-release \ && apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 \ && echo "deb https://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ && apt-get update \ diff --git a/hedera-mirror-rosetta/build/postgresql-restore.conf b/hedera-mirror-rosetta/build/postgresql-restore.conf new file mode 100644 index 00000000000..bd1a2a7a9fa --- /dev/null +++ b/hedera-mirror-rosetta/build/postgresql-restore.conf @@ -0,0 +1,8 @@ +checkpoint_timeout = 30min +listen_addresses = '*' +maintenance_work_mem = 2GB +max_parallel_maintenance_workers = 4 +max_wal_size = 512GB +password_encryption = scram-sha-256 +temp_file_limit = 2147483647kB +work_mem = 256MB diff --git a/hedera-mirror-rosetta/build/run_supervisord.sh b/hedera-mirror-rosetta/build/run_supervisord.sh index 046b817d06b..d75c7742c7f 100755 --- a/hedera-mirror-rosetta/build/run_supervisord.sh +++ b/hedera-mirror-rosetta/build/run_supervisord.sh @@ -1,24 +1,73 @@ #!/bin/bash +set -eo pipefail function run_offline_mode() { - supervisord --configuration supervisord-offline.conf + echo "Running in offline mode" + supervisord --configuration /app/supervisord-offline.conf } function run_online_mode() { - supervisord + echo "Running in online mode" + supervisord --configuration /app/supervisord.conf +} + +function cleanup() { + /etc/init.d/postgresql stop || true + cp /app/postgresql.conf "${PGCONF}/conf.d" + rm -rf "${TMPDIR}" + cd /app + echo "Cleanup complete" +} + +function restore() { + if [[ -z "${RESTORE}" ]]; then + echo "Skipping database restore" + return + fi + + DATA_DIR="data_dump" + TMPDIR=$(mktemp -d) + export PGPASSWORD="${HEDERA_MIRROR_IMPORTER_DB_OWNERPASSWORD:-mirror_node_pass}" + + cp /app/postgresql-restore.conf "${PGCONF}/conf.d/postgresql.conf" + /etc/init.d/postgresql start + + if (psql -h localhost -d mirror_node -U mirror_node -c 'select count(*) from flyway_schema_history' > /dev/null); then + echo "Skipping restore since database already contains data" + cleanup + return + fi + + echo "Downloading database backup: ${RESTORE}" + cd "${TMPDIR}" + curl --fail -L --retry 3 "${RESTORE}" | tar -xvf - + + if [[ ! -d "${DATA_DIR}" ]]; then + echo "Database dump does not contain the required '${DATA_DIR}' directory" + cleanup + exit 1 + fi + + echo "Restoring from database backup" + pg_restore -h localhost -U mirror_node --exit-on-error --format=directory --no-owner --no-acl -j 6 -d mirror_node "${DATA_DIR}" + + echo "Restoration complete" + cleanup } function main() { - if [[ -n "$NETWORK" ]]; then - export HEDERA_MIRROR_IMPORTER_NETWORK=$NETWORK - export HEDERA_MIRROR_ROSETTA_NETWORK=$NETWORK + if [[ -n "${NETWORK}" ]]; then + export HEDERA_MIRROR_IMPORTER_NETWORK="${NETWORK}" + export HEDERA_MIRROR_ROSETTA_NETWORK="${NETWORK}" fi - case $MODE in + + case "${MODE}" in "offline") run_offline_mode ;; *) + restore run_online_mode ;; esac diff --git a/hedera-mirror-rosetta/build/supervisord.conf b/hedera-mirror-rosetta/build/supervisord.conf index a1e4c6c5d0e..88b56e9728e 100644 --- a/hedera-mirror-rosetta/build/supervisord.conf +++ b/hedera-mirror-rosetta/build/supervisord.conf @@ -8,6 +8,7 @@ nodaemon=true ; start in foreground if true; default false [unix_http_server] file=/tmp/supervisor.sock +user=dummy [program:postgres] command=postgres @@ -26,7 +27,7 @@ redirect_stderr=true stdout_logfile=/dev/fd/1 stdout_logfile_maxbytes=0 priority=10 -environment=HEDERA_MIRROR_IMPORTER_PARSER_RECORD_ENTITY_PERSIST_NONFEETRANSFERS=true,HEDERA_MIRROR_IMPORTER_PARSER_RECORD_ENTITY_REDIS_ENABLED=false +environment=HEDERA_MIRROR_IMPORTER_PARSER_RECORD_ENTITY_PERSIST_NONFEETRANSFERS=true,HEDERA_MIRROR_IMPORTER_PARSER_RECORD_ENTITY_REDIS_ENABLED=false,HEDERA_MIRROR_IMPORTER_STARTDATE=1970-01-01T00:00:00Z [program:rosetta] command=/app/rosetta/hedera-mirror-rosetta