From b74886b78e55450e820ddb1fa4b112842082e458 Mon Sep 17 00:00:00 2001 From: Paul Steckler Date: Thu, 9 Dec 2021 16:38:35 -0800 Subject: [PATCH 1/4] Fix pending blocks balance query --- .../missing_blocks_auditor.ml | 29 +++++++------------ src/app/rosetta/lib/account.ml | 28 ++++++++++++------ 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/src/app/missing_blocks_auditor/missing_blocks_auditor.ml b/src/app/missing_blocks_auditor/missing_blocks_auditor.ml index 3b3ddb7b6e2..bb1dcc56b70 100644 --- a/src/app/missing_blocks_auditor/missing_blocks_auditor.ml +++ b/src/app/missing_blocks_auditor/missing_blocks_auditor.ml @@ -3,27 +3,21 @@ open Core_kernel open Async -(* prime numbers; multiply by them to derive error exit code - check divisibility by any one of them to see if a particular error occurred -*) -let no_error = 1 +(* bits in error code *) -let missing_blocks_error = 2 +let missing_blocks_error = 0 -let pending_blocks_error = 3 +let pending_blocks_error = 1 -let chain_length_error = 5 +let chain_length_error = 2 -let chain_status_error = 7 +let chain_status_error = 3 -let add_error, get_error_exit = - let error_exit = ref no_error in - let add_error n = - (* multiply by n, if n not already a factor *) - if !error_exit mod n > 0 then error_exit := !error_exit * n - in - let get_error_exit () = !error_exit in - (add_error, get_error_exit) +let add_error, get_exit_code = + let exit_code = ref 0 in + let add_error n = exit_code := !exit_code lor (1 lsl n) in + let get_exit_code () = !exit_code in + (add_error, get_exit_code) let main ~archive_uri () = let logger = Logger.create () in @@ -148,8 +142,7 @@ let main ~archive_uri () = ; ("state_hash", `String state_hash) ; ("chain_status", `String chain_status) ]) ; - let error_exit = get_error_exit () in - if error_exit <> no_error then Core.exit error_exit else Core.exit 0 + Core.exit (get_exit_code ()) let () = Command.( diff --git a/src/app/rosetta/lib/account.ml b/src/app/rosetta/lib/account.ml index bed847c1cde..d604c65ef1d 100644 --- a/src/app/rosetta/lib/account.ml +++ b/src/app/rosetta/lib/account.ml @@ -36,7 +36,7 @@ module Sql = struct Caqti_request.find_opt Caqti_type.(tup2 string int64) Caqti_type.(tup2 int64 int64) - {sql| WITH RECURSIVE chain AS ( + {sql| WITH RECURSIVE pending_chain AS ( (SELECT id, state_hash, parent_id, height, global_slot_since_genesis, timestamp, chain_status @@ -50,24 +50,34 @@ module Sql = struct SELECT b.id, b.state_hash, b.parent_id, b.height, b.global_slot_since_genesis, b.timestamp, b.chain_status FROM blocks b - INNER JOIN chain - ON b.id = chain.parent_id AND chain.id <> chain.parent_id - AND chain.chain_status <> 'canonical' + INNER JOIN pending_chain + ON b.id = pending_chain.parent_id AND pending_chain.id <> pending_chain.parent_id + AND pending_chain.chain_status <> 'canonical' ) - SELECT chain.global_slot_since_genesis AS block_global_slot_since_genesis,balance + SELECT full_chain.global_slot_since_genesis AS block_global_slot_since_genesis,balance - FROM chain - INNER JOIN balances bal ON chain.id = bal.block_id + FROM (SELECT + id, state_hash, parent_id, height, global_slot_since_genesis, timestamp, chain_status + FROM pending_chain + + UNION ALL + + SELECT id, state_hash, parent_id, height, global_slot_since_genesis, timestamp, chain_status + + FROM blocks b + WHERE chain_status = 'canonical') AS full_chain + + INNER JOIN balances bal ON full_chain.id = bal.block_id INNER JOIN public_keys pks ON bal.public_key_id = pks.id WHERE pks.value = $1 - AND chain.height <= $2 + AND full_chain.height <= $2 ORDER BY (bal.block_height, bal.block_sequence_no, bal.block_secondary_sequence_no) DESC LIMIT 1 - |sql} + |sql} let query_canonical = Caqti_request.find_opt From 2551476658018a06cca7cf16db47d36838d28676 Mon Sep 17 00:00:00 2001 From: Paul Steckler Date: Thu, 9 Dec 2021 18:23:14 -0800 Subject: [PATCH 2/4] Use db for current block --- src/app/rosetta/lib/network.ml | 62 +++++++++++++++++++++++----------- src/app/rosetta/rosetta.conf | 4 +-- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/src/app/rosetta/lib/network.ml b/src/app/rosetta/lib/network.ml index 458b2fecfc5..05e178e6a3c 100644 --- a/src/app/rosetta/lib/network.ml +++ b/src/app/rosetta/lib/network.ml @@ -33,6 +33,23 @@ module Get_status = } |}] +module Sql = struct + let oldest_block_query = + Caqti_request.find Caqti_type.unit + Caqti_type.(tup2 int64 string) + "SELECT height, state_hash FROM blocks ORDER BY timestamp ASC LIMIT 1" + + let latest_block_query = + Caqti_request.find + Caqti_type.unit + Caqti_type.(tup2 int64 string) + {sql| SELECT height, state_hash FROM blocks b + WHERE height = (select MAX(height) from blocks) + ORDER BY timestamp ASC + LIMIT 1 + |sql} +end + let sync_status_to_string = function | `BOOTSTRAP -> "Bootstrap" @@ -85,12 +102,6 @@ module Get_network_memoized = struct end end - -let oldest_block_query = - Caqti_request.find Caqti_type.unit - (Caqti_type.tup2 Caqti_type.int64 Caqti_type.string) - "SELECT height, state_hash FROM blocks ORDER BY timestamp ASC LIMIT 1" - (* TODO: Update this when we have a new chainId *) let mainnet_chain_id = "5f704cc0c82e0ed70e873f0893d7e06f148524e3f0bdae2afb02e7819a0c24d1" @@ -218,6 +229,7 @@ module Status = struct type 'gql t = { gql: unit -> ('gql, Errors.t) M.t ; db_oldest_block: unit -> (int64 * string, Errors.t) M.t + ; db_latest_block: unit -> (int64 * string, Errors.t) M.t ; validate_network_choice: network_identifier:Network_identifier.t -> graphql_uri:Uri.t -> (unit, Errors.t) M.t } end @@ -239,10 +251,14 @@ module Status = struct | None -> let%map result = Errors.Lift.sql ~context:"Oldest block query" - @@ Db.find oldest_block_query () + @@ Db.find Sql.oldest_block_query () in Result.iter result ~f:(fun oldest_block -> oldest_block_ref := Some oldest_block) ; result ) + ; db_latest_block= + (fun () -> + Errors.Lift.sql ~context:"Latest db block query" + @@ Db.find Sql.latest_block_query ()) ; validate_network_choice= Validate_choice.Real.validate } end @@ -254,7 +270,7 @@ module Status = struct env.validate_network_choice ~graphql_uri ~network_identifier:network.network_identifier in - let%bind latest_block = + let%bind latest_node_block = match res#bestChain with | None | Some [||] -> M.fail (Errors.create `Chain_info_missing) @@ -262,21 +278,20 @@ module Status = struct M.return (Array.last chain) in let genesis_block_state_hash = (res#genesisBlock)#stateHash in - let%map oldest_block = env.db_oldest_block () in + let%bind (latest_db_block_height,latest_db_block_hash) = env.db_latest_block () in + let%map (oldest_db_block_height,oldest_db_block_hash) = env.db_oldest_block () in { Network_status_response.current_block_identifier= - Block_identifier.create - ((latest_block#protocolState)#consensusState)#blockHeight - latest_block#stateHash + Block_identifier.create latest_db_block_height latest_db_block_hash ; current_block_timestamp= - ((latest_block#protocolState)#blockchainState)#utcDate + ((latest_node_block#protocolState)#blockchainState)#utcDate ; genesis_block_identifier= Block_identifier.create genesis_block_height genesis_block_state_hash ; oldest_block_identifier= - ( if String.equal (snd oldest_block) genesis_block_state_hash then + ( if String.equal oldest_db_block_hash genesis_block_state_hash then None else Some - (Block_identifier.create (fst oldest_block) (snd oldest_block)) + (Block_identifier.create oldest_db_block_height oldest_db_block_hash) ) ; peers= (let peer_objs = (res#daemonStatus)#peers |> Array.to_list in @@ -284,7 +299,7 @@ module Status = struct ; sync_status= Some { Sync_status.current_index= - Some ((latest_block#protocolState)#consensusState)#blockHeight + Some ((latest_node_block#protocolState)#consensusState)#blockHeight ; target_index= None ; stage= Some (sync_status_to_string res#syncStatus) ; synced = None @@ -342,7 +357,10 @@ module Status = struct { gql= (fun () -> Result.return @@ build ~best_chain_missing:true) ; validate_network_choice= Validate_choice.Mock.succeed ; db_oldest_block= - (fun () -> Result.return (Int64.of_int_exn 1, "GENESIS_HASH")) } + (fun () -> Result.return (Int64.of_int_exn 1, "GENESIS_HASH")) + ; db_latest_block= + (fun () -> Result.return (Int64.max_value, "LATEST_BLOCK_HASH")) + } let%test_unit "chain info missing" = Test.assert_ ~f:Network_status_response.to_yojson @@ -353,7 +371,10 @@ module Status = struct { gql= (fun () -> Result.return @@ build ~best_chain_missing:false) ; validate_network_choice= Validate_choice.Mock.succeed ; db_oldest_block= - (fun () -> Result.return (Int64.of_int_exn 1, "GENESIS_HASH")) } + (fun () -> Result.return (Int64.of_int_exn 1, "GENESIS_HASH")) + ; db_latest_block= + (fun () -> Result.return (Int64.max_value, "LATEST_BLOCK_HASH")) + } let%test_unit "oldest block is genesis" = Test.assert_ ~f:Network_status_response.to_yojson @@ -382,7 +403,10 @@ module Status = struct { gql= (fun () -> Result.return @@ build ~best_chain_missing:false) ; validate_network_choice= Validate_choice.Mock.succeed ; db_oldest_block= - (fun () -> Result.return (Int64.of_int_exn 3, "SOME_HASH")) } + (fun () -> Result.return (Int64.of_int_exn 3, "SOME_HASH")) + ; db_latest_block= + (fun () -> Result.return (Int64.of_int_exn 10000, "ANOTHER_HASH")) + } let%test_unit "oldest block is different" = Test.assert_ ~f:Network_status_response.to_yojson diff --git a/src/app/rosetta/rosetta.conf b/src/app/rosetta/rosetta.conf index 7a1e1868232..a826063afd5 100644 --- a/src/app/rosetta/rosetta.conf +++ b/src/app/rosetta/rosetta.conf @@ -4,7 +4,7 @@ "network": "mainnet" }, "online_url": "http://localhost:3087", - "data_directory": "/data/rosetta-cli", + "data_directory": "rosetta-cli", "http_timeout": 500, "max_sync_concurrency": 64, "retry_elapsed_time": 0, @@ -168,7 +168,7 @@ "inactive_discrepency_search_disabled": false, "balance_tracking_disabled": false, "coin_tracking_disabled": false, - "results_output_file": "/data/rosetta-cli/results", + "results_output_file": "rosetta-cli/results", "end_conditions": { "tip": true } From 33a9f67f031fa1e0321b18c69a38da2145610dee Mon Sep 17 00:00:00 2001 From: Paul Steckler Date: Thu, 9 Dec 2021 20:42:45 -0800 Subject: [PATCH 3/4] revert debug code --- src/app/rosetta/rosetta.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/rosetta/rosetta.conf b/src/app/rosetta/rosetta.conf index a826063afd5..7a1e1868232 100644 --- a/src/app/rosetta/rosetta.conf +++ b/src/app/rosetta/rosetta.conf @@ -4,7 +4,7 @@ "network": "mainnet" }, "online_url": "http://localhost:3087", - "data_directory": "rosetta-cli", + "data_directory": "/data/rosetta-cli", "http_timeout": 500, "max_sync_concurrency": 64, "retry_elapsed_time": 0, @@ -168,7 +168,7 @@ "inactive_discrepency_search_disabled": false, "balance_tracking_disabled": false, "coin_tracking_disabled": false, - "results_output_file": "rosetta-cli/results", + "results_output_file": "/data/rosetta-cli/results", "end_conditions": { "tip": true } From e4921dad3eac74a0f66d7b88cb3fd78371a6d7a2 Mon Sep 17 00:00:00 2001 From: Paul Steckler Date: Thu, 9 Dec 2021 21:07:12 -0800 Subject: [PATCH 4/4] Use timestamp from db for latest block --- src/app/rosetta/lib/network.ml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/app/rosetta/lib/network.ml b/src/app/rosetta/lib/network.ml index 05e178e6a3c..6d6c2f1e636 100644 --- a/src/app/rosetta/lib/network.ml +++ b/src/app/rosetta/lib/network.ml @@ -42,8 +42,8 @@ module Sql = struct let latest_block_query = Caqti_request.find Caqti_type.unit - Caqti_type.(tup2 int64 string) - {sql| SELECT height, state_hash FROM blocks b + Caqti_type.(tup3 int64 string int64) + {sql| SELECT height, state_hash, timestamp FROM blocks b WHERE height = (select MAX(height) from blocks) ORDER BY timestamp ASC LIMIT 1 @@ -229,7 +229,7 @@ module Status = struct type 'gql t = { gql: unit -> ('gql, Errors.t) M.t ; db_oldest_block: unit -> (int64 * string, Errors.t) M.t - ; db_latest_block: unit -> (int64 * string, Errors.t) M.t + ; db_latest_block: unit -> (int64 * string * int64, Errors.t) M.t ; validate_network_choice: network_identifier:Network_identifier.t -> graphql_uri:Uri.t -> (unit, Errors.t) M.t } end @@ -278,12 +278,11 @@ module Status = struct M.return (Array.last chain) in let genesis_block_state_hash = (res#genesisBlock)#stateHash in - let%bind (latest_db_block_height,latest_db_block_hash) = env.db_latest_block () in + let%bind (latest_db_block_height,latest_db_block_hash, latest_db_block_timestamp) = env.db_latest_block () in let%map (oldest_db_block_height,oldest_db_block_hash) = env.db_oldest_block () in { Network_status_response.current_block_identifier= Block_identifier.create latest_db_block_height latest_db_block_hash - ; current_block_timestamp= - ((latest_node_block#protocolState)#blockchainState)#utcDate + ; current_block_timestamp= latest_db_block_timestamp ; genesis_block_identifier= Block_identifier.create genesis_block_height genesis_block_state_hash ; oldest_block_identifier= @@ -359,7 +358,7 @@ module Status = struct ; db_oldest_block= (fun () -> Result.return (Int64.of_int_exn 1, "GENESIS_HASH")) ; db_latest_block= - (fun () -> Result.return (Int64.max_value, "LATEST_BLOCK_HASH")) + (fun () -> Result.return (Int64.max_value, "LATEST_BLOCK_HASH", Int64.max_value)) } let%test_unit "chain info missing" = @@ -373,7 +372,7 @@ module Status = struct ; db_oldest_block= (fun () -> Result.return (Int64.of_int_exn 1, "GENESIS_HASH")) ; db_latest_block= - (fun () -> Result.return (Int64.max_value, "LATEST_BLOCK_HASH")) + (fun () -> Result.return (Int64.max_value, "LATEST_BLOCK_HASH", Int64.max_value)) } let%test_unit "oldest block is genesis" = @@ -405,7 +404,7 @@ module Status = struct ; db_oldest_block= (fun () -> Result.return (Int64.of_int_exn 3, "SOME_HASH")) ; db_latest_block= - (fun () -> Result.return (Int64.of_int_exn 10000, "ANOTHER_HASH")) + (fun () -> Result.return (Int64.of_int_exn 10000, "ANOTHER_HASH", Int64.of_int_exn 20000)) } let%test_unit "oldest block is different" =