From b03e8bcf151bbf4c20a7d7faa782366fc3554d58 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 24 Dec 2021 09:54:07 +0100 Subject: [PATCH] Inner hashing of value in state trie (runtime versioning). (#9732) * starting * Updated from other branch. * setting flag * flag in storage struct * fix flagging to access and insert. * added todo to fix * also missing serialize meta to storage proof * extract meta. * Isolate old trie layout. * failing test that requires storing in meta when old hash scheme is used. * old hash compatibility * Db migrate. * runing tests with both states when interesting. * fix chain spec test with serde default. * export state (missing trie function). * Pending using new branch, lacking genericity on layout resolution. * extract and set global meta * Update to branch 4 * fix iterator with root flag (no longer insert node). * fix trie root hashing of root * complete basic backend. * Remove old_hash meta from proof that do not use inner_hashing. * fix trie test for empty (force layout on empty deltas). * Root update fix. * debug on meta * Use trie key iteration that do not include value in proofs. * switch default test ext to use inner hash. * small integration test, and fix tx cache mgmt in ext. test failing * Proof scenario at state-machine level. * trace for db upgrade * try different param * act more like iter_from. * Bigger batches. * Update trie dependency. * drafting codec changes and refact * before removing unused branch no value alt hashing. more work todo rename all flag var to alt_hash, and remove extrinsic replace by storage query at every storage_root call. * alt hashing only for branch with value. * fix trie tests * Hash of value include the encoded size. * removing fields(broken) * fix trie_stream to also include value length in inner hash. * triedbmut only using alt type if inner hashing. * trie_stream to also only use alt hashing type when actually alt hashing. * Refactor meta state, logic should work with change of trie treshold. * Remove NoMeta variant. * Remove state_hashed trigger specific functions. * pending switching to using threshold, new storage root api does not make much sense. * refactoring to use state from backend (not possible payload changes). * Applying from previous state * Remove default from storage, genesis need a special build. * rem empty space * Catch problem: when using triedb with default: we should not revert nodes: otherwhise thing as trie codec cannot decode-encode without changing state. * fix compilation * Right logic to avoid switch on reencode when default layout. * Clean up some todos * remove trie meta from root upstream * update upstream and fix benches. * split some long lines. * UPdate trie crate to work with new design. * Finish update to refactored upstream. * update to latest triedb changes. * Clean up. * fix executor test. * rust fmt from master. * rust format. * rustfmt * fix * start host function driven versioning * update state-machine part * still need access to state version from runtime * state hash in mem: wrong * direction likely correct, but passing call to code exec for genesis init seem awkward. * state version serialize in runtime, wrong approach, just initialize it with no threshold for core api < 4 seems more proper. * stateversion from runtime version (core api >= 4). * update trie, fix tests * unused import * clean some TODOs * Require RuntimeVersionOf for executor * use RuntimeVersionOf to resolve genesis state version. * update runtime version test * fix state-machine tests * TODO * Use runtime version from storage wasm with fast sync. * rustfmt * fmt * fix test * revert useless changes. * clean some unused changes * fmt * removing useless trait function. * remove remaining reference to state_hash * fix some imports * Follow chain state version management. * trie update, fix and constant threshold for trie layouts. * update deps * Update to latest trie pr changes. * fix benches * Verify proof requires right layout. * update trie_root * Update trie deps to latest * Update to latest trie versioning * Removing patch * update lock * extrinsic for sc-service-test using layout v0. * Adding RuntimeVersionOf to CallExecutor works. * fmt * error when resolving version and no wasm in storage. * use existing utils to instantiate runtime code. * Patch to delay runtime switch. * Revert "Patch to delay runtime switch." This reverts commit d35f273b7d67b1b85a9e72973cab13c5c156c1d3. * useless closure * remove remaining state_hash variables. * Remove outdated comment * useless inner hash * fmt * fmt and opt-in feature to apply state change. * feature gate core version, use new test feature for node and test node * Use a 'State' api version instead of Core one. * fix merge of test function * use blake macro. * Fix state api (require declaring the api in runtime). * Opt out feature, fix macro for io to select a given version instead of latest. * run test nodes on new state. * fix * Apply review change (docs and error). * fmt * use explicit runtime_interface in doc test * fix ui test * fix doc test * fmt * use default for path and specname when resolving version. * small review related changes. * doc value size requirement. * rename old_state feature * Remove macro changes * feature rename * state version as host function parameter * remove flag for client api * fix tests * switch storage chain proof to V1 * host functions, pass by state version enum * use WrappedRuntimeCode * start * state_version in runtime version * rust fmt * Update storage proof of max size. * fix runtime version rpc test * right intent of convert from compat * fix doc test * fix doc test * split proof * decode without replay, and remove some reexports. * Decode with compatibility by default. * switch state_version to u8. And remove RuntimeVersionBasis. * test * use api when reading embedded version * fix decode with apis * extract core version instead * test fix * unused import * review changes. Co-authored-by: kianenigma --- Cargo.lock | 34 ++- bin/node-template/runtime/src/lib.rs | 1 + bin/node/bench/src/generator.rs | 5 +- bin/node/bench/src/trie.rs | 6 +- bin/node/executor/benches/bench.rs | 4 +- bin/node/executor/tests/common.rs | 2 +- bin/node/runtime/src/lib.rs | 1 + client/api/src/backend.rs | 9 +- client/api/src/call_executor.rs | 4 +- client/api/src/in_mem.rs | 15 +- client/block-builder/src/lib.rs | 1 + client/db/src/bench.rs | 13 +- client/db/src/lib.rs | 108 +++++--- client/db/src/storage_cache.rs | 22 +- client/executor/runtime-test/src/lib.rs | 1 + client/executor/src/integration_tests/mod.rs | 11 +- client/executor/src/wasm_runtime.rs | 88 ++++--- client/network/src/protocol/sync.rs | 6 +- client/rpc/src/state/tests.rs | 4 +- client/service/src/client/call_executor.rs | 16 +- client/service/src/client/client.rs | 64 ++++- client/service/src/client/genesis.rs | 6 +- client/service/test/src/client/mod.rs | 16 +- frame/benchmarking/src/lib.rs | 5 +- frame/executive/src/lib.rs | 29 ++- frame/session/src/historical/mod.rs | 4 +- frame/support/src/lib.rs | 10 +- frame/support/src/storage/child.rs | 7 +- frame/support/test/compile_pass/src/lib.rs | 1 + frame/system/src/lib.rs | 11 +- frame/system/src/mock.rs | 1 + frame/transaction-storage/src/benchmarking.rs | 116 +++++---- frame/transaction-storage/src/lib.rs | 3 +- .../api/proc-macro/src/impl_runtime_apis.rs | 7 + primitives/api/src/lib.rs | 45 +--- primitives/externalities/src/lib.rs | 10 +- primitives/io/src/lib.rs | 144 ++++++++++- primitives/runtime-interface/src/impls.rs | 4 + primitives/runtime/src/lib.rs | 1 + primitives/runtime/src/traits.rs | 22 +- primitives/state-machine/Cargo.toml | 4 +- primitives/state-machine/src/backend.rs | 18 +- primitives/state-machine/src/basic.rs | 30 ++- primitives/state-machine/src/ext.rs | 169 +++++++----- .../state-machine/src/in_memory_backend.rs | 65 +++-- primitives/state-machine/src/lib.rs | 159 +++++++++--- .../src/overlayed_changes/mod.rs | 16 +- .../state-machine/src/proving_backend.rs | 99 ++++--- primitives/state-machine/src/read_only.rs | 10 +- primitives/state-machine/src/testing.rs | 58 ++++- primitives/state-machine/src/trie_backend.rs | 157 ++++++++---- .../state-machine/src/trie_backend_essence.rs | 157 +++++++----- primitives/storage/src/lib.rs | 51 ++++ primitives/tasks/src/async_externalities.rs | 10 +- .../transaction-storage-proof/src/lib.rs | 2 +- primitives/trie/Cargo.toml | 8 +- primitives/trie/benches/bench.rs | 4 +- primitives/trie/src/lib.rs | 242 ++++++++++++------ primitives/trie/src/node_codec.rs | 114 +++++++-- primitives/trie/src/node_header.rs | 93 +++++-- primitives/trie/src/storage_proof.rs | 11 +- primitives/trie/src/trie_codec.rs | 6 +- primitives/trie/src/trie_stream.rs | 81 +++--- primitives/version/Cargo.toml | 1 + .../proc-macro/src/decl_runtime_version.rs | 24 +- primitives/version/src/lib.rs | 78 +++++- test-utils/client/src/client_ext.rs | 2 +- test-utils/client/src/lib.rs | 4 +- test-utils/runtime/Cargo.toml | 2 +- test-utils/runtime/client/src/lib.rs | 2 + test-utils/runtime/src/genesismap.rs | 2 + test-utils/runtime/src/lib.rs | 14 +- test-utils/runtime/src/system.rs | 10 +- .../cli/src/commands/execute_block.rs | 2 +- .../cli/src/commands/follow_chain.rs | 17 +- .../cli/src/commands/offchain_worker.rs | 2 +- .../cli/src/commands/on_runtime_upgrade.rs | 2 +- utils/frame/try-runtime/cli/src/lib.rs | 7 +- 78 files changed, 1811 insertions(+), 779 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 96146f6f8bece..c6dd25ffd74aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4179,6 +4179,17 @@ dependencies = [ "parity-util-mem", ] +[[package]] +name = "memory-db" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d505169b746dacf02f7d14d8c80b34edfd8212159c63d23c977739a0d960c626" +dependencies = [ + "hash-db", + "hashbrown 0.11.2", + "parity-util-mem", +] + [[package]] name = "memory_units" version = "0.3.0" @@ -9908,7 +9919,7 @@ dependencies = [ "criterion", "hash-db", "hex-literal", - "memory-db", + "memory-db 0.28.0", "parity-scale-codec", "scale-info", "sp-core", @@ -9929,6 +9940,7 @@ dependencies = [ "parity-wasm 0.42.2", "scale-info", "serde", + "sp-core-hashing-proc-macro", "sp-runtime", "sp-std", "sp-version-proc-macro", @@ -10201,7 +10213,7 @@ dependencies = [ "frame-system-rpc-runtime-api", "futures 0.3.16", "log 0.4.14", - "memory-db", + "memory-db 0.27.0", "pallet-babe", "pallet-timestamp", "parity-scale-codec", @@ -10840,14 +10852,14 @@ checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" [[package]] name = "trie-bench" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4edd9bdf0c2e08fd77c0fb2608179cac7ebed997ae18f58d47a2d96425ff51f0" +checksum = "36ac46f6503d0fa976193db46f9dbb1d454e5dbde76495f1316f576c7f3f0e6b" dependencies = [ "criterion", "hash-db", "keccak-hasher", - "memory-db", + "memory-db 0.28.0", "parity-scale-codec", "trie-db", "trie-root", @@ -10856,9 +10868,9 @@ dependencies = [ [[package]] name = "trie-db" -version = "0.22.6" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eac131e334e81b6b3be07399482042838adcd7957aa0010231d0813e39e02fa" +checksum = "e3ddae50680c12ef75bfbf58416ca6622fa43d879553f6cb2ed1a817346e1ffe" dependencies = [ "hash-db", "hashbrown 0.11.2", @@ -10869,9 +10881,9 @@ dependencies = [ [[package]] name = "trie-root" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "652931506d2c1244d7217a70b99f56718a7b4161b37f04e7cd868072a99f68cd" +checksum = "9a36c5ca3911ed3c9a5416ee6c679042064b93fc637ded67e25f92e68d783891" dependencies = [ "hash-db", ] @@ -10986,8 +10998,8 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f559b464de2e2bdabcac6a210d12e9b5a5973c251e102c44c585c71d51bd78e" dependencies = [ - "cfg-if 1.0.0", - "rand 0.8.4", + "cfg-if 0.1.10", + "rand 0.6.5", "static_assertions", ] diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index 5c277a1bb7942..08c6c8fb8bcba 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -102,6 +102,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { impl_version: 1, apis: RUNTIME_API_VERSIONS, transaction_version: 1, + state_version: 1, }; /// This determines the average expected block time that we are targeting. diff --git a/bin/node/bench/src/generator.rs b/bin/node/bench/src/generator.rs index e3aa1192b5d1f..d57142893f38f 100644 --- a/bin/node/bench/src/generator.rs +++ b/bin/node/bench/src/generator.rs @@ -20,7 +20,7 @@ use std::{collections::HashMap, sync::Arc}; use kvdb::KeyValueDB; use node_primitives::Hash; -use sp_trie::{trie_types::TrieDBMut, TrieMut}; +use sp_trie::{trie_types::TrieDBMutV1, TrieMut}; use crate::simple_trie::SimpleTrie; @@ -43,8 +43,7 @@ pub fn generate_trie( ); let mut trie = SimpleTrie { db, overlay: &mut overlay }; { - let mut trie_db = TrieDBMut::new(&mut trie, &mut root); - + let mut trie_db = TrieDBMutV1::::new(&mut trie, &mut root); for (key, value) in key_values { trie_db.insert(&key, &value).expect("trie insertion failed"); } diff --git a/bin/node/bench/src/trie.rs b/bin/node/bench/src/trie.rs index a17e386ca879b..374ed3568475e 100644 --- a/bin/node/bench/src/trie.rs +++ b/bin/node/bench/src/trie.rs @@ -23,7 +23,7 @@ use kvdb::KeyValueDB; use lazy_static::lazy_static; use rand::Rng; use sp_state_machine::Backend as _; -use sp_trie::{trie_types::TrieDBMut, TrieMut as _}; +use sp_trie::{trie_types::TrieDBMutV1, TrieMut as _}; use std::{borrow::Cow, collections::HashMap, sync::Arc}; use node_primitives::Hash; @@ -286,8 +286,8 @@ impl core::Benchmark for TrieWriteBenchmark { let mut overlay = HashMap::new(); let mut trie = SimpleTrie { db: kvdb.clone(), overlay: &mut overlay }; - let mut trie_db_mut = - TrieDBMut::from_existing(&mut trie, &mut new_root).expect("Failed to create TrieDBMut"); + let mut trie_db_mut = TrieDBMutV1::from_existing(&mut trie, &mut new_root) + .expect("Failed to create TrieDBMut"); for (warmup_key, warmup_value) in self.warmup_keys.iter() { let value = trie_db_mut diff --git a/bin/node/executor/benches/bench.rs b/bin/node/executor/benches/bench.rs index aea37b68d55ba..f1f950182f705 100644 --- a/bin/node/executor/benches/bench.rs +++ b/bin/node/executor/benches/bench.rs @@ -83,14 +83,14 @@ fn construct_block( parent_hash: Hash, extrinsics: Vec, ) -> (Vec, Hash) { - use sp_trie::{trie_types::Layout, TrieConfiguration}; + use sp_trie::{LayoutV0, TrieConfiguration}; // sign extrinsics. let extrinsics = extrinsics.into_iter().map(sign).collect::>(); // calculate the header fields that we can. let extrinsics_root = - Layout::::ordered_trie_root(extrinsics.iter().map(Encode::encode)) + LayoutV0::::ordered_trie_root(extrinsics.iter().map(Encode::encode)) .to_fixed_bytes() .into(); diff --git a/bin/node/executor/tests/common.rs b/bin/node/executor/tests/common.rs index 84b5728a1a840..8d21f086795a4 100644 --- a/bin/node/executor/tests/common.rs +++ b/bin/node/executor/tests/common.rs @@ -142,7 +142,7 @@ pub fn construct_block( extrinsics: Vec, babe_slot: Slot, ) -> (Vec, Hash) { - use sp_trie::{trie_types::Layout, TrieConfiguration}; + use sp_trie::{LayoutV1 as Layout, TrieConfiguration}; // sign extrinsics. let extrinsics = extrinsics.into_iter().map(sign).collect::>(); diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index a79d8997f4ce1..1a4dad1071e15 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -126,6 +126,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 2, + state_version: 1, }; /// The BABE epoch configuration at genesis. diff --git a/client/api/src/backend.rs b/client/api/src/backend.rs index f300e31f826c9..76f2f569814ff 100644 --- a/client/api/src/backend.rs +++ b/client/api/src/backend.rs @@ -29,7 +29,7 @@ use sp_core::offchain::OffchainStorage; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, HashFor, NumberFor}, - Justification, Justifications, Storage, + Justification, Justifications, StateVersion, Storage, }; use sp_state_machine::{ ChildStorageCollection, IndexOperation, OffchainChangesCollection, StorageCollection, @@ -166,10 +166,15 @@ pub trait BlockImportOperation { &mut self, storage: Storage, commit: bool, + state_version: StateVersion, ) -> sp_blockchain::Result; /// Inject storage data into the database replacing any existing data. - fn reset_storage(&mut self, storage: Storage) -> sp_blockchain::Result; + fn reset_storage( + &mut self, + storage: Storage, + state_version: StateVersion, + ) -> sp_blockchain::Result; /// Set storage changes. fn update_storage( diff --git a/client/api/src/call_executor.rs b/client/api/src/call_executor.rs index 22af495c06542..b1fd731ca088c 100644 --- a/client/api/src/call_executor.rs +++ b/client/api/src/call_executor.rs @@ -19,7 +19,7 @@ //! A method call executor interface. use codec::{Decode, Encode}; -use sc_executor::RuntimeVersion; +use sc_executor::{RuntimeVersion, RuntimeVersionOf}; use sp_core::NativeOrEncoded; use sp_externalities::Extensions; use sp_runtime::{generic::BlockId, traits::Block as BlockT}; @@ -42,7 +42,7 @@ pub trait ExecutorProvider { } /// Method call executor. -pub trait CallExecutor { +pub trait CallExecutor: RuntimeVersionOf { /// Externalities error type. type Error: sp_state_machine::Error; diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 39fe9e063d20b..c9650bf6db748 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -26,7 +26,7 @@ use sp_core::{ use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor, Zero}, - Justification, Justifications, Storage, + Justification, Justifications, StateVersion, Storage, }; use sp_state_machine::{ Backend as StateBackend, ChildStorageCollection, InMemoryBackend, IndexOperation, @@ -506,6 +506,7 @@ where &mut self, storage: Storage, commit: bool, + state_version: StateVersion, ) -> sp_blockchain::Result { check_genesis_storage(&storage)?; @@ -519,6 +520,7 @@ where let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, + state_version, ); if commit { @@ -566,12 +568,17 @@ where &mut self, storage: Storage, commit: bool, + state_version: StateVersion, ) -> sp_blockchain::Result { - self.apply_storage(storage, commit) + self.apply_storage(storage, commit, state_version) } - fn reset_storage(&mut self, storage: Storage) -> sp_blockchain::Result { - self.apply_storage(storage, true) + fn reset_storage( + &mut self, + storage: Storage, + state_version: StateVersion, + ) -> sp_blockchain::Result { + self.apply_storage(storage, true, state_version) } fn insert_aux(&mut self, ops: I) -> sp_blockchain::Result<()> diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index 01afaca0cacf4..c3e91fa055400 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -232,6 +232,7 @@ where header.extrinsics_root().clone(), HashFor::::ordered_trie_root( self.extrinsics.iter().map(Encode::encode).collect(), + sp_runtime::StateVersion::V0, ), ); diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index d46aca8e8ff78..cafafca9b635c 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -34,7 +34,7 @@ use sp_core::{ }; use sp_runtime::{ traits::{Block as BlockT, HashFor}, - Storage, + StateVersion, Storage, }; use sp_state_machine::{ backend::Backend as StateBackend, ChildStorageCollection, DBValue, ProofRecorder, @@ -73,6 +73,7 @@ impl sp_state_machine::Storage> for StorageDb { root: Cell, @@ -105,9 +106,10 @@ impl BenchmarkingState { record_proof: bool, enable_tracking: bool, ) -> Result { + let state_version = sp_runtime::StateVersion::default(); let mut root = B::Hash::default(); let mut mdb = MemoryDB::>::default(); - sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); + sp_state_machine::TrieDBMutV1::>::new(&mut mdb, &mut root); let mut state = BenchmarkingState { state: RefCell::new(None), @@ -138,6 +140,7 @@ impl BenchmarkingState { state.state.borrow_mut().as_mut().unwrap().full_storage_root( genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))), child_delta, + state_version, ); state.genesis = transaction.clone().drain(); state.genesis_root = root.clone(); @@ -415,6 +418,7 @@ impl StateBackend> for BenchmarkingState { fn storage_root<'a>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord, @@ -422,13 +426,14 @@ impl StateBackend> for BenchmarkingState { self.state .borrow() .as_ref() - .map_or(Default::default(), |s| s.storage_root(delta)) + .map_or(Default::default(), |s| s.storage_root(delta, state_version)) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord, @@ -436,7 +441,7 @@ impl StateBackend> for BenchmarkingState { self.state .borrow() .as_ref() - .map_or(Default::default(), |s| s.child_storage_root(child_info, delta)) + .map_or(Default::default(), |s| s.child_storage_root(child_info, delta, state_version)) } fn pairs(&self) -> Vec<(Vec, Vec)> { diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 7d46b63da5bb4..137c26fb82188 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -82,7 +82,7 @@ use sp_runtime::{ Block as BlockT, Hash, HashFor, Header as HeaderT, NumberFor, One, SaturatedConversion, Zero, }, - Justification, Justifications, Storage, + Justification, Justifications, StateVersion, Storage, }; use sp_state_machine::{ backend::Backend as StateBackend, ChildStorageCollection, DBValue, IndexOperation, @@ -235,22 +235,24 @@ impl StateBackend> for RefTrackingState { fn storage_root<'a>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord, { - self.state.storage_root(delta) + self.state.storage_root(delta, state_version) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord, { - self.state.child_storage_root(child_info, delta) + self.state.child_storage_root(child_info, delta, state_version) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -760,7 +762,11 @@ impl BlockImportOperation { } } - fn apply_new_state(&mut self, storage: Storage) -> ClientResult { + fn apply_new_state( + &mut self, + storage: Storage, + state_version: StateVersion, + ) -> ClientResult { if storage.top.keys().any(|k| well_known_keys::is_child_storage_key(&k)) { return Err(sp_blockchain::Error::InvalidState.into()) } @@ -775,6 +781,7 @@ impl BlockImportOperation { let (root, transaction) = self.old_state.full_storage_root( storage.top.iter().map(|(k, v)| (&k[..], Some(&v[..]))), child_delta, + state_version, ); self.db_updates = transaction; @@ -814,14 +821,23 @@ impl sc_client_api::backend::BlockImportOperation Ok(()) } - fn reset_storage(&mut self, storage: Storage) -> ClientResult { - let root = self.apply_new_state(storage)?; + fn reset_storage( + &mut self, + storage: Storage, + state_version: StateVersion, + ) -> ClientResult { + let root = self.apply_new_state(storage, state_version)?; self.commit_state = true; Ok(root) } - fn set_genesis_state(&mut self, storage: Storage, commit: bool) -> ClientResult { - let root = self.apply_new_state(storage)?; + fn set_genesis_state( + &mut self, + storage: Storage, + commit: bool, + state_version: StateVersion, + ) -> ClientResult { + let root = self.apply_new_state(storage, state_version)?; self.commit_state = commit; Ok(root) } @@ -924,7 +940,8 @@ impl EmptyStorage { pub fn new() -> Self { let mut root = Block::Hash::default(); let mut mdb = MemoryDB::>::default(); - sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); + // both triedbmut are the same on empty storage. + sp_state_machine::TrieDBMutV1::>::new(&mut mdb, &mut root); EmptyStorage(root) } } @@ -2262,7 +2279,7 @@ pub(crate) mod tests { use sp_runtime::{ testing::{Block as RawBlock, ExtrinsicWrapper, Header}, traits::{BlakeTwo256, Hash}, - ConsensusEngineId, + ConsensusEngineId, StateVersion, }; const CONS0_ENGINE_ID: ConsensusEngineId = *b"CON0"; @@ -2295,7 +2312,7 @@ pub(crate) mod tests { let header = Header { number, parent_hash, - state_root: BlakeTwo256::trie_root(Vec::new()), + state_root: BlakeTwo256::trie_root(Vec::new(), StateVersion::V1), digest, extrinsics_root, }; @@ -2375,6 +2392,10 @@ pub(crate) mod tests { #[test] fn set_state_data() { + set_state_data_inner(StateVersion::V0); + set_state_data_inner(StateVersion::V1); + } + fn set_state_data_inner(state_version: StateVersion) { let db = Backend::::new_test(2, 0); let hash = { let mut op = db.begin_operation().unwrap(); @@ -2390,15 +2411,18 @@ pub(crate) mod tests { header.state_root = op .old_state - .storage_root(storage.iter().map(|(x, y)| (&x[..], Some(&y[..])))) + .storage_root(storage.iter().map(|(x, y)| (&x[..], Some(&y[..]))), state_version) .0 .into(); let hash = header.hash(); - op.reset_storage(Storage { - top: storage.into_iter().collect(), - children_default: Default::default(), - }) + op.reset_storage( + Storage { + top: storage.into_iter().collect(), + children_default: Default::default(), + }, + state_version, + ) .unwrap(); op.set_block_data(header.clone(), Some(vec![]), None, None, NewBlockState::Best) .unwrap(); @@ -2427,9 +2451,10 @@ pub(crate) mod tests { let storage = vec![(vec![1, 3, 5], None), (vec![5, 5, 5], Some(vec![4, 5, 6]))]; - let (root, overlay) = op - .old_state - .storage_root(storage.iter().map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..])))); + let (root, overlay) = op.old_state.storage_root( + storage.iter().map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))), + state_version, + ); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); @@ -2450,6 +2475,7 @@ pub(crate) mod tests { #[test] fn delete_only_when_negative_rc() { sp_tracing::try_init_simple(); + let state_version = StateVersion::default(); let key; let backend = Backend::::new_test(1, 0); @@ -2466,13 +2492,14 @@ pub(crate) mod tests { extrinsics_root: Default::default(), }; - header.state_root = op.old_state.storage_root(std::iter::empty()).0.into(); + header.state_root = + op.old_state.storage_root(std::iter::empty(), state_version).0.into(); let hash = header.hash(); - op.reset_storage(Storage { - top: Default::default(), - children_default: Default::default(), - }) + op.reset_storage( + Storage { top: Default::default(), children_default: Default::default() }, + state_version, + ) .unwrap(); key = op.db_updates.insert(EMPTY_PREFIX, b"hello"); @@ -2506,7 +2533,7 @@ pub(crate) mod tests { header.state_root = op .old_state - .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y)))) + .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y))), state_version) .0 .into(); let hash = header.hash(); @@ -2543,7 +2570,7 @@ pub(crate) mod tests { header.state_root = op .old_state - .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y)))) + .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y))), state_version) .0 .into(); let hash = header.hash(); @@ -2577,7 +2604,7 @@ pub(crate) mod tests { header.state_root = op .old_state - .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y)))) + .storage_root(storage.iter().cloned().map(|(x, y)| (x, Some(y))), state_version) .0 .into(); @@ -2912,6 +2939,7 @@ pub(crate) mod tests { #[test] fn storage_hash_is_cached_correctly() { + let state_version = StateVersion::default(); let backend = Backend::::new_test(10, 10); let hash0 = { @@ -2931,15 +2959,18 @@ pub(crate) mod tests { header.state_root = op .old_state - .storage_root(storage.iter().map(|(x, y)| (&x[..], Some(&y[..])))) + .storage_root(storage.iter().map(|(x, y)| (&x[..], Some(&y[..]))), state_version) .0 .into(); let hash = header.hash(); - op.reset_storage(Storage { - top: storage.into_iter().collect(), - children_default: Default::default(), - }) + op.reset_storage( + Storage { + top: storage.into_iter().collect(), + children_default: Default::default(), + }, + state_version, + ) .unwrap(); op.set_block_data(header.clone(), Some(vec![]), None, None, NewBlockState::Best) .unwrap(); @@ -2968,9 +2999,10 @@ pub(crate) mod tests { let storage = vec![(b"test".to_vec(), Some(b"test2".to_vec()))]; - let (root, overlay) = op - .old_state - .storage_root(storage.iter().map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..])))); + let (root, overlay) = op.old_state.storage_root( + storage.iter().map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))), + state_version, + ); op.update_db_storage(overlay).unwrap(); header.state_root = root.into(); let hash = header.hash(); @@ -3212,7 +3244,7 @@ pub(crate) mod tests { let header = Header { number: 1, parent_hash: block0, - state_root: BlakeTwo256::trie_root(Vec::new()), + state_root: BlakeTwo256::trie_root(Vec::new(), StateVersion::V1), digest: Default::default(), extrinsics_root: Default::default(), }; @@ -3224,7 +3256,7 @@ pub(crate) mod tests { let header = Header { number: 2, parent_hash: block1, - state_root: BlakeTwo256::trie_root(Vec::new()), + state_root: BlakeTwo256::trie_root(Vec::new(), StateVersion::V1), digest: Default::default(), extrinsics_root: Default::default(), }; @@ -3247,7 +3279,7 @@ pub(crate) mod tests { let header = Header { number: 1, parent_hash: block0, - state_root: BlakeTwo256::trie_root(Vec::new()), + state_root: BlakeTwo256::trie_root(Vec::new(), StateVersion::V1), digest: Default::default(), extrinsics_root: Default::default(), }; diff --git a/client/db/src/storage_cache.rs b/client/db/src/storage_cache.rs index 5fef0e5b12d08..579703ea9db85 100644 --- a/client/db/src/storage_cache.rs +++ b/client/db/src/storage_cache.rs @@ -26,7 +26,10 @@ use linked_hash_map::{Entry, LinkedHashMap}; use log::trace; use parking_lot::{RwLock, RwLockUpgradableReadGuard}; use sp_core::{hexdisplay::HexDisplay, storage::ChildInfo}; -use sp_runtime::traits::{Block as BlockT, HashFor, Header, NumberFor}; +use sp_runtime::{ + traits::{Block as BlockT, HashFor, Header, NumberFor}, + StateVersion, +}; use sp_state_machine::{ backend::Backend as StateBackend, ChildStorageCollection, StorageCollection, StorageKey, StorageValue, TrieBackend, @@ -673,22 +676,24 @@ impl>, B: BlockT> StateBackend> for Cachin fn storage_root<'a>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord, { - self.state.storage_root(delta) + self.state.storage_root(delta, state_version) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord, { - self.state.child_storage_root(child_info, delta) + self.state.child_storage_root(child_info, delta, state_version) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -871,22 +876,24 @@ impl>, B: BlockT> StateBackend> fn storage_root<'a>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, Self::Transaction) where B::Hash: Ord, { - self.caching_state().storage_root(delta) + self.caching_state().storage_root(delta, state_version) } fn child_storage_root<'a>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord, { - self.caching_state().child_storage_root(child_info, delta) + self.caching_state().child_storage_root(child_info, delta, state_version) } fn pairs(&self) -> Vec<(Vec, Vec)> { @@ -1182,7 +1189,10 @@ mod tests { let shared = new_shared_cache::(256 * 1024, (0, 1)); let mut backend = InMemoryBackend::::default(); - backend.insert(std::iter::once((None, vec![(key.clone(), Some(vec![1]))]))); + backend.insert( + std::iter::once((None, vec![(key.clone(), Some(vec![1]))])), + Default::default(), + ); let mut s = CachingState::new(backend.clone(), shared.clone(), Some(root_parent)); s.cache.sync_cache( diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs index 01e46ab946354..0655160c1ab7a 100644 --- a/client/executor/runtime-test/src/lib.rs +++ b/client/executor/runtime-test/src/lib.rs @@ -180,6 +180,7 @@ sp_core::wasm_export_functions! { b"one"[..].into(), b"two"[..].into(), ], + sp_core::storage::StateVersion::V1, ).as_ref().to_vec() } diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 89648e9bac696..c480ad74ade67 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -33,7 +33,7 @@ use sp_core::{ }; use sp_runtime::traits::BlakeTwo256; use sp_state_machine::TestExternalities as CoreTestExternalities; -use sp_trie::{trie_types::Layout, TrieConfiguration}; +use sp_trie::{LayoutV1 as Layout, TrieConfiguration}; use std::sync::Arc; use tracing_subscriber::layer::SubscriberExt; @@ -215,21 +215,22 @@ fn panicking_should_work(wasm_method: WasmExecutionMethod) { test_wasm_execution!(storage_should_work); fn storage_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); + // Test value must be bigger than 32 bytes + // to test the trie versioning. + let value = vec![7u8; 60]; { let mut ext = ext.ext(); ext.set_storage(b"foo".to_vec(), b"bar".to_vec()); - let output = - call_in_wasm("test_data_in", &b"Hello world".to_vec().encode(), wasm_method, &mut ext) - .unwrap(); + let output = call_in_wasm("test_data_in", &value.encode(), wasm_method, &mut ext).unwrap(); assert_eq!(output, b"all ok!".to_vec().encode()); } let expected = TestExternalities::new(sp_core::storage::Storage { top: map![ - b"input".to_vec() => b"Hello world".to_vec(), + b"input".to_vec() => value, b"foo".to_vec() => b"bar".to_vec(), b"baz".to_vec() => b"bar".to_vec() ], diff --git a/client/executor/src/wasm_runtime.rs b/client/executor/src/wasm_runtime.rs index cebfc7b01b8d8..94c4f431ef45a 100644 --- a/client/executor/src/wasm_runtime.rs +++ b/client/executor/src/wasm_runtime.rs @@ -334,22 +334,11 @@ where } fn decode_version(mut version: &[u8]) -> Result { - let v: RuntimeVersion = sp_api::OldRuntimeVersion::decode(&mut &version[..]) - .map_err(|_| { - WasmError::Instantiation( - "failed to decode \"Core_version\" result using old runtime version".into(), - ) - })? - .into(); - - let core_api_id = sp_core_hashing_proc_macro::blake2b_64!(b"Core"); - if v.has_api_with(&core_api_id, |v| v >= 3) { - sp_api::RuntimeVersion::decode(&mut version).map_err(|_| { - WasmError::Instantiation("failed to decode \"Core_version\" result".into()) - }) - } else { - Ok(v) - } + Decode::decode(&mut version).map_err(|_| { + WasmError::Instantiation( + "failed to decode \"Core_version\" result using old runtime version".into(), + ) + }) } fn decode_runtime_apis(apis: &[u8]) -> Result, WasmError> { @@ -373,17 +362,25 @@ fn decode_runtime_apis(apis: &[u8]) -> Result, WasmError> { /// sections, `Err` will be returned. pub fn read_embedded_version(blob: &RuntimeBlob) -> Result, WasmError> { if let Some(mut version_section) = blob.custom_section_contents("runtime_version") { - // We do not use `decode_version` here because the runtime_version section is not supposed - // to ever contain a legacy version. Apart from that `decode_version` relies on presence - // of a special API in the `apis` field to treat the input as a non-legacy version. However - // the structure found in the `runtime_version` always contain an empty `apis` field. - // Therefore the version read will be mistakenly treated as an legacy one. - let mut decoded_version = sp_api::RuntimeVersion::decode(&mut version_section) - .map_err(|_| WasmError::Instantiation("failed to decode version section".into()))?; - - // Don't stop on this and check if there is a special section that encodes all runtime APIs. - if let Some(apis_section) = blob.custom_section_contents("runtime_apis") { - decoded_version.apis = decode_runtime_apis(apis_section)?.into(); + let apis = blob + .custom_section_contents("runtime_apis") + .map(decode_runtime_apis) + .transpose()? + .map(Into::into); + + let core_version = apis.as_ref().and_then(|apis| sp_version::core_version_from_apis(apis)); + // We do not use `RuntimeVersion::decode` here because that `decode_version` relies on + // presence of a special API in the `apis` field to treat the input as a non-legacy version. + // However the structure found in the `runtime_version` always contain an empty `apis` + // field. Therefore the version read will be mistakenly treated as an legacy one. + let mut decoded_version = sp_version::RuntimeVersion::decode_with_version_hint( + &mut version_section, + core_version, + ) + .map_err(|_| WasmError::Instantiation("failed to decode version section".into()))?; + + if let Some(apis) = apis { + decoded_version.apis = apis; } Ok(Some(decoded_version)) @@ -455,9 +452,20 @@ mod tests { use super::*; use codec::Encode; use sp_api::{Core, RuntimeApiInfo}; + use sp_runtime::RuntimeString; use sp_wasm_interface::HostFunctions; use substrate_test_runtime::Block; + #[derive(Encode)] + pub struct OldRuntimeVersion { + pub spec_name: RuntimeString, + pub impl_name: RuntimeString, + pub authoring_version: u32, + pub spec_version: u32, + pub impl_version: u32, + pub apis: sp_version::ApisVec, + } + #[test] fn host_functions_are_equal() { let host_functions = sp_io::SubstrateHostFunctions::host_functions(); @@ -468,7 +476,7 @@ mod tests { #[test] fn old_runtime_version_decodes() { - let old_runtime_version = sp_api::OldRuntimeVersion { + let old_runtime_version = OldRuntimeVersion { spec_name: "test".into(), impl_name: "test".into(), authoring_version: 1, @@ -479,11 +487,12 @@ mod tests { let version = decode_version(&old_runtime_version.encode()).unwrap(); assert_eq!(1, version.transaction_version); + assert_eq!(0, version.state_version); } #[test] fn old_runtime_version_decodes_fails_with_version_3() { - let old_runtime_version = sp_api::OldRuntimeVersion { + let old_runtime_version = OldRuntimeVersion { spec_name: "test".into(), impl_name: "test".into(), authoring_version: 1, @@ -505,10 +514,27 @@ mod tests { impl_version: 1, apis: sp_api::create_apis_vec!([(>::ID, 3)]), transaction_version: 3, + state_version: 4, + }; + + let version = decode_version(&old_runtime_version.encode()).unwrap(); + assert_eq!(3, version.transaction_version); + assert_eq!(0, version.state_version); + + let old_runtime_version = sp_api::RuntimeVersion { + spec_name: "test".into(), + impl_name: "test".into(), + authoring_version: 1, + spec_version: 1, + impl_version: 1, + apis: sp_api::create_apis_vec!([(>::ID, 4)]), + transaction_version: 3, + state_version: 4, }; let version = decode_version(&old_runtime_version.encode()).unwrap(); assert_eq!(3, version.transaction_version); + assert_eq!(4, version.state_version); } #[test] @@ -518,15 +544,15 @@ mod tests { sp_maybe_compressed_blob::CODE_BLOB_BOMB_LIMIT, ) .expect("Decompressing works"); - let runtime_version = RuntimeVersion { spec_name: "test_replace".into(), impl_name: "test_replace".into(), authoring_version: 100, spec_version: 100, impl_version: 100, - apis: sp_api::create_apis_vec!([(>::ID, 3)]), + apis: sp_api::create_apis_vec!([(>::ID, 4)]), transaction_version: 100, + state_version: 1, }; let embedded = sp_version::embed::embed_runtime_version(&wasm, runtime_version.clone()) diff --git a/client/network/src/protocol/sync.rs b/client/network/src/protocol/sync.rs index 37646875e3b16..a84f5131242ef 100644 --- a/client/network/src/protocol/sync.rs +++ b/client/network/src/protocol/sync.rs @@ -2539,8 +2539,10 @@ fn validate_blocks( } if let (Some(header), Some(body)) = (&b.header, &b.body) { let expected = *header.extrinsics_root(); - let got = - HashFor::::ordered_trie_root(body.iter().map(Encode::encode).collect()); + let got = HashFor::::ordered_trie_root( + body.iter().map(Encode::encode).collect(), + sp_runtime::StateVersion::V0, + ); if expected != got { debug!( target:"sync", diff --git a/client/rpc/src/state/tests.rs b/client/rpc/src/state/tests.rs index d360701c88b2a..12e1c449d3bd4 100644 --- a/client/rpc/src/state/tests.rs +++ b/client/rpc/src/state/tests.rs @@ -526,11 +526,11 @@ fn should_return_runtime_version() { ); let result = "{\"specName\":\"test\",\"implName\":\"parity-test\",\"authoringVersion\":1,\ - \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",3],\ + \"specVersion\":2,\"implVersion\":2,\"apis\":[[\"0xdf6acb689907609b\",4],\ [\"0x37e397fc7c91f5e4\",1],[\"0xd2bc9897eed08f15\",3],[\"0x40fe3ad401f8959a\",5],\ [\"0xc6e9a76309f39b09\",1],[\"0xdd718d5cc53262d4\",1],[\"0xcbca25e39f142387\",2],\ [\"0xf78b278be53f454c\",2],[\"0xab3c0572291feb8b\",1],[\"0xbc9d89904f5b923f\",1]],\ - \"transactionVersion\":1}"; + \"transactionVersion\":1,\"stateVersion\":1}"; let runtime_version = executor::block_on(api.runtime_version(None.into())).unwrap(); let serialized = serde_json::to_string(&runtime_version).unwrap(); diff --git a/client/service/src/client/call_executor.rs b/client/service/src/client/call_executor.rs index 9f56424daf2f5..2acbbb75dcb1d 100644 --- a/client/service/src/client/call_executor.rs +++ b/client/service/src/client/call_executor.rs @@ -90,7 +90,7 @@ where Block: BlockT, B: backend::Backend, { - let spec = self.runtime_version(id)?; + let spec = CallExecutor::runtime_version(self, id)?; let code = if let Some(d) = self .wasm_override .as_ref() @@ -321,6 +321,20 @@ where } } +impl RuntimeVersionOf for LocalCallExecutor +where + E: RuntimeVersionOf, + Block: BlockT, +{ + fn runtime_version( + &self, + ext: &mut dyn sp_externalities::Externalities, + runtime_code: &sp_core::traits::RuntimeCode, + ) -> Result { + RuntimeVersionOf::runtime_version(&self.executor, ext, runtime_code) + } +} + impl sp_version::GetRuntimeVersionAt for LocalCallExecutor where B: backend::Backend, diff --git a/client/service/src/client/client.rs b/client/service/src/client/client.rs index 833d8b7bc2257..1f76fe017ae07 100644 --- a/client/service/src/client/client.rs +++ b/client/service/src/client/client.rs @@ -45,7 +45,7 @@ use sc_client_api::{ use sc_consensus::{ BlockCheckParams, BlockImportParams, ForkChoiceStrategy, ImportResult, StateAction, }; -use sc_executor::RuntimeVersion; +use sc_executor::{RuntimeVersion, RuntimeVersionOf}; use sc_telemetry::{telemetry, TelemetryHandle, SUBSTRATE_INFO}; use sp_api::{ ApiExt, ApiRef, CallApiAt, CallApiAtParams, ConstructRuntimeApi, Core as CoreApi, @@ -60,8 +60,8 @@ use sp_consensus::{BlockOrigin, BlockStatus, Error as ConsensusError}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_core::{ storage::{ - well_known_keys, ChildInfo, ChildType, PrefixedStorageKey, StorageChild, StorageData, - StorageKey, + well_known_keys, ChildInfo, ChildType, PrefixedStorageKey, Storage, StorageChild, + StorageData, StorageKey, }, NativeOrEncoded, }; @@ -72,7 +72,7 @@ use sp_runtime::{ traits::{ Block as BlockT, HashFor, Header as HeaderT, NumberFor, One, SaturatedConversion, Zero, }, - BuildStorage, Digest, Justification, Justifications, + BuildStorage, Digest, Justification, Justifications, StateVersion, }; use sp_state_machine::{ prove_child_read, prove_range_read_with_child_with_size, prove_read, @@ -81,7 +81,7 @@ use sp_state_machine::{ }; use sp_trie::{CompactProof, StorageProof}; use std::{ - collections::{HashMap, HashSet}, + collections::{hash_map::DefaultHasher, HashMap, HashSet}, marker::PhantomData, panic::UnwindSafe, path::PathBuf, @@ -93,7 +93,6 @@ use std::{ use { super::call_executor::LocalCallExecutor, sc_client_api::in_mem, - sc_executor::RuntimeVersionOf, sp_core::traits::{CodeExecutor, SpawnNamed}, }; @@ -334,8 +333,11 @@ where if info.finalized_state.is_none() { let genesis_storage = build_genesis_storage.build_storage().map_err(sp_blockchain::Error::Storage)?; + let genesis_state_version = + Self::resolve_state_version_from_wasm(&genesis_storage, &executor)?; let mut op = backend.begin_operation()?; - let state_root = op.set_genesis_state(genesis_storage, !config.no_genesis)?; + let state_root = + op.set_genesis_state(genesis_storage, !config.no_genesis, genesis_state_version)?; let genesis_block = genesis::construct_genesis_block::(state_root.into()); info!( "🔨 Initializing Genesis block/state (state: {}, header-hash: {})", @@ -403,7 +405,7 @@ where /// Get the RuntimeVersion at a given block. pub fn runtime_version_at(&self, id: &BlockId) -> sp_blockchain::Result { - self.executor.runtime_version(id) + CallExecutor::runtime_version(&self.executor, id) } /// Apply a checked and validated block to an operation. If a justification is provided @@ -606,7 +608,11 @@ where } } - let state_root = operation.op.reset_storage(storage)?; + // This is use by fast sync for runtime version to be resolvable from + // changes. + let state_version = + Self::resolve_state_version_from_wasm(&storage, &self.executor)?; + let state_root = operation.op.reset_storage(storage, state_version)?; if state_root != *import_headers.post().state_root() { // State root mismatch when importing state. This should not happen in // safe fast sync mode, but may happen in unsafe mode. @@ -1041,6 +1047,35 @@ where trace!("Collected {} uncles", uncles.len()); Ok(uncles) } + + fn resolve_state_version_from_wasm( + storage: &Storage, + executor: &E, + ) -> sp_blockchain::Result { + if let Some(wasm) = storage.top.get(well_known_keys::CODE) { + let mut ext = sp_state_machine::BasicExternalities::new_empty(); // just to read runtime version. + + let code_fetcher = sp_core::traits::WrappedRuntimeCode(wasm.as_slice().into()); + let runtime_code = sp_core::traits::RuntimeCode { + code_fetcher: &code_fetcher, + heap_pages: None, + hash: { + use std::hash::{Hash, Hasher}; + let mut state = DefaultHasher::new(); + wasm.hash(&mut state); + state.finish().to_le_bytes().to_vec() + }, + }; + let runtime_version = + RuntimeVersionOf::runtime_version(executor, &mut ext, &runtime_code) + .map_err(|e| sp_blockchain::Error::VersionInvalid(format!("{:?}", e)))?; + Ok(runtime_version.state_version()) + } else { + Err(sp_blockchain::Error::VersionInvalid( + "Runtime missing from initial storage, could not read state version.".to_string(), + )) + } + } } impl UsageProvider for Client @@ -1095,12 +1130,14 @@ where size_limit: usize, ) -> sp_blockchain::Result<(CompactProof, u32)> { let state = self.state_at(id)?; - let root = state.storage_root(std::iter::empty()).0; + // this is a read proof, using version V0 or V1 is equivalent. + let root = state.storage_root(std::iter::empty(), StateVersion::V0).0; let (proof, count) = prove_range_read_with_child_with_size::<_, HashFor>( state, size_limit, start_key, )?; - let proof = sp_trie::encode_compact::>>(proof, root) + // This is read proof only, we can use either LayoutV0 or LayoutV1. + let proof = sp_trie::encode_compact::>>(proof, root) .map_err(|e| sp_blockchain::Error::from_state(Box::new(e)))?; Ok((proof, count)) } @@ -1225,7 +1262,8 @@ where start_key: &[Vec], ) -> sp_blockchain::Result<(KeyValueStates, usize)> { let mut db = sp_state_machine::MemoryDB::>::new(&[]); - let _ = sp_trie::decode_compact::>, _, _>( + // Compact encoding + let _ = sp_trie::decode_compact::>, _, _>( &mut db, proof.iter_compact_encoded_nodes(), Some(&root), @@ -1594,7 +1632,7 @@ where } fn runtime_version_at(&self, at: &BlockId) -> Result { - self.runtime_version_at(at).map_err(Into::into) + CallExecutor::runtime_version(&self.executor, at).map_err(Into::into) } } diff --git a/client/service/src/client/genesis.rs b/client/service/src/client/genesis.rs index e764e8e24f105..b48ca2464b54e 100644 --- a/client/service/src/client/genesis.rs +++ b/client/service/src/client/genesis.rs @@ -22,8 +22,10 @@ use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header as HeaderT, Zero /// Create a genesis block, given the initial storage. pub fn construct_genesis_block(state_root: Block::Hash) -> Block { - let extrinsics_root = - <<::Header as HeaderT>::Hashing as HashT>::trie_root(Vec::new()); + let extrinsics_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( + Vec::new(), + sp_runtime::StateVersion::V0, + ); Block::new( <::Header as HeaderT>::new( diff --git a/client/service/test/src/client/mod.rs b/client/service/test/src/client/mod.rs index 286d819a6ce5e..cd4ce132b6634 100644 --- a/client/service/test/src/client/mod.rs +++ b/client/service/test/src/client/mod.rs @@ -34,13 +34,13 @@ use sp_core::{testing::TaskExecutor, H256}; use sp_runtime::{ generic::BlockId, traits::{BlakeTwo256, Block as BlockT, Header as HeaderT}, - ConsensusEngineId, Justifications, + ConsensusEngineId, Justifications, StateVersion, }; use sp_state_machine::{ backend::Backend as _, ExecutionStrategy, InMemoryBackend, OverlayedChanges, StateMachine, }; use sp_storage::{ChildInfo, StorageKey}; -use sp_trie::{trie_types::Layout, TrieConfiguration}; +use sp_trie::{LayoutV0, TrieConfiguration}; use std::{collections::HashSet, sync::Arc}; use substrate_test_runtime::TestAPI; use substrate_test_runtime_client::{ @@ -90,7 +90,7 @@ fn construct_block( let transactions = txs.into_iter().map(|tx| tx.into_signed_tx()).collect::>(); let iter = transactions.iter().map(Encode::encode); - let extrinsics_root = Layout::::ordered_trie_root(iter).into(); + let extrinsics_root = LayoutV0::::ordered_trie_root(iter).into(); let mut header = Header { parent_hash, @@ -177,7 +177,7 @@ fn construct_genesis_should_work_with_native() { .genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); - let backend = InMemoryBackend::from(storage); + let backend = InMemoryBackend::from((storage, StateVersion::default())); let (b1data, _b1hash) = block1(genesis_hash, &backend); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); @@ -210,7 +210,7 @@ fn construct_genesis_should_work_with_wasm() { .genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); - let backend = InMemoryBackend::from(storage); + let backend = InMemoryBackend::from((storage, StateVersion::default())); let (b1data, _b1hash) = block1(genesis_hash, &backend); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); @@ -243,7 +243,7 @@ fn construct_genesis_with_bad_transaction_should_panic() { .genesis_map(); let genesis_hash = insert_genesis_block(&mut storage); - let backend = InMemoryBackend::from(storage); + let backend = InMemoryBackend::from((storage, StateVersion::default())); let (b1data, _b1hash) = block1(genesis_hash, &backend); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); @@ -418,8 +418,8 @@ fn uncles_with_multiple_forks() { // block tree: // G -> A1 -> A2 -> A3 -> A4 -> A5 // A1 -> B2 -> B3 -> B4 - // B2 -> C3 - // A1 -> D2 + // B2 -> C3 + // A1 -> D2 let mut client = substrate_test_runtime_client::new(); // G -> A1 diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index 56545914dcba3..3e1ed1ed2f11d 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -42,8 +42,9 @@ pub use sp_io::storage::root as storage_root; #[doc(hidden)] pub use sp_runtime::traits::Zero; #[doc(hidden)] -pub use sp_std::{self, boxed::Box, prelude::Vec, str, vec}; +pub use sp_runtime::StateVersion; #[doc(hidden)] +pub use sp_std::{self, boxed::Box, prelude::Vec, str, vec}; pub use sp_storage::TrackedStorageKey; pub use utils::*; @@ -1079,7 +1080,7 @@ macro_rules! impl_benchmark { // Time the storage root recalculation. let start_storage_root = $crate::benchmarking::current_time(); - $crate::storage_root(); + $crate::storage_root($crate::StateVersion::V1); let finish_storage_root = $crate::benchmarking::current_time(); let elapsed_storage_root = finish_storage_root - start_storage_root; diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index cc013d4c333f4..1b74bdc2fea0a 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -671,7 +671,7 @@ mod tests { #[pallet::weight(0)] pub fn calculate_storage_root(_origin: OriginFor) -> DispatchResult { - let root = sp_io::storage::root(); + let root = sp_io::storage::root(sp_runtime::StateVersion::V1); sp_io::storage::set("storage_root".as_bytes(), &root); Ok(()) } @@ -897,17 +897,32 @@ mod tests { t.into() } + fn new_test_ext_v0(balance_factor: Balance) -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); + pallet_balances::GenesisConfig:: { balances: vec![(1, 111 * balance_factor)] } + .assimilate_storage(&mut t) + .unwrap(); + (t, sp_runtime::StateVersion::V0).into() + } + #[test] fn block_import_works() { - new_test_ext(1).execute_with(|| { + block_import_works_inner( + new_test_ext_v0(1), + hex!("1039e1a4bd0cf5deefe65f313577e70169c41c7773d6acf31ca8d671397559f5").into(), + ); + block_import_works_inner( + new_test_ext(1), + hex!("75e7d8f360d375bbe91bcf8019c01ab6362448b4a89e3b329717eb9d910340e5").into(), + ); + } + fn block_import_works_inner(mut ext: sp_io::TestExternalities, state_root: H256) { + ext.execute_with(|| { Executive::execute_block(Block { header: Header { parent_hash: [69u8; 32].into(), number: 1, - state_root: hex!( - "1039e1a4bd0cf5deefe65f313577e70169c41c7773d6acf31ca8d671397559f5" - ) - .into(), + state_root, extrinsics_root: hex!( "03170a2e7597b7b7e3d84c05391d139a62b157e78786d8c082f29dcf4c111314" ) @@ -948,7 +963,7 @@ mod tests { parent_hash: [69u8; 32].into(), number: 1, state_root: hex!( - "49cd58a254ccf6abc4a023d9a22dcfc421e385527a250faec69f8ad0d8ed3e48" + "75e7d8f360d375bbe91bcf8019c01ab6362448b4a89e3b329717eb9d910340e5" ) .into(), extrinsics_root: [0u8; 32].into(), diff --git a/frame/session/src/historical/mod.rs b/frame/session/src/historical/mod.rs index a3e64f4f9efa4..fa7cb9ee91c03 100644 --- a/frame/session/src/historical/mod.rs +++ b/frame/session/src/historical/mod.rs @@ -39,7 +39,7 @@ use sp_session::{MembershipProof, ValidatorCount}; use sp_staking::SessionIndex; use sp_std::prelude::*; use sp_trie::{ - trie_types::{TrieDB, TrieDBMut}, + trie_types::{TrieDB, TrieDBMutV0}, MemoryDB, Recorder, Trie, TrieMut, EMPTY_PREFIX, }; @@ -236,7 +236,7 @@ impl ProvingTrie { let mut root = Default::default(); { - let mut trie = TrieDBMut::new(&mut db, &mut root); + let mut trie = TrieDBMutV0::new(&mut db, &mut root); for (i, (validator, full_id)) in validators.into_iter().enumerate() { let i = i as u32; let keys = match >::load_keys(&validator) { diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 2adcd8ce4efcf..2a23d203adf63 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -46,7 +46,7 @@ pub use sp_core_hashing_proc_macro; #[doc(hidden)] pub use sp_io::{self, storage::root as storage_root}; #[doc(hidden)] -pub use sp_runtime::RuntimeDebug; +pub use sp_runtime::{RuntimeDebug, StateVersion}; #[cfg(feature = "std")] #[doc(hidden)] pub use sp_state_machine::BasicExternalities; @@ -751,9 +751,9 @@ macro_rules! assert_noop { $x:expr, $y:expr $(,)? ) => { - let h = $crate::storage_root(); + let h = $crate::storage_root($crate::StateVersion::V1); $crate::assert_err!($x, $y); - assert_eq!(h, $crate::storage_root()); + assert_eq!(h, $crate::storage_root($crate::StateVersion::V1)); }; } @@ -766,9 +766,9 @@ macro_rules! assert_storage_noop { ( $x:expr ) => { - let h = $crate::storage_root(); + let h = $crate::storage_root($crate::StateVersion::V1); $x; - assert_eq!(h, $crate::storage_root()); + assert_eq!(h, $crate::storage_root($crate::StateVersion::V1)); }; } diff --git a/frame/support/src/storage/child.rs b/frame/support/src/storage/child.rs index 5497f642067cf..19b00d1051361 100644 --- a/frame/support/src/storage/child.rs +++ b/frame/support/src/storage/child.rs @@ -24,7 +24,7 @@ pub use crate::sp_io::KillStorageResult; use crate::sp_std::prelude::*; use codec::{Codec, Decode, Encode}; -pub use sp_core::storage::{ChildInfo, ChildType}; +pub use sp_core::storage::{ChildInfo, ChildType, StateVersion}; /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. pub fn get(child_info: &ChildInfo, key: &[u8]) -> Option { @@ -167,9 +167,10 @@ pub fn put_raw(child_info: &ChildInfo, key: &[u8], value: &[u8]) { } /// Calculate current child root value. -pub fn root(child_info: &ChildInfo) -> Vec { +pub fn root(child_info: &ChildInfo, version: StateVersion) -> Vec { match child_info.child_type() { - ChildType::ParentKeyId => sp_io::default_child_storage::root(child_info.storage_key()), + ChildType::ParentKeyId => + sp_io::default_child_storage::root(child_info.storage_key(), version), } } diff --git a/frame/support/test/compile_pass/src/lib.rs b/frame/support/test/compile_pass/src/lib.rs index a5586996215e9..47b81e1020693 100644 --- a/frame/support/test/compile_pass/src/lib.rs +++ b/frame/support/test/compile_pass/src/lib.rs @@ -41,6 +41,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { impl_version: 0, apis: sp_version::create_apis_vec!([]), transaction_version: 0, + state_version: 0, }; pub type Signature = sr25519::Signature; diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index e167f44ffd88c..8026f77902ba4 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -127,13 +127,19 @@ pub use extensions::check_mortality::CheckMortality as CheckEra; pub use weights::WeightInfo; /// Compute the trie root of a list of extrinsics. +/// +/// The merkle proof is using the same trie as runtime state with +/// `state_version` 0. pub fn extrinsics_root(extrinsics: &[E]) -> H::Output { extrinsics_data_root::(extrinsics.iter().map(codec::Encode::encode).collect()) } /// Compute the trie root of a list of extrinsics. +/// +/// The merkle proof is using the same trie as runtime state with +/// `state_version` 0. pub fn extrinsics_data_root(xts: Vec>) -> H::Output { - H::ordered_trie_root(xts) + H::ordered_trie_root(xts, sp_core::storage::StateVersion::V0) } /// An object to track the currently used extrinsic weight in a block. @@ -1355,7 +1361,8 @@ impl Pallet { >::remove(to_remove); } - let storage_root = T::Hash::decode(&mut &sp_io::storage::root()[..]) + let version = T::Version::get().state_version(); + let storage_root = T::Hash::decode(&mut &sp_io::storage::root(version)[..]) .expect("Node is configured to use the same hash; qed"); ::new( diff --git a/frame/system/src/mock.rs b/frame/system/src/mock.rs index d8d9e7323aed6..5d10962a8f402 100644 --- a/frame/system/src/mock.rs +++ b/frame/system/src/mock.rs @@ -53,6 +53,7 @@ parameter_types! { impl_version: 1, apis: sp_version::create_apis_vec!([]), transaction_version: 1, + state_version: 1, }; pub const DbWeight: RuntimeDbWeight = RuntimeDbWeight { read: 10, diff --git a/frame/transaction-storage/src/benchmarking.rs b/frame/transaction-storage/src/benchmarking.rs index 0f90b51087ae9..b03c75148fa12 100644 --- a/frame/transaction-storage/src/benchmarking.rs +++ b/frame/transaction-storage/src/benchmarking.rs @@ -29,54 +29,77 @@ use sp_transaction_storage_proof::TransactionStorageProof; use crate::Pallet as TransactionStorage; +// Proof generated from max size storage: +// ``` +// let mut transactions = Vec::new(); +// let tx_size = DEFAULT_MAX_TRANSACTION_SIZE; +// for _ in 0..DEFAULT_MAX_BLOCK_TRANSACTIONS { +// transactions.push(vec![0; tx_size]); +// } +// let hash = vec![0; 32]; +// build_proof(hash.as_slice(), transactions).unwrap().encode() +// ``` +// while hardforcing target chunk key in `build_proof` to [22, 21, 1, 0]. const PROOF: &[u8] = &hex_literal::hex!( " - 0104000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 000000000000000000000000000000014cd0780ffff80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe8 - 7d12a3662c4c0080e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb - 13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2 - f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f - 1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f - 3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a47 - 8e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cf - f93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e31 - 6a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f - 53cff93f3ca2f3dfe87d12a3662c4c80e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4c8 - 0e316a478e2f1fcb13cf22fd0b2dbb54a6f53cff93f3ca2f3dfe87d12a3662c4cbd05807777809a5d7a720ce5f9d9a012 - fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bcbf8a3dc2f6b9e957d129e610c06d411e11743062dc1cf - 3ac289390ae4c8008592aa2d915f52941036afbe72bac4ebe7ce186c4ddc53f118e0ddd4decd8cc809a5d7a720ce5f9d9 - a012fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bcbf8a3dc2f6b9e957d129e610c06d411e11743062d - c1cf3ac289390ae4c00809a5d7a720ce5f9d9a012fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bcbf8a - 3dc2f6b9e957d129e610c06d411e11743062dc1cf3ac289390ae4c8008592aa2d915f52941036afbe72bac4ebe7ce186c - 4ddc53f118e0ddd4decd8cc809a5d7a720ce5f9d9a012fbf25e92c30e732dadba8f312b05e02976313ea64d9f807d43bc - bf8a3dc2f6b9e957d129e610c06d411e11743062dc1cf3ac289390ae4c8008592aa2d915f52941036afbe72bac4ebe7ce - 186c4ddc53f118e0ddd4decd8cccd0780ffff8081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb0 - 3bdb31008081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253 - 515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa139 - 8e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5 - f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3a - a1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2b - a8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f32 - 2d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa - 9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f0 - 2f322d3aa1398e0cb03bdb318081b825bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb318081b82 - 5bfa9b2ba8f5f253515e7db09eb1ad3d4f02f322d3aa1398e0cb03bdb31cd0780ffff80b4f23ac50c8e67d9b280f2b31a - 5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd1885 - 44c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2 - b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd - 188544c5f9b0080b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9 - b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84 - d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e - 67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977aca - ac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac5 - 0c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b89297 - 7acaac84d530bd188544c5f9b80b4f23ac50c8e67d9b280f2b31a5707d52b892977acaac84d530bd188544c5f9b104401 - 0000 -" + 0104000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000 + 0000000000000000000000000000000000000000000000000000000000000000000000000000 + 00000000000000000000000000000000000000000000000000000000000014cd0780ffff8030 + 2eb0a6d2f63b834d15f1e729d1c1004657e3048cf206d697eeb153f61a30ba0080302eb0a6d2 + f63b834d15f1e729d1c1004657e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15 + f1e729d1c1004657e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c1 + 004657e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c1004657e304 + 8cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c1004657e3048cf206d697 + eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c1004657e3048cf206d697eeb153f61a + 30ba80302eb0a6d2f63b834d15f1e729d1c1004657e3048cf206d697eeb153f61a30ba80302e + b0a6d2f63b834d15f1e729d1c1004657e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b + 834d15f1e729d1c1004657e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e7 + 29d1c1004657e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c10046 + 57e3048cf206d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c1004657e3048cf2 + 06d697eeb153f61a30ba80302eb0a6d2f63b834d15f1e729d1c1004657e3048cf206d697eeb1 + 53f61a30ba80302eb0a6d2f63b834d15f1e729d1c1004657e3048cf206d697eeb153f61a30ba + bd058077778010fd81bc1359802f0b871aeb95e4410a8ec92b93af10ea767a2027cf4734e8de + 808da338e6b722f7bf2051901bd5bccee5e71d5cf6b1faff338ad7120b0256c28380221ce17f + 19117affa96e077905fe48a99723a065969c638593b7d9ab57b538438010fd81bc1359802f0b + 871aeb95e4410a8ec92b93af10ea767a2027cf4734e8de808da338e6b722f7bf2051901bd5bc + cee5e71d5cf6b1faff338ad7120b0256c283008010fd81bc1359802f0b871aeb95e4410a8ec9 + 2b93af10ea767a2027cf4734e8de808da338e6b722f7bf2051901bd5bccee5e71d5cf6b1faff + 338ad7120b0256c28380221ce17f19117affa96e077905fe48a99723a065969c638593b7d9ab + 57b538438010fd81bc1359802f0b871aeb95e4410a8ec92b93af10ea767a2027cf4734e8de80 + 8da338e6b722f7bf2051901bd5bccee5e71d5cf6b1faff338ad7120b0256c28380221ce17f19 + 117affa96e077905fe48a99723a065969c638593b7d9ab57b53843cd0780ffff804509f59593 + fd47b1a97189127ba65a5649cfb0346637f9836e155eaf891a939c00804509f59593fd47b1a9 + 7189127ba65a5649cfb0346637f9836e155eaf891a939c804509f59593fd47b1a97189127ba6 + 5a5649cfb0346637f9836e155eaf891a939c804509f59593fd47b1a97189127ba65a5649cfb0 + 346637f9836e155eaf891a939c804509f59593fd47b1a97189127ba65a5649cfb0346637f983 + 6e155eaf891a939c804509f59593fd47b1a97189127ba65a5649cfb0346637f9836e155eaf89 + 1a939c804509f59593fd47b1a97189127ba65a5649cfb0346637f9836e155eaf891a939c8045 + 09f59593fd47b1a97189127ba65a5649cfb0346637f9836e155eaf891a939c804509f59593fd + 47b1a97189127ba65a5649cfb0346637f9836e155eaf891a939c804509f59593fd47b1a97189 + 127ba65a5649cfb0346637f9836e155eaf891a939c804509f59593fd47b1a97189127ba65a56 + 49cfb0346637f9836e155eaf891a939c804509f59593fd47b1a97189127ba65a5649cfb03466 + 37f9836e155eaf891a939c804509f59593fd47b1a97189127ba65a5649cfb0346637f9836e15 + 5eaf891a939c804509f59593fd47b1a97189127ba65a5649cfb0346637f9836e155eaf891a93 + 9c804509f59593fd47b1a97189127ba65a5649cfb0346637f9836e155eaf891a939ccd0780ff + ff8078916e776c64ccea05e958559f015c082d9d06feafa3610fc44a5b2ef543cb818078916e + 776c64ccea05e958559f015c082d9d06feafa3610fc44a5b2ef543cb818078916e776c64ccea + 05e958559f015c082d9d06feafa3610fc44a5b2ef543cb818078916e776c64ccea05e958559f + 015c082d9d06feafa3610fc44a5b2ef543cb818078916e776c64ccea05e958559f015c082d9d + 06feafa3610fc44a5b2ef543cb81008078916e776c64ccea05e958559f015c082d9d06feafa3 + 610fc44a5b2ef543cb818078916e776c64ccea05e958559f015c082d9d06feafa3610fc44a5b + 2ef543cb818078916e776c64ccea05e958559f015c082d9d06feafa3610fc44a5b2ef543cb81 + 8078916e776c64ccea05e958559f015c082d9d06feafa3610fc44a5b2ef543cb818078916e77 + 6c64ccea05e958559f015c082d9d06feafa3610fc44a5b2ef543cb818078916e776c64ccea05 + e958559f015c082d9d06feafa3610fc44a5b2ef543cb818078916e776c64ccea05e958559f01 + 5c082d9d06feafa3610fc44a5b2ef543cb818078916e776c64ccea05e958559f015c082d9d06 + feafa3610fc44a5b2ef543cb818078916e776c64ccea05e958559f015c082d9d06feafa3610f + c44a5b2ef543cb818078916e776c64ccea05e958559f015c082d9d06feafa3610fc44a5b2ef5 + 43cb811044010000 + " ); type BalanceOf = @@ -136,7 +159,6 @@ benchmarks! { )?; } run_to_block::(StoragePeriod::::get() + T::BlockNumber::one()); - let random_hash = [0u8]; let mut encoded_proof = PROOF; let proof = TransactionStorageProof::decode(&mut encoded_proof).unwrap(); }: check_proof(RawOrigin::None, proof) diff --git a/frame/transaction-storage/src/lib.rs b/frame/transaction-storage/src/lib.rs index 94937c04ed5d8..2cd47b1cd0db2 100644 --- a/frame/transaction-storage/src/lib.rs +++ b/frame/transaction-storage/src/lib.rs @@ -188,7 +188,7 @@ pub mod pallet { // Chunk data and compute storage root let chunk_count = num_chunks(data.len() as u32); let chunks = data.chunks(CHUNK_SIZE).map(|c| c.to_vec()).collect(); - let root = sp_io::trie::blake2_256_ordered_root(chunks); + let root = sp_io::trie::blake2_256_ordered_root(chunks, sp_runtime::StateVersion::V1); let content_hash = sp_io::hashing::blake2_256(&data); let extrinsic_index = >::extrinsic_index() @@ -300,6 +300,7 @@ pub mod pallet { &proof.proof, &encode_index(chunk_index), &proof.chunk, + sp_runtime::StateVersion::V1, ), Error::::InvalidProof ); diff --git a/primitives/api/proc-macro/src/impl_runtime_apis.rs b/primitives/api/proc-macro/src/impl_runtime_apis.rs index dd2a7f6c14909..db3ab3b76850e 100644 --- a/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -293,10 +293,17 @@ fn generate_runtime_api_base_structures() -> Result { #crate_::StorageChanges, String > where Self: Sized { + let at = #crate_::BlockId::Hash(parent_hash.clone()); + let state_version = self.call + .runtime_version_at(&at) + .map(|v| v.state_version()) + .map_err(|e| format!("Failed to get state version: {:?}", e))?; + self.changes.replace(Default::default()).into_storage_changes( backend, parent_hash, self.storage_transaction_cache.replace(Default::default()), + state_version, ) } } diff --git a/primitives/api/src/lib.rs b/primitives/api/src/lib.rs index 7f64e191941f7..41c0151d6c538 100644 --- a/primitives/api/src/lib.rs +++ b/primitives/api/src/lib.rs @@ -84,6 +84,8 @@ pub use sp_core::NativeOrEncoded; use sp_core::OpaqueMetadata; #[doc(hidden)] pub use sp_core::{offchain, ExecutionContext}; +#[cfg(feature = "std")] +pub use sp_runtime::StateVersion; #[doc(hidden)] pub use sp_runtime::{ generic::BlockId, @@ -269,6 +271,7 @@ pub use sp_api_proc_macro::decl_runtime_apis; /// // Here we are exposing the runtime api versions. /// apis: RUNTIME_API_VERSIONS, /// transaction_version: 1, +/// state_version: 1, /// }; /// /// # fn main() {} @@ -654,53 +657,13 @@ pub fn deserialize_runtime_api_info(bytes: [u8; RUNTIME_API_INFO_SIZE]) -> ([u8; (id, version) } -#[derive(codec::Encode, codec::Decode)] -pub struct OldRuntimeVersion { - pub spec_name: RuntimeString, - pub impl_name: RuntimeString, - pub authoring_version: u32, - pub spec_version: u32, - pub impl_version: u32, - pub apis: ApisVec, -} - -impl From for RuntimeVersion { - fn from(x: OldRuntimeVersion) -> Self { - Self { - spec_name: x.spec_name, - impl_name: x.impl_name, - authoring_version: x.authoring_version, - spec_version: x.spec_version, - impl_version: x.impl_version, - apis: x.apis, - transaction_version: 1, - } - } -} - -impl From for OldRuntimeVersion { - fn from(x: RuntimeVersion) -> Self { - Self { - spec_name: x.spec_name, - impl_name: x.impl_name, - authoring_version: x.authoring_version, - spec_version: x.spec_version, - impl_version: x.impl_version, - apis: x.apis, - } - } -} - decl_runtime_apis! { /// The `Core` runtime api that every Substrate runtime needs to implement. #[core_trait] - #[api_version(3)] + #[api_version(4)] pub trait Core { /// Returns the version of the runtime. fn version() -> RuntimeVersion; - /// Returns the version of the runtime. - #[changed_in(3)] - fn version() -> OldRuntimeVersion; /// Execute the given block. fn execute_block(block: Block); /// Initialize a block with the given header. diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index aac45234deadd..6d560c3c7f11b 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -31,7 +31,7 @@ use sp_std::{ vec::Vec, }; -use sp_storage::{ChildInfo, TrackedStorageKey}; +use sp_storage::{ChildInfo, StateVersion, TrackedStorageKey}; pub use extensions::{Extension, ExtensionStore, Extensions}; pub use scope_limited::{set_and_run_with_externalities, with_externalities}; @@ -157,7 +157,7 @@ pub trait Externalities: ExtensionStore { /// This will also update all child storage keys in the top-level storage map. /// /// The returned hash is defined by the `Block` and is SCALE encoded. - fn storage_root(&mut self) -> Vec; + fn storage_root(&mut self, state_version: StateVersion) -> Vec; /// Get the trie root of a child storage map. /// @@ -165,7 +165,11 @@ pub trait Externalities: ExtensionStore { /// /// If the storage root equals the default hash as defined by the trie, the key in the top-level /// storage map will be removed. - fn child_storage_root(&mut self, child_info: &ChildInfo) -> Vec; + fn child_storage_root( + &mut self, + child_info: &ChildInfo, + state_version: StateVersion, + ) -> Vec; /// Append storage item. /// diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 1edf87703fbe6..9025ae3d76f43 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -51,11 +51,13 @@ use sp_core::{ offchain::{ HttpError, HttpRequestId, HttpRequestStatus, OpaqueNetworkState, StorageKind, Timestamp, }, - sr25519, LogLevel, LogLevelFilter, OpaquePeerId, H256, + sr25519, + storage::StateVersion, + LogLevel, LogLevelFilter, OpaquePeerId, H256, }; #[cfg(feature = "std")] -use sp_trie::{trie_types::Layout, TrieConfiguration}; +use sp_trie::{LayoutV0, LayoutV1, TrieConfiguration}; use sp_runtime_interface::{ pass_by::{PassBy, PassByCodec}, @@ -192,7 +194,17 @@ pub trait Storage { /// /// Returns a `Vec` that holds the SCALE encoded hash. fn root(&mut self) -> Vec { - self.storage_root() + self.storage_root(StateVersion::V0) + } + + /// "Commit" all existing operations and compute the resulting storage root. + /// + /// The hashing algorithm is defined by the `Block`. + /// + /// Returns a `Vec` that holds the SCALE encoded hash. + #[version(2)] + fn root(&mut self, version: StateVersion) -> Vec { + self.storage_root(version) } /// Always returns `None`. This function exists for compatibility reasons. @@ -373,7 +385,19 @@ pub trait DefaultChildStorage { /// Returns a `Vec` that holds the SCALE encoded hash. fn root(&mut self, storage_key: &[u8]) -> Vec { let child_info = ChildInfo::new_default(storage_key); - self.child_storage_root(&child_info) + self.child_storage_root(&child_info, StateVersion::V0) + } + + /// Default child root calculation. + /// + /// "Commit" all existing operations and compute the resulting child storage root. + /// The hashing algorithm is defined by the `Block`. + /// + /// Returns a `Vec` that holds the SCALE encoded hash. + #[version(2)] + fn root(&mut self, storage_key: &[u8], version: StateVersion) -> Vec { + let child_info = ChildInfo::new_default(storage_key); + self.child_storage_root(&child_info, version) } /// Child storage key iteration. @@ -390,27 +414,63 @@ pub trait DefaultChildStorage { pub trait Trie { /// A trie root formed from the iterated items. fn blake2_256_root(input: Vec<(Vec, Vec)>) -> H256 { - Layout::::trie_root(input) + LayoutV0::::trie_root(input) + } + + /// A trie root formed from the iterated items. + #[version(2)] + fn blake2_256_root(input: Vec<(Vec, Vec)>, version: StateVersion) -> H256 { + match version { + StateVersion::V0 => LayoutV0::::trie_root(input), + StateVersion::V1 => LayoutV1::::trie_root(input), + } } /// A trie root formed from the enumerated items. fn blake2_256_ordered_root(input: Vec>) -> H256 { - Layout::::ordered_trie_root(input) + LayoutV0::::ordered_trie_root(input) + } + + /// A trie root formed from the enumerated items. + #[version(2)] + fn blake2_256_ordered_root(input: Vec>, version: StateVersion) -> H256 { + match version { + StateVersion::V0 => LayoutV0::::ordered_trie_root(input), + StateVersion::V1 => LayoutV1::::ordered_trie_root(input), + } } /// A trie root formed from the iterated items. fn keccak_256_root(input: Vec<(Vec, Vec)>) -> H256 { - Layout::::trie_root(input) + LayoutV0::::trie_root(input) + } + + /// A trie root formed from the iterated items. + #[version(2)] + fn keccak_256_root(input: Vec<(Vec, Vec)>, version: StateVersion) -> H256 { + match version { + StateVersion::V0 => LayoutV0::::trie_root(input), + StateVersion::V1 => LayoutV1::::trie_root(input), + } } /// A trie root formed from the enumerated items. fn keccak_256_ordered_root(input: Vec>) -> H256 { - Layout::::ordered_trie_root(input) + LayoutV0::::ordered_trie_root(input) + } + + /// A trie root formed from the enumerated items. + #[version(2)] + fn keccak_256_ordered_root(input: Vec>, version: StateVersion) -> H256 { + match version { + StateVersion::V0 => LayoutV0::::ordered_trie_root(input), + StateVersion::V1 => LayoutV1::::ordered_trie_root(input), + } } /// Verify trie proof fn blake2_256_verify_proof(root: H256, proof: &[Vec], key: &[u8], value: &[u8]) -> bool { - sp_trie::verify_trie_proof::, _, _, _>( + sp_trie::verify_trie_proof::, _, _, _>( &root, proof, &[(key, Some(value))], @@ -418,15 +478,69 @@ pub trait Trie { .is_ok() } + /// Verify trie proof + #[version(2)] + fn blake2_256_verify_proof( + root: H256, + proof: &[Vec], + key: &[u8], + value: &[u8], + version: StateVersion, + ) -> bool { + match version { + StateVersion::V0 => sp_trie::verify_trie_proof::< + LayoutV0, + _, + _, + _, + >(&root, proof, &[(key, Some(value))]) + .is_ok(), + StateVersion::V1 => sp_trie::verify_trie_proof::< + LayoutV1, + _, + _, + _, + >(&root, proof, &[(key, Some(value))]) + .is_ok(), + } + } + /// Verify trie proof fn keccak_256_verify_proof(root: H256, proof: &[Vec], key: &[u8], value: &[u8]) -> bool { - sp_trie::verify_trie_proof::, _, _, _>( + sp_trie::verify_trie_proof::, _, _, _>( &root, proof, &[(key, Some(value))], ) .is_ok() } + + /// Verify trie proof + #[version(2)] + fn keccak_256_verify_proof( + root: H256, + proof: &[Vec], + key: &[u8], + value: &[u8], + version: StateVersion, + ) -> bool { + match version { + StateVersion::V0 => sp_trie::verify_trie_proof::< + LayoutV0, + _, + _, + _, + >(&root, proof, &[(key, Some(value))]) + .is_ok(), + StateVersion::V1 => sp_trie::verify_trie_proof::< + LayoutV1, + _, + _, + _, + >(&root, proof, &[(key, Some(value))]) + .is_ok(), + } + } } /// Interface that provides miscellaneous functions for communicating between the runtime and the @@ -1570,6 +1684,16 @@ mod tests { assert_eq!(storage::get(b"hello"), None); assert_eq!(storage::get(b"foo"), Some(b"bar".to_vec())); }); + + let value = vec![7u8; 35]; + let storage = + Storage { top: map![b"foo00".to_vec() => value.clone()], children_default: map![] }; + t = BasicExternalities::new(storage); + + t.execute_with(|| { + assert_eq!(storage::get(b"hello"), None); + assert_eq!(storage::get(b"foo00"), Some(value.clone())); + }); } #[test] diff --git a/primitives/runtime-interface/src/impls.rs b/primitives/runtime-interface/src/impls.rs index 35badd27fa992..16eecbd021e00 100644 --- a/primitives/runtime-interface/src/impls.rs +++ b/primitives/runtime-interface/src/impls.rs @@ -544,3 +544,7 @@ impl PassBy for sp_wasm_interface::Value { impl PassBy for sp_storage::TrackedStorageKey { type PassBy = Codec; } + +impl PassBy for sp_storage::StateVersion { + type PassBy = Enum; +} diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index deea4a73c8e2d..7fdc5dda5c74b 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -40,6 +40,7 @@ pub use paste; #[doc(hidden)] pub use sp_application_crypto as app_crypto; +pub use sp_core::storage::StateVersion; #[cfg(feature = "std")] pub use sp_core::storage::{Storage, StorageChild}; diff --git a/primitives/runtime/src/traits.rs b/primitives/runtime/src/traits.rs index 0ddd8cae93ea7..cc83f66ea4630 100644 --- a/primitives/runtime/src/traits.rs +++ b/primitives/runtime/src/traits.rs @@ -36,7 +36,7 @@ pub use sp_arithmetic::traits::{ CheckedShr, CheckedSub, IntegerSquareRoot, One, SaturatedConversion, Saturating, UniqueSaturatedFrom, UniqueSaturatedInto, Zero, }; -use sp_core::{self, Hasher, RuntimeDebug, TypeId}; +use sp_core::{self, storage::StateVersion, Hasher, RuntimeDebug, TypeId}; use sp_std::{ self, convert::{TryFrom, TryInto}, @@ -459,10 +459,10 @@ pub trait Hash: } /// The ordered Patricia tree root of the given `input`. - fn ordered_trie_root(input: Vec>) -> Self::Output; + fn ordered_trie_root(input: Vec>, state_version: StateVersion) -> Self::Output; /// The Patricia tree root of the given mapping. - fn trie_root(input: Vec<(Vec, Vec)>) -> Self::Output; + fn trie_root(input: Vec<(Vec, Vec)>, state_version: StateVersion) -> Self::Output; } /// Blake2-256 Hash implementation. @@ -483,12 +483,12 @@ impl Hasher for BlakeTwo256 { impl Hash for BlakeTwo256 { type Output = sp_core::H256; - fn trie_root(input: Vec<(Vec, Vec)>) -> Self::Output { - sp_io::trie::blake2_256_root(input) + fn trie_root(input: Vec<(Vec, Vec)>, version: StateVersion) -> Self::Output { + sp_io::trie::blake2_256_root(input, version) } - fn ordered_trie_root(input: Vec>) -> Self::Output { - sp_io::trie::blake2_256_ordered_root(input) + fn ordered_trie_root(input: Vec>, version: StateVersion) -> Self::Output { + sp_io::trie::blake2_256_ordered_root(input, version) } } @@ -510,12 +510,12 @@ impl Hasher for Keccak256 { impl Hash for Keccak256 { type Output = sp_core::H256; - fn trie_root(input: Vec<(Vec, Vec)>) -> Self::Output { - sp_io::trie::keccak_256_root(input) + fn trie_root(input: Vec<(Vec, Vec)>, version: StateVersion) -> Self::Output { + sp_io::trie::keccak_256_root(input, version) } - fn ordered_trie_root(input: Vec>) -> Self::Output { - sp_io::trie::keccak_256_ordered_root(input) + fn ordered_trie_root(input: Vec>, version: StateVersion) -> Self::Output { + sp_io::trie::keccak_256_ordered_root(input, version) } } diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 2c84b9186d0ab..b9e5c1d4f3f0f 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -18,8 +18,8 @@ log = { version = "0.4.11", optional = true } thiserror = { version = "1.0.30", optional = true } parking_lot = { version = "0.11.2", optional = true } hash-db = { version = "0.15.2", default-features = false } -trie-db = { version = "0.22.6", default-features = false } -trie-root = { version = "0.16.0", default-features = false } +trie-db = { version = "0.23.0", default-features = false } +trie-root = { version = "0.17.0", default-features = false } sp-trie = { version = "4.0.0", path = "../trie", default-features = false } sp-core = { version = "4.1.0-dev", path = "../core", default-features = false } sp-panic-handler = { version = "4.0.0", path = "../panic-handler", optional = true } diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index eb6e2939b83fc..8a6ed4d3dba81 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -23,7 +23,7 @@ use crate::{ }; use codec::Encode; use hash_db::Hasher; -use sp_core::storage::{ChildInfo, TrackedStorageKey}; +use sp_core::storage::{ChildInfo, StateVersion, TrackedStorageKey}; #[cfg(feature = "std")] use sp_core::traits::RuntimeCode; use sp_std::vec::Vec; @@ -140,6 +140,7 @@ pub trait Backend: sp_std::fmt::Debug { fn storage_root<'a>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (H::Out, Self::Transaction) where H::Out: Ord; @@ -151,6 +152,7 @@ pub trait Backend: sp_std::fmt::Debug { &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord; @@ -176,7 +178,6 @@ pub trait Backend: sp_std::fmt::Debug { fn as_trie_backend(&self) -> Option<&TrieBackend> { None } - /// Calculate the storage root, with given delta over what is already stored /// in the backend, and produce a "transaction" that can be used to commit. /// Does include child storage updates. @@ -186,6 +187,7 @@ pub trait Backend: sp_std::fmt::Debug { child_deltas: impl Iterator< Item = (&'a ChildInfo, impl Iterator)>), >, + state_version: StateVersion, ) -> (H::Out, Self::Transaction) where H::Out: Ord + Encode, @@ -194,7 +196,8 @@ pub trait Backend: sp_std::fmt::Debug { let mut child_roots: Vec<_> = Default::default(); // child first for (child_info, child_delta) in child_deltas { - let (child_root, empty, child_txs) = self.child_storage_root(&child_info, child_delta); + let (child_root, empty, child_txs) = + self.child_storage_root(&child_info, child_delta, state_version); let prefixed_storage_key = child_info.prefixed_storage_key(); txs.consolidate(child_txs); if empty { @@ -205,8 +208,9 @@ pub trait Backend: sp_std::fmt::Debug { } let (root, parent_txs) = self.storage_root( delta - .map(|(k, v)| (k, v.as_ref().map(|v| &v[..]))) + .map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..]))) .chain(child_roots.iter().map(|(k, v)| (&k[..], v.as_ref().map(|v| &v[..])))), + state_version, ); txs.consolidate(parent_txs); (root, txs) @@ -286,7 +290,11 @@ impl Consolidate for Vec<(Option, StorageCollection)> { } } -impl> Consolidate for sp_trie::GenericMemoryDB { +impl Consolidate for sp_trie::GenericMemoryDB +where + H: Hasher, + KF: sp_trie::KeyFunction, +{ fn consolidate(&mut self, other: Self) { sp_trie::GenericMemoryDB::consolidate(self, other) } diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 3774adc5b0368..c0bec8d08f000 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -23,13 +23,14 @@ use hash_db::Hasher; use log::warn; use sp_core::{ storage::{ - well_known_keys::is_child_storage_key, ChildInfo, Storage, StorageChild, TrackedStorageKey, + well_known_keys::is_child_storage_key, ChildInfo, StateVersion, Storage, StorageChild, + TrackedStorageKey, }, traits::Externalities, Blake2Hasher, }; use sp_externalities::{Extension, Extensions}; -use sp_trie::{empty_child_trie_root, trie_types::Layout, TrieConfiguration}; +use sp_trie::{empty_child_trie_root, LayoutV0, LayoutV1, TrieConfiguration}; use std::{ any::{Any, TypeId}, collections::BTreeMap, @@ -273,7 +274,7 @@ impl Externalities for BasicExternalities { crate::ext::StorageAppend::new(current).append(value); } - fn storage_root(&mut self) -> Vec { + fn storage_root(&mut self, state_version: StateVersion) -> Vec { let mut top = self.inner.top.clone(); let prefixed_keys: Vec<_> = self .inner @@ -284,9 +285,9 @@ impl Externalities for BasicExternalities { // Single child trie implementation currently allows using the same child // empty root for all child trie. Using null storage key until multiple // type of child trie support. - let empty_hash = empty_child_trie_root::>(); + let empty_hash = empty_child_trie_root::>(); for (prefixed_storage_key, child_info) in prefixed_keys { - let child_root = self.child_storage_root(&child_info); + let child_root = self.child_storage_root(&child_info, state_version); if &empty_hash[..] == &child_root[..] { top.remove(prefixed_storage_key.as_slice()); } else { @@ -294,17 +295,26 @@ impl Externalities for BasicExternalities { } } - Layout::::trie_root(self.inner.top.clone()).as_ref().into() + match state_version { + StateVersion::V0 => + LayoutV0::::trie_root(self.inner.top.clone()).as_ref().into(), + StateVersion::V1 => + LayoutV1::::trie_root(self.inner.top.clone()).as_ref().into(), + } } - fn child_storage_root(&mut self, child_info: &ChildInfo) -> Vec { + fn child_storage_root( + &mut self, + child_info: &ChildInfo, + state_version: StateVersion, + ) -> Vec { if let Some(child) = self.inner.children_default.get(child_info.storage_key()) { let delta = child.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))); crate::in_memory_backend::new_in_mem::() - .child_storage_root(&child.child_info, delta) + .child_storage_root(&child.child_info, delta, state_version) .0 } else { - empty_child_trie_root::>() + empty_child_trie_root::>() } .encode() } @@ -389,7 +399,7 @@ mod tests { const ROOT: [u8; 32] = hex!("39245109cef3758c2eed2ccba8d9b370a917850af3824bc8348d505df2c298fa"); - assert_eq!(&ext.storage_root()[..], &ROOT); + assert_eq!(&ext.storage_root(StateVersion::default())[..], &ROOT); } #[test] diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 8f914ab3eee64..f5158acbc74d1 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -24,9 +24,11 @@ use codec::{Decode, Encode, EncodeAppend}; use hash_db::Hasher; #[cfg(feature = "std")] use sp_core::hexdisplay::HexDisplay; -use sp_core::storage::{well_known_keys::is_child_storage_key, ChildInfo, TrackedStorageKey}; +use sp_core::storage::{ + well_known_keys::is_child_storage_key, ChildInfo, StateVersion, TrackedStorageKey, +}; use sp_externalities::{Extension, ExtensionStore, Externalities}; -use sp_trie::{empty_child_trie_root, trie_types::Layout}; +use sp_trie::{empty_child_trie_root, LayoutV1}; use crate::{log_error, trace, warn, StorageTransactionCache}; use sp_std::{ @@ -505,7 +507,7 @@ where StorageAppend::new(current_value).append(value); } - fn storage_root(&mut self) -> Vec { + fn storage_root(&mut self, state_version: StateVersion) -> Vec { let _guard = guard(); if let Some(ref root) = self.storage_transaction_cache.transaction_storage_root { trace!( @@ -518,7 +520,9 @@ where return root.encode() } - let root = self.overlay.storage_root(self.backend, self.storage_transaction_cache); + let root = + self.overlay + .storage_root(self.backend, self.storage_transaction_cache, state_version); trace!( target: "state", method = "StorageRoot", @@ -529,7 +533,11 @@ where root.encode() } - fn child_storage_root(&mut self, child_info: &ChildInfo) -> Vec { + fn child_storage_root( + &mut self, + child_info: &ChildInfo, + state_version: StateVersion, + ) -> Vec { let _guard = guard(); let storage_key = child_info.storage_key(); let prefixed_storage_key = child_info.prefixed_storage_key(); @@ -537,7 +545,8 @@ where let root = self .storage(prefixed_storage_key.as_slice()) .and_then(|k| Decode::decode(&mut &k[..]).ok()) - .unwrap_or_else(|| empty_child_trie_root::>()); + // V1 is equivalent to V0 on empty root. + .unwrap_or_else(|| empty_child_trie_root::>()); trace!( target: "state", method = "ChildStorageRoot", @@ -550,7 +559,7 @@ where } else { let root = if let Some((changes, info)) = self.overlay.child_changes(storage_key) { let delta = changes.map(|(k, v)| (k.as_ref(), v.value().map(AsRef::as_ref))); - Some(self.backend.child_storage_root(info, delta)) + Some(self.backend.child_storage_root(info, delta, state_version)) } else { None }; @@ -583,7 +592,8 @@ where let root = self .storage(prefixed_storage_key.as_slice()) .and_then(|k| Decode::decode(&mut &k[..]).ok()) - .unwrap_or_else(|| empty_child_trie_root::>()); + // V1 is equivalent to V0 on empty root. + .unwrap_or_else(|| empty_child_trie_root::>()); trace!( target: "state", @@ -648,7 +658,12 @@ where self.overlay.rollback_transaction().expect(BENCHMARKING_FN); } self.overlay - .drain_storage_changes(self.backend, Default::default(), self.storage_transaction_cache) + .drain_storage_changes( + self.backend, + Default::default(), + self.storage_transaction_cache, + Default::default(), // using any state + ) .expect(EXT_NOT_ALLOWED_TO_FAIL); self.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL); self.mark_dirty(); @@ -658,12 +673,19 @@ where } fn commit(&mut self) { + // Bench always use latest state. + let state_version = StateVersion::default(); for _ in 0..self.overlay.transaction_depth() { self.overlay.commit_transaction().expect(BENCHMARKING_FN); } let changes = self .overlay - .drain_storage_changes(self.backend, Default::default(), self.storage_transaction_cache) + .drain_storage_changes( + self.backend, + Default::default(), + self.storage_transaction_cache, + state_version, + ) .expect(EXT_NOT_ALLOWED_TO_FAIL); self.backend .commit( @@ -878,15 +900,18 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_storage(vec![20], None); overlay.set_storage(vec![30], Some(vec![31])); - let backend = Storage { - top: map![ - vec![10] => vec![10], - vec![20] => vec![20], - vec![40] => vec![40] - ], - children_default: map![], - } - .into(); + let backend = ( + Storage { + top: map![ + vec![10] => vec![10], + vec![20] => vec![20], + vec![40] => vec![40] + ], + children_default: map![], + }, + StateVersion::default(), + ) + .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None); @@ -924,13 +949,16 @@ mod tests { overlay.set_storage(vec![27], None); overlay.set_storage(vec![28], None); overlay.set_storage(vec![29], None); - let backend = Storage { - top: map![ - vec![30] => vec![30] - ], - children_default: map![], - } - .into(); + let backend = ( + Storage { + top: map![ + vec![30] => vec![30] + ], + children_default: map![], + }, + StateVersion::default(), + ) + .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None); @@ -948,20 +976,23 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(child_info, vec![20], None); overlay.set_child_storage(child_info, vec![30], Some(vec![31])); - let backend = Storage { - top: map![], - children_default: map![ - child_info.storage_key().to_vec() => StorageChild { - data: map![ - vec![10] => vec![10], - vec![20] => vec![20], - vec![40] => vec![40] - ], - child_info: child_info.to_owned(), - } - ], - } - .into(); + let backend = ( + Storage { + top: map![], + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { + data: map![ + vec![10] => vec![10], + vec![20] => vec![20], + vec![40] => vec![40] + ], + child_info: child_info.to_owned(), + } + ], + }, + StateVersion::default(), + ) + .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None); @@ -993,20 +1024,23 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(child_info, vec![20], None); overlay.set_child_storage(child_info, vec![30], Some(vec![31])); - let backend = Storage { - top: map![], - children_default: map![ - child_info.storage_key().to_vec() => StorageChild { - data: map![ - vec![10] => vec![10], - vec![20] => vec![20], - vec![30] => vec![40] - ], - child_info: child_info.to_owned(), - } - ], - } - .into(); + let backend = ( + Storage { + top: map![], + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { + data: map![ + vec![10] => vec![10], + vec![20] => vec![20], + vec![30] => vec![40] + ], + child_info: child_info.to_owned(), + } + ], + }, + StateVersion::default(), + ) + .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None); @@ -1032,18 +1066,21 @@ mod tests { let child_info = &child_info; let mut cache = StorageTransactionCache::default(); let mut overlay = OverlayedChanges::default(); - let backend = Storage { - top: map![], - children_default: map![ - child_info.storage_key().to_vec() => StorageChild { - data: map![ - vec![30] => vec![40] - ], - child_info: child_info.to_owned(), - } - ], - } - .into(); + let backend = ( + Storage { + top: map![], + children_default: map![ + child_info.storage_key().to_vec() => StorageChild { + data: map![ + vec![30] => vec![40] + ], + child_info: child_info.to_owned(), + } + ], + }, + StateVersion::default(), + ) + .into(); let ext = TestExt::new(&mut overlay, &mut cache, &backend, None); diff --git a/primitives/state-machine/src/in_memory_backend.rs b/primitives/state-machine/src/in_memory_backend.rs index f9f94c0c50d60..f4ae0d01d5fb9 100644 --- a/primitives/state-machine/src/in_memory_backend.rs +++ b/primitives/state-machine/src/in_memory_backend.rs @@ -22,8 +22,8 @@ use crate::{ }; use codec::Codec; use hash_db::Hasher; -use sp_core::storage::{ChildInfo, Storage}; -use sp_trie::{empty_trie_root, Layout, MemoryDB}; +use sp_core::storage::{ChildInfo, StateVersion, Storage}; +use sp_trie::{empty_trie_root, LayoutV1, MemoryDB}; use std::collections::{BTreeMap, HashMap}; /// Create a new empty instance of in-memory backend. @@ -32,7 +32,8 @@ where H::Out: Codec + Ord, { let db = MemoryDB::default(); - TrieBackend::new(db, empty_trie_root::>()) + // V1 is same as V0 for an empty trie. + TrieBackend::new(db, empty_trie_root::>()) } impl TrieBackend, H> @@ -43,9 +44,10 @@ where pub fn update, StorageCollection)>>( &self, changes: T, + state_version: StateVersion, ) -> Self { let mut clone = self.clone(); - clone.insert(changes); + clone.insert(changes, state_version); clone } @@ -53,6 +55,7 @@ where pub fn insert, StorageCollection)>>( &mut self, changes: T, + state_version: StateVersion, ) { let (top, child) = changes.into_iter().partition::, _>(|v| v.0.is_none()); let (root, transaction) = self.full_storage_root( @@ -60,6 +63,7 @@ where child.iter().filter_map(|v| { v.0.as_ref().map(|c| (c, v.1.iter().map(|(k, v)| (&k[..], v.as_deref())))) }), + state_version, ); self.apply_transaction(root, transaction); @@ -103,53 +107,63 @@ where } } -impl From, BTreeMap>> +impl From<(HashMap, BTreeMap>, StateVersion)> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(inner: HashMap, BTreeMap>) -> Self { + fn from( + (inner, state_version): ( + HashMap, BTreeMap>, + StateVersion, + ), + ) -> Self { let mut backend = new_in_mem(); backend.insert( inner .into_iter() .map(|(k, m)| (k, m.into_iter().map(|(k, v)| (k, Some(v))).collect())), + state_version, ); backend } } -impl From for TrieBackend, H> +impl From<(Storage, StateVersion)> for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(inners: Storage) -> Self { + fn from((inners, state_version): (Storage, StateVersion)) -> Self { let mut inner: HashMap, BTreeMap> = inners .children_default .into_iter() .map(|(_k, c)| (Some(c.child_info), c.data)) .collect(); inner.insert(None, inners.top); - inner.into() + (inner, state_version).into() } } -impl From> for TrieBackend, H> +impl From<(BTreeMap, StateVersion)> + for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(inner: BTreeMap) -> Self { + fn from((inner, state_version): (BTreeMap, StateVersion)) -> Self { let mut expanded = HashMap::new(); expanded.insert(None, inner); - expanded.into() + (expanded, state_version).into() } } -impl From, StorageCollection)>> for TrieBackend, H> +impl From<(Vec<(Option, StorageCollection)>, StateVersion)> + for TrieBackend, H> where H::Out: Codec + Ord, { - fn from(inner: Vec<(Option, StorageCollection)>) -> Self { + fn from( + (inner, state_version): (Vec<(Option, StorageCollection)>, StateVersion), + ) -> Self { let mut expanded: HashMap, BTreeMap> = HashMap::new(); for (child_info, key_values) in inner { @@ -160,7 +174,7 @@ where } } } - expanded.into() + (expanded, state_version).into() } } @@ -168,16 +182,20 @@ where mod tests { use super::*; use crate::backend::Backend; + use sp_core::storage::StateVersion; use sp_runtime::traits::BlakeTwo256; /// Assert in memory backend with only child trie keys works as trie backend. #[test] fn in_memory_with_child_trie_only() { + let state_version = StateVersion::default(); let storage = new_in_mem::(); let child_info = ChildInfo::new_default(b"1"); let child_info = &child_info; - let storage = storage - .update(vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])]); + let storage = storage.update( + vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])], + state_version, + ); let trie_backend = storage.as_trie_backend().unwrap(); assert_eq!(trie_backend.child_storage(child_info, b"2").unwrap(), Some(b"3".to_vec())); let storage_key = child_info.prefixed_storage_key(); @@ -186,13 +204,18 @@ mod tests { #[test] fn insert_multiple_times_child_data_works() { + let state_version = StateVersion::default(); let mut storage = new_in_mem::(); let child_info = ChildInfo::new_default(b"1"); - storage - .insert(vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])]); - storage - .insert(vec![(Some(child_info.clone()), vec![(b"1".to_vec(), Some(b"3".to_vec()))])]); + storage.insert( + vec![(Some(child_info.clone()), vec![(b"2".to_vec(), Some(b"3".to_vec()))])], + state_version, + ); + storage.insert( + vec![(Some(child_info.clone()), vec![(b"1".to_vec(), Some(b"3".to_vec()))])], + state_version, + ); assert_eq!(storage.child_storage(&child_info, &b"2"[..]), Ok(Some(b"3".to_vec()))); assert_eq!(storage.child_storage(&child_info, &b"1"[..]), Ok(Some(b"3".to_vec()))); diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index e5c19f3bb0d57..dff4999816342 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -126,6 +126,7 @@ impl sp_std::fmt::Display for DefaultError { pub use crate::{ backend::Backend, + error::{Error, ExecutionError}, ext::Ext, overlayed_changes::{ ChildStorageCollection, IndexOperation, OffchainChangesCollection, @@ -136,7 +137,6 @@ pub use crate::{ trie_backend::TrieBackend, trie_backend_essence::{Storage, TrieBackendStorage}, }; -pub use error::{Error, ExecutionError}; #[cfg(feature = "std")] mod std_reexport { @@ -151,8 +151,8 @@ mod std_reexport { testing::TestExternalities, }; pub use sp_trie::{ - trie_types::{Layout, TrieDBMut}, - CompactProof, DBValue, MemoryDB, StorageProof, TrieMut, + trie_types::{TrieDBMutV0, TrieDBMutV1}, + CompactProof, DBValue, LayoutV0, LayoutV1, MemoryDB, StorageProof, TrieMut, }; } @@ -1353,7 +1353,7 @@ mod tests { use codec::{Decode, Encode}; use sp_core::{ map, - storage::ChildInfo, + storage::{ChildInfo, StateVersion}, testing::TaskExecutor, traits::{CodeExecutor, Externalities, RuntimeCode}, NativeOrEncoded, NeverNativeValue, @@ -1416,7 +1416,11 @@ mod tests { #[test] fn execute_works() { - let backend = trie_backend::tests::test_trie(); + execute_works_inner(StateVersion::V0); + execute_works_inner(StateVersion::V1); + } + fn execute_works_inner(state_version: StateVersion) { + let backend = trie_backend::tests::test_trie(state_version); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); @@ -1440,7 +1444,11 @@ mod tests { #[test] fn execute_works_with_native_else_wasm() { - let backend = trie_backend::tests::test_trie(); + execute_works_with_native_else_wasm_inner(StateVersion::V0); + execute_works_with_native_else_wasm_inner(StateVersion::V1); + } + fn execute_works_with_native_else_wasm_inner(state_version: StateVersion) { + let backend = trie_backend::tests::test_trie(state_version); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); @@ -1464,8 +1472,12 @@ mod tests { #[test] fn dual_execution_strategy_detects_consensus_failure() { + dual_execution_strategy_detects_consensus_failure_inner(StateVersion::V0); + dual_execution_strategy_detects_consensus_failure_inner(StateVersion::V1); + } + fn dual_execution_strategy_detects_consensus_failure_inner(state_version: StateVersion) { let mut consensus_failed = false; - let backend = trie_backend::tests::test_trie(); + let backend = trie_backend::tests::test_trie(state_version); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); @@ -1498,6 +1510,10 @@ mod tests { #[test] fn prove_execution_and_proof_check_works() { + prove_execution_and_proof_check_works_inner(StateVersion::V0); + prove_execution_and_proof_check_works_inner(StateVersion::V1); + } + fn prove_execution_and_proof_check_works_inner(state_version: StateVersion) { let executor = DummyCodeExecutor { native_available: true, native_succeeds: true, @@ -1505,8 +1521,8 @@ mod tests { }; // fetch execution proof from 'remote' full node - let mut remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(std::iter::empty()).0; + let mut remote_backend = trie_backend::tests::test_trie(state_version); + let remote_root = remote_backend.storage_root(std::iter::empty(), state_version).0; let (remote_result, remote_proof) = prove_execution( &mut remote_backend, &mut Default::default(), @@ -1544,7 +1560,7 @@ mod tests { b"abc".to_vec() => b"2".to_vec(), b"bbb".to_vec() => b"3".to_vec() ]; - let state = InMemoryBackend::::from(initial); + let state = InMemoryBackend::::from((initial, StateVersion::default())); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); @@ -1613,7 +1629,7 @@ mod tests { b"d".to_vec() => b"3".to_vec() ], ]; - let backend = InMemoryBackend::::from(initial); + let backend = InMemoryBackend::::from((initial, StateVersion::default())); let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(&child_info, b"1".to_vec(), Some(b"1312".to_vec())); @@ -1655,7 +1671,7 @@ mod tests { b"d".to_vec() => b"3".to_vec() ], ]; - let backend = InMemoryBackend::::from(initial); + let backend = InMemoryBackend::::from((initial, StateVersion::default())); let mut overlay = OverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new(&mut overlay, &mut cache, &backend, None); @@ -1789,13 +1805,17 @@ mod tests { #[test] fn prove_read_and_proof_check_works() { + prove_read_and_proof_check_works_inner(StateVersion::V0); + prove_read_and_proof_check_works_inner(StateVersion::V1); + } + fn prove_read_and_proof_check_works_inner(state_version: StateVersion) { let child_info = ChildInfo::new_default(b"sub1"); let missing_child_info = ChildInfo::new_default(b"sub1sub2"); // key will include other child root to proof. let child_info = &child_info; let missing_child_info = &missing_child_info; // fetch read proof from 'remote' full node - let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(std::iter::empty()).0; + let remote_backend = trie_backend::tests::test_trie(state_version); + let remote_root = remote_backend.storage_root(std::iter::empty(), state_version).0; let remote_proof = prove_read(remote_backend, &[b"value2"]).unwrap(); let remote_proof = test_compact(remote_proof, &remote_root); // check proof locally @@ -1812,8 +1832,8 @@ mod tests { ); assert_eq!(local_result2, false); // on child trie - let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(std::iter::empty()).0; + let remote_backend = trie_backend::tests::test_trie(state_version); + let remote_root = remote_backend.storage_root(std::iter::empty(), state_version).0; let remote_proof = prove_child_read(remote_backend, child_info, &[b"value3"]).unwrap(); let remote_proof = test_compact(remote_proof, &remote_root); let local_result1 = read_child_proof_check::( @@ -1877,7 +1897,8 @@ mod tests { storage.insert(Some(child_info), items); } - let trie: InMemoryBackend = storage.clone().into(); + let trie: InMemoryBackend = + (storage.clone(), StateVersion::default()).into(); let trie_root = trie.root().clone(); let backend = crate::ProvingBackend::new(&trie); let mut queries = Vec::new(); @@ -1940,15 +1961,16 @@ mod tests { #[test] fn prove_read_with_size_limit_works() { - let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(::std::iter::empty()).0; + let state_version = StateVersion::V0; + let remote_backend = trie_backend::tests::test_trie(state_version); + let remote_root = remote_backend.storage_root(::std::iter::empty(), state_version).0; let (proof, count) = prove_range_read_with_size(remote_backend, None, None, 0, None).unwrap(); // Always contains at least some nodes. assert_eq!(proof.into_memory_db::().drain().len(), 3); assert_eq!(count, 1); - let remote_backend = trie_backend::tests::test_trie(); + let remote_backend = trie_backend::tests::test_trie(state_version); let (proof, count) = prove_range_read_with_size(remote_backend, None, None, 800, Some(&[])).unwrap(); assert_eq!(proof.clone().into_memory_db::().drain().len(), 9); @@ -1971,7 +1993,7 @@ mod tests { assert_eq!(results.len() as u32, 101); assert_eq!(completed, false); - let remote_backend = trie_backend::tests::test_trie(); + let remote_backend = trie_backend::tests::test_trie(state_version); let (proof, count) = prove_range_read_with_size(remote_backend, None, None, 50000, Some(&[])).unwrap(); assert_eq!(proof.clone().into_memory_db::().drain().len(), 11); @@ -1989,10 +2011,66 @@ mod tests { assert_eq!(completed, true); } + #[test] + fn inner_state_versioning_switch_proofs() { + let mut state_version = StateVersion::V0; + let (mut mdb, mut root) = trie_backend::tests::test_db(state_version); + { + let mut trie = TrieDBMutV0::from_existing(&mut mdb, &mut root).unwrap(); + trie.insert(b"foo", vec![1u8; 1_000].as_slice()) // big inner hash + .expect("insert failed"); + trie.insert(b"foo2", vec![3u8; 16].as_slice()) // no inner hash + .expect("insert failed"); + trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash + .expect("insert failed"); + } + + let check_proof = |mdb, root, state_version| -> StorageProof { + let remote_backend = TrieBackend::new(mdb, root); + let remote_root = remote_backend.storage_root(std::iter::empty(), state_version).0; + let remote_proof = prove_read(remote_backend, &[b"foo222"]).unwrap(); + // check proof locally + let local_result1 = + read_proof_check::(remote_root, remote_proof.clone(), &[b"foo222"]) + .unwrap(); + // check that results are correct + assert_eq!( + local_result1.into_iter().collect::>(), + vec![(b"foo222".to_vec(), Some(vec![5u8; 100]))], + ); + remote_proof + }; + + let remote_proof = check_proof(mdb.clone(), root.clone(), state_version); + // check full values in proof + assert!(remote_proof.encode().len() > 1_100); + assert!(remote_proof.encoded_size() > 1_100); + let root1 = root.clone(); + + // do switch + state_version = StateVersion::V1; + { + let mut trie = TrieDBMutV1::from_existing(&mut mdb, &mut root).unwrap(); + trie.insert(b"foo222", vec![5u8; 100].as_slice()) // inner hash + .expect("insert failed"); + // update with same value do change + trie.insert(b"foo", vec![1u8; 1000].as_slice()) // inner hash + .expect("insert failed"); + } + let root3 = root.clone(); + assert!(root1 != root3); + let remote_proof = check_proof(mdb.clone(), root.clone(), state_version); + // nodes foo is replaced by its hashed value form. + assert!(remote_proof.encode().len() < 1000); + assert!(remote_proof.encoded_size() < 1000); + assert_eq!(remote_proof.encode().len(), remote_proof.encoded_size()); + } + #[test] fn prove_range_with_child_works() { - let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(::std::iter::empty()).0; + let state_version = StateVersion::V0; + let remote_backend = trie_backend::tests::test_trie(state_version); + let remote_root = remote_backend.storage_root(std::iter::empty(), state_version).0; let mut start_at = smallvec::SmallVec::<[Vec; 2]>::new(); let trie_backend = remote_backend.as_trie_backend().unwrap(); let max_iter = 1000; @@ -2030,20 +2108,34 @@ mod tests { #[test] fn compact_multiple_child_trie() { + let size_no_inner_hash = compact_multiple_child_trie_inner(StateVersion::V0); + let size_inner_hash = compact_multiple_child_trie_inner(StateVersion::V1); + assert!(size_inner_hash < size_no_inner_hash); + } + fn compact_multiple_child_trie_inner(state_version: StateVersion) -> usize { // this root will be queried let child_info1 = ChildInfo::new_default(b"sub1"); // this root will not be include in proof let child_info2 = ChildInfo::new_default(b"sub2"); // this root will be include in proof let child_info3 = ChildInfo::new_default(b"sub"); - let remote_backend = trie_backend::tests::test_trie(); + let remote_backend = trie_backend::tests::test_trie(state_version); + let long_vec: Vec = (0..1024usize).map(|_| 8u8).collect(); let (remote_root, transaction) = remote_backend.full_storage_root( std::iter::empty(), vec![ ( &child_info1, - vec![(&b"key1"[..], Some(&b"val2"[..])), (&b"key2"[..], Some(&b"val3"[..]))] - .into_iter(), + vec![ + // a inner hashable node + (&b"k"[..], Some(&long_vec[..])), + // need to ensure this is not an inline node + // otherwhise we do not know what is accessed when + // storing proof. + (&b"key1"[..], Some(&vec![5u8; 32][..])), + (&b"key2"[..], Some(&b"val3"[..])), + ] + .into_iter(), ), ( &child_info2, @@ -2057,11 +2149,13 @@ mod tests { ), ] .into_iter(), + state_version, ); let mut remote_storage = remote_backend.into_storage(); remote_storage.consolidate(transaction); let remote_backend = TrieBackend::new(remote_storage, remote_root); let remote_proof = prove_child_read(remote_backend, &child_info1, &[b"key1"]).unwrap(); + let size = remote_proof.encoded_size(); let remote_proof = test_compact(remote_proof, &remote_root); let local_result1 = read_child_proof_check::( remote_root, @@ -2071,11 +2165,13 @@ mod tests { ) .unwrap(); assert_eq!(local_result1.len(), 1); - assert_eq!(local_result1.get(&b"key1"[..]), Some(&Some(b"val2".to_vec()))); + assert_eq!(local_result1.get(&b"key1"[..]), Some(&Some(vec![5u8; 32]))); + size } #[test] fn child_storage_uuid() { + let state_version = StateVersion::V0; let child_info_1 = ChildInfo::new_default(b"sub_test1"); let child_info_2 = ChildInfo::new_default(b"sub_test2"); @@ -2083,12 +2179,12 @@ mod tests { let mut overlay = OverlayedChanges::default(); let mut transaction = { - let backend = test_trie(); + let backend = test_trie(state_version); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new(&mut overlay, &mut cache, &backend, None); ext.set_child_storage(&child_info_1, b"abc".to_vec(), b"def".to_vec()); ext.set_child_storage(&child_info_2, b"abc".to_vec(), b"def".to_vec()); - ext.storage_root(); + ext.storage_root(state_version); cache.transaction.unwrap() }; let mut duplicate = false; @@ -2108,7 +2204,7 @@ mod tests { b"aaa".to_vec() => b"0".to_vec(), b"bbb".to_vec() => b"".to_vec() ]; - let state = InMemoryBackend::::from(initial); + let state = InMemoryBackend::::from((initial, StateVersion::default())); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); @@ -2134,12 +2230,13 @@ mod tests { #[test] fn runtime_registered_extensions_are_removed_after_execution() { + let state_version = StateVersion::default(); use sp_externalities::ExternalitiesExt; sp_externalities::decl_extension! { struct DummyExt(u32); } - let backend = trie_backend::tests::test_trie(); + let backend = trie_backend::tests::test_trie(state_version); let mut overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); diff --git a/primitives/state-machine/src/overlayed_changes/mod.rs b/primitives/state-machine/src/overlayed_changes/mod.rs index b7a535792aae6..659b52e0a708f 100644 --- a/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/primitives/state-machine/src/overlayed_changes/mod.rs @@ -27,7 +27,7 @@ use hash_db::Hasher; pub use offchain::OffchainOverlayedChanges; use sp_core::{ offchain::OffchainOverlayedChange, - storage::{well_known_keys::EXTRINSIC_INDEX, ChildInfo}, + storage::{well_known_keys::EXTRINSIC_INDEX, ChildInfo, StateVersion}, }; #[cfg(feature = "std")] use sp_externalities::{Extension, Extensions}; @@ -502,11 +502,12 @@ impl OverlayedChanges { backend: &B, parent_hash: H::Out, mut cache: StorageTransactionCache, + state_version: StateVersion, ) -> Result, DefaultError> where H::Out: Ord + Encode + 'static, { - self.drain_storage_changes(backend, parent_hash, &mut cache) + self.drain_storage_changes(backend, parent_hash, &mut cache, state_version) } /// Drain all changes into a [`StorageChanges`] instance. Leave empty overlay in place. @@ -515,13 +516,14 @@ impl OverlayedChanges { backend: &B, _parent_hash: H::Out, mut cache: &mut StorageTransactionCache, + state_version: StateVersion, ) -> Result, DefaultError> where H::Out: Ord + Encode + 'static, { // If the transaction does not exist, we generate it. if cache.transaction.is_none() { - self.storage_root(backend, &mut cache); + self.storage_root(backend, &mut cache, state_version); } let (transaction, transaction_storage_root) = cache @@ -580,6 +582,7 @@ impl OverlayedChanges { &self, backend: &B, cache: &mut StorageTransactionCache, + state_version: StateVersion, ) -> H::Out where H::Out: Ord + Encode, @@ -589,7 +592,7 @@ impl OverlayedChanges { (info, changes.map(|(k, v)| (&k[..], v.value().map(|v| &v[..])))) }); - let (root, transaction) = backend.full_storage_root(delta, child_delta); + let (root, transaction) = backend.full_storage_root(delta, child_delta, state_version); cache.transaction = Some(transaction); cache.transaction_storage_root = Some(root); @@ -830,6 +833,7 @@ mod tests { #[test] fn overlayed_storage_root_works() { + let state_version = StateVersion::default(); let initial: BTreeMap<_, _> = vec![ (b"doe".to_vec(), b"reindeer".to_vec()), (b"dog".to_vec(), b"puppyXXX".to_vec()), @@ -838,7 +842,7 @@ mod tests { ] .into_iter() .collect(); - let backend = InMemoryBackend::::from(initial); + let backend = InMemoryBackend::::from((initial, state_version)); let mut overlay = OverlayedChanges::default(); overlay.start_transaction(); @@ -856,7 +860,7 @@ mod tests { const ROOT: [u8; 32] = hex!("39245109cef3758c2eed2ccba8d9b370a917850af3824bc8348d505df2c298fa"); - assert_eq!(&ext.storage_root()[..], &ROOT); + assert_eq!(&ext.storage_root(state_version)[..], &ROOT); } #[test] diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index a354adaf697d6..4e1fd64da2b8e 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -26,14 +26,11 @@ use codec::{Codec, Decode, Encode}; use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; use log::debug; use parking_lot::RwLock; -use sp_core::storage::ChildInfo; +use sp_core::storage::{ChildInfo, StateVersion}; +pub use sp_trie::trie_types::TrieError; use sp_trie::{ empty_child_trie_root, read_child_trie_value_with, read_trie_value_with, record_all_keys, - MemoryDB, StorageProof, -}; -pub use sp_trie::{ - trie_types::{Layout, TrieError}, - Recorder, + LayoutV1, MemoryDB, Recorder, StorageProof, }; use std::{ collections::{hash_map::Entry, HashMap}, @@ -59,7 +56,8 @@ where let map_e = |e| format!("Trie lookup error: {}", e); - read_trie_value_with::, _, Ephemeral>( + // V1 is equivalent to V0 on read. + read_trie_value_with::, _, Ephemeral>( &eph, self.backend.root(), key, @@ -78,14 +76,16 @@ where let root = self .storage(storage_key)? .and_then(|r| Decode::decode(&mut &r[..]).ok()) - .unwrap_or_else(|| empty_child_trie_root::>()); + // V1 is equivalent to V0 on empty trie + .unwrap_or_else(|| empty_child_trie_root::>()); let mut read_overlay = S::Overlay::default(); let eph = Ephemeral::new(self.backend.backend_storage(), &mut read_overlay); let map_e = |e| format!("Trie lookup error: {}", e); - read_child_trie_value_with::, _, _>( + // V1 is equivalent to V0 on read + read_child_trie_value_with::, _, _>( child_info.keyspace(), &eph, &root.as_ref(), @@ -102,7 +102,8 @@ where let mut iter = move || -> Result<(), Box>> { let root = self.backend.root(); - record_all_keys::, _>(&eph, root, &mut *self.proof_recorder) + // V1 and V is equivalent to V0 on read and recorder is key read. + record_all_keys::, _>(&eph, root, &mut *self.proof_recorder) }; if let Err(e) = iter() { @@ -338,22 +339,24 @@ where fn storage_root<'b>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (H::Out, Self::Transaction) where H::Out: Ord, { - self.0.storage_root(delta) + self.0.storage_root(delta, state_version) } fn child_storage_root<'b>( &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord, { - self.0.child_storage_root(child_info, delta) + self.0.child_storage_root(child_info, delta, state_version) } fn register_overlay_stats(&self, _stats: &crate::stats::StateMachineStats) {} @@ -401,13 +404,21 @@ mod tests { #[test] fn proof_is_empty_until_value_is_read() { - let trie_backend = test_trie(); + proof_is_empty_until_value_is_read_inner(StateVersion::V0); + proof_is_empty_until_value_is_read_inner(StateVersion::V1); + } + fn proof_is_empty_until_value_is_read_inner(test_hash: StateVersion) { + let trie_backend = test_trie(test_hash); assert!(test_proving(&trie_backend).extract_proof().is_empty()); } #[test] fn proof_is_non_empty_after_value_is_read() { - let trie_backend = test_trie(); + proof_is_non_empty_after_value_is_read_inner(StateVersion::V0); + proof_is_non_empty_after_value_is_read_inner(StateVersion::V1); + } + fn proof_is_non_empty_after_value_is_read_inner(test_hash: StateVersion) { + let trie_backend = test_trie(test_hash); let backend = test_proving(&trie_backend); assert_eq!(backend.storage(b"key").unwrap(), Some(b"value".to_vec())); assert!(!backend.extract_proof().is_empty()); @@ -425,58 +436,82 @@ mod tests { #[test] fn passes_through_backend_calls() { - let trie_backend = test_trie(); + passes_through_backend_calls_inner(StateVersion::V0); + passes_through_backend_calls_inner(StateVersion::V1); + } + fn passes_through_backend_calls_inner(state_version: StateVersion) { + let trie_backend = test_trie(state_version); let proving_backend = test_proving(&trie_backend); assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap()); assert_eq!(trie_backend.pairs(), proving_backend.pairs()); - let (trie_root, mut trie_mdb) = trie_backend.storage_root(std::iter::empty()); - let (proving_root, mut proving_mdb) = proving_backend.storage_root(std::iter::empty()); + let (trie_root, mut trie_mdb) = + trie_backend.storage_root(std::iter::empty(), state_version); + let (proving_root, mut proving_mdb) = + proving_backend.storage_root(std::iter::empty(), state_version); assert_eq!(trie_root, proving_root); assert_eq!(trie_mdb.drain(), proving_mdb.drain()); } #[test] - fn proof_recorded_and_checked() { - let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>(); + fn proof_recorded_and_checked_top() { + proof_recorded_and_checked_inner(StateVersion::V0); + proof_recorded_and_checked_inner(StateVersion::V1); + } + fn proof_recorded_and_checked_inner(state_version: StateVersion) { + let size_content = 34; // above hashable value treshold. + let value_range = 0..64; + let contents = value_range + .clone() + .map(|i| (vec![i], Some(vec![i; size_content]))) + .collect::>(); let in_memory = InMemoryBackend::::default(); - let in_memory = in_memory.update(vec![(None, contents)]); - let in_memory_root = in_memory.storage_root(::std::iter::empty()).0; - (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); + let in_memory = in_memory.update(vec![(None, contents)], state_version); + let in_memory_root = in_memory.storage_root(std::iter::empty(), state_version).0; + value_range.clone().for_each(|i| { + assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i; size_content]) + }); let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(::std::iter::empty()).0; + let trie_root = trie.storage_root(std::iter::empty(), state_version).0; assert_eq!(in_memory_root, trie_root); - (0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i])); + value_range + .clone() + .for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i; size_content])); let proving = ProvingBackend::new(trie); - assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]); + assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); let proof = proving.extract_proof(); let proof_check = create_proof_check_backend::(in_memory_root.into(), proof).unwrap(); - assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]); + assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42; size_content]); } #[test] fn proof_recorded_and_checked_with_child() { + proof_recorded_and_checked_with_child_inner(StateVersion::V0); + proof_recorded_and_checked_with_child_inner(StateVersion::V1); + } + fn proof_recorded_and_checked_with_child_inner(state_version: StateVersion) { let child_info_1 = ChildInfo::new_default(b"sub1"); let child_info_2 = ChildInfo::new_default(b"sub2"); let child_info_1 = &child_info_1; let child_info_2 = &child_info_2; let contents = vec![ - (None, (0..64).map(|i| (vec![i], Some(vec![i]))).collect()), + (None, (0..64).map(|i| (vec![i], Some(vec![i]))).collect::>()), (Some(child_info_1.clone()), (28..65).map(|i| (vec![i], Some(vec![i]))).collect()), (Some(child_info_2.clone()), (10..15).map(|i| (vec![i], Some(vec![i]))).collect()), ]; let in_memory = InMemoryBackend::::default(); - let in_memory = in_memory.update(contents); + let in_memory = in_memory.update(contents, state_version); let child_storage_keys = vec![child_info_1.to_owned(), child_info_2.to_owned()]; let in_memory_root = in_memory .full_storage_root( std::iter::empty(), child_storage_keys.iter().map(|k| (k, std::iter::empty())), + state_version, ) .0; (0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i])); @@ -488,7 +523,7 @@ mod tests { }); let trie = in_memory.as_trie_backend().unwrap(); - let trie_root = trie.storage_root(std::iter::empty()).0; + let trie_root = trie.storage_root(std::iter::empty(), state_version).0; assert_eq!(in_memory_root, trie_root); (0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i])); @@ -516,7 +551,11 @@ mod tests { #[test] fn storage_proof_encoded_size_estimation_works() { - let trie_backend = test_trie(); + storage_proof_encoded_size_estimation_works_inner(StateVersion::V0); + storage_proof_encoded_size_estimation_works_inner(StateVersion::V1); + } + fn storage_proof_encoded_size_estimation_works_inner(state_version: StateVersion) { + let trie_backend = test_trie(state_version); let backend = test_proving(&trie_backend); let check_estimation = diff --git a/primitives/state-machine/src/read_only.rs b/primitives/state-machine/src/read_only.rs index b3e43d4c46e7f..2039b39a06038 100644 --- a/primitives/state-machine/src/read_only.rs +++ b/primitives/state-machine/src/read_only.rs @@ -21,7 +21,7 @@ use crate::{Backend, StorageKey, StorageValue}; use codec::Encode; use hash_db::Hasher; use sp_core::{ - storage::{ChildInfo, TrackedStorageKey}, + storage::{ChildInfo, StateVersion, TrackedStorageKey}, traits::Externalities, Blake2Hasher, }; @@ -145,11 +145,15 @@ impl<'a, H: Hasher, B: 'a + Backend> Externalities for ReadOnlyExternalities< unimplemented!("storage_append is not supported in ReadOnlyExternalities") } - fn storage_root(&mut self) -> Vec { + fn storage_root(&mut self, _state_version: StateVersion) -> Vec { unimplemented!("storage_root is not supported in ReadOnlyExternalities") } - fn child_storage_root(&mut self, _child_info: &ChildInfo) -> Vec { + fn child_storage_root( + &mut self, + _child_info: &ChildInfo, + _state_version: StateVersion, + ) -> Vec { unimplemented!("child_storage_root is not supported in ReadOnlyExternalities") } diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index 890137c43d881..dbb0a25d3c396 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -32,7 +32,7 @@ use sp_core::{ offchain::testing::TestPersistentOffchainDB, storage::{ well_known_keys::{is_child_storage_key, CODE}, - Storage, + StateVersion, Storage, }, testing::TaskExecutor, traits::TaskExecutorExt, @@ -54,6 +54,8 @@ where pub backend: InMemoryBackend, /// Extensions. pub extensions: Extensions, + /// State version to use during tests. + pub state_version: StateVersion, } impl TestExternalities @@ -72,18 +74,31 @@ where /// Create a new instance of `TestExternalities` with storage. pub fn new(storage: Storage) -> Self { - Self::new_with_code(&[], storage) + Self::new_with_code_and_state(&[], storage, Default::default()) + } + + /// Create a new instance of `TestExternalities` with storage for a given state version. + pub fn new_with_state_version(storage: Storage, state_version: StateVersion) -> Self { + Self::new_with_code_and_state(&[], storage, state_version) } /// New empty test externalities. pub fn new_empty() -> Self { - Self::new_with_code(&[], Storage::default()) + Self::new_with_code_and_state(&[], Storage::default(), Default::default()) } /// Create a new instance of `TestExternalities` with code and storage. - pub fn new_with_code(code: &[u8], mut storage: Storage) -> Self { - let overlay = OverlayedChanges::default(); + pub fn new_with_code(code: &[u8], storage: Storage) -> Self { + Self::new_with_code_and_state(code, storage, Default::default()) + } + /// Create a new instance of `TestExternalities` with code and storage for a given state + /// version. + pub fn new_with_code_and_state( + code: &[u8], + mut storage: Storage, + state_version: StateVersion, + ) -> Self { assert!(storage.top.keys().all(|key| !is_child_storage_key(key))); assert!(storage.children_default.keys().all(|key| is_child_storage_key(key))); @@ -94,12 +109,15 @@ where let offchain_db = TestPersistentOffchainDB::new(); + let backend = (storage, state_version).into(); + TestExternalities { - overlay, + overlay: OverlayedChanges::default(), offchain_db, extensions, - backend: storage.into(), + backend, storage_transaction_cache: Default::default(), + state_version, } } @@ -120,14 +138,14 @@ where /// Insert key/value into backend pub fn insert(&mut self, k: StorageKey, v: StorageValue) { - self.backend.insert(vec![(None, vec![(k, Some(v))])]); + self.backend.insert(vec![(None, vec![(k, Some(v))])], self.state_version); } /// Insert key/value into backend. /// /// This only supports inserting keys in child tries. pub fn insert_child(&mut self, c: sp_core::storage::ChildInfo, k: StorageKey, v: StorageValue) { - self.backend.insert(vec![(Some(c), vec![(k, Some(v))])]); + self.backend.insert(vec![(Some(c), vec![(k, Some(v))])], self.state_version); } /// Registers the given extension for this instance. @@ -151,7 +169,7 @@ where )) } - self.backend.update(transaction) + self.backend.update(transaction, self.state_version) } /// Commit all pending changes to the underlying backend. @@ -164,6 +182,7 @@ where &self.backend, Default::default(), &mut Default::default(), + self.state_version, )?; self.backend @@ -240,7 +259,8 @@ where H::Out: Ord + 'static + codec::Codec, { fn default() -> Self { - Self::new(Default::default()) + // default to default version. + Self::new_with_state_version(Storage::default(), Default::default()) } } @@ -249,7 +269,16 @@ where H::Out: Ord + 'static + codec::Codec, { fn from(storage: Storage) -> Self { - Self::new(storage) + Self::new_with_state_version(storage, Default::default()) + } +} + +impl From<(Storage, StateVersion)> for TestExternalities +where + H::Out: Ord + 'static + codec::Codec, +{ + fn from((storage, state_version): (Storage, StateVersion)) -> Self { + Self::new_with_state_version(storage, state_version) } } @@ -309,14 +338,15 @@ mod tests { #[test] fn commit_should_work() { - let mut ext = TestExternalities::::default(); + let storage = Storage::default(); // avoid adding the trie threshold. + let mut ext = TestExternalities::::from((storage, Default::default())); let mut ext = ext.ext(); ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec()); let root = H256::from(hex!("ed4d8c799d996add422395a6abd7545491d40bd838d738afafa1b8a4de625489")); - assert_eq!(H256::from_slice(ext.storage_root().as_slice()), root); + assert_eq!(H256::from_slice(ext.storage_root(Default::default()).as_slice()), root); } #[test] diff --git a/primitives/state-machine/src/trie_backend.rs b/primitives/state-machine/src/trie_backend.rs index 7f9a02e055251..616a5512ea40b 100644 --- a/primitives/state-machine/src/trie_backend.rs +++ b/primitives/state-machine/src/trie_backend.rs @@ -24,12 +24,12 @@ use crate::{ }; use codec::{Codec, Decode}; use hash_db::Hasher; -use sp_core::storage::{ChildInfo, ChildType}; +use sp_core::storage::{ChildInfo, ChildType, StateVersion}; use sp_std::{boxed::Box, vec::Vec}; use sp_trie::{ child_delta_trie_root, delta_trie_root, empty_child_trie_root, - trie_types::{Layout, TrieDB, TrieError}, - Trie, + trie_types::{TrieDB, TrieError}, + LayoutV0, LayoutV1, Trie, }; /// Patricia trie-based backend. Transaction type is an overlay of changes to commit. @@ -186,6 +186,7 @@ where fn storage_root<'a>( &self, delta: impl Iterator)>, + state_version: StateVersion, ) -> (H::Out, Self::Transaction) where H::Out: Ord, @@ -195,8 +196,14 @@ where { let mut eph = Ephemeral::new(self.essence.backend_storage(), &mut write_overlay); - - match delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta) { + let res = match state_version { + StateVersion::V0 => + delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta), + StateVersion::V1 => + delta_trie_root::, _, _, _, _, _>(&mut eph, root, delta), + }; + + match res { Ok(ret) => root = ret, Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), } @@ -209,14 +216,14 @@ where &self, child_info: &ChildInfo, delta: impl Iterator)>, + state_version: StateVersion, ) -> (H::Out, bool, Self::Transaction) where H::Out: Ord, { let default_root = match child_info.child_type() { - ChildType::ParentKeyId => empty_child_trie_root::>(), + ChildType::ParentKeyId => empty_child_trie_root::>(), }; - let mut write_overlay = S::Overlay::default(); let prefixed_storage_key = child_info.prefixed_storage_key(); let mut root = match self.storage(prefixed_storage_key.as_slice()) { @@ -231,13 +238,20 @@ where { let mut eph = Ephemeral::new(self.essence.backend_storage(), &mut write_overlay); - - match child_delta_trie_root::, _, _, _, _, _, _>( - child_info.keyspace(), - &mut eph, - root, - delta, - ) { + match match state_version { + StateVersion::V0 => child_delta_trie_root::, _, _, _, _, _, _>( + child_info.keyspace(), + &mut eph, + root, + delta, + ), + StateVersion::V1 => child_delta_trie_root::, _, _, _, _, _, _>( + child_info.keyspace(), + &mut eph, + root, + delta, + ), + } { Ok(ret) => root = ret, Err(e) => warn!(target: "trie", "Failed to write to trie: {}", e), } @@ -269,52 +283,91 @@ pub mod tests { use codec::Encode; use sp_core::H256; use sp_runtime::traits::BlakeTwo256; - use sp_trie::{trie_types::TrieDBMut, KeySpacedDBMut, PrefixedMemoryDB, TrieMut}; + use sp_trie::{ + trie_types::{TrieDBMutV0, TrieDBMutV1}, + KeySpacedDBMut, PrefixedMemoryDB, TrieMut, + }; use std::{collections::HashSet, iter}; const CHILD_KEY_1: &[u8] = b"sub1"; - fn test_db() -> (PrefixedMemoryDB, H256) { + pub(crate) fn test_db(state_version: StateVersion) -> (PrefixedMemoryDB, H256) { let child_info = ChildInfo::new_default(CHILD_KEY_1); let mut root = H256::default(); let mut mdb = PrefixedMemoryDB::::default(); { let mut mdb = KeySpacedDBMut::new(&mut mdb, child_info.keyspace()); - let mut trie = TrieDBMut::new(&mut mdb, &mut root); - trie.insert(b"value3", &[142; 33]).expect("insert failed"); - trie.insert(b"value4", &[124; 33]).expect("insert failed"); + match state_version { + StateVersion::V0 => { + let mut trie = TrieDBMutV0::new(&mut mdb, &mut root); + trie.insert(b"value3", &[142; 33]).expect("insert failed"); + trie.insert(b"value4", &[124; 33]).expect("insert failed"); + }, + StateVersion::V1 => { + let mut trie = TrieDBMutV1::new(&mut mdb, &mut root); + trie.insert(b"value3", &[142; 33]).expect("insert failed"); + trie.insert(b"value4", &[124; 33]).expect("insert failed"); + }, + }; }; { let mut sub_root = Vec::new(); root.encode_to(&mut sub_root); - let mut trie = TrieDBMut::new(&mut mdb, &mut root); - trie.insert(child_info.prefixed_storage_key().as_slice(), &sub_root[..]) - .expect("insert failed"); - trie.insert(b"key", b"value").expect("insert failed"); - trie.insert(b"value1", &[42]).expect("insert failed"); - trie.insert(b"value2", &[24]).expect("insert failed"); - trie.insert(b":code", b"return 42").expect("insert failed"); - for i in 128u8..255u8 { - trie.insert(&[i], &[i]).unwrap(); + + fn build( + mut trie: sp_trie::TrieDBMut, + child_info: &ChildInfo, + sub_root: &[u8], + ) { + trie.insert(child_info.prefixed_storage_key().as_slice(), sub_root) + .expect("insert failed"); + trie.insert(b"key", b"value").expect("insert failed"); + trie.insert(b"value1", &[42]).expect("insert failed"); + trie.insert(b"value2", &[24]).expect("insert failed"); + trie.insert(b":code", b"return 42").expect("insert failed"); + for i in 128u8..255u8 { + trie.insert(&[i], &[i]).unwrap(); + } } + + match state_version { + StateVersion::V0 => { + let trie = TrieDBMutV0::new(&mut mdb, &mut root); + build(trie, &child_info, &sub_root[..]) + }, + StateVersion::V1 => { + let trie = TrieDBMutV1::new(&mut mdb, &mut root); + build(trie, &child_info, &sub_root[..]) + }, + }; } (mdb, root) } - pub(crate) fn test_trie() -> TrieBackend, BlakeTwo256> { - let (mdb, root) = test_db(); + pub(crate) fn test_trie( + hashed_value: StateVersion, + ) -> TrieBackend, BlakeTwo256> { + let (mdb, root) = test_db(hashed_value); TrieBackend::new(mdb, root) } #[test] fn read_from_storage_returns_some() { - assert_eq!(test_trie().storage(b"key").unwrap(), Some(b"value".to_vec())); + read_from_storage_returns_some_inner(StateVersion::V0); + read_from_storage_returns_some_inner(StateVersion::V1); + } + fn read_from_storage_returns_some_inner(state_version: StateVersion) { + assert_eq!(test_trie(state_version).storage(b"key").unwrap(), Some(b"value".to_vec())); } #[test] fn read_from_child_storage_returns_some() { - let test_trie = test_trie(); + read_from_child_storage_returns_some_inner(StateVersion::V0); + read_from_child_storage_returns_some_inner(StateVersion::V1); + } + fn read_from_child_storage_returns_some_inner(state_version: StateVersion) { + let test_trie = test_trie(state_version); assert_eq!( test_trie .child_storage(&ChildInfo::new_default(CHILD_KEY_1), b"value3") @@ -341,12 +394,20 @@ pub mod tests { #[test] fn read_from_storage_returns_none() { - assert_eq!(test_trie().storage(b"non-existing-key").unwrap(), None); + read_from_storage_returns_none_inner(StateVersion::V0); + read_from_storage_returns_none_inner(StateVersion::V1); + } + fn read_from_storage_returns_none_inner(state_version: StateVersion) { + assert_eq!(test_trie(state_version).storage(b"non-existing-key").unwrap(), None); } #[test] fn pairs_are_not_empty_on_non_empty_storage() { - assert!(!test_trie().pairs().is_empty()); + pairs_are_not_empty_on_non_empty_storage_inner(StateVersion::V0); + pairs_are_not_empty_on_non_empty_storage_inner(StateVersion::V1); + } + fn pairs_are_not_empty_on_non_empty_storage_inner(state_version: StateVersion) { + assert!(!test_trie(state_version).pairs().is_empty()); } #[test] @@ -361,25 +422,35 @@ pub mod tests { #[test] fn storage_root_is_non_default() { - assert!(test_trie().storage_root(iter::empty()).0 != H256::repeat_byte(0)); + storage_root_is_non_default_inner(StateVersion::V0); + storage_root_is_non_default_inner(StateVersion::V1); } - - #[test] - fn storage_root_transaction_is_empty() { - assert!(test_trie().storage_root(iter::empty()).1.drain().is_empty()); + fn storage_root_is_non_default_inner(state_version: StateVersion) { + assert!( + test_trie(state_version).storage_root(iter::empty(), state_version).0 != + H256::repeat_byte(0) + ); } #[test] fn storage_root_transaction_is_non_empty() { - let (new_root, mut tx) = - test_trie().storage_root(iter::once((&b"new-key"[..], Some(&b"new-value"[..])))); + storage_root_transaction_is_non_empty_inner(StateVersion::V0); + storage_root_transaction_is_non_empty_inner(StateVersion::V1); + } + fn storage_root_transaction_is_non_empty_inner(state_version: StateVersion) { + let (new_root, mut tx) = test_trie(state_version) + .storage_root(iter::once((&b"new-key"[..], Some(&b"new-value"[..]))), state_version); assert!(!tx.drain().is_empty()); - assert!(new_root != test_trie().storage_root(iter::empty()).0); + assert!(new_root != test_trie(state_version).storage_root(iter::empty(), state_version).0); } #[test] fn prefix_walking_works() { - let trie = test_trie(); + prefix_walking_works_inner(StateVersion::V0); + prefix_walking_works_inner(StateVersion::V1); + } + fn prefix_walking_works_inner(state_version: StateVersion) { + let trie = test_trie(state_version); let mut seen = HashSet::new(); trie.for_keys_with_prefix(b"value", |key| { diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 6c575f0d76bc7..945fd05ebffb4 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -20,20 +20,23 @@ use crate::{backend::Consolidate, debug, warn, StorageKey, StorageValue}; use codec::Encode; -use hash_db::{self, Hasher, Prefix}; +use hash_db::{self, AsHashDB, HashDB, HashDBRef, Hasher, Prefix}; #[cfg(feature = "std")] use parking_lot::RwLock; use sp_core::storage::ChildInfo; use sp_std::{boxed::Box, vec::Vec}; use sp_trie::{ empty_child_trie_root, read_child_trie_value, read_trie_value, - trie_types::{Layout, TrieDB, TrieError}, - DBValue, KeySpacedDB, MemoryDB, PrefixedMemoryDB, Trie, TrieDBIterator, + trie_types::{TrieDB, TrieError}, + DBValue, KeySpacedDB, PrefixedMemoryDB, Trie, TrieDBIterator, TrieDBKeyIterator, }; #[cfg(feature = "std")] use std::collections::HashMap; #[cfg(feature = "std")] use std::sync::Arc; +// In this module, we only use layout for read operation and empty root, +// where V1 and V0 are equivalent. +use sp_trie::LayoutV1 as Layout; #[cfg(not(feature = "std"))] macro_rules! format { @@ -178,7 +181,7 @@ where child_info: Option<&ChildInfo>, key: &[u8], ) -> Result> { - let dyn_eph: &dyn hash_db::HashDBRef<_, _>; + let dyn_eph: &dyn HashDBRef<_, _>; let keyspace_eph; if let Some(child_info) = child_info.as_ref() { keyspace_eph = KeySpacedDB::new(self, child_info.keyspace()); @@ -189,7 +192,7 @@ where let trie = TrieDB::::new(dyn_eph, root).map_err(|e| format!("TrieDB creation error: {}", e))?; - let mut iter = trie.iter().map_err(|e| format!("TrieDB iteration error: {}", e))?; + let mut iter = trie.key_iter().map_err(|e| format!("TrieDB iteration error: {}", e))?; // The key just after the one given in input, basically `key++0`. // Note: We are sure this is the next key if: @@ -205,7 +208,7 @@ where let next_element = iter.next(); let next_key = if let Some(next_element) = next_element { - let (next_key, _) = + let next_key = next_element.map_err(|e| format!("TrieDB iterator next error: {}", e))?; Some(next_key) } else { @@ -291,17 +294,7 @@ where &self.root }; - let _ = self.trie_iter_inner( - root, - prefix, - |k, _v| { - f(&k); - true - }, - child_info, - None, - false, - ); + self.trie_iter_key_inner(root, prefix, |k| f(k), child_info) } /// Execute given closure for all keys starting with prefix. @@ -320,32 +313,70 @@ where }; let mut root = H::Out::default(); root.as_mut().copy_from_slice(&root_vec); - let _ = self.trie_iter_inner( + self.trie_iter_key_inner( &root, Some(prefix), - |k, _v| { - f(&k); + |k| { + f(k); true }, Some(child_info), - None, - false, - ); + ) } /// Execute given closure for all keys starting with prefix. pub fn for_keys_with_prefix(&self, prefix: &[u8], mut f: F) { - let _ = self.trie_iter_inner( + self.trie_iter_key_inner( &self.root, Some(prefix), - |k, _v| { - f(&k); + |k| { + f(k); true }, None, - None, - false, - ); + ) + } + + fn trie_iter_key_inner bool>( + &self, + root: &H::Out, + prefix: Option<&[u8]>, + mut f: F, + child_info: Option<&ChildInfo>, + ) { + let mut iter = move |db| -> sp_std::result::Result<(), Box>> { + let trie = TrieDB::::new(db, root)?; + let iter = if let Some(prefix) = prefix.as_ref() { + TrieDBKeyIterator::new_prefixed(&trie, prefix)? + } else { + TrieDBKeyIterator::new(&trie)? + }; + + for x in iter { + let key = x?; + + debug_assert!(prefix + .as_ref() + .map(|prefix| key.starts_with(prefix)) + .unwrap_or(true)); + + if !f(&key) { + break + } + } + + Ok(()) + }; + + let result = if let Some(child_info) = child_info { + let db = KeySpacedDB::new(self, child_info.keyspace()); + iter(&db) + } else { + iter(self) + }; + if let Err(e) = result { + debug!(target: "trie", "Error while iterating by prefix: {}", e); + } } fn trie_iter_inner, Vec) -> bool>( @@ -414,13 +445,13 @@ pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> { overlay: &'a mut S::Overlay, } -impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> hash_db::AsHashDB +impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> AsHashDB for Ephemeral<'a, S, H> { - fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { + fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { self } } @@ -435,7 +466,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - if let Some(val) = hash_db::HashDB::get(self.overlay, key, prefix) { + if let Some(val) = HashDB::get(self.overlay, key, prefix) { Some(val) } else { match self.storage.get(&key, prefix) { @@ -449,38 +480,37 @@ impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDB } fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - hash_db::HashDB::get(self, key, prefix).is_some() + HashDB::get(self, key, prefix).is_some() } fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { - hash_db::HashDB::insert(self.overlay, prefix, value) + HashDB::insert(self.overlay, prefix, value) } fn emplace(&mut self, key: H::Out, prefix: Prefix, value: DBValue) { - hash_db::HashDB::emplace(self.overlay, key, prefix, value) + HashDB::emplace(self.overlay, key, prefix, value) } fn remove(&mut self, key: &H::Out, prefix: Prefix) { - hash_db::HashDB::remove(self.overlay, key, prefix) + HashDB::remove(self.overlay, key, prefix) } } -impl<'a, S: 'a + TrieBackendStorage, H: Hasher> hash_db::HashDBRef - for Ephemeral<'a, S, H> -{ +impl<'a, S: 'a + TrieBackendStorage, H: Hasher> HashDBRef for Ephemeral<'a, S, H> { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - hash_db::HashDB::get(self, key, prefix) + HashDB::get(self, key, prefix) } fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - hash_db::HashDB::contains(self, key, prefix) + HashDB::contains(self, key, prefix) } } /// Key-value pairs storage that is used by trie backend essence. pub trait TrieBackendStorage: Send + Sync { /// Type of in-memory overlay. - type Overlay: hash_db::HashDB + Default + Consolidate; + type Overlay: HashDB + Default + Consolidate; + /// Get the value stored at key. fn get(&self, key: &H::Out, prefix: Prefix) -> Result>; } @@ -495,35 +525,28 @@ impl TrieBackendStorage for Arc> { } } -// This implementation is used by test storage trie clients. -impl TrieBackendStorage for PrefixedMemoryDB { - type Overlay = PrefixedMemoryDB; - - fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { - Ok(hash_db::HashDB::get(self, key, prefix)) - } -} - -impl TrieBackendStorage for MemoryDB { - type Overlay = MemoryDB; +impl TrieBackendStorage for sp_trie::GenericMemoryDB +where + H: Hasher, + KF: sp_trie::KeyFunction + Send + Sync, +{ + type Overlay = Self; fn get(&self, key: &H::Out, prefix: Prefix) -> Result> { Ok(hash_db::HashDB::get(self, key, prefix)) } } -impl, H: Hasher> hash_db::AsHashDB - for TrieBackendEssence -{ - fn as_hash_db<'b>(&'b self) -> &'b (dyn hash_db::HashDB + 'b) { +impl, H: Hasher> AsHashDB for TrieBackendEssence { + fn as_hash_db<'b>(&'b self) -> &'b (dyn HashDB + 'b) { self } - fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn hash_db::HashDB + 'b) { + fn as_hash_db_mut<'b>(&'b mut self) -> &'b mut (dyn HashDB + 'b) { self } } -impl, H: Hasher> hash_db::HashDB for TrieBackendEssence { +impl, H: Hasher> HashDB for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { if *key == self.empty { return Some([0u8].to_vec()) @@ -538,7 +561,7 @@ impl, H: Hasher> hash_db::HashDB for TrieBa } fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - hash_db::HashDB::get(self, key, prefix).is_some() + HashDB::get(self, key, prefix).is_some() } fn insert(&mut self, _prefix: Prefix, _value: &[u8]) -> H::Out { @@ -554,15 +577,13 @@ impl, H: Hasher> hash_db::HashDB for TrieBa } } -impl, H: Hasher> hash_db::HashDBRef - for TrieBackendEssence -{ +impl, H: Hasher> HashDBRef for TrieBackendEssence { fn get(&self, key: &H::Out, prefix: Prefix) -> Option { - hash_db::HashDB::get(self, key, prefix) + HashDB::get(self, key, prefix) } fn contains(&self, key: &H::Out, prefix: Prefix) -> bool { - hash_db::HashDB::contains(self, key, prefix) + HashDB::contains(self, key, prefix) } } @@ -570,7 +591,9 @@ impl, H: Hasher> hash_db::HashDBRef mod test { use super::*; use sp_core::{Blake2Hasher, H256}; - use sp_trie::{trie_types::TrieDBMut, KeySpacedDBMut, PrefixedMemoryDB, TrieMut}; + use sp_trie::{ + trie_types::TrieDBMutV1 as TrieDBMut, KeySpacedDBMut, PrefixedMemoryDB, TrieMut, + }; #[test] fn next_storage_key_and_next_child_storage_key_work() { diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index c655a9bdc1cf0..eff74c4aac0b2 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -237,6 +237,9 @@ pub mod well_known_keys { } } +/// Threshold size to start using trie value nodes in state. +pub const TRIE_VALUE_NODE_THRESHOLD: u32 = 33; + /// Information related to a child state. #[derive(Debug, Clone)] #[cfg_attr(feature = "std", derive(PartialEq, Eq, Hash, PartialOrd, Ord, Encode, Decode))] @@ -401,6 +404,54 @@ impl ChildTrieParentKeyId { } } +/// Different possible state version. +/// +/// V0 and V1 uses a same trie implementation, but V1 will write external value node in the trie for +/// value with size at least `TRIE_VALUE_NODE_THRESHOLD`. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum StateVersion { + /// Old state version, no value nodes. + V0 = 0, + /// New state version can use value nodes. + V1 = 1, +} + +impl Default for StateVersion { + fn default() -> Self { + StateVersion::V1 + } +} + +impl From for u8 { + fn from(version: StateVersion) -> u8 { + version as u8 + } +} + +impl sp_std::convert::TryFrom for StateVersion { + type Error = (); + fn try_from(val: u8) -> sp_std::result::Result { + match val { + 0 => Ok(StateVersion::V0), + 1 => Ok(StateVersion::V1), + _ => Err(()), + } + } +} + +impl StateVersion { + /// If defined, values in state of size bigger or equal + /// to this threshold will use a separate trie node. + /// Otherwhise, value will be inlined in branch or leaf + /// node. + pub fn state_value_threshold(&self) -> Option { + match self { + StateVersion::V0 => None, + StateVersion::V1 => Some(TRIE_VALUE_NODE_THRESHOLD), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/primitives/tasks/src/async_externalities.rs b/primitives/tasks/src/async_externalities.rs index 59733490a18ff..9f2fec299f598 100644 --- a/primitives/tasks/src/async_externalities.rs +++ b/primitives/tasks/src/async_externalities.rs @@ -19,7 +19,7 @@ //! Async externalities. use sp_core::{ - storage::{ChildInfo, TrackedStorageKey}, + storage::{ChildInfo, StateVersion, TrackedStorageKey}, traits::{Externalities, RuntimeSpawn, RuntimeSpawnExt, SpawnNamed, TaskExecutorExt}, }; use sp_externalities::{Extensions, ExternalitiesExt as _}; @@ -126,11 +126,15 @@ impl Externalities for AsyncExternalities { panic!("`storage_append`: should not be used in async externalities!") } - fn storage_root(&mut self) -> Vec { + fn storage_root(&mut self, _state_version: StateVersion) -> Vec { panic!("`storage_root`: should not be used in async externalities!") } - fn child_storage_root(&mut self, _child_info: &ChildInfo) -> Vec { + fn child_storage_root( + &mut self, + _child_info: &ChildInfo, + _state_version: StateVersion, + ) -> Vec { panic!("`child_storage_root`: should not be used in async externalities!") } diff --git a/primitives/transaction-storage-proof/src/lib.rs b/primitives/transaction-storage-proof/src/lib.rs index 4b01a8d45d454..f76ae7faac08b 100644 --- a/primitives/transaction-storage-proof/src/lib.rs +++ b/primitives/transaction-storage-proof/src/lib.rs @@ -143,7 +143,7 @@ pub mod registration { use sp_trie::TrieMut; type Hasher = sp_core::Blake2Hasher; - type TrieLayout = sp_trie::Layout; + type TrieLayout = sp_trie::LayoutV1; /// Create a new inherent data provider instance for a given parent block hash. pub fn new_data_provider( diff --git a/primitives/trie/Cargo.toml b/primitives/trie/Cargo.toml index 02aaa75aa939e..8e54035b3ae7d 100644 --- a/primitives/trie/Cargo.toml +++ b/primitives/trie/Cargo.toml @@ -22,13 +22,13 @@ codec = { package = "parity-scale-codec", version = "2.0.0", default-features = scale-info = { version = "1.0", default-features = false, features = ["derive"] } sp-std = { version = "4.0.0", default-features = false, path = "../std" } hash-db = { version = "0.15.2", default-features = false } -trie-db = { version = "0.22.6", default-features = false } -trie-root = { version = "0.16.0", default-features = false } -memory-db = { version = "0.27.0", default-features = false } +trie-db = { version = "0.23.0", default-features = false } +trie-root = { version = "0.17.0", default-features = false } +memory-db = { version = "0.28.0", default-features = false } sp-core = { version = "4.1.0-dev", default-features = false, path = "../core" } [dev-dependencies] -trie-bench = "0.28.0" +trie-bench = "0.29.0" trie-standardmap = "0.15.2" criterion = "0.3.3" hex-literal = "0.3.4" diff --git a/primitives/trie/benches/bench.rs b/primitives/trie/benches/bench.rs index 8c84c6354f2c3..d78ceadff7283 100644 --- a/primitives/trie/benches/bench.rs +++ b/primitives/trie/benches/bench.rs @@ -21,11 +21,11 @@ criterion_main!(benches); fn benchmark(c: &mut Criterion) { trie_bench::standard_benchmark::< - sp_trie::Layout, + sp_trie::LayoutV1, sp_trie::TrieStream, >(c, "substrate-blake2"); trie_bench::standard_benchmark::< - sp_trie::Layout, + sp_trie::LayoutV1, sp_trie::TrieStream, >(c, "substrate-keccak"); } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 8ba13284d379f..399484143dbc4 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -45,31 +45,43 @@ pub use trie_db::proof::VerifyError; use trie_db::proof::{generate_proof, verify_proof}; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ - nibble_ops, CError, DBValue, Query, Recorder, Trie, TrieConfiguration, TrieDBIterator, + nibble_ops, + node::{NodePlan, ValuePlan}, + CError, DBValue, Query, Recorder, Trie, TrieConfiguration, TrieDBIterator, TrieDBKeyIterator, TrieLayout, TrieMut, }; /// The Substrate format implementation of `TrieStream`. pub use trie_stream::TrieStream; -#[derive(Default)] /// substrate trie layout -pub struct Layout(sp_std::marker::PhantomData); +pub struct LayoutV0(sp_std::marker::PhantomData); -impl TrieLayout for Layout { +/// substrate trie layout, with external value nodes. +pub struct LayoutV1(sp_std::marker::PhantomData); + +impl TrieLayout for LayoutV0 +where + H: Hasher, +{ const USE_EXTENSION: bool = false; const ALLOW_EMPTY: bool = true; + const MAX_INLINE_VALUE: Option = None; + type Hash = H; type Codec = NodeCodec; } -impl TrieConfiguration for Layout { +impl TrieConfiguration for LayoutV0 +where + H: Hasher, +{ fn trie_root(input: I) -> ::Out where I: IntoIterator, A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::trie_root_no_extension::(input) + trie_root::trie_root_no_extension::(input, Self::MAX_INLINE_VALUE) } fn trie_root_unhashed(input: I) -> Vec @@ -78,7 +90,52 @@ impl TrieConfiguration for Layout { A: AsRef<[u8]> + Ord, B: AsRef<[u8]>, { - trie_root::unhashed_trie_no_extension::(input) + trie_root::unhashed_trie_no_extension::( + input, + Self::MAX_INLINE_VALUE, + ) + } + + fn encode_index(input: u32) -> Vec { + codec::Encode::encode(&codec::Compact(input)) + } +} + +impl TrieLayout for LayoutV1 +where + H: Hasher, +{ + const USE_EXTENSION: bool = false; + const ALLOW_EMPTY: bool = true; + const MAX_INLINE_VALUE: Option = Some(sp_core::storage::TRIE_VALUE_NODE_THRESHOLD); + + type Hash = H; + type Codec = NodeCodec; +} + +impl TrieConfiguration for LayoutV1 +where + H: Hasher, +{ + fn trie_root(input: I) -> ::Out + where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + { + trie_root::trie_root_no_extension::(input, Self::MAX_INLINE_VALUE) + } + + fn trie_root_unhashed(input: I) -> Vec + where + I: IntoIterator, + A: AsRef<[u8]> + Ord, + B: AsRef<[u8]>, + { + trie_root::unhashed_trie_no_extension::( + input, + Self::MAX_INLINE_VALUE, + ) } fn encode_index(input: u32) -> Vec { @@ -118,17 +175,22 @@ pub type TrieDBMut<'a, L> = trie_db::TrieDBMut<'a, L>; pub type Lookup<'a, L, Q> = trie_db::Lookup<'a, L, Q>; /// Hash type for a trie layout. pub type TrieHash = <::Hash as Hasher>::Out; - /// This module is for non generic definition of trie type. /// Only the `Hasher` trait is generic in this case. pub mod trie_types { - pub type Layout = super::Layout; + use super::*; + /// Persistent trie database read-access interface for the a given hasher. - pub type TrieDB<'a, H> = super::TrieDB<'a, Layout>; + /// Read only V1 and V0 are compatible, thus we always use V1. + pub type TrieDB<'a, H> = super::TrieDB<'a, LayoutV1>; /// Persistent trie database write-access interface for the a given hasher. - pub type TrieDBMut<'a, H> = super::TrieDBMut<'a, Layout>; + pub type TrieDBMutV0<'a, H> = super::TrieDBMut<'a, LayoutV0>; + /// Persistent trie database write-access interface for the a given hasher. + pub type TrieDBMutV1<'a, H> = super::TrieDBMut<'a, LayoutV1>; + /// Querying interface, as in `trie_db` but less generic. + pub type LookupV0<'a, H, Q> = trie_db::Lookup<'a, LayoutV0, Q>; /// Querying interface, as in `trie_db` but less generic. - pub type Lookup<'a, H, Q> = trie_db::Lookup<'a, Layout, Q>; + pub type LookupV1<'a, H, Q> = trie_db::Lookup<'a, LayoutV1, Q>; /// As in `trie_db`, but less generic, error type for the crate. pub type TrieError = trie_db::TrieError; } @@ -141,16 +203,18 @@ pub mod trie_types { /// For a key `K` that is included in the `db` a proof of inclusion is generated. /// For a key `K` that is not included in the `db` a proof of non-inclusion is generated. /// These can be later checked in `verify_trie_proof`. -pub fn generate_trie_proof<'a, L: TrieConfiguration, I, K, DB>( +pub fn generate_trie_proof<'a, L, I, K, DB>( db: &DB, root: TrieHash, keys: I, ) -> Result>, Box>> where + L: TrieConfiguration, I: IntoIterator, K: 'a + AsRef<[u8]>, DB: hash_db::HashDBRef, { + // Can use default layout (read only). let trie = TrieDB::::new(db, &root)?; generate_proof(&trie, keys) } @@ -163,17 +227,18 @@ where /// checked for inclusion in the proof. /// If the value is omitted (`(key, None)`), this key will be checked for non-inclusion in the /// proof. -pub fn verify_trie_proof<'a, L: TrieConfiguration, I, K, V>( +pub fn verify_trie_proof<'a, L, I, K, V>( root: &TrieHash, proof: &[Vec], items: I, -) -> Result<(), VerifyError, error::Error>> +) -> Result<(), VerifyError, CError>> where + L: TrieConfiguration, I: IntoIterator)>, K: 'a + AsRef<[u8]>, V: 'a + AsRef<[u8]>, { - verify_proof::, _, _, _>(root, proof, items) + verify_proof::(root, proof, items) } /// Determine a trie root given a hash DB and delta values. @@ -207,28 +272,33 @@ where } /// Read a value from the trie. -pub fn read_trie_value>( +pub fn read_trie_value( db: &DB, root: &TrieHash, key: &[u8], -) -> Result>, Box>> { - TrieDB::::new(&*db, root)?.get(key).map(|x| x.map(|val| val.to_vec())) +) -> Result>, Box>> +where + L: TrieConfiguration, + DB: hash_db::HashDBRef, +{ + Ok(TrieDB::::new(&*db, root)?.get(key).map(|x| x.map(|val| val.to_vec()))?) } /// Read a value from the trie with given Query. -pub fn read_trie_value_with< - L: TrieConfiguration, - Q: Query, - DB: hash_db::HashDBRef, ->( +pub fn read_trie_value_with( db: &DB, root: &TrieHash, key: &[u8], query: Q, -) -> Result>, Box>> { - TrieDB::::new(&*db, root)? +) -> Result>, Box>> +where + L: TrieConfiguration, + Q: Query, + DB: hash_db::HashDBRef, +{ + Ok(TrieDB::::new(&*db, root)? .get_with(key, query) - .map(|x| x.map(|val| val.to_vec())) + .map(|x| x.map(|val| val.to_vec()))?) } /// Determine the empty trie root. @@ -319,7 +389,7 @@ where } /// Read a value from the child trie with given query. -pub fn read_child_trie_value_with, DB>( +pub fn read_child_trie_value_with( keyspace: &[u8], db: &DB, root_slice: &[u8], @@ -327,6 +397,8 @@ pub fn read_child_trie_value_with Result>, Box>> where + L: TrieConfiguration, + Q: Query, DB: hash_db::HashDBRef, { let mut root = TrieHash::::default(); @@ -444,11 +516,15 @@ where /// Constants used into trie simplification codec. mod trie_constants { - pub const EMPTY_TRIE: u8 = 0; - pub const NIBBLE_SIZE_BOUND: usize = u16::MAX as usize; + const FIRST_PREFIX: u8 = 0b_00 << 6; + pub const NIBBLE_SIZE_BOUND: usize = u16::max_value() as usize; pub const LEAF_PREFIX_MASK: u8 = 0b_01 << 6; pub const BRANCH_WITHOUT_MASK: u8 = 0b_10 << 6; pub const BRANCH_WITH_MASK: u8 = 0b_11 << 6; + pub const EMPTY_TRIE: u8 = FIRST_PREFIX | (0b_00 << 4); + pub const ALT_HASHING_LEAF_PREFIX_MASK: u8 = FIRST_PREFIX | (0b_1 << 5); + pub const ALT_HASHING_BRANCH_WITH_MASK: u8 = FIRST_PREFIX | (0b_01 << 4); + pub const ESCAPE_COMPACT_HEADER: u8 = EMPTY_TRIE | 0b_00_01; } #[cfg(test)] @@ -461,7 +537,11 @@ mod tests { use trie_db::{DBValue, NodeCodec as NodeCodecT, Trie, TrieMut}; use trie_standardmap::{Alphabet, StandardMap, ValueMode}; - type Layout = super::Layout; + type LayoutV0 = super::LayoutV0; + type LayoutV1 = super::LayoutV1; + + type MemoryDBMeta = + memory_db::MemoryDB, trie_db::DBValue, MemTracker>; fn hashed_null_node() -> TrieHash { ::hashed_null_node() @@ -473,7 +553,7 @@ mod tests { let d = T::trie_root_unhashed(input.clone()); println!("Data: {:#x?}, {:#x?}", d, Blake2Hasher::hash(&d[..])); let persistent = { - let mut memdb = MemoryDB::default(); + let mut memdb = MemoryDBMeta::default(); let mut root = Default::default(); let mut t = TrieDBMut::::new(&mut memdb, &mut root); for (x, y) in input.iter().rev() { @@ -486,7 +566,7 @@ mod tests { } fn check_iteration(input: &Vec<(&[u8], &[u8])>) { - let mut memdb = MemoryDB::default(); + let mut memdb = MemoryDBMeta::default(); let mut root = Default::default(); { let mut t = TrieDBMut::::new(&mut memdb, &mut root); @@ -506,14 +586,21 @@ mod tests { } } + fn check_input(input: &Vec<(&[u8], &[u8])>) { + check_equivalent::(input); + check_iteration::(input); + check_equivalent::(input); + check_iteration::(input); + } + #[test] fn default_trie_root() { let mut db = MemoryDB::default(); - let mut root = TrieHash::::default(); - let mut empty = TrieDBMut::::new(&mut db, &mut root); + let mut root = TrieHash::::default(); + let mut empty = TrieDBMut::::new(&mut db, &mut root); empty.commit(); let root1 = empty.root().as_ref().to_vec(); - let root2: Vec = Layout::trie_root::<_, Vec, Vec>(std::iter::empty()) + let root2: Vec = LayoutV1::trie_root::<_, Vec, Vec>(std::iter::empty()) .as_ref() .iter() .cloned() @@ -525,31 +612,27 @@ mod tests { #[test] fn empty_is_equivalent() { let input: Vec<(&[u8], &[u8])> = vec![]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] fn leaf_is_equivalent() { let input: Vec<(&[u8], &[u8])> = vec![(&[0xaa][..], &[0xbb][..])]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] fn branch_is_equivalent() { let input: Vec<(&[u8], &[u8])> = vec![(&[0xaa][..], &[0x10][..]), (&[0xba][..], &[0x11][..])]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] fn extension_and_branch_is_equivalent() { let input: Vec<(&[u8], &[u8])> = vec![(&[0xaa][..], &[0x10][..]), (&[0xab][..], &[0x11][..])]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -564,8 +647,7 @@ mod tests { let mut d = st.make(); d.sort_by(|&(ref a, _), &(ref b, _)| a.cmp(b)); let dr = d.iter().map(|v| (&v.0[..], &v.1[..])).collect(); - check_equivalent::(&dr); - check_iteration::(&dr); + check_input(&dr); } #[test] @@ -575,8 +657,7 @@ mod tests { (&[0xaa, 0xaa][..], &[0xaa][..]), (&[0xaa, 0xbb][..], &[0xab][..]), ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -589,8 +670,7 @@ mod tests { (&[0xbb, 0xbb][..], &[0xbb][..]), (&[0xbb, 0xcc][..], &[0xbc][..]), ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -602,8 +682,7 @@ mod tests { ), (&[0xba][..], &[0x11][..]), ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } #[test] @@ -618,15 +697,17 @@ mod tests { &b"ABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABCABC"[..], ), ]; - check_equivalent::(&input); - check_iteration::(&input); + check_input(&input); } - fn populate_trie<'db, T: TrieConfiguration>( + fn populate_trie<'db, T>( db: &'db mut dyn HashDB, root: &'db mut TrieHash, v: &[(Vec, Vec)], - ) -> TrieDBMut<'db, T> { + ) -> TrieDBMut<'db, T> + where + T: TrieConfiguration, + { let mut t = TrieDBMut::::new(db, root); for i in 0..v.len() { let key: &[u8] = &v[i].0; @@ -648,8 +729,12 @@ mod tests { #[test] fn random_should_work() { + random_should_work_inner::(); + random_should_work_inner::(); + } + fn random_should_work_inner() { let mut seed = ::Out::zero(); - for test_i in 0..10000 { + for test_i in 0..10_000 { if test_i % 50 == 0 { println!("{:?} of 10000 stress tests done", test_i); } @@ -662,10 +747,11 @@ mod tests { } .make_with(seed.as_fixed_bytes_mut()); - let real = Layout::trie_root(x.clone()); + let real = L::trie_root(x.clone()); let mut memdb = MemoryDB::default(); let mut root = Default::default(); - let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); + + let mut memtrie = populate_trie::(&mut memdb, &mut root, &x); memtrie.commit(); if *memtrie.root() != real { @@ -677,9 +763,9 @@ mod tests { } } assert_eq!(*memtrie.root(), real); - unpopulate_trie::(&mut memtrie, &x); + unpopulate_trie::(&mut memtrie, &x); memtrie.commit(); - let hashed_null_node = hashed_null_node::(); + let hashed_null_node = hashed_null_node::(); if *memtrie.root() != hashed_null_node { println!("- TRIE MISMATCH"); println!(""); @@ -699,7 +785,7 @@ mod tests { #[test] fn codec_trie_empty() { let input: Vec<(&[u8], &[u8])> = vec![]; - let trie = Layout::trie_root_unhashed::<_, _, _>(input); + let trie = LayoutV1::trie_root_unhashed(input); println!("trie: {:#x?}", trie); assert_eq!(trie, vec![0x0]); } @@ -707,7 +793,7 @@ mod tests { #[test] fn codec_trie_single_tuple() { let input = vec![(vec![0xaa], vec![0xbb])]; - let trie = Layout::trie_root_unhashed::<_, _, _>(input); + let trie = LayoutV1::trie_root_unhashed(input); println!("trie: {:#x?}", trie); assert_eq!( trie, @@ -723,7 +809,7 @@ mod tests { #[test] fn codec_trie_two_tuples_disjoint_keys() { let input = vec![(&[0x48, 0x19], &[0xfe]), (&[0x13, 0x14], &[0xff])]; - let trie = Layout::trie_root_unhashed::<_, _, _>(input); + let trie = LayoutV1::trie_root_unhashed(input); println!("trie: {:#x?}", trie); let mut ex = Vec::::new(); ex.push(0x80); // branch, no value (0b_10..) no nibble @@ -747,6 +833,10 @@ mod tests { #[test] fn iterator_works() { + iterator_works_inner::(); + iterator_works_inner::(); + } + fn iterator_works_inner() { let pairs = vec![ (hex!("0103000000000000000464").to_vec(), hex!("0400000000").to_vec()), (hex!("0103000000000000000469").to_vec(), hex!("0401000000").to_vec()), @@ -777,15 +867,15 @@ mod tests { let mut memdb = MemoryDB::default(); let mut root = Default::default(); - populate_trie::(&mut memdb, &mut root, &pairs); + populate_trie::(&mut memdb, &mut root, &pairs); let non_included_key: Vec = hex!("0909").to_vec(); let proof = - generate_trie_proof::(&memdb, root, &[non_included_key.clone()]) + generate_trie_proof::(&memdb, root, &[non_included_key.clone()]) .unwrap(); // Verifying that the K was not included into the trie should work. - assert!(verify_trie_proof::>( + assert!(verify_trie_proof::>( &root, &proof, &[(non_included_key.clone(), None)], @@ -793,7 +883,7 @@ mod tests { .is_ok()); // Verifying that the K was included into the trie should fail. - assert!(verify_trie_proof::>( + assert!(verify_trie_proof::>( &root, &proof, &[(non_included_key, Some(hex!("1010").to_vec()))], @@ -810,13 +900,13 @@ mod tests { let mut memdb = MemoryDB::default(); let mut root = Default::default(); - populate_trie::(&mut memdb, &mut root, &pairs); + populate_trie::(&mut memdb, &mut root, &pairs); let proof = - generate_trie_proof::(&memdb, root, &[pairs[0].0.clone()]).unwrap(); + generate_trie_proof::(&memdb, root, &[pairs[0].0.clone()]).unwrap(); // Check that a K, V included into the proof are verified. - assert!(verify_trie_proof::( + assert!(verify_trie_proof::( &root, &proof, &[(pairs[0].0.clone(), Some(pairs[0].1.clone()))] @@ -824,7 +914,7 @@ mod tests { .is_ok()); // Absence of the V is not verified with the proof that has K, V included. - assert!(verify_trie_proof::>( + assert!(verify_trie_proof::>( &root, &proof, &[(pairs[0].0.clone(), None)] @@ -832,7 +922,7 @@ mod tests { .is_err()); // K not included into the trie is not verified. - assert!(verify_trie_proof::( + assert!(verify_trie_proof::( &root, &proof, &[(hex!("4242").to_vec(), Some(pairs[0].1.clone()))] @@ -840,7 +930,7 @@ mod tests { .is_err()); // K included into the trie but not included into the proof is not verified. - assert!(verify_trie_proof::( + assert!(verify_trie_proof::( &root, &proof, &[(pairs[1].0.clone(), Some(pairs[1].1.clone()))] @@ -865,13 +955,13 @@ mod tests { .unwrap(); let proof_db = proof.into_memory_db::(); - let first_storage_root = delta_trie_root::( + let first_storage_root = delta_trie_root::( &mut proof_db.clone(), storage_root, valid_delta, ) .unwrap(); - let second_storage_root = delta_trie_root::( + let second_storage_root = delta_trie_root::( &mut proof_db.clone(), storage_root, invalid_delta, diff --git a/primitives/trie/src/node_codec.rs b/primitives/trie/src/node_codec.rs index d5ffb3219cf68..53a5de270a79a 100644 --- a/primitives/trie/src/node_codec.rs +++ b/primitives/trie/src/node_codec.rs @@ -24,7 +24,7 @@ use hash_db::Hasher; use sp_std::{borrow::Borrow, marker::PhantomData, ops::Range, vec::Vec}; use trie_db::{ self, nibble_ops, - node::{NibbleSlicePlan, NodeHandlePlan, NodePlan}, + node::{NibbleSlicePlan, NodeHandlePlan, NodePlan, Value, ValuePlan}, ChildReference, NodeCodec as NodeCodecT, Partial, }; @@ -80,7 +80,11 @@ impl<'a> Input for ByteSliceInput<'a> { #[derive(Default, Clone)] pub struct NodeCodec(PhantomData); -impl NodeCodecT for NodeCodec { +impl NodeCodecT for NodeCodec +where + H: Hasher, +{ + const ESCAPE_HEADER: Option = Some(trie_constants::ESCAPE_COMPACT_HEADER); type Error = Error; type HashOut = H::Out; @@ -88,11 +92,22 @@ impl NodeCodecT for NodeCodec { H::hash(::empty_node()) } - fn decode_plan(data: &[u8]) -> sp_std::result::Result { + fn decode_plan(data: &[u8]) -> Result { let mut input = ByteSliceInput::new(data); - match NodeHeader::decode(&mut input)? { + + let header = NodeHeader::decode(&mut input)?; + let contains_hash = header.contains_hash_of_value(); + + let branch_has_value = if let NodeHeader::Branch(has_value, _) = &header { + *has_value + } else { + // hashed_value_branch + true + }; + + match header { NodeHeader::Null => Ok(NodePlan::Empty), - NodeHeader::Branch(has_value, nibble_count) => { + NodeHeader::HashedValueBranch(nibble_count) | NodeHeader::Branch(_, nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -105,9 +120,13 @@ impl NodeCodecT for NodeCodec { let partial_padding = nibble_ops::number_padding(nibble_count); let bitmap_range = input.take(BITMAP_LENGTH)?; let bitmap = Bitmap::decode(&data[bitmap_range])?; - let value = if has_value { - let count = >::decode(&mut input)?.0 as usize; - Some(input.take(count)?) + let value = if branch_has_value { + Some(if contains_hash { + ValuePlan::Node(input.take(H::LENGTH)?) + } else { + let count = >::decode(&mut input)?.0 as usize; + ValuePlan::Inline(input.take(count)?) + }) } else { None }; @@ -132,7 +151,7 @@ impl NodeCodecT for NodeCodec { children, }) }, - NodeHeader::Leaf(nibble_count) => { + NodeHeader::HashedValueLeaf(nibble_count) | NodeHeader::Leaf(nibble_count) => { let padding = nibble_count % nibble_ops::NIBBLE_PER_BYTE != 0; // check that the padding is valid (if any) if padding && nibble_ops::pad_left(data[input.offset]) != 0 { @@ -143,10 +162,16 @@ impl NodeCodecT for NodeCodec { nibble_ops::NIBBLE_PER_BYTE, )?; let partial_padding = nibble_ops::number_padding(nibble_count); - let count = >::decode(&mut input)?.0 as usize; + let value = if contains_hash { + ValuePlan::Node(input.take(H::LENGTH)?) + } else { + let count = >::decode(&mut input)?.0 as usize; + ValuePlan::Inline(input.take(count)?) + }; + Ok(NodePlan::Leaf { partial: NibbleSlicePlan::new(partial, partial_padding), - value: input.take(count)?, + value, }) }, } @@ -160,9 +185,23 @@ impl NodeCodecT for NodeCodec { &[trie_constants::EMPTY_TRIE] } - fn leaf_node(partial: Partial, value: &[u8]) -> Vec { - let mut output = partial_encode(partial, NodeKind::Leaf); - value.encode_to(&mut output); + fn leaf_node(partial: Partial, value: Value) -> Vec { + let contains_hash = matches!(&value, Value::Node(..)); + let mut output = if contains_hash { + partial_encode(partial, NodeKind::HashedValueLeaf) + } else { + partial_encode(partial, NodeKind::Leaf) + }; + match value { + Value::Inline(value) => { + Compact(value.len() as u32).encode_to(&mut output); + output.extend_from_slice(value); + }, + Value::Node(hash, _) => { + debug_assert!(hash.len() == H::LENGTH); + output.extend_from_slice(hash); + }, + } output } @@ -171,33 +210,46 @@ impl NodeCodecT for NodeCodec { _nbnibble: usize, _child: ChildReference<::Out>, ) -> Vec { - unreachable!() + unreachable!("No extension codec.") } fn branch_node( _children: impl Iterator::Out>>>>, - _maybe_value: Option<&[u8]>, + _maybe_value: Option, ) -> Vec { - unreachable!() + unreachable!("No extension codec.") } fn branch_node_nibbled( partial: impl Iterator, number_nibble: usize, children: impl Iterator::Out>>>>, - maybe_value: Option<&[u8]>, + value: Option, ) -> Vec { - let mut output = if maybe_value.is_some() { - partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue) - } else { - partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue) + let contains_hash = matches!(&value, Some(Value::Node(..))); + let mut output = match (&value, contains_hash) { + (&None, _) => + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchNoValue), + (_, false) => + partial_from_iterator_encode(partial, number_nibble, NodeKind::BranchWithValue), + (_, true) => + partial_from_iterator_encode(partial, number_nibble, NodeKind::HashedValueBranch), }; + let bitmap_index = output.len(); let mut bitmap: [u8; BITMAP_LENGTH] = [0; BITMAP_LENGTH]; (0..BITMAP_LENGTH).for_each(|_| output.push(0)); - if let Some(value) = maybe_value { - value.encode_to(&mut output); - }; + match value { + Some(Value::Inline(value)) => { + Compact(value.len() as u32).encode_to(&mut output); + output.extend_from_slice(value); + }, + Some(Value::Node(hash, _)) => { + debug_assert!(hash.len() == H::LENGTH); + output.extend_from_slice(hash); + }, + None => (), + } Bitmap::encode( children.map(|maybe_child| match maybe_child.borrow() { Some(ChildReference::Hash(h)) => { @@ -229,11 +281,15 @@ fn partial_from_iterator_encode>( ) -> Vec { let nibble_count = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, nibble_count); - let mut output = Vec::with_capacity(3 + (nibble_count / nibble_ops::NIBBLE_PER_BYTE)); + let mut output = Vec::with_capacity(4 + (nibble_count / nibble_ops::NIBBLE_PER_BYTE)); match node_kind { NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), + NodeKind::HashedValueLeaf => + NodeHeader::HashedValueLeaf(nibble_count).encode_to(&mut output), + NodeKind::HashedValueBranch => + NodeHeader::HashedValueBranch(nibble_count).encode_to(&mut output), }; output.extend(partial); output @@ -247,11 +303,15 @@ fn partial_encode(partial: Partial, node_kind: NodeKind) -> Vec { let nibble_count = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, nibble_count); - let mut output = Vec::with_capacity(3 + partial.1.len()); + let mut output = Vec::with_capacity(4 + partial.1.len()); match node_kind { NodeKind::Leaf => NodeHeader::Leaf(nibble_count).encode_to(&mut output), NodeKind::BranchWithValue => NodeHeader::Branch(true, nibble_count).encode_to(&mut output), NodeKind::BranchNoValue => NodeHeader::Branch(false, nibble_count).encode_to(&mut output), + NodeKind::HashedValueLeaf => + NodeHeader::HashedValueLeaf(nibble_count).encode_to(&mut output), + NodeKind::HashedValueBranch => + NodeHeader::HashedValueBranch(nibble_count).encode_to(&mut output), }; if number_nibble_encoded > 0 { output.push(nibble_ops::pad_right((partial.0).1)); diff --git a/primitives/trie/src/node_header.rs b/primitives/trie/src/node_header.rs index 9f05113a35935..895713ffd1f8d 100644 --- a/primitives/trie/src/node_header.rs +++ b/primitives/trie/src/node_header.rs @@ -25,8 +25,20 @@ use sp_std::iter::once; #[derive(Copy, Clone, PartialEq, Eq, sp_core::RuntimeDebug)] pub(crate) enum NodeHeader { Null, + // contains wether there is a value and nibble count Branch(bool, usize), + // contains nibble count Leaf(usize), + // contains nibble count. + HashedValueBranch(usize), + // contains nibble count. + HashedValueLeaf(usize), +} + +impl NodeHeader { + pub(crate) fn contains_hash_of_value(&self) -> bool { + matches!(self, NodeHeader::HashedValueBranch(_) | NodeHeader::HashedValueLeaf(_)) + } } /// NodeHeader without content @@ -34,6 +46,8 @@ pub(crate) enum NodeKind { Leaf, BranchNoValue, BranchWithValue, + HashedValueLeaf, + HashedValueBranch, } impl Encode for NodeHeader { @@ -41,11 +55,27 @@ impl Encode for NodeHeader { match self { NodeHeader::Null => output.push_byte(trie_constants::EMPTY_TRIE), NodeHeader::Branch(true, nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITH_MASK, output), - NodeHeader::Branch(false, nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITHOUT_MASK, output), + encode_size_and_prefix(*nibble_count, trie_constants::BRANCH_WITH_MASK, 2, output), + NodeHeader::Branch(false, nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::BRANCH_WITHOUT_MASK, + 2, + output, + ), NodeHeader::Leaf(nibble_count) => - encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, output), + encode_size_and_prefix(*nibble_count, trie_constants::LEAF_PREFIX_MASK, 2, output), + NodeHeader::HashedValueBranch(nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::ALT_HASHING_BRANCH_WITH_MASK, + 4, + output, + ), + NodeHeader::HashedValueLeaf(nibble_count) => encode_size_and_prefix( + *nibble_count, + trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, + 3, + output, + ), } } } @@ -59,13 +89,22 @@ impl Decode for NodeHeader { return Ok(NodeHeader::Null) } match i & (0b11 << 6) { - trie_constants::LEAF_PREFIX_MASK => Ok(NodeHeader::Leaf(decode_size(i, input)?)), - trie_constants::BRANCH_WITHOUT_MASK => - Ok(NodeHeader::Branch(false, decode_size(i, input)?)), + trie_constants::LEAF_PREFIX_MASK => Ok(NodeHeader::Leaf(decode_size(i, input, 2)?)), trie_constants::BRANCH_WITH_MASK => - Ok(NodeHeader::Branch(true, decode_size(i, input)?)), - // do not allow any special encoding - _ => Err("Unallowed encoding".into()), + Ok(NodeHeader::Branch(true, decode_size(i, input, 2)?)), + trie_constants::BRANCH_WITHOUT_MASK => + Ok(NodeHeader::Branch(false, decode_size(i, input, 2)?)), + trie_constants::EMPTY_TRIE => { + if i & (0b111 << 5) == trie_constants::ALT_HASHING_LEAF_PREFIX_MASK { + Ok(NodeHeader::HashedValueLeaf(decode_size(i, input, 3)?)) + } else if i & (0b1111 << 4) == trie_constants::ALT_HASHING_BRANCH_WITH_MASK { + Ok(NodeHeader::HashedValueBranch(decode_size(i, input, 4)?)) + } else { + // do not allow any special encoding + Err("Unallowed encoding".into()) + } + }, + _ => unreachable!(), } } } @@ -73,12 +112,20 @@ impl Decode for NodeHeader { /// Returns an iterator over encoded bytes for node header and size. /// Size encoding allows unlimited, length inefficient, representation, but /// is bounded to 16 bit maximum value to avoid possible DOS. -pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8) -> impl Iterator { +pub(crate) fn size_and_prefix_iterator( + size: usize, + prefix: u8, + prefix_mask: usize, +) -> impl Iterator { let size = sp_std::cmp::min(trie_constants::NIBBLE_SIZE_BOUND, size); - let l1 = sp_std::cmp::min(62, size); - let (first_byte, mut rem) = - if size == l1 { (once(prefix + l1 as u8), 0) } else { (once(prefix + 63), size - l1) }; + let max_value = 255u8 >> prefix_mask; + let l1 = sp_std::cmp::min((max_value as usize).saturating_sub(1), size); + let (first_byte, mut rem) = if size == l1 { + (once(prefix + l1 as u8), 0) + } else { + (once(prefix + max_value as u8), size - l1) + }; let next_bytes = move || { if rem > 0 { if rem < 256 { @@ -97,16 +144,24 @@ pub(crate) fn size_and_prefix_iterator(size: usize, prefix: u8) -> impl Iterator } /// Encodes size and prefix to a stream output. -fn encode_size_and_prefix(size: usize, prefix: u8, out: &mut W) { - for b in size_and_prefix_iterator(size, prefix) { +fn encode_size_and_prefix(size: usize, prefix: u8, prefix_mask: usize, out: &mut W) +where + W: Output + ?Sized, +{ + for b in size_and_prefix_iterator(size, prefix, prefix_mask) { out.push_byte(b) } } /// Decode size only from stream input and header byte. -fn decode_size(first: u8, input: &mut impl Input) -> Result { - let mut result = (first & 255u8 >> 2) as usize; - if result < 63 { +fn decode_size( + first: u8, + input: &mut impl Input, + prefix_mask: usize, +) -> Result { + let max_value = 255u8 >> prefix_mask; + let mut result = (first & max_value) as usize; + if result < max_value as usize { return Ok(result) } result -= 1; diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index 91f2159f2957e..cd65fa5b26f2f 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -19,6 +19,9 @@ use codec::{Decode, Encode}; use hash_db::{HashDB, Hasher}; use scale_info::TypeInfo; use sp_std::vec::Vec; +// Note that `LayoutV1` usage here (proof compaction) is compatible +// with `LayoutV0`. +use crate::LayoutV1 as Layout; /// A proof that some set of key-value pairs are included in the storage trie. The proof contains /// the storage values so that the partial storage backend can be reconstructed by a verifier that @@ -95,8 +98,8 @@ impl StorageProof { pub fn into_compact_proof( self, root: H::Out, - ) -> Result>> { - crate::encode_compact::>(self, root) + ) -> Result>> { + crate::encode_compact::>(self, root) } /// Returns the estimated encoded size of the compact proof. @@ -124,9 +127,9 @@ impl CompactProof { pub fn to_storage_proof( &self, expected_root: Option<&H::Out>, - ) -> Result<(StorageProof, H::Out), crate::CompactProofError>> { + ) -> Result<(StorageProof, H::Out), crate::CompactProofError>> { let mut db = crate::MemoryDB::::new(&[]); - let root = crate::decode_compact::, _, _>( + let root = crate::decode_compact::, _, _>( &mut db, self.iter_compact_encoded_nodes(), expected_root, diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index 1596229f2b5de..f485ee13e3329 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -112,8 +112,7 @@ where I: IntoIterator, { let mut nodes_iter = encoded.into_iter(); - let (top_root, _nb_used) = - trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; + let (top_root, _nb_used) = trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; // Only check root if expected root is passed as argument. if let Some(expected_root) = expected_root { @@ -164,8 +163,7 @@ where let mut nodes_iter = nodes_iter.peekable(); for child_root in child_tries.into_iter() { if previous_extracted_child_trie.is_none() && nodes_iter.peek().is_some() { - let (top_root, _) = - trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; + let (top_root, _) = trie_db::decode_compact_from_iter::(db, &mut nodes_iter)?; previous_extracted_child_trie = Some(top_root); } diff --git a/primitives/trie/src/trie_stream.rs b/primitives/trie/src/trie_stream.rs index e0e26fea67c2e..20f607c840d32 100644 --- a/primitives/trie/src/trie_stream.rs +++ b/primitives/trie/src/trie_stream.rs @@ -18,21 +18,18 @@ //! `TrieStream` implementation for Substrate's trie format. use crate::{ - node_codec::Bitmap, node_header::{size_and_prefix_iterator, NodeKind}, trie_constants, }; -use codec::Encode; +use codec::{Compact, Encode}; use hash_db::Hasher; use sp_std::vec::Vec; use trie_root; -const BRANCH_NODE_NO_VALUE: u8 = 254; -const BRANCH_NODE_WITH_VALUE: u8 = 255; - #[derive(Default, Clone)] /// Codec-flavored TrieStream. pub struct TrieStream { + /// Current node buffer. buffer: Vec, } @@ -60,51 +57,76 @@ fn fuse_nibbles_node<'a>(nibbles: &'a [u8], kind: NodeKind) -> impl Iterator size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK), + NodeKind::Leaf => size_and_prefix_iterator(size, trie_constants::LEAF_PREFIX_MASK, 2), NodeKind::BranchNoValue => - size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK), + size_and_prefix_iterator(size, trie_constants::BRANCH_WITHOUT_MASK, 2), NodeKind::BranchWithValue => - size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK), + size_and_prefix_iterator(size, trie_constants::BRANCH_WITH_MASK, 2), + NodeKind::HashedValueLeaf => + size_and_prefix_iterator(size, trie_constants::ALT_HASHING_LEAF_PREFIX_MASK, 3), + NodeKind::HashedValueBranch => + size_and_prefix_iterator(size, trie_constants::ALT_HASHING_BRANCH_WITH_MASK, 4), }; iter_start .chain(if nibbles.len() % 2 == 1 { Some(nibbles[0]) } else { None }) .chain(nibbles[nibbles.len() % 2..].chunks(2).map(|ch| ch[0] << 4 | ch[1])) } +use trie_root::Value as TrieStreamValue; impl trie_root::TrieStream for TrieStream { fn new() -> Self { - TrieStream { buffer: Vec::new() } + Self { buffer: Vec::new() } } fn append_empty_data(&mut self) { self.buffer.push(trie_constants::EMPTY_TRIE); } - fn append_leaf(&mut self, key: &[u8], value: &[u8]) { - self.buffer.extend(fuse_nibbles_node(key, NodeKind::Leaf)); - value.encode_to(&mut self.buffer); + fn append_leaf(&mut self, key: &[u8], value: TrieStreamValue) { + let kind = match &value { + TrieStreamValue::Inline(..) => NodeKind::Leaf, + TrieStreamValue::Node(..) => NodeKind::HashedValueLeaf, + }; + self.buffer.extend(fuse_nibbles_node(key, kind)); + match &value { + TrieStreamValue::Inline(value) => { + Compact(value.len() as u32).encode_to(&mut self.buffer); + self.buffer.extend_from_slice(value); + }, + TrieStreamValue::Node(hash) => { + self.buffer.extend_from_slice(hash.as_slice()); + }, + }; } fn begin_branch( &mut self, maybe_partial: Option<&[u8]>, - maybe_value: Option<&[u8]>, + maybe_value: Option, has_children: impl Iterator, ) { if let Some(partial) = maybe_partial { - if maybe_value.is_some() { - self.buffer.extend(fuse_nibbles_node(partial, NodeKind::BranchWithValue)); - } else { - self.buffer.extend(fuse_nibbles_node(partial, NodeKind::BranchNoValue)); - } + let kind = match &maybe_value { + None => NodeKind::BranchNoValue, + Some(TrieStreamValue::Inline(..)) => NodeKind::BranchWithValue, + Some(TrieStreamValue::Node(..)) => NodeKind::HashedValueBranch, + }; + + self.buffer.extend(fuse_nibbles_node(partial, kind)); let bm = branch_node_bit_mask(has_children); self.buffer.extend([bm.0, bm.1].iter()); } else { - debug_assert!(false, "trie stream codec only for no extension trie"); - self.buffer.extend(&branch_node(maybe_value.is_some(), has_children)); + unreachable!("trie stream codec only for no extension trie"); } - if let Some(value) = maybe_value { - value.encode_to(&mut self.buffer); + match maybe_value { + None => (), + Some(TrieStreamValue::Inline(value)) => { + Compact(value.len() as u32).encode_to(&mut self.buffer); + self.buffer.extend_from_slice(value); + }, + Some(TrieStreamValue::Node(hash)) => { + self.buffer.extend_from_slice(hash.as_slice()); + }, } } @@ -124,18 +146,3 @@ impl trie_root::TrieStream for TrieStream { self.buffer } } - -fn branch_node(has_value: bool, has_children: impl Iterator) -> [u8; 3] { - let mut result = [0, 0, 0]; - branch_node_buffered(has_value, has_children, &mut result[..]); - result -} - -fn branch_node_buffered(has_value: bool, has_children: I, output: &mut [u8]) -where - I: Iterator, -{ - let first = if has_value { BRANCH_NODE_WITH_VALUE } else { BRANCH_NODE_NO_VALUE }; - output[0] = first; - Bitmap::encode(has_children, &mut output[1..]); -} diff --git a/primitives/version/Cargo.toml b/primitives/version/Cargo.toml index 585d089a54b5c..7fb55295088ea 100644 --- a/primitives/version/Cargo.toml +++ b/primitives/version/Cargo.toml @@ -22,6 +22,7 @@ sp-std = { version = "4.0.0", default-features = false, path = "../std" } sp-runtime = { version = "4.0.0", default-features = false, path = "../runtime" } sp-version-proc-macro = { version = "4.0.0-dev", default-features = false, path = "proc-macro" } parity-wasm = { version = "0.42.2", optional = true } +sp-core-hashing-proc-macro = { version = "4.0.0-dev", path = "../core/hashing/proc-macro" } thiserror = { version = "1.0.30", optional = true } [features] diff --git a/primitives/version/proc-macro/src/decl_runtime_version.rs b/primitives/version/proc-macro/src/decl_runtime_version.rs index eef6314be4c81..7fcb716bbf607 100644 --- a/primitives/version/proc-macro/src/decl_runtime_version.rs +++ b/primitives/version/proc-macro/src/decl_runtime_version.rs @@ -63,6 +63,7 @@ struct RuntimeVersion { impl_version: u32, apis: u8, transaction_version: u32, + state_version: u8, } #[derive(Default, Debug)] @@ -73,6 +74,7 @@ struct ParseRuntimeVersion { spec_version: Option, impl_version: Option, transaction_version: Option, + state_version: Option, } impl ParseRuntimeVersion { @@ -122,6 +124,8 @@ impl ParseRuntimeVersion { parse_once(&mut self.impl_version, field_value, Self::parse_num_literal)?; } else if field_name == "transaction_version" { parse_once(&mut self.transaction_version, field_value, Self::parse_num_literal)?; + } else if field_name == "state_version" { + parse_once(&mut self.state_version, field_value, Self::parse_num_literal_u8)?; } else if field_name == "apis" { // Intentionally ignored // @@ -147,6 +151,18 @@ impl ParseRuntimeVersion { lit.base10_parse::() } + fn parse_num_literal_u8(expr: &Expr) -> Result { + let lit = match *expr { + Expr::Lit(ExprLit { lit: Lit::Int(ref lit), .. }) => lit, + _ => + return Err(Error::new( + expr.span(), + "only numeric literals (e.g. `10`) are supported here", + )), + }; + lit.base10_parse::() + } + fn parse_str_literal(expr: &Expr) -> Result { let mac = match *expr { Expr::Macro(syn::ExprMacro { ref mac, .. }) => mac, @@ -182,6 +198,7 @@ impl ParseRuntimeVersion { spec_version, impl_version, transaction_version, + state_version, } = self; Ok(RuntimeVersion { @@ -191,6 +208,7 @@ impl ParseRuntimeVersion { spec_version: required!(spec_version), impl_version: required!(impl_version), transaction_version: required!(transaction_version), + state_version: required!(state_version), apis: 0, }) } @@ -210,7 +228,6 @@ fn generate_emit_link_section_decl(contents: &[u8], section_name: &str) -> Token #[cfg(test)] mod tests { use super::*; - use codec::DecodeAll; use std::borrow::Cow; #[test] @@ -223,11 +240,13 @@ mod tests { impl_version: 1, apis: 0, transaction_version: 2, + state_version: 1, } .encode(); assert_eq!( - sp_version::RuntimeVersion::decode_all(&mut &version_bytes[..]).unwrap(), + sp_version::RuntimeVersion::decode_with_version_hint(&mut &version_bytes[..], Some(4)) + .unwrap(), sp_version::RuntimeVersion { spec_name: "hello".into(), impl_name: "world".into(), @@ -236,6 +255,7 @@ mod tests { impl_version: 1, apis: Cow::Owned(vec![]), transaction_version: 2, + state_version: 1, }, ); } diff --git a/primitives/version/src/lib.rs b/primitives/version/src/lib.rs index 0b72865e7b690..74251aa4320fb 100644 --- a/primitives/version/src/lib.rs +++ b/primitives/version/src/lib.rs @@ -40,10 +40,10 @@ use std::collections::HashSet; #[cfg(feature = "std")] use std::fmt; -use codec::{Decode, Encode}; +use codec::{Decode, Encode, Input}; use scale_info::TypeInfo; -pub use sp_runtime::create_runtime_str; use sp_runtime::RuntimeString; +pub use sp_runtime::{create_runtime_str, StateVersion}; #[doc(hidden)] pub use sp_std; @@ -79,6 +79,7 @@ pub mod embed; /// impl_version: 1, /// apis: RUNTIME_API_VERSIONS, /// transaction_version: 2, +/// state_version: 1, /// }; /// /// # const RUNTIME_API_VERSIONS: sp_version::ApisVec = sp_version::create_apis_vec!([]); @@ -154,7 +155,7 @@ macro_rules! create_apis_vec { /// In particular: bug fixes should result in an increment of `spec_version` and possibly /// `authoring_version`, absolutely not `impl_version` since they change the semantics of the /// runtime. -#[derive(Clone, PartialEq, Eq, Encode, Decode, Default, sp_runtime::RuntimeDebug, TypeInfo)] +#[derive(Clone, PartialEq, Eq, Encode, Default, sp_runtime::RuntimeDebug, TypeInfo)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] #[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] pub struct RuntimeVersion { @@ -207,6 +208,53 @@ pub struct RuntimeVersion { /// /// It need *not* change when a new module is added or when a dispatchable is added. pub transaction_version: u32, + + /// Version of the state implementation used by this runtime. + /// Use of an incorrect version is consensus breaking. + pub state_version: u8, +} + +impl RuntimeVersion { + /// `Decode` while giving a "version hint" + /// + /// There exists multiple versions of [`RuntimeVersion`] and they are versioned using the `Core` + /// runtime api: + /// - `Core` version < 3 is a runtime version without a transaction version and state version. + /// - `Core` version 3 is a runtime version without a state version. + /// - `Core` version 4 is the latest runtime version. + pub fn decode_with_version_hint( + input: &mut I, + core_version: Option, + ) -> Result { + let spec_name = Decode::decode(input)?; + let impl_name = Decode::decode(input)?; + let authoring_version = Decode::decode(input)?; + let spec_version = Decode::decode(input)?; + let impl_version = Decode::decode(input)?; + let apis = Decode::decode(input)?; + let core_version = + if core_version.is_some() { core_version } else { core_version_from_apis(&apis) }; + let transaction_version = + if core_version.map(|v| v >= 3).unwrap_or(false) { Decode::decode(input)? } else { 1 }; + let state_version = + if core_version.map(|v| v >= 4).unwrap_or(false) { Decode::decode(input)? } else { 0 }; + Ok(RuntimeVersion { + spec_name, + impl_name, + authoring_version, + spec_version, + impl_version, + apis, + transaction_version, + state_version, + }) + } +} + +impl Decode for RuntimeVersion { + fn decode(input: &mut I) -> Result { + Self::decode_with_version_hint(input, None) + } } #[cfg(feature = "std")] @@ -225,6 +273,16 @@ impl fmt::Display for RuntimeVersion { } } +fn has_api_with bool>(apis: &ApisVec, id: &ApiId, predicate: P) -> bool { + apis.iter().any(|(s, v)| s == id && predicate(*v)) +} + +/// Returns the version of the `Core` runtime api. +pub fn core_version_from_apis(apis: &ApisVec) -> Option { + let id = sp_core_hashing_proc_macro::blake2b_64!(b"Core"); + apis.iter().find(|(s, _v)| s == &id).map(|(_s, v)| *v) +} + #[cfg(feature = "std")] impl RuntimeVersion { /// Check if this version matches other version for calling into runtime. @@ -237,7 +295,7 @@ impl RuntimeVersion { /// Check if the given api with `api_id` is implemented and the version passes the given /// `predicate`. pub fn has_api_with bool>(&self, id: &ApiId, predicate: P) -> bool { - self.apis.iter().any(|(s, v)| s == id && predicate(*v)) + has_api_with(&self.apis, id, predicate) } /// Returns the api version found for api with `id`. @@ -246,6 +304,18 @@ impl RuntimeVersion { } } +impl RuntimeVersion { + /// Returns state version to use for update. + /// + /// For runtime with core api version less than 4, + /// V0 trie version will be applied to state. + /// Otherwhise, V1 trie version will be use. + pub fn state_version(&self) -> StateVersion { + // If version > than 1, keep using latest version. + self.state_version.try_into().unwrap_or(StateVersion::V1) + } +} + /// The version of the native runtime. /// /// In contrast to the bare [`RuntimeVersion`] this also carries a list of `spec_version`s of diff --git a/test-utils/client/src/client_ext.rs b/test-utils/client/src/client_ext.rs index bf1c9898972ca..72828fdcc9188 100644 --- a/test-utils/client/src/client_ext.rs +++ b/test-utils/client/src/client_ext.rs @@ -69,7 +69,7 @@ pub trait ClientBlockImportExt: Sized { impl ClientExt for Client where B: sc_client_api::backend::Backend, - E: sc_client_api::CallExecutor + 'static, + E: sc_client_api::CallExecutor + sc_executor::RuntimeVersionOf + 'static, Self: BlockImport, Block: BlockT, { diff --git a/test-utils/client/src/lib.rs b/test-utils/client/src/lib.rs index 86231bb34c506..6ba28d8c124f2 100644 --- a/test-utils/client/src/lib.rs +++ b/test-utils/client/src/lib.rs @@ -208,13 +208,13 @@ impl sc_consensus::LongestChain, ) where - ExecutorDispatch: sc_client_api::CallExecutor + 'static, + ExecutorDispatch: + sc_client_api::CallExecutor + sc_executor::RuntimeVersionOf + 'static, Backend: sc_client_api::backend::Backend, >::OffchainStorage: 'static, { let storage = { let mut storage = self.genesis_init.genesis_storage(); - // Add some child storage keys. for (key, child_content) in self.child_storage_extension { storage.children_default.insert( diff --git a/test-utils/runtime/Cargo.toml b/test-utils/runtime/Cargo.toml index 35c1291f38fb5..c32586834f30b 100644 --- a/test-utils/runtime/Cargo.toml +++ b/test-utils/runtime/Cargo.toml @@ -39,7 +39,7 @@ pallet-timestamp = { version = "4.0.0-dev", default-features = false, path = ".. sp-finality-grandpa = { version = "4.0.0-dev", default-features = false, path = "../../primitives/finality-grandpa" } sp-trie = { version = "4.0.0", default-features = false, path = "../../primitives/trie" } sp-transaction-pool = { version = "4.0.0-dev", default-features = false, path = "../../primitives/transaction-pool" } -trie-db = { version = "0.22.6", default-features = false } +trie-db = { version = "0.23.0", default-features = false } parity-util-mem = { version = "0.10.2", default-features = false, features = ["primitive-types"] } sc-service = { version = "0.10.0-dev", default-features = false, optional = true, features = ["test-helpers"], path = "../../client/service" } sp-state-machine = { version = "0.10.0", default-features = false, path = "../../primitives/state-machine" } diff --git a/test-utils/runtime/client/src/lib.rs b/test-utils/runtime/client/src/lib.rs index a5d1e1966e307..d5c8a4fcb8a46 100644 --- a/test-utils/runtime/client/src/lib.rs +++ b/test-utils/runtime/client/src/lib.rs @@ -138,6 +138,7 @@ impl substrate_test_client::GenesisInit for GenesisParameters { let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( child_content.data.clone().into_iter().collect(), + sp_runtime::StateVersion::V1, ); let prefixed_storage_key = child_content.child_info.prefixed_storage_key(); (prefixed_storage_key.into_inner(), state_root.encode()) @@ -145,6 +146,7 @@ impl substrate_test_client::GenesisInit for GenesisParameters { let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( storage.top.clone().into_iter().chain(child_roots).collect(), + sp_runtime::StateVersion::V1, ); let block: runtime::Block = client::genesis::construct_genesis_block(state_root); storage.top.extend(additional_storage_with_genesis(&block)); diff --git a/test-utils/runtime/src/genesismap.rs b/test-utils/runtime/src/genesismap.rs index a06d9f310fb04..b81ed91ca6df0 100644 --- a/test-utils/runtime/src/genesismap.rs +++ b/test-utils/runtime/src/genesismap.rs @@ -95,6 +95,7 @@ pub fn insert_genesis_block(storage: &mut Storage) -> sp_core::hash::H256 { let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( child_content.data.clone().into_iter().collect(), + sp_runtime::StateVersion::V1, ); (sk.clone(), state_root.encode()) }); @@ -102,6 +103,7 @@ pub fn insert_genesis_block(storage: &mut Storage) -> sp_core::hash::H256 { storage.top.extend(child_roots); let state_root = <<::Header as HeaderT>::Hashing as HashT>::trie_root( storage.top.clone().into_iter().collect(), + sp_runtime::StateVersion::V1, ); let block: crate::Block = genesis::construct_genesis_block(state_root); let genesis_hash = block.header.hash(); diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index 03e39be324e90..bf224b43946c7 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -29,10 +29,7 @@ use sp_std::{marker::PhantomData, prelude::*}; use sp_application_crypto::{ecdsa, ed25519, sr25519, RuntimeAppPublic}; use sp_core::{offchain::KeyTypeId, OpaqueMetadata, RuntimeDebug}; -use sp_trie::{ - trie_types::{TrieDB, TrieDBMut}, - PrefixedMemoryDB, StorageProof, -}; +use sp_trie::{trie_types::TrieDB, PrefixedMemoryDB, StorageProof}; use trie_db::{Trie, TrieMut}; use cfg_if::cfg_if; @@ -62,6 +59,8 @@ use sp_runtime::{ #[cfg(any(feature = "std", test))] use sp_version::NativeVersion; use sp_version::RuntimeVersion; +// bench on latest state. +use sp_trie::trie_types::TrieDBMutV1 as TrieDBMut; // Ensure Babe and Aura use the same crypto to simplify things a bit. pub use sp_consensus_babe::{AllowedSlots, AuthorityId, Slot}; @@ -105,6 +104,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { impl_version: 2, apis: RUNTIME_API_VERSIONS, transaction_version: 1, + state_version: 1, }; fn version() -> RuntimeVersion { @@ -1268,9 +1268,9 @@ fn test_witness(proof: StorageProof, root: crate::Hash) { None, ); assert!(ext.storage(b"value3").is_some()); - assert!(ext.storage_root().as_slice() == &root[..]); + assert!(ext.storage_root(Default::default()).as_slice() == &root[..]); ext.place_storage(vec![0], Some(vec![1])); - assert!(ext.storage_root().as_slice() != &root[..]); + assert!(ext.storage_root(Default::default()).as_slice() != &root[..]); } #[cfg(test)] @@ -1334,7 +1334,7 @@ mod tests { let mut root = crate::Hash::default(); let mut mdb = sp_trie::MemoryDB::::default(); { - let mut trie = sp_trie::trie_types::TrieDBMut::new(&mut mdb, &mut root); + let mut trie = sp_trie::trie_types::TrieDBMutV1::new(&mut mdb, &mut root); trie.insert(b"value3", &[142]).expect("insert failed"); trie.insert(b"value4", &[124]).expect("insert failed"); }; diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index 0c72c083baae2..57eae0aecf04a 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -194,9 +194,10 @@ pub fn execute_transaction(utx: Extrinsic) -> ApplyExtrinsicResult { /// Finalize the block. pub fn finalize_block() -> Header { + use sp_core::storage::StateVersion; let extrinsic_index: u32 = storage::unhashed::take(well_known_keys::EXTRINSIC_INDEX).unwrap(); let txs: Vec<_> = (0..extrinsic_index).map(ExtrinsicData::take).collect(); - let extrinsics_root = trie::blake2_256_ordered_root(txs).into(); + let extrinsics_root = trie::blake2_256_ordered_root(txs, StateVersion::V0).into(); let number = ::take().expect("Number is set by `initialize_block`"); let parent_hash = ::take(); let mut digest = ::take().expect("StorageDigest is set by `initialize_block`"); @@ -205,8 +206,8 @@ pub fn finalize_block() -> Header { // This MUST come after all changes to storage are done. Otherwise we will fail the // “Storage root does not match that calculated” assertion. - let storage_root = - Hash::decode(&mut &storage_root()[..]).expect("`storage_root` is a valid hash"); + let storage_root = Hash::decode(&mut &storage_root(StateVersion::V1)[..]) + .expect("`storage_root` is a valid hash"); if let Some(new_authorities) = o_new_authorities { digest.push(generic::DigestItem::Consensus(*b"aura", new_authorities.encode())); @@ -351,6 +352,7 @@ mod tests { Sr25519Keyring::Bob.to_raw_public(), Sr25519Keyring::Charlie.to_raw_public(), ]; + TestExternalities::new_with_code( wasm_binary_unwrap(), sp_core::storage::Storage { @@ -359,7 +361,7 @@ mod tests { twox_128(b"sys:auth").to_vec() => authorities.encode(), blake2_256(&AccountKeyring::Alice.to_raw_public().to_keyed_vec(b"balance:")).to_vec() => { vec![111u8, 0, 0, 0, 0, 0, 0, 0] - } + }, ], children_default: map![], }, diff --git a/utils/frame/try-runtime/cli/src/commands/execute_block.rs b/utils/frame/try-runtime/cli/src/commands/execute_block.rs index a717b410c2bf4..f092f6eaf4716 100644 --- a/utils/frame/try-runtime/cli/src/commands/execute_block.rs +++ b/utils/frame/try-runtime/cli/src/commands/execute_block.rs @@ -157,7 +157,7 @@ where header.digest_mut().pop(); let block = Block::new(header, extrinsics); - let (expected_spec_name, expected_spec_version) = + let (expected_spec_name, expected_spec_version, _) = local_spec::(&ext, &executor); ensure_matching_spec::( block_ws_uri.clone(), diff --git a/utils/frame/try-runtime/cli/src/commands/follow_chain.rs b/utils/frame/try-runtime/cli/src/commands/follow_chain.rs index 09f541c887536..dbc1c3005f06a 100644 --- a/utils/frame/try-runtime/cli/src/commands/follow_chain.rs +++ b/utils/frame/try-runtime/cli/src/commands/follow_chain.rs @@ -123,7 +123,7 @@ where new_ext.as_backend().root() ); - let (expected_spec_name, expected_spec_version) = + let (expected_spec_name, expected_spec_version, spec_state_version) = local_spec::(&new_ext, &executor); ensure_matching_spec::( command.uri.clone(), @@ -133,10 +133,10 @@ where ) .await; - maybe_state_ext = Some(new_ext); + maybe_state_ext = Some((new_ext, spec_state_version)); } - let state_ext = + let (state_ext, spec_state_version) = maybe_state_ext.as_mut().expect("state_ext either existed or was just created"); let (mut changes, encoded_result) = state_machine_call_with_proof::( @@ -152,7 +152,16 @@ where .map_err(|e| format!("failed to decode output: {:?}", e))?; let storage_changes = changes - .drain_storage_changes(&state_ext.backend, Default::default(), &mut Default::default()) + .drain_storage_changes( + &state_ext.backend, + Default::default(), + &mut Default::default(), + // Note that in case a block contains a runtime upgrade, + // state version could potentially be incorrect here, + // this is very niche and would only result in unaligned + // roots, so this use case is ignored for now. + *spec_state_version, + ) .unwrap(); state_ext.backend.apply_transaction( storage_changes.transaction_storage_root, diff --git a/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs b/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs index 22120ef4b5fe4..8839c5556900e 100644 --- a/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs +++ b/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs @@ -140,7 +140,7 @@ where builder.build().await? }; - let (expected_spec_name, expected_spec_version) = + let (expected_spec_name, expected_spec_version, _) = local_spec::(&ext, &executor); ensure_matching_spec::( header_ws_uri, diff --git a/utils/frame/try-runtime/cli/src/commands/on_runtime_upgrade.rs b/utils/frame/try-runtime/cli/src/commands/on_runtime_upgrade.rs index 6343b2b2e3f0d..3b5a3db72a8f1 100644 --- a/utils/frame/try-runtime/cli/src/commands/on_runtime_upgrade.rs +++ b/utils/frame/try-runtime/cli/src/commands/on_runtime_upgrade.rs @@ -58,7 +58,7 @@ where }; if let Some(uri) = command.state.live_uri() { - let (expected_spec_name, expected_spec_version) = + let (expected_spec_name, expected_spec_version, _) = local_spec::(&ext, &executor); ensure_matching_spec::( uri, diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index 3872f84a62a4b..9f8af6906a351 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -799,7 +799,7 @@ pub(crate) fn state_machine_call_with_proof( ext: &TestExternalities, executor: &NativeElseWasmExecutor, -) -> (String, u32) { +) -> (String, u32, sp_core::storage::StateVersion) { let (_, encoded) = state_machine_call::( &ext, &executor, @@ -811,6 +811,9 @@ pub(crate) fn local_spec( .expect("all runtimes should have version; qed"); ::decode(&mut &*encoded) .map_err(|e| format!("failed to decode output: {:?}", e)) - .map(|v| (v.spec_name.into(), v.spec_version)) + .map(|v| { + let state_version = v.state_version(); + (v.spec_name.into(), v.spec_version, state_version) + }) .expect("all runtimes should have version; qed") }