From 488d0a93e0d56c789c9d8f49267ffb2a15a5e421 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Fri, 14 Aug 2020 12:34:55 +0200 Subject: [PATCH 01/19] Reset code, almost ready for PR --- Cargo.lock | 6 +- bin/node/cli/src/service.rs | 2 + client/api/src/in_mem.rs | 18 +++--- client/api/src/light.rs | 17 +++--- client/chain-spec/Cargo.toml | 1 + client/chain-spec/src/chain_spec.rs | 89 +++++++++++++++++++++++++---- client/chain-spec/src/lib.rs | 6 +- client/db/src/light.rs | 60 +++++++++++++------ client/rpc-api/Cargo.toml | 1 + client/rpc-api/src/system/error.rs | 23 +++++++- client/rpc-api/src/system/mod.rs | 5 ++ client/rpc/src/system/mod.rs | 18 +++++- client/service/src/builder.rs | 13 +++-- client/service/src/lib.rs | 88 +++++++++++++++++++++++++++- 14 files changed, 293 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc664f97ddeb5..21266728dce79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6134,6 +6134,7 @@ name = "sc-chain-spec" version = "2.0.0-rc5" dependencies = [ "impl-trait-for-tuples", + "parity-scale-codec", "sc-chain-spec-derive", "sc-network", "sc-telemetry", @@ -6918,6 +6919,7 @@ dependencies = [ "parking_lot 0.10.2", "serde", "serde_json", + "sp-blockchain", "sp-chain-spec", "sp-core", "sp-rpc", @@ -8624,8 +8626,8 @@ name = "substrate-test-utils-derive" version = "0.8.0-rc5" dependencies = [ "proc-macro-crate", - "quote 1.0.6", - "syn 1.0.33", + "quote", + "syn", ] [[package]] diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index cd98c26809644..c7c4dfff3636f 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -175,6 +175,7 @@ pub fn new_full_base( sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), + backend: backend.clone(), transaction_pool: transaction_pool.clone(), spawn_handle: task_manager.spawn_handle(), import_queue, @@ -394,6 +395,7 @@ pub fn new_light_base(config: Configuration) -> Result<( sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), + backend: backend.clone(), transaction_pool: transaction_pool.clone(), spawn_handle: task_manager.spawn_handle(), import_queue, diff --git a/client/api/src/in_mem.rs b/client/api/src/in_mem.rs index 306c3c2b2f10c..13ff7a01f9193 100644 --- a/client/api/src/in_mem.rs +++ b/client/api/src/in_mem.rs @@ -447,6 +447,16 @@ impl light::Storage for Blockchain Blockchain::finalize_header(self, id, None) } + fn cache(&self) -> Option>> { + None + } + + fn usage_info(&self) -> Option { + None + } +} + +impl light::ChtRootStorage for Blockchain { fn header_cht_root( &self, _cht_size: NumberFor, @@ -466,14 +476,6 @@ impl light::Storage for Blockchain .ok_or_else(|| sp_blockchain::Error::Backend(format!("Changes trie CHT for block {} not exists", block))) .map(Some) } - - fn cache(&self) -> Option>> { - None - } - - fn usage_info(&self) -> Option { - None - } } /// In-memory operation. diff --git a/client/api/src/light.rs b/client/api/src/light.rs index b359c1149eea6..ed816048fb852 100644 --- a/client/api/src/light.rs +++ b/client/api/src/light.rs @@ -232,7 +232,7 @@ pub trait FetchChecker: Send + Sync { /// Light client blockchain storage. -pub trait Storage: AuxStore + HeaderBackend + HeaderMetadata { +pub trait Storage: AuxStore + HeaderBackend + HeaderMetadata + ChtRootStorage { /// Store new header. Should refuse to revert any finalized blocks. /// /// Takes new authorities, the leaf state of the new block, and @@ -254,6 +254,15 @@ pub trait Storage: AuxStore + HeaderBackend + HeaderMetada /// Get last finalized header. fn last_finalized(&self) -> ClientResult; + /// Get storage cache. + fn cache(&self) -> Option>>; + + /// Get storage usage statistics. + fn usage_info(&self) -> Option; +} + +/// Light client CHT root storage. +pub trait ChtRootStorage { /// Get headers CHT root for given block. Returns None if the block is not pruned (not a part of any CHT). fn header_cht_root( &self, @@ -267,12 +276,6 @@ pub trait Storage: AuxStore + HeaderBackend + HeaderMetada cht_size: NumberFor, block: NumberFor, ) -> ClientResult>; - - /// Get storage cache. - fn cache(&self) -> Option>>; - - /// Get storage usage statistics. - fn usage_info(&self) -> Option; } /// Remote header. diff --git a/client/chain-spec/Cargo.toml b/client/chain-spec/Cargo.toml index 5a4759cbf17ec..bb609a35654bd 100644 --- a/client/chain-spec/Cargo.toml +++ b/client/chain-spec/Cargo.toml @@ -21,3 +21,4 @@ serde_json = "1.0.41" sp-runtime = { version = "2.0.0-rc5", path = "../../primitives/runtime" } sp-chain-spec = { version = "2.0.0-rc5", path = "../../primitives/chain-spec" } sc-telemetry = { version = "2.0.0-rc5", path = "../telemetry" } +codec = { package = "parity-scale-codec", version = "1.3.4" } diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 52414f8687c96..d9e62cdd2e198 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -17,6 +17,7 @@ // along with this program. If not, see . //! Substrate chain configurations. +#![warn(missing_docs)] use std::{borrow::Cow, fs::File, path::PathBuf, sync::Arc, collections::HashMap}; use serde::{Serialize, Deserialize}; @@ -26,6 +27,7 @@ use serde_json as json; use crate::{RuntimeGenesis, ChainType, extension::GetExtension, Properties}; use sc_network::config::MultiaddrWithPeerId; use sc_telemetry::TelemetryEndpoints; +use sp_runtime::traits::Block as BlockT; enum GenesisSource { File(PathBuf), @@ -157,6 +159,7 @@ struct ClientSpec { consensus_engine: (), #[serde(skip_serializing)] genesis: serde::de::IgnoredAny, + hardcoded_sync: Option, } /// A type denoting empty extensions. @@ -245,6 +248,7 @@ impl ChainSpec { extensions, consensus_engine: (), genesis: Default::default(), + hardcoded_sync: None, }; ChainSpec { @@ -257,6 +261,11 @@ impl ChainSpec { fn chain_type(&self) -> ChainType { self.client_spec.chain_type.clone() } + + /// Hardcode infomation to allow light clients to sync quickly into the chain spec. + fn set_hardcoded_sync(&mut self, hardcoded_sync: SerializableHardcodedSync) { + self.client_spec.hardcoded_sync = Some(hardcoded_sync); + } } impl ChainSpec { @@ -284,16 +293,15 @@ impl ChainSpec { } } -impl ChainSpec { - /// Dump to json string. - pub fn as_json(&self, raw: bool) -> Result { - #[derive(Serialize, Deserialize)] - struct Container { - #[serde(flatten)] - client_spec: ClientSpec, - genesis: Genesis, +#[derive(Serialize, Deserialize)] +struct JsonContainer { + #[serde(flatten)] + client_spec: ClientSpec, + genesis: Genesis, +} - }; +impl ChainSpec { + fn json_container(&self, raw: bool) -> Result, String> { let genesis = match (raw, self.genesis.resolve()?) { (true, Genesis::Runtime(g)) => { let storage = g.build_storage()?; @@ -313,13 +321,25 @@ impl ChainSpec { }, (_, genesis) => genesis, }; - let container = Container { + Ok(JsonContainer { client_spec: self.client_spec.clone(), genesis, - }; + }) + } + + /// Dump to json string. + pub fn as_json(&self, raw: bool) -> Result { + let container = self.json_container(raw)?; json::to_string_pretty(&container) .map_err(|e| format!("Error generating spec json: {}", e)) } + + /// Dump to json value + pub fn as_json_value(&self, raw: bool) -> Result { + let container = self.json_container(raw)?; + json::to_value(container) + .map_err(|e| format!("Error generating spec json: {}", e)) + } } impl crate::ChainSpec for ChainSpec @@ -367,6 +387,10 @@ where ChainSpec::as_json(self, raw) } + fn as_json_value(&self, raw: bool) -> Result { + ChainSpec::as_json_value(self, raw) + } + fn as_storage_builder(&self) -> &dyn BuildStorage { self } @@ -378,6 +402,49 @@ where fn set_storage(&mut self, storage: Storage) { self.genesis = GenesisSource::Storage(storage); } + + fn set_hardcoded_sync(&mut self, hardcoded_sync: SerializableHardcodedSync) { + ChainSpec::set_hardcoded_sync(self, hardcoded_sync) + } +} + +/// Hardcoded infomation that allows light clients to sync quickly. +pub struct HardcodedSync { + /// The header of the best finalized block. + pub header: ::Header, + /// A list of all CHTs in the chain. + pub chts: Vec<::Hash>, +} + +impl HardcodedSync { + /// Convert into a `SerializableHardcodecSync`. + pub fn to_serializable(&self) -> SerializableHardcodedSync { + use codec::Encode; + + SerializableHardcodedSync { + header: StorageData(self.header.encode()), + chts: self.chts.iter().map(|hash| StorageData(hash.encode())).collect(), + } + } + + /// Convert from a `SerializableHardcodecSync`. + pub fn from_serializable(serialized: &SerializableHardcodedSync) -> Result { + Ok(Self { + header: codec::Decode::decode(&mut &serialized.header.0[..])?, + chts: serialized.chts.iter() + .map(|cht| codec::Decode::decode(&mut &cht.0[..])) + .collect::>()?, + }) + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +#[serde(deny_unknown_fields)] +/// The serializable form of `HardcodedSync`. Created using `HardcodedSync::serialize`. +pub struct SerializableHardcodedSync { + header: StorageData, + chts: Vec, } #[cfg(test)] diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index 8901a9a682224..b5f61c538f0f0 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -108,7 +108,7 @@ mod chain_spec; mod extension; -pub use chain_spec::{ChainSpec as GenericChainSpec, NoExtension}; +pub use chain_spec::{ChainSpec as GenericChainSpec, NoExtension, HardcodedSync, SerializableHardcodedSync}; pub use extension::{Group, Fork, Forks, Extension, GetExtension, get_extension}; pub use sc_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup}; pub use sp_chain_spec::{Properties, ChainType}; @@ -147,6 +147,8 @@ pub trait ChainSpec: BuildStorage + Send { fn add_boot_node(&mut self, addr: MultiaddrWithPeerId); /// Return spec as JSON. fn as_json(&self, raw: bool) -> Result; + /// Return spec as JSON value. + fn as_json_value(&self, raw: bool) -> Result; /// Return StorageBuilder for this spec. fn as_storage_builder(&self) -> &dyn BuildStorage; /// Returns a cloned `Box`. @@ -155,6 +157,8 @@ pub trait ChainSpec: BuildStorage + Send { /// /// This will be used as storage at genesis. fn set_storage(&mut self, storage: Storage); + /// Hardcode infomation to allow light clients to sync quickly into the chain spec. + fn set_hardcoded_sync(&mut self, hardcoded_sync: SerializableHardcodedSync); } impl std::fmt::Debug for dyn ChainSpec { diff --git a/client/db/src/light.rs b/client/db/src/light.rs index 139ecf3b22c72..effd70cd45147 100644 --- a/client/db/src/light.rs +++ b/client/db/src/light.rs @@ -27,7 +27,7 @@ use sc_client_api::{ blockchain::{ BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo, }, - Storage + Storage, ChtRootStorage, }; use sp_blockchain::{ CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache, @@ -523,22 +523,6 @@ impl Storage for LightStorage } } - fn header_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult> { - self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block) - } - - fn changes_trie_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult> { - self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block) - } - fn finalize_header(&self, id: BlockId) -> ClientResult<()> { if let Some(header) = self.header(id)? { let mut transaction = Transaction::new(); @@ -612,6 +596,48 @@ impl Storage for LightStorage } } +impl ChtRootStorage for LightStorage + where Block: BlockT, +{ + fn header_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult> { + self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block) + } + + fn changes_trie_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult> { + self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block) + } +} + + +impl<'a, Block> ChtRootStorage for &'a LightStorage + where Block: BlockT, +{ + fn header_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult> { + self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block) + } + + fn changes_trie_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult> { + self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block) + } +} + + /// Build the key for inserting header-CHT at given block. fn cht_key>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> { let mut key = [cht_type; 5]; diff --git a/client/rpc-api/Cargo.toml b/client/rpc-api/Cargo.toml index 7701befbf71a4..b88c2e3eb401e 100644 --- a/client/rpc-api/Cargo.toml +++ b/client/rpc-api/Cargo.toml @@ -29,3 +29,4 @@ serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" sp-transaction-pool = { version = "2.0.0-rc5", path = "../../primitives/transaction-pool" } sp-rpc = { version = "2.0.0-rc5", path = "../../primitives/rpc" } +sp-blockchain = { version = "2.0.0-rc5", path = "../../primitives/blockchain" } diff --git a/client/rpc-api/src/system/error.rs b/client/rpc-api/src/system/error.rs index 4897aa485cbe4..1d04e8fda2941 100644 --- a/client/rpc-api/src/system/error.rs +++ b/client/rpc-api/src/system/error.rs @@ -32,6 +32,12 @@ pub enum Error { NotHealthy(Health), /// Peer argument is malformatted. MalformattedPeerArg(String), + /// Failed to serialize chain spec. + ChainSpec(serde_json::error::Error), + /// Backend doesn't store CHT roots. + #[display(fmt = "Backend doesn't store CHT roots. Make sure you're calling this on a light client.")] + BackendNoChtRoots, + BlockchainError(sp_blockchain::Error), } impl std::error::Error for Error {} @@ -51,7 +57,22 @@ impl From for rpc::Error { code :rpc::ErrorCode::ServerError(BASE_ERROR + 2), message: e.clone(), data: None, - } + }, + Error::ChainSpec(ref s) => rpc::Error { + code :rpc::ErrorCode::ServerError(BASE_ERROR + 3), + message: s.to_string(), + data: None, + }, + Error::BackendNoChtRoots => rpc::Error { + code :rpc::ErrorCode::ServerError(BASE_ERROR + 4), + message: format!("{}", e), + data: None, + }, + Error::BlockchainError(ref b) => rpc::Error { + code :rpc::ErrorCode::ServerError(BASE_ERROR + 5), + message: b.to_string(), + data: None, + }, } } } diff --git a/client/rpc-api/src/system/mod.rs b/client/rpc-api/src/system/mod.rs index a7b746ee1b114..3c28aa89bf3de 100644 --- a/client/rpc-api/src/system/mod.rs +++ b/client/rpc-api/src/system/mod.rs @@ -103,4 +103,9 @@ pub trait SystemApi { /// Returns the roles the node is running as. #[rpc(name = "system_nodeRoles", returns = "Vec")] fn system_node_roles(&self) -> Receiver>; + + /// Returns the json-serialized chainspec running the node. + #[rpc(name = "system_genChainSpec", returns = "jsonrpc_core::Value")] + fn system_gen_chain_spec(&self, raw: bool, hardcode_sync: bool) + -> Compat>>; } diff --git a/client/rpc/src/system/mod.rs b/client/rpc/src/system/mod.rs index 4b8abbe144462..966cd2b968bfe 100644 --- a/client/rpc/src/system/mod.rs +++ b/client/rpc/src/system/mod.rs @@ -66,7 +66,9 @@ pub enum Request { /// Must return any potential parse error. NetworkRemoveReservedPeer(String, oneshot::Sender>), /// Must return the node role. - NodeRoles(oneshot::Sender>) + NodeRoles(oneshot::Sender>), + /// Must return either a chain spec encoded as a `Value` or an error. + GenChainSpec(bool, bool, oneshot::Sender>), } impl System { @@ -189,4 +191,18 @@ impl SystemApi::Number> for Sy let _ = self.send_back.unbounded_send(Request::NodeRoles(tx)); Receiver(Compat::new(rx)) } + + fn system_gen_chain_spec(&self, raw: bool, hardcode_sync: bool) + -> Compat>> + { + let (tx, rx) = oneshot::channel(); + let _ = self.send_back.unbounded_send(Request::GenChainSpec(raw, hardcode_sync, tx)); + async move { + match rx.await { + Ok(Ok(value)) => Ok(value), + Ok(Err(e)) => Err(rpc::Error::from(e)), + Err(_) => Err(rpc::Error::internal_error()), + } + }.boxed().compat() + } } diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index eedc4582299d3..48e07d0b5a829 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -791,11 +791,13 @@ fn gen_handler( } /// Parameters to pass into `build_network`. -pub struct BuildNetworkParams<'a, TBl: BlockT, TExPool, TImpQu, TCl> { +pub struct BuildNetworkParams<'a, TBl: BlockT, TExPool, TImpQu, TCl, TBackend> { /// The service configuration. pub config: &'a Configuration, /// A shared client returned by `new_full_parts`/`new_light_parts`. pub client: Arc, + /// A shared backend returned by `new_full_parts`/`new_light_parts`. + pub backend: Arc, /// A shared transaction pool. pub transaction_pool: Arc, /// A handle for spawning tasks. @@ -815,8 +817,8 @@ pub struct BuildNetworkParams<'a, TBl: BlockT, TExPool, TImpQu, TCl> { } /// Build the network service, the network status sinks and an RPC sender. -pub fn build_network( - params: BuildNetworkParams +pub fn build_network( + params: BuildNetworkParams ) -> Result< ( Arc::Hash>>, @@ -833,9 +835,10 @@ pub fn build_network( HeaderBackend + BlockchainEvents + 'static, TExPool: MaintainedTransactionPool::Hash> + 'static, TImpQu: ImportQueue + 'static, + TBackend: crate::MaybeChtRootStorageProvider + Send + Sync + 'static, { let BuildNetworkParams { - config, client, transaction_pool, spawn_handle, import_queue, on_demand, + config, client, backend, transaction_pool, spawn_handle, import_queue, on_demand, block_announce_validator_builder, finality_proof_request_builder, finality_proof_provider, } = params; @@ -893,8 +896,10 @@ pub fn build_network( let future = build_network_future( config.role.clone(), + config.chain_spec.cloned_box(), network_mut, client, + backend.clone(), network_status_sinks.clone(), system_rpc_rx, has_bootnodes, diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 415a5de4f930d..5fbad3d62ff95 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -46,9 +46,10 @@ use sc_network::{NetworkStatus, network_state::NetworkState, PeerId}; use log::{warn, debug, error}; use codec::{Encode, Decode}; use sp_runtime::generic::BlockId; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero, One}; use parity_util_mem::MallocSizeOf; use sp_utils::{status_sinks, mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}}; +use sp_blockchain::HeaderBackend; pub use self::error::Error; pub use self::builder::{ @@ -153,6 +154,24 @@ impl TelemetryConnectionSinks { } } +/// Something that might allow access to a `ChtRootStorage`. +pub trait MaybeChtRootStorageProvider { + /// Potentially get a reference to a `ChtRootStorage`. + fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage>; +} + +impl MaybeChtRootStorageProvider for TFullBackend { + fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage> { + None + } +} + +impl MaybeChtRootStorageProvider for TLightBackend { + fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage> { + Some(self.blockchain().storage()) + } +} + /// An imcomplete set of chain components, but enough to run the chain ops subcommands. pub struct PartialComponents { /// A shared client instance. @@ -180,12 +199,15 @@ pub struct PartialComponents, + C: BlockchainEvents + HeaderBackend, + BE: MaybeChtRootStorageProvider, H: sc_network::ExHashT > ( role: Role, + chain_spec: Box, mut network: sc_network::NetworkWorker, client: Arc, + backend: Arc, status_sinks: NetworkStatusSinks, mut rpc_rx: TracingUnboundedReceiver>, should_have_peers: bool, @@ -304,6 +326,23 @@ async fn build_network_future< }; let _ = sender.send(vec![node_role]); + }, + sc_rpc::system::Request::GenChainSpec(raw, hardcode_sync, sender) => { + let mut chain_spec = chain_spec.cloned_box(); + + let hardcoded_sync_result = if hardcode_sync { + build_hardcoded_sync(client.clone(), backend.clone()) + .map(|hardcoded_sync| chain_spec.set_hardcoded_sync(hardcoded_sync.serialize())) + } else { + Ok(()) + }; + + let value = match hardcoded_sync_result { + Ok(()) => chain_spec.as_json_value(raw).map_err(|err| err.into()), + Err(error) => Err(error) + }; + + let _ = sender.send(value); } } } @@ -332,6 +371,51 @@ async fn build_network_future< } } +fn build_hardcoded_sync( + client: Arc, + backend: Arc, +) -> Result, sc_rpc::system::error::Error> + where + TBl: BlockT, + TCl: HeaderBackend, + TBackend: MaybeChtRootStorageProvider, +{ + let storage = backend.cht_root_storage().ok_or(sc_rpc::system::error::Error::BackendNoChtRoots)?; + + let finalized_hash = client.info().finalized_hash; + let finalized_number = client.info().finalized_number; + + let header = client.header(BlockId::Hash(finalized_hash))?.unwrap(); + + use sc_client_api::cht; + + let mut chts = Vec::new(); + + if finalized_number > cht::size::>() * NumberFor::::from(2) { + let max = cht::block_to_cht_number(cht::size(), finalized_number).unwrap(); + + let mut i = NumberFor::::zero(); + + log::warn!("{:?} {:?}", i, max); + + while i <= max { + let number = (i * cht::size()) + NumberFor::::one(); + + match storage.header_cht_root(cht::size(), number)? { + Some(cht_root) => chts.push(cht_root), + None => log::warn!("No CHT found for block {}", number), + } + + i += NumberFor::::one(); + } + } + + Ok(sc_chain_spec::HardcodedSync:: { + header, + chts, + }) +} + #[cfg(not(target_os = "unknown"))] // Wrapper for HTTP and WS servers that makes sure they are properly shut down. mod waiting { From 15dcb2817e8bf26a58e84cd785a970922439d55a Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Fri, 14 Aug 2020 15:35:14 +0200 Subject: [PATCH 02/19] Improved build_hardcoded_spec --- bin/node-template/node/src/service.rs | 2 ++ client/rpc-api/src/system/error.rs | 1 + client/service/src/lib.rs | 29 ++++++++++----------------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 4eba4fdd093e9..b56cc865fa9de 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -87,6 +87,7 @@ pub fn new_full(config: Configuration) -> Result { sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), + backend: backend.clone(), transaction_pool: transaction_pool.clone(), spawn_handle: task_manager.spawn_handle(), import_queue, @@ -258,6 +259,7 @@ pub fn new_light(config: Configuration) -> Result { sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), + backend: backend.clone(), transaction_pool: transaction_pool.clone(), spawn_handle: task_manager.spawn_handle(), import_queue, diff --git a/client/rpc-api/src/system/error.rs b/client/rpc-api/src/system/error.rs index 1d04e8fda2941..cf4eab87e38ad 100644 --- a/client/rpc-api/src/system/error.rs +++ b/client/rpc-api/src/system/error.rs @@ -37,6 +37,7 @@ pub enum Error { /// Backend doesn't store CHT roots. #[display(fmt = "Backend doesn't store CHT roots. Make sure you're calling this on a light client.")] BackendNoChtRoots, + /// Generic blockchain backend error. BlockchainError(sp_blockchain::Error), } diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 95c62aacc9546..5fbe75e39e2aa 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -46,7 +46,7 @@ use sc_network::{NetworkStatus, network_state::NetworkState, PeerId}; use log::{warn, debug, error}; use codec::{Encode, Decode}; use sp_runtime::generic::BlockId; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero, One}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor, Saturating, One}; use parity_util_mem::MallocSizeOf; use sp_utils::{status_sinks, mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}}; use sp_blockchain::HeaderBackend; @@ -387,33 +387,26 @@ fn build_hardcoded_sync( let finalized_hash = client.info().finalized_hash; let finalized_number = client.info().finalized_number; - let header = client.header(BlockId::Hash(finalized_hash))?.unwrap(); - use sc_client_api::cht; let mut chts = Vec::new(); - if finalized_number > cht::size::>() * NumberFor::::from(2) { - let max = cht::block_to_cht_number(cht::size(), finalized_number).unwrap(); - - let mut i = NumberFor::::zero(); - - log::warn!("{:?} {:?}", i, max); + /// We can't fetch a CHT root later than `finalized_number - 2 * cht_size`. + let cht_size_x_2 = cht::size::>() * NumberFor::::from(2); - while i <= max { - let number = (i * cht::size()) + NumberFor::::one(); + let mut number = NumberFor::::one(); - match storage.header_cht_root(cht::size(), number)? { - Some(cht_root) => chts.push(cht_root), - None => log::warn!("No CHT found for block {}", number), - } - - i += NumberFor::::one(); + while number <= finalized_number.saturating_sub(cht_size_x_2) { + match storage.header_cht_root(cht::size(), number)? { + Some(cht_root) => chts.push(cht_root), + None => log::error!("No CHT found for block {}", number), } + + number += cht::size(); } Ok(sc_chain_spec::HardcodedSync:: { - header, + header: client.header(BlockId::Hash(finalized_hash))?.unwrap(), chts, }) } From ecb2f56e4ca11890647509330f2dbe2846d68fc5 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Fri, 14 Aug 2020 15:46:02 +0200 Subject: [PATCH 03/19] Fix line widths --- client/api/src/light.rs | 4 +++- client/chain-spec/src/lib.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/client/api/src/light.rs b/client/api/src/light.rs index ed816048fb852..f9aa002841caa 100644 --- a/client/api/src/light.rs +++ b/client/api/src/light.rs @@ -232,7 +232,9 @@ pub trait FetchChecker: Send + Sync { /// Light client blockchain storage. -pub trait Storage: AuxStore + HeaderBackend + HeaderMetadata + ChtRootStorage { +pub trait Storage: AuxStore + HeaderBackend + + HeaderMetadata + ChtRootStorage +{ /// Store new header. Should refuse to revert any finalized blocks. /// /// Takes new authorities, the leaf state of the new block, and diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index b5f61c538f0f0..93eeca8369895 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -108,7 +108,9 @@ mod chain_spec; mod extension; -pub use chain_spec::{ChainSpec as GenericChainSpec, NoExtension, HardcodedSync, SerializableHardcodedSync}; +pub use chain_spec::{ + ChainSpec as GenericChainSpec, NoExtension, HardcodedSync, SerializableHardcodedSync, +}; pub use extension::{Group, Fork, Forks, Extension, GetExtension, get_extension}; pub use sc_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup}; pub use sp_chain_spec::{Properties, ChainType}; From 837c5f7ed50a18bd068c2d8e0bea073ce2b15c4d Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Fri, 14 Aug 2020 16:01:19 +0200 Subject: [PATCH 04/19] Fix tests --- client/rpc/src/system/tests.rs | 9 ++++++--- client/service/src/lib.rs | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs index dfe1fcc415159..cb5ea9fba3857 100644 --- a/client/rpc/src/system/tests.rs +++ b/client/rpc/src/system/tests.rs @@ -79,7 +79,7 @@ fn api>>(sync: T) -> System { }); } let _ = sender.send(peers); - } + }, Request::NetworkState(sender) => { let _ = sender.send(serde_json::to_value(&sc_network::network_state::NetworkState { peer_id: String::new(), @@ -103,10 +103,13 @@ fn api>>(sync: T) -> System { Ok(_) => sender.send(Ok(())), Err(s) => sender.send(Err(error::Error::MalformattedPeerArg(s.to_string()))), }; - } + }, Request::NodeRoles(sender) => { let _ = sender.send(vec![NodeRole::Authority]); - } + }, + Request::GenChainSpec(_, _, sender) => { + let _ = sender.send(Ok(serde_json::Value::Null)); + }, }; future::ready(()) diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 5fbe75e39e2aa..93d0d275ff2db 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -391,7 +391,7 @@ fn build_hardcoded_sync( let mut chts = Vec::new(); - /// We can't fetch a CHT root later than `finalized_number - 2 * cht_size`. + // We can't fetch a CHT root later than `finalized_number - 2 * cht_size`. let cht_size_x_2 = cht::size::>() * NumberFor::::from(2); let mut number = NumberFor::::one(); From 42d88d5ef9a4ec6a357d4923ee746de69464ed4e Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Fri, 14 Aug 2020 16:09:54 +0200 Subject: [PATCH 05/19] Fix sc-service-test --- client/service/test/src/client/light.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/client/service/test/src/client/light.rs b/client/service/test/src/client/light.rs index ffc84ad47b8f3..515d7d1532677 100644 --- a/client/service/test/src/client/light.rs +++ b/client/service/test/src/client/light.rs @@ -42,7 +42,7 @@ use sc_executor::{NativeExecutor, WasmExecutionMethod, RuntimeVersion, NativeVer use sp_core::{H256, NativeOrEncoded, testing::TaskExecutor}; use sc_client_api::{ blockchain::Info, backend::NewBlockState, Backend as ClientBackend, ProofProvider, - in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, + in_mem::{Backend as InMemBackend, Blockchain as InMemoryBlockchain}, ChtRootStorage, AuxStore, Storage, CallExecutor, cht, ExecutionStrategy, StorageProof, BlockImportOperation, RemoteCallRequest, StorageProvider, ChangesProof, RemoteBodyRequest, RemoteReadRequest, RemoteChangesRequest, FetchChecker, RemoteReadChildRequest, RemoteHeaderRequest, BlockBackend, @@ -164,6 +164,16 @@ impl Storage for DummyStorage { Err(ClientError::Backend("Test error".into())) } + fn cache(&self) -> Option>> { + None + } + + fn usage_info(&self) -> Option { + None + } +} + +impl ChtRootStorage for DummyStorage { fn header_cht_root(&self, _cht_size: u64, _block: u64) -> ClientResult> { Err(ClientError::Backend("Test error".into())) } @@ -177,14 +187,6 @@ impl Storage for DummyStorage { ).into()) .map(Some) } - - fn cache(&self) -> Option>> { - None - } - - fn usage_info(&self) -> Option { - None - } } struct DummyCallExecutor; From 4288dc47e02604fb1ecc09037dd56589fc2420ca Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Fri, 14 Aug 2020 21:07:06 +0200 Subject: [PATCH 06/19] Suggestions from code review --- client/chain-spec/src/chain_spec.rs | 2 +- client/db/src/light.rs | 22 ---------------------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index d9e62cdd2e198..3c48c2126aaaa 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -438,10 +438,10 @@ impl HardcodedSync { } } +/// The serializable form of `HardcodedSync`. Created using `HardcodedSync::serialize`. #[derive(Serialize, Deserialize, Clone, Debug)] #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] -/// The serializable form of `HardcodedSync`. Created using `HardcodedSync::serialize`. pub struct SerializableHardcodedSync { header: StorageData, chts: Vec, diff --git a/client/db/src/light.rs b/client/db/src/light.rs index effd70cd45147..adf9a98d35e2a 100644 --- a/client/db/src/light.rs +++ b/client/db/src/light.rs @@ -616,28 +616,6 @@ impl ChtRootStorage for LightStorage } } - -impl<'a, Block> ChtRootStorage for &'a LightStorage - where Block: BlockT, -{ - fn header_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult> { - self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block) - } - - fn changes_trie_cht_root( - &self, - cht_size: NumberFor, - block: NumberFor, - ) -> ClientResult> { - self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block) - } -} - - /// Build the key for inserting header-CHT at given block. fn cht_key>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> { let mut key = [cht_type; 5]; From 44ff9f944042f920689a344759fff6764e665daa Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Fri, 14 Aug 2020 22:03:12 +0200 Subject: [PATCH 07/19] Rename to LightSyncState --- client/chain-spec/src/chain_spec.rs | 30 ++++++++++++++--------------- client/chain-spec/src/lib.rs | 4 ++-- client/service/src/lib.rs | 16 +++++++-------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index 3c48c2126aaaa..b7e3e9572953c 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -159,7 +159,7 @@ struct ClientSpec { consensus_engine: (), #[serde(skip_serializing)] genesis: serde::de::IgnoredAny, - hardcoded_sync: Option, + light_sync_state: Option, } /// A type denoting empty extensions. @@ -248,7 +248,7 @@ impl ChainSpec { extensions, consensus_engine: (), genesis: Default::default(), - hardcoded_sync: None, + light_sync_state: None, }; ChainSpec { @@ -263,8 +263,8 @@ impl ChainSpec { } /// Hardcode infomation to allow light clients to sync quickly into the chain spec. - fn set_hardcoded_sync(&mut self, hardcoded_sync: SerializableHardcodedSync) { - self.client_spec.hardcoded_sync = Some(hardcoded_sync); + fn set_light_sync_state(&mut self, light_sync_state: SerializableLightSyncState) { + self.client_spec.light_sync_state = Some(light_sync_state); } } @@ -403,32 +403,32 @@ where self.genesis = GenesisSource::Storage(storage); } - fn set_hardcoded_sync(&mut self, hardcoded_sync: SerializableHardcodedSync) { - ChainSpec::set_hardcoded_sync(self, hardcoded_sync) + fn set_light_sync_state(&mut self, light_sync_state: SerializableLightSyncState) { + ChainSpec::set_light_sync_state(self, light_sync_state) } } /// Hardcoded infomation that allows light clients to sync quickly. -pub struct HardcodedSync { +pub struct LightSyncState { /// The header of the best finalized block. pub header: ::Header, /// A list of all CHTs in the chain. pub chts: Vec<::Hash>, } -impl HardcodedSync { - /// Convert into a `SerializableHardcodecSync`. - pub fn to_serializable(&self) -> SerializableHardcodedSync { +impl LightSyncState { + /// Convert into a `SerializableLightSyncState`. + pub fn to_serializable(&self) -> SerializableLightSyncState { use codec::Encode; - SerializableHardcodedSync { + SerializableLightSyncState { header: StorageData(self.header.encode()), chts: self.chts.iter().map(|hash| StorageData(hash.encode())).collect(), } } - /// Convert from a `SerializableHardcodecSync`. - pub fn from_serializable(serialized: &SerializableHardcodedSync) -> Result { + /// Convert from a `SerializableLightSyncState`. + pub fn from_serializable(serialized: &SerializableLightSyncState) -> Result { Ok(Self { header: codec::Decode::decode(&mut &serialized.header.0[..])?, chts: serialized.chts.iter() @@ -438,11 +438,11 @@ impl HardcodedSync { } } -/// The serializable form of `HardcodedSync`. Created using `HardcodedSync::serialize`. +/// The serializable form of `LightSyncState`. Created using `LightSyncState::serialize`. #[derive(Serialize, Deserialize, Clone, Debug)] #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] -pub struct SerializableHardcodedSync { +pub struct SerializableLightSyncState { header: StorageData, chts: Vec, } diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index 93eeca8369895..a1802d5b523e3 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -109,7 +109,7 @@ mod chain_spec; mod extension; pub use chain_spec::{ - ChainSpec as GenericChainSpec, NoExtension, HardcodedSync, SerializableHardcodedSync, + ChainSpec as GenericChainSpec, NoExtension, LightSyncState, SerializableLightSyncState, }; pub use extension::{Group, Fork, Forks, Extension, GetExtension, get_extension}; pub use sc_chain_spec_derive::{ChainSpecExtension, ChainSpecGroup}; @@ -160,7 +160,7 @@ pub trait ChainSpec: BuildStorage + Send { /// This will be used as storage at genesis. fn set_storage(&mut self, storage: Storage); /// Hardcode infomation to allow light clients to sync quickly into the chain spec. - fn set_hardcoded_sync(&mut self, hardcoded_sync: SerializableHardcodedSync); + fn set_light_sync_state(&mut self, light_sync_state: SerializableLightSyncState); } impl std::fmt::Debug for dyn ChainSpec { diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 93d0d275ff2db..727134c819237 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -330,16 +330,16 @@ async fn build_network_future< sc_rpc::system::Request::GenChainSpec(raw, hardcode_sync, sender) => { let mut chain_spec = chain_spec.cloned_box(); - let hardcoded_sync_result = if hardcode_sync { - build_hardcoded_sync(client.clone(), backend.clone()) - .map(|hardcoded_sync| { - chain_spec.set_hardcoded_sync(hardcoded_sync.to_serializable()) + let light_sync_state_result = if hardcode_sync { + build_light_sync_state(client.clone(), backend.clone()) + .map(|light_sync_state| { + chain_spec.set_light_sync_state(light_sync_state.to_serializable()) }) } else { Ok(()) }; - let value = match hardcoded_sync_result { + let value = match light_sync_state_result { Ok(()) => chain_spec.as_json_value(raw).map_err(|err| err.into()), Err(error) => Err(error) }; @@ -373,10 +373,10 @@ async fn build_network_future< } } -fn build_hardcoded_sync( +fn build_light_sync_state( client: Arc, backend: Arc, -) -> Result, sc_rpc::system::error::Error> +) -> Result, sc_rpc::system::error::Error> where TBl: BlockT, TCl: HeaderBackend, @@ -405,7 +405,7 @@ fn build_hardcoded_sync( number += cht::size(); } - Ok(sc_chain_spec::HardcodedSync:: { + Ok(sc_chain_spec::LightSyncState { header: client.header(BlockId::Hash(finalized_hash))?.unwrap(), chts, }) From a6b5491cc34db6d2ed363b71a2ae1161cd3f29a0 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Mon, 17 Aug 2020 15:35:20 +0200 Subject: [PATCH 08/19] It's not syncing :^( --- Cargo.lock | 1 + bin/node/cli/src/browser.rs | 2 +- bin/node/cli/src/command.rs | 18 +++- bin/node/cli/src/service.rs | 12 ++- client/cli/Cargo.toml | 1 + .../cli/src/commands/export_sync_state_cmd.rs | 96 +++++++++++++++++++ client/cli/src/commands/mod.rs | 7 +- client/cli/src/runner.rs | 33 ++++--- client/service/src/lib.rs | 2 +- 9 files changed, 148 insertions(+), 24 deletions(-) create mode 100644 client/cli/src/commands/export_sync_state_cmd.rs diff --git a/Cargo.lock b/Cargo.lock index 109f6b47f332d..6bd4f340cc051 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6189,6 +6189,7 @@ dependencies = [ "serde", "serde_json", "sp-blockchain", + "sp-consensus", "sp-core", "sp-keyring", "sp-panic-handler", diff --git a/bin/node/cli/src/browser.rs b/bin/node/cli/src/browser.rs index 3aec486c10370..4cd517a43236c 100644 --- a/bin/node/cli/src/browser.rs +++ b/bin/node/cli/src/browser.rs @@ -54,7 +54,7 @@ async fn start_inner(chain_spec: Option, log_level: String) -> Result Result<()> { } Some(Subcommand::Base(subcommand)) => { let runner = cli.create_runner(subcommand)?; - runner.run_subcommand(subcommand, |config| { - let PartialComponents { client, backend, task_manager, import_queue, ..} - = new_partial(&config)?; - Ok((client, backend, import_queue, task_manager)) - }) + runner.run_subcommand( + subcommand, + |config| { + let PartialComponents { client, backend, task_manager, import_queue, ..} + = new_partial(&config)?; + Ok((client, backend, import_queue, task_manager)) + }, + |config| { + let (task_manager, client, backend, _, network, ..) = + crate::service::new_light_base(config)?; + Ok((client, backend, network, task_manager)) + } + ) } } } diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index c7c4dfff3636f..47d7ebe7e4d41 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -43,6 +43,7 @@ type FullSelectChain = sc_consensus::LongestChain; type FullGrandpaBlockImport = grandpa::GrandpaBlockImport; type LightClient = sc_service::TLightClient; +type LightBackend = sc_service::TLightBackend; pub fn new_partial(config: &Configuration) -> Result Result<( - TaskManager, Arc, Arc, - Arc::Hash>>, + TaskManager, Arc, Arc, + Arc, Arc::Hash>>, Arc>> ), ServiceError> { let (client, backend, keystore, mut task_manager, on_demand) = @@ -427,19 +428,20 @@ pub fn new_light_base(config: Configuration) -> Result<( remote_blockchain: Some(backend.remote_blockchain()), rpc_extensions_builder: Box::new(sc_service::NoopRpcExtensionBuilder(rpc_extensions)), client: client.clone(), + backend: backend.clone(), transaction_pool: transaction_pool.clone(), - config, keystore, backend, network_status_sinks, system_rpc_tx, + config, keystore, network_status_sinks, system_rpc_tx, network: network.clone(), telemetry_connection_sinks: sc_service::TelemetryConnectionSinks::default(), task_manager: &mut task_manager, })?; - Ok((task_manager, rpc_handlers, client, network, transaction_pool)) + Ok((task_manager, client, backend, rpc_handlers, network, transaction_pool)) } /// Builds a new service for a light client. pub fn new_light(config: Configuration) -> Result { - new_light_base(config).map(|(task_manager, _, _, _, _)| { + new_light_base(config).map(|(task_manager, ..)| { task_manager }) } diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index 85a1eb0fe0a4c..e29e5269ebb64 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -36,6 +36,7 @@ sp-core = { version = "2.0.0-rc5", path = "../../primitives/core" } sc-service = { version = "0.8.0-rc5", default-features = false, path = "../service" } sp-state-machine = { version = "0.8.0-rc5", path = "../../primitives/state-machine" } sc-telemetry = { version = "2.0.0-rc5", path = "../telemetry" } +sp-consensus = { version = "0.8.0-rc5", path = "../../primitives/consensus/common" } substrate-prometheus-endpoint = { path = "../../utils/prometheus" , version = "0.8.0-rc5"} sp-keyring = { version = "2.0.0-rc5", path = "../../primitives/keyring" } names = "0.11.0" diff --git a/client/cli/src/commands/export_sync_state_cmd.rs b/client/cli/src/commands/export_sync_state_cmd.rs new file mode 100644 index 0000000000000..0f50ec10b609d --- /dev/null +++ b/client/cli/src/commands/export_sync_state_cmd.rs @@ -0,0 +1,96 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use crate::error; +use crate::params::NodeKeyParams; +use crate::params::SharedParams; +use crate::CliConfiguration; +use crate::Role; +use log::info; +use sc_network::config::build_multiaddr; +use sc_service::ChainSpec; +use structopt::StructOpt; +use std::io::Write; +use std::sync::Arc; +use std::task::Poll; +use sp_runtime::traits::Block as BlockT; + +#[derive(Debug, StructOpt)] +pub struct ExportSyncStateCmd { + /// Force raw genesis storage output. + #[structopt(long = "raw")] + pub raw: bool, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub node_key_params: NodeKeyParams, +} + +impl ExportSyncStateCmd { + pub async fn run( + &self, + mut spec: Box, + client: Arc, + backend: Arc, + mut sync_oracle: SO, + ) -> error::Result<()> + where + B: BlockT, + CL: sp_blockchain::HeaderBackend, + BA: sc_service::MaybeChtRootStorageProvider, + SO: sp_consensus::SyncOracle, + { + futures::future::poll_fn(|_| { + match sync_oracle.is_major_syncing() || sync_oracle.is_offline() { + true => Poll::Pending, + false => Poll::Ready(()) + } + }).await; + + info!("Building chain spec"); + + let light_sync_state = sc_service::build_light_sync_state(client, backend) + .map_err(|err| err.to_string())?; + + spec.set_light_sync_state(light_sync_state.to_serializable()); + + let json = sc_service::chain_ops::build_spec(&*spec, self.raw)?; + if std::io::stdout().write_all(json.as_bytes()).is_err() { + let _ = std::io::stderr().write_all(b"Error writing to stdout\n"); + } + Ok(()) + } +} + +impl CliConfiguration for ExportSyncStateCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } + + fn node_key_params(&self) -> Option<&NodeKeyParams> { + Some(&self.node_key_params) + } + + fn role(&self, _is_dev: bool) -> error::Result { + Ok(Role::Light) + } +} diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs index 04cce66bef80d..227e17a7348d1 100644 --- a/client/cli/src/commands/mod.rs +++ b/client/cli/src/commands/mod.rs @@ -23,6 +23,7 @@ mod import_blocks_cmd; mod purge_chain_cmd; mod revert_cmd; mod run_cmd; +mod export_sync_state_cmd; pub use self::build_spec_cmd::BuildSpecCmd; pub use self::check_block_cmd::CheckBlockCmd; @@ -32,6 +33,7 @@ pub use self::import_blocks_cmd::ImportBlocksCmd; pub use self::purge_chain_cmd::PurgeChainCmd; pub use self::revert_cmd::RevertCmd; pub use self::run_cmd::RunCmd; +pub use self::export_sync_state_cmd::ExportSyncStateCmd; use std::fmt::Debug; use structopt::StructOpt; @@ -62,6 +64,9 @@ pub enum Subcommand { /// Export state as raw chain spec. ExportState(ExportStateCmd), + + /// Export the chain spec containing a state to sync the light client. + ExportSyncState(ExportSyncStateCmd), } // TODO: move to config.rs? @@ -413,5 +418,5 @@ macro_rules! substrate_cli_subcommands { } substrate_cli_subcommands!( - Subcommand => BuildSpec, ExportBlocks, ImportBlocks, CheckBlock, Revert, PurgeChain, ExportState + Subcommand => BuildSpec, ExportBlocks, ImportBlocks, CheckBlock, Revert, PurgeChain, ExportState, ExportSyncState ); diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index bdbf55eb8326a..8e49ac42abbde 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -30,6 +30,7 @@ use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use sp_utils::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL}; use std::{fmt::Debug, marker::PhantomData, str::FromStr, sync::Arc}; use sc_client_api::{UsageProvider, BlockBackend, StorageProvider}; +use sp_blockchain::HeaderBackend; #[cfg(target_family = "unix")] async fn main(func: F) -> std::result::Result<(), Box> @@ -177,19 +178,25 @@ impl Runner { /// A helper function that runs a future with tokio and stops if the process receives the signal /// `SIGTERM` or `SIGINT`. - pub fn run_subcommand(self, subcommand: &Subcommand, builder: BU) - -> Result<()> + pub fn run_subcommand( + self, subcommand: &Subcommand, full_builder: FB, light_builder: LB, + ) -> Result<()> where - BU: FnOnce(Configuration) - -> sc_service::error::Result<(Arc, Arc, IQ, TaskManager)>, + FB: FnOnce(Configuration) + -> sc_service::error::Result<(Arc, Arc, IQ, TaskManager)>, + LB: FnOnce(Configuration) + -> sc_service::error::Result<(Arc, Arc, SO, TaskManager)>, B: BlockT + for<'de> serde::Deserialize<'de>, - BA: sc_client_api::backend::Backend + 'static, + FBA: sc_client_api::backend::Backend + 'static, IQ: sc_service::ImportQueue + 'static, ::Hash: FromStr, <::Hash as FromStr>::Err: Debug, <<::Header as HeaderT>::Number as FromStr>::Err: Debug, - CL: UsageProvider + BlockBackend + StorageProvider + Send + Sync + + FCL: UsageProvider + BlockBackend + StorageProvider + Send + Sync + 'static, + LCL: HeaderBackend, + LBA: sc_service::MaybeChtRootStorageProvider, + SO: sp_consensus::SyncOracle, { let chain_spec = self.config.chain_spec.cloned_box(); let network_config = self.config.network.clone(); @@ -198,26 +205,30 @@ impl Runner { match subcommand { Subcommand::BuildSpec(cmd) => cmd.run(chain_spec, network_config), Subcommand::ExportBlocks(cmd) => { - let (client, _, _, task_manager) = builder(self.config)?; + let (client, _, _, task_manager) = full_builder(self.config)?; run_until_exit(self.tokio_runtime, cmd.run(client, db_config), task_manager) } Subcommand::ImportBlocks(cmd) => { - let (client, _, import_queue, task_manager) = builder(self.config)?; + let (client, _, import_queue, task_manager) = full_builder(self.config)?; run_until_exit(self.tokio_runtime, cmd.run(client, import_queue), task_manager) } Subcommand::CheckBlock(cmd) => { - let (client, _, import_queue, task_manager) = builder(self.config)?; + let (client, _, import_queue, task_manager) = full_builder(self.config)?; run_until_exit(self.tokio_runtime, cmd.run(client, import_queue), task_manager) } Subcommand::Revert(cmd) => { - let (client, backend, _, task_manager) = builder(self.config)?; + let (client, backend, _, task_manager) = full_builder(self.config)?; run_until_exit(self.tokio_runtime, cmd.run(client, backend), task_manager) }, Subcommand::PurgeChain(cmd) => cmd.run(db_config), Subcommand::ExportState(cmd) => { - let (client, _, _, task_manager) = builder(self.config)?; + let (client, _, _, task_manager) = full_builder(self.config)?; run_until_exit(self.tokio_runtime, cmd.run(client, chain_spec), task_manager) }, + Subcommand::ExportSyncState(cmd) => { + let (client, backend, sync_oracle, task_manager) = light_builder(self.config)?; + run_until_exit(self.tokio_runtime, cmd.run(chain_spec, client, backend, sync_oracle), task_manager) + } } } diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 727134c819237..7f16f00b50d5c 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -373,7 +373,7 @@ async fn build_network_future< } } -fn build_light_sync_state( +pub fn build_light_sync_state( client: Arc, backend: Arc, ) -> Result, sc_rpc::system::error::Error> From 346b7113fb6f12f9c2995bc8dff871a8fa432b58 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Mon, 17 Aug 2020 16:02:38 +0200 Subject: [PATCH 09/19] It syncs! --- Cargo.lock | 1 + bin/node-template/node/Cargo.toml | 1 + bin/node-template/node/src/command.rs | 20 +++++++++---- bin/node-template/node/src/service.rs | 30 +++++++++++++------ bin/node/cli/src/command.rs | 6 ++-- .../cli/src/commands/export_sync_state_cmd.rs | 14 ++++----- client/service/src/lib.rs | 1 + 7 files changed, 48 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f62bcd638a454..5b5ca1a0998eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3822,6 +3822,7 @@ dependencies = [ "sc-consensus-aura", "sc-executor", "sc-finality-grandpa", + "sc-network", "sc-rpc", "sc-rpc-api", "sc-service", diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index 0c988ebd1a22b..f4f524fdb7e35 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -32,6 +32,7 @@ sc-consensus = { version = "0.8.0-rc5", path = "../../../client/consensus/common sc-finality-grandpa = { version = "0.8.0-rc5", path = "../../../client/finality-grandpa" } sp-finality-grandpa = { version = "2.0.0-rc5", path = "../../../primitives/finality-grandpa" } sc-client-api = { version = "2.0.0-rc5", path = "../../../client/api" } +sc-network = { version = "0.8.0-rc5", path = "../../../client/network" } sp-runtime = { version = "2.0.0-rc5", path = "../../../primitives/runtime" } # These dependencies are used for the node template's RPCs diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index b3f1cfaf11f55..055872f72aebd 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -20,7 +20,7 @@ use crate::cli::Cli; use crate::service; use sc_cli::{SubstrateCli, RuntimeVersion, Role, ChainSpec}; use sc_service::PartialComponents; -use crate::service::new_partial; +use crate::service::{new_partial, new_light_base}; impl SubstrateCli for Cli { fn impl_name() -> String { @@ -69,11 +69,19 @@ pub fn run() -> sc_cli::Result<()> { match &cli.subcommand { Some(subcommand) => { let runner = cli.create_runner(subcommand)?; - runner.run_subcommand(subcommand, |config| { - let PartialComponents { client, backend, task_manager, import_queue, .. } - = new_partial(&config)?; - Ok((client, backend, import_queue, task_manager)) - }) + runner.run_subcommand( + subcommand, + |config| { + let PartialComponents { client, backend, task_manager, import_queue, .. } + = new_partial(&config)?; + Ok((client, backend, import_queue, task_manager)) + }, + |config| { + let (task_manager, client, backend, network) + = new_light_base(config)?; + Ok((client, backend, network, task_manager)) + }, + ) } None => { let runner = cli.create_runner(&cli.run)?; diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index d754b7bfce615..c7bb394759e87 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -10,6 +10,8 @@ use sc_executor::native_executor_instance; pub use sc_executor::NativeExecutor; use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair}; use sc_finality_grandpa::{FinalityProofProvider as GrandpaFinalityProofProvider, SharedVoterState}; +use sp_runtime::traits::Block as BlockT; +use sc_network::NetworkService; // Our native executor instance. native_executor_instance!( @@ -21,6 +23,8 @@ native_executor_instance!( type FullClient = sc_service::TFullClient; type FullBackend = sc_service::TFullBackend; type FullSelectChain = sc_consensus::LongestChain; +type LightClient = sc_service::TLightClient; +type LightBackend = sc_service::TLightBackend; pub fn new_partial(config: &Configuration) -> Result Result { Ok(task_manager) } -/// Builds a new service for a light client. -pub fn new_light(config: Configuration) -> Result { +pub(crate) fn new_light_base(config: Configuration) -> Result<( + TaskManager, + Arc, Arc,Arc::Hash>>, +), ServiceError> { let (client, backend, keystore, mut task_manager, on_demand) = - sc_service::new_light_parts::(&config)?; + sc_service::new_light_parts::(&config)?; let transaction_pool = Arc::new(sc_transaction_pool::BasicPool::new_light( config.transaction_pool.clone(), @@ -285,15 +291,21 @@ pub fn new_light(config: Configuration) -> Result { rpc_extensions_builder: Box::new(|_, _| ()), telemetry_connection_sinks: sc_service::TelemetryConnectionSinks::default(), config, - client, + client: client.clone(), keystore, - backend, - network, + backend: backend.clone(), + network: network.clone(), network_status_sinks, system_rpc_tx, - })?; + })?; - network_starter.start_network(); + network_starter.start_network(); - Ok(task_manager) + Ok((task_manager, client, backend, network)) +} + +/// Builds a new service for a light client. +pub fn new_light(config: Configuration) -> Result { + let (task_manager, ..) = new_light_base(config)?; + Ok(task_manager) } diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 7f8eed66dc47f..b73db4a585cf6 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -21,7 +21,7 @@ use node_executor::Executor; use node_runtime::{Block, RuntimeApi}; use sc_cli::{Result, SubstrateCli, RuntimeVersion, Role, ChainSpec}; use sc_service::PartialComponents; -use crate::service::new_partial; +use crate::service::{new_partial, new_light_base}; impl SubstrateCli for Cli { fn impl_name() -> String { @@ -104,9 +104,9 @@ pub fn run() -> Result<()> { }, |config| { let (task_manager, client, backend, _, network, ..) = - crate::service::new_light_base(config)?; + new_light_base(config)?; Ok((client, backend, network, task_manager)) - } + }, ) } } diff --git a/client/cli/src/commands/export_sync_state_cmd.rs b/client/cli/src/commands/export_sync_state_cmd.rs index 0f50ec10b609d..9c316c471a560 100644 --- a/client/cli/src/commands/export_sync_state_cmd.rs +++ b/client/cli/src/commands/export_sync_state_cmd.rs @@ -17,12 +17,10 @@ // along with this program. If not, see . use crate::error; -use crate::params::NodeKeyParams; -use crate::params::SharedParams; +use crate::params::{SharedParams, NetworkParams}; use crate::CliConfiguration; use crate::Role; use log::info; -use sc_network::config::build_multiaddr; use sc_service::ChainSpec; use structopt::StructOpt; use std::io::Write; @@ -30,9 +28,10 @@ use std::sync::Arc; use std::task::Poll; use sp_runtime::traits::Block as BlockT; +/// Export the chain spec containing a state to sync the light client. #[derive(Debug, StructOpt)] pub struct ExportSyncStateCmd { - /// Force raw genesis storage output. + /// Force raw genesis storage output in the chain spec. #[structopt(long = "raw")] pub raw: bool, @@ -42,10 +41,11 @@ pub struct ExportSyncStateCmd { #[allow(missing_docs)] #[structopt(flatten)] - pub node_key_params: NodeKeyParams, + pub network_params: NetworkParams, } impl ExportSyncStateCmd { + /// Run the `export-sync-state` command pub async fn run( &self, mut spec: Box, @@ -86,8 +86,8 @@ impl CliConfiguration for ExportSyncStateCmd { &self.shared_params } - fn node_key_params(&self) -> Option<&NodeKeyParams> { - Some(&self.node_key_params) + fn network_params(&self) -> Option<&NetworkParams> { + Some(&self.network_params) } fn role(&self, _is_dev: bool) -> error::Result { diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index acfbf0e78e874..5276bf7826054 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -379,6 +379,7 @@ async fn build_network_future< } } +/// Build a `LightSyncState` from the CHT roots stored in a backend. pub fn build_light_sync_state( client: Arc, backend: Arc, From 6447b85d849702f8c2498745c048dcd77c35ddc9 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Mon, 17 Aug 2020 16:09:18 +0200 Subject: [PATCH 10/19] Remove rpc call --- Cargo.lock | 1 - bin/node-template/node/src/service.rs | 2 -- bin/node/cli/src/service.rs | 2 -- client/rpc-api/Cargo.toml | 1 - client/rpc-api/src/system/error.rs | 24 +--------------------- client/rpc-api/src/system/mod.rs | 5 ----- client/rpc/src/system/mod.rs | 18 +---------------- client/rpc/src/system/tests.rs | 9 +++------ client/service/src/builder.rs | 13 ++++-------- client/service/src/lib.rs | 29 ++++----------------------- 10 files changed, 13 insertions(+), 91 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b5ca1a0998eb..c4e2ec64698c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6928,7 +6928,6 @@ dependencies = [ "parking_lot 0.10.2", "serde", "serde_json", - "sp-blockchain", "sp-chain-spec", "sp-core", "sp-rpc", diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index c7bb394759e87..43cdaf2be44b3 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -92,7 +92,6 @@ pub fn new_full(config: Configuration) -> Result { sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), - backend: backend.clone(), transaction_pool: transaction_pool.clone(), spawn_handle: task_manager.spawn_handle(), import_queue, @@ -267,7 +266,6 @@ pub(crate) fn new_light_base(config: Configuration) -> Result<( sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), - backend: backend.clone(), transaction_pool: transaction_pool.clone(), spawn_handle: task_manager.spawn_handle(), import_queue, diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 62325a825aae8..74e00343dd3ec 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -177,7 +177,6 @@ pub fn new_full_base( sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), - backend: backend.clone(), transaction_pool: transaction_pool.clone(), spawn_handle: task_manager.spawn_handle(), import_queue, @@ -398,7 +397,6 @@ pub fn new_light_base(config: Configuration) -> Result<( sc_service::build_network(sc_service::BuildNetworkParams { config: &config, client: client.clone(), - backend: backend.clone(), transaction_pool: transaction_pool.clone(), spawn_handle: task_manager.spawn_handle(), import_queue, diff --git a/client/rpc-api/Cargo.toml b/client/rpc-api/Cargo.toml index b88c2e3eb401e..7701befbf71a4 100644 --- a/client/rpc-api/Cargo.toml +++ b/client/rpc-api/Cargo.toml @@ -29,4 +29,3 @@ serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" sp-transaction-pool = { version = "2.0.0-rc5", path = "../../primitives/transaction-pool" } sp-rpc = { version = "2.0.0-rc5", path = "../../primitives/rpc" } -sp-blockchain = { version = "2.0.0-rc5", path = "../../primitives/blockchain" } diff --git a/client/rpc-api/src/system/error.rs b/client/rpc-api/src/system/error.rs index cf4eab87e38ad..4897aa485cbe4 100644 --- a/client/rpc-api/src/system/error.rs +++ b/client/rpc-api/src/system/error.rs @@ -32,13 +32,6 @@ pub enum Error { NotHealthy(Health), /// Peer argument is malformatted. MalformattedPeerArg(String), - /// Failed to serialize chain spec. - ChainSpec(serde_json::error::Error), - /// Backend doesn't store CHT roots. - #[display(fmt = "Backend doesn't store CHT roots. Make sure you're calling this on a light client.")] - BackendNoChtRoots, - /// Generic blockchain backend error. - BlockchainError(sp_blockchain::Error), } impl std::error::Error for Error {} @@ -58,22 +51,7 @@ impl From for rpc::Error { code :rpc::ErrorCode::ServerError(BASE_ERROR + 2), message: e.clone(), data: None, - }, - Error::ChainSpec(ref s) => rpc::Error { - code :rpc::ErrorCode::ServerError(BASE_ERROR + 3), - message: s.to_string(), - data: None, - }, - Error::BackendNoChtRoots => rpc::Error { - code :rpc::ErrorCode::ServerError(BASE_ERROR + 4), - message: format!("{}", e), - data: None, - }, - Error::BlockchainError(ref b) => rpc::Error { - code :rpc::ErrorCode::ServerError(BASE_ERROR + 5), - message: b.to_string(), - data: None, - }, + } } } } diff --git a/client/rpc-api/src/system/mod.rs b/client/rpc-api/src/system/mod.rs index 3c28aa89bf3de..a7b746ee1b114 100644 --- a/client/rpc-api/src/system/mod.rs +++ b/client/rpc-api/src/system/mod.rs @@ -103,9 +103,4 @@ pub trait SystemApi { /// Returns the roles the node is running as. #[rpc(name = "system_nodeRoles", returns = "Vec")] fn system_node_roles(&self) -> Receiver>; - - /// Returns the json-serialized chainspec running the node. - #[rpc(name = "system_genChainSpec", returns = "jsonrpc_core::Value")] - fn system_gen_chain_spec(&self, raw: bool, hardcode_sync: bool) - -> Compat>>; } diff --git a/client/rpc/src/system/mod.rs b/client/rpc/src/system/mod.rs index 966cd2b968bfe..4b8abbe144462 100644 --- a/client/rpc/src/system/mod.rs +++ b/client/rpc/src/system/mod.rs @@ -66,9 +66,7 @@ pub enum Request { /// Must return any potential parse error. NetworkRemoveReservedPeer(String, oneshot::Sender>), /// Must return the node role. - NodeRoles(oneshot::Sender>), - /// Must return either a chain spec encoded as a `Value` or an error. - GenChainSpec(bool, bool, oneshot::Sender>), + NodeRoles(oneshot::Sender>) } impl System { @@ -191,18 +189,4 @@ impl SystemApi::Number> for Sy let _ = self.send_back.unbounded_send(Request::NodeRoles(tx)); Receiver(Compat::new(rx)) } - - fn system_gen_chain_spec(&self, raw: bool, hardcode_sync: bool) - -> Compat>> - { - let (tx, rx) = oneshot::channel(); - let _ = self.send_back.unbounded_send(Request::GenChainSpec(raw, hardcode_sync, tx)); - async move { - match rx.await { - Ok(Ok(value)) => Ok(value), - Ok(Err(e)) => Err(rpc::Error::from(e)), - Err(_) => Err(rpc::Error::internal_error()), - } - }.boxed().compat() - } } diff --git a/client/rpc/src/system/tests.rs b/client/rpc/src/system/tests.rs index cb5ea9fba3857..dfe1fcc415159 100644 --- a/client/rpc/src/system/tests.rs +++ b/client/rpc/src/system/tests.rs @@ -79,7 +79,7 @@ fn api>>(sync: T) -> System { }); } let _ = sender.send(peers); - }, + } Request::NetworkState(sender) => { let _ = sender.send(serde_json::to_value(&sc_network::network_state::NetworkState { peer_id: String::new(), @@ -103,13 +103,10 @@ fn api>>(sync: T) -> System { Ok(_) => sender.send(Ok(())), Err(s) => sender.send(Err(error::Error::MalformattedPeerArg(s.to_string()))), }; - }, + } Request::NodeRoles(sender) => { let _ = sender.send(vec![NodeRole::Authority]); - }, - Request::GenChainSpec(_, _, sender) => { - let _ = sender.send(Ok(serde_json::Value::Null)); - }, + } }; future::ready(()) diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index dc45afa9b08da..8ad95511f77d3 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -791,13 +791,11 @@ fn gen_handler( } /// Parameters to pass into `build_network`. -pub struct BuildNetworkParams<'a, TBl: BlockT, TExPool, TImpQu, TCl, TBackend> { +pub struct BuildNetworkParams<'a, TBl: BlockT, TExPool, TImpQu, TCl> { /// The service configuration. pub config: &'a Configuration, /// A shared client returned by `new_full_parts`/`new_light_parts`. pub client: Arc, - /// A shared backend returned by `new_full_parts`/`new_light_parts`. - pub backend: Arc, /// A shared transaction pool. pub transaction_pool: Arc, /// A handle for spawning tasks. @@ -817,8 +815,8 @@ pub struct BuildNetworkParams<'a, TBl: BlockT, TExPool, TImpQu, TCl, TBackend> { } /// Build the network service, the network status sinks and an RPC sender. -pub fn build_network( - params: BuildNetworkParams +pub fn build_network( + params: BuildNetworkParams ) -> Result< ( Arc::Hash>>, @@ -835,10 +833,9 @@ pub fn build_network( HeaderBackend + BlockchainEvents + 'static, TExPool: MaintainedTransactionPool::Hash> + 'static, TImpQu: ImportQueue + 'static, - TBackend: crate::MaybeChtRootStorageProvider + Send + Sync + 'static, { let BuildNetworkParams { - config, client, backend, transaction_pool, spawn_handle, import_queue, on_demand, + config, client, transaction_pool, spawn_handle, import_queue, on_demand, block_announce_validator_builder, finality_proof_request_builder, finality_proof_provider, } = params; @@ -896,10 +893,8 @@ pub fn build_network( let future = build_network_future( config.role.clone(), - config.chain_spec.cloned_box(), network_mut, client, - backend.clone(), network_status_sinks.clone(), system_rpc_rx, has_bootnodes, diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 5276bf7826054..19ea22be2767d 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -205,15 +205,12 @@ pub struct PartialComponents + HeaderBackend, - BE: MaybeChtRootStorageProvider, + C: BlockchainEvents, H: sc_network::ExHashT > ( role: Role, - chain_spec: Box, mut network: sc_network::NetworkWorker, client: Arc, - backend: Arc, status_sinks: NetworkStatusSinks, mut rpc_rx: TracingUnboundedReceiver>, should_have_peers: bool, @@ -332,25 +329,6 @@ async fn build_network_future< }; let _ = sender.send(vec![node_role]); - }, - sc_rpc::system::Request::GenChainSpec(raw, hardcode_sync, sender) => { - let mut chain_spec = chain_spec.cloned_box(); - - let light_sync_state_result = if hardcode_sync { - build_light_sync_state(client.clone(), backend.clone()) - .map(|light_sync_state| { - chain_spec.set_light_sync_state(light_sync_state.to_serializable()) - }) - } else { - Ok(()) - }; - - let value = match light_sync_state_result { - Ok(()) => chain_spec.as_json_value(raw).map_err(|err| err.into()), - Err(error) => Err(error) - }; - - let _ = sender.send(value); } } } @@ -383,13 +361,14 @@ async fn build_network_future< pub fn build_light_sync_state( client: Arc, backend: Arc, -) -> Result, sc_rpc::system::error::Error> +) -> Result, sp_blockchain::Error> where TBl: BlockT, TCl: HeaderBackend, TBackend: MaybeChtRootStorageProvider, { - let storage = backend.cht_root_storage().ok_or(sc_rpc::system::error::Error::BackendNoChtRoots)?; + let cht_root_error = "Backend doesn't store CHT roots. Make sure you're calling this on a light client."; + let storage = backend.cht_root_storage().ok_or(cht_root_error)?; let finalized_hash = client.info().finalized_hash; let finalized_number = client.info().finalized_number; From 1adad2e524e8cd27c8d4b54e70b990ff0b284cd6 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Mon, 17 Aug 2020 16:13:32 +0200 Subject: [PATCH 11/19] Convert spaces to tabs --- .../cli/src/commands/export_sync_state_cmd.rs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/client/cli/src/commands/export_sync_state_cmd.rs b/client/cli/src/commands/export_sync_state_cmd.rs index 9c316c471a560..9391629035c8b 100644 --- a/client/cli/src/commands/export_sync_state_cmd.rs +++ b/client/cli/src/commands/export_sync_state_cmd.rs @@ -33,9 +33,9 @@ use sp_runtime::traits::Block as BlockT; pub struct ExportSyncStateCmd { /// Force raw genesis storage output in the chain spec. #[structopt(long = "raw")] - pub raw: bool, - - #[allow(missing_docs)] + pub raw: bool, + + #[allow(missing_docs)] #[structopt(flatten)] pub shared_params: SharedParams, @@ -49,16 +49,16 @@ impl ExportSyncStateCmd { pub async fn run( &self, mut spec: Box, - client: Arc, + client: Arc, backend: Arc, mut sync_oracle: SO, - ) -> error::Result<()> - where - B: BlockT, - CL: sp_blockchain::HeaderBackend, + ) -> error::Result<()> + where + B: BlockT, + CL: sp_blockchain::HeaderBackend, BA: sc_service::MaybeChtRootStorageProvider, SO: sp_consensus::SyncOracle, - { + { futures::future::poll_fn(|_| { match sync_oracle.is_major_syncing() || sync_oracle.is_offline() { true => Poll::Pending, @@ -66,12 +66,12 @@ impl ExportSyncStateCmd { } }).await; - info!("Building chain spec"); + info!("Building chain spec"); - let light_sync_state = sc_service::build_light_sync_state(client, backend) - .map_err(|err| err.to_string())?; + let light_sync_state = sc_service::build_light_sync_state(client, backend) + .map_err(|err| err.to_string())?; - spec.set_light_sync_state(light_sync_state.to_serializable()); + spec.set_light_sync_state(light_sync_state.to_serializable()); let json = sc_service::chain_ops::build_spec(&*spec, self.raw)?; if std::io::stdout().write_all(json.as_bytes()).is_err() { From 937266732d8ded27409774625a325f9eefde5c15 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Mon, 17 Aug 2020 16:22:35 +0200 Subject: [PATCH 12/19] Moved sc-service things to export_sync_state.rs --- .../cli/src/commands/export_sync_state_cmd.rs | 6 +- client/cli/src/runner.rs | 2 +- .../src/chain_ops/export_sync_state.rs | 79 +++++++++++++++++++ client/service/src/chain_ops/mod.rs | 2 + client/service/src/lib.rs | 61 +------------- 5 files changed, 86 insertions(+), 64 deletions(-) create mode 100644 client/service/src/chain_ops/export_sync_state.rs diff --git a/client/cli/src/commands/export_sync_state_cmd.rs b/client/cli/src/commands/export_sync_state_cmd.rs index 9391629035c8b..6f6d149cc131e 100644 --- a/client/cli/src/commands/export_sync_state_cmd.rs +++ b/client/cli/src/commands/export_sync_state_cmd.rs @@ -22,6 +22,7 @@ use crate::CliConfiguration; use crate::Role; use log::info; use sc_service::ChainSpec; +use sc_service::chain_ops::{MaybeChtRootStorageProvider, build_light_sync_state}; use structopt::StructOpt; use std::io::Write; use std::sync::Arc; @@ -56,7 +57,7 @@ impl ExportSyncStateCmd { where B: BlockT, CL: sp_blockchain::HeaderBackend, - BA: sc_service::MaybeChtRootStorageProvider, + BA: MaybeChtRootStorageProvider, SO: sp_consensus::SyncOracle, { futures::future::poll_fn(|_| { @@ -68,8 +69,7 @@ impl ExportSyncStateCmd { info!("Building chain spec"); - let light_sync_state = sc_service::build_light_sync_state(client, backend) - .map_err(|err| err.to_string())?; + let light_sync_state = build_light_sync_state(client, backend)?; spec.set_light_sync_state(light_sync_state.to_serializable()); diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index 8e49ac42abbde..08735a0bdb806 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -195,7 +195,7 @@ impl Runner { FCL: UsageProvider + BlockBackend + StorageProvider + Send + Sync + 'static, LCL: HeaderBackend, - LBA: sc_service::MaybeChtRootStorageProvider, + LBA: sc_service::chain_ops::MaybeChtRootStorageProvider, SO: sp_consensus::SyncOracle, { let chain_spec = self.config.chain_spec.cloned_box(); diff --git a/client/service/src/chain_ops/export_sync_state.rs b/client/service/src/chain_ops/export_sync_state.rs new file mode 100644 index 0000000000000..ae65861eb5fc5 --- /dev/null +++ b/client/service/src/chain_ops/export_sync_state.rs @@ -0,0 +1,79 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use sp_runtime::traits::{Block as BlockT, NumberFor, Saturating, One}; +use sp_blockchain::HeaderBackend; +use crate::{TFullBackend, TLightBackend}; +use std::sync::Arc; +use sp_runtime::generic::BlockId; + +/// Something that might allow access to a `ChtRootStorage`. +pub trait MaybeChtRootStorageProvider { + /// Potentially get a reference to a `ChtRootStorage`. + fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage>; +} + +impl MaybeChtRootStorageProvider for TFullBackend { + fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage> { + None + } +} + +impl MaybeChtRootStorageProvider for TLightBackend { + fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage> { + Some(self.blockchain().storage()) + } +} + +/// Build a `LightSyncState` from the CHT roots stored in a backend. +pub fn build_light_sync_state( + client: Arc, + backend: Arc, +) -> Result, sp_blockchain::Error> + where + TBl: BlockT, + TCl: HeaderBackend, + TBackend: MaybeChtRootStorageProvider, +{ + let cht_root_error = "Backend doesn't store CHT roots. Make sure you're calling this on a light client."; + let storage = backend.cht_root_storage().ok_or(cht_root_error)?; + + let finalized_hash = client.info().finalized_hash; + let finalized_number = client.info().finalized_number; + + use sc_client_api::cht; + + let mut chts = Vec::new(); + + // We can't fetch a CHT root later than `finalized_number - 2 * cht_size`. + let cht_size_x_2 = cht::size::>() * NumberFor::::from(2); + + let mut number = NumberFor::::one(); + + while number <= finalized_number.saturating_sub(cht_size_x_2) { + match storage.header_cht_root(cht::size(), number)? { + Some(cht_root) => chts.push(cht_root), + None => log::error!("No CHT found for block {}", number), + } + + number += cht::size(); + } + + Ok(sc_chain_spec::LightSyncState { + header: client.header(BlockId::Hash(finalized_hash))?.unwrap(), + chts, + }) +} diff --git a/client/service/src/chain_ops/mod.rs b/client/service/src/chain_ops/mod.rs index af6e6f632fc06..942ec82fae653 100644 --- a/client/service/src/chain_ops/mod.rs +++ b/client/service/src/chain_ops/mod.rs @@ -21,9 +21,11 @@ mod export_blocks; mod export_raw_state; mod import_blocks; mod revert_chain; +mod export_sync_state; pub use check_block::*; pub use export_blocks::*; pub use export_raw_state::*; pub use import_blocks::*; pub use revert_chain::*; +pub use export_sync_state::*; diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 19ea22be2767d..d19b9f5ea247d 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -46,10 +46,9 @@ use sc_network::{NetworkStatus, network_state::NetworkState, PeerId}; use log::{warn, debug, error}; use codec::{Encode, Decode}; use sp_runtime::generic::BlockId; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor, Saturating, One}; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use parity_util_mem::MallocSizeOf; use sp_utils::{status_sinks, mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}}; -use sp_blockchain::HeaderBackend; pub use self::error::Error; pub use self::builder::{ @@ -160,24 +159,6 @@ impl TelemetryConnectionSinks { } } -/// Something that might allow access to a `ChtRootStorage`. -pub trait MaybeChtRootStorageProvider { - /// Potentially get a reference to a `ChtRootStorage`. - fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage>; -} - -impl MaybeChtRootStorageProvider for TFullBackend { - fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage> { - None - } -} - -impl MaybeChtRootStorageProvider for TLightBackend { - fn cht_root_storage(&self) -> Option<&dyn sc_client_api::light::ChtRootStorage> { - Some(self.blockchain().storage()) - } -} - /// An imcomplete set of chain components, but enough to run the chain ops subcommands. pub struct PartialComponents { /// A shared client instance. @@ -357,46 +338,6 @@ async fn build_network_future< } } -/// Build a `LightSyncState` from the CHT roots stored in a backend. -pub fn build_light_sync_state( - client: Arc, - backend: Arc, -) -> Result, sp_blockchain::Error> - where - TBl: BlockT, - TCl: HeaderBackend, - TBackend: MaybeChtRootStorageProvider, -{ - let cht_root_error = "Backend doesn't store CHT roots. Make sure you're calling this on a light client."; - let storage = backend.cht_root_storage().ok_or(cht_root_error)?; - - let finalized_hash = client.info().finalized_hash; - let finalized_number = client.info().finalized_number; - - use sc_client_api::cht; - - let mut chts = Vec::new(); - - // We can't fetch a CHT root later than `finalized_number - 2 * cht_size`. - let cht_size_x_2 = cht::size::>() * NumberFor::::from(2); - - let mut number = NumberFor::::one(); - - while number <= finalized_number.saturating_sub(cht_size_x_2) { - match storage.header_cht_root(cht::size(), number)? { - Some(cht_root) => chts.push(cht_root), - None => log::error!("No CHT found for block {}", number), - } - - number += cht::size(); - } - - Ok(sc_chain_spec::LightSyncState { - header: client.header(BlockId::Hash(finalized_hash))?.unwrap(), - chts, - }) -} - #[cfg(not(target_os = "unknown"))] // Wrapper for HTTP and WS servers that makes sure they are properly shut down. mod waiting { From db2da9606c30c6122a04f618883883e6e08c4fec Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Mon, 17 Aug 2020 17:20:37 +0200 Subject: [PATCH 13/19] Fix tests --- bin/node/cli/src/chain_spec.rs | 2 +- bin/node/cli/src/service.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index e323f7956f169..a179bb38577f8 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -435,7 +435,7 @@ pub(crate) mod tests { Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) }, |config| { - let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; + let (keep_alive, client, _, _, network, transaction_pool) = new_light_base(config)?; Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) } ); diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 74e00343dd3ec..5c60f5b7cac8d 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -517,7 +517,7 @@ mod tests { Ok((node, (inherent_data_providers, setup_handles.unwrap()))) }, |config| { - let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; + let (keep_alive, client, _, _, network, transaction_pool) = new_light_base(config)?; Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) }, |service, &mut (ref inherent_data_providers, (ref mut block_import, ref babe_link))| { @@ -667,7 +667,7 @@ mod tests { Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) }, |config| { - let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; + let (keep_alive, client, _, _, network, transaction_pool) = new_light_base(config)?; Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) }, vec![ From b53014790910d86e9101bee2d9af54fab90126e4 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Mon, 17 Aug 2020 18:25:09 +0200 Subject: [PATCH 14/19] Wait for syncing with network_status_sinks --- Cargo.lock | 1 - bin/node-template/node/src/command.rs | 4 +- bin/node-template/node/src/service.rs | 9 +- bin/node/cli/src/command.rs | 4 +- bin/node/cli/src/service.rs | 9 +- client/cli/Cargo.toml | 1 - client/cli/src/commands/build_spec_cmd.rs | 53 +++++++++- .../cli/src/commands/export_sync_state_cmd.rs | 96 ------------------- client/cli/src/commands/mod.rs | 7 +- client/cli/src/runner.rs | 23 +++-- .../{export_sync_state.rs => build_spec.rs} | 0 client/service/src/chain_ops/mod.rs | 4 +- 12 files changed, 79 insertions(+), 132 deletions(-) delete mode 100644 client/cli/src/commands/export_sync_state_cmd.rs rename client/service/src/chain_ops/{export_sync_state.rs => build_spec.rs} (100%) diff --git a/Cargo.lock b/Cargo.lock index c4e2ec64698c0..d521e48b5c797 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6190,7 +6190,6 @@ dependencies = [ "serde", "serde_json", "sp-blockchain", - "sp-consensus", "sp-core", "sp-keyring", "sp-panic-handler", diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index 055872f72aebd..4e5f45828cd8f 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -77,9 +77,9 @@ pub fn run() -> sc_cli::Result<()> { Ok((client, backend, import_queue, task_manager)) }, |config| { - let (task_manager, client, backend, network) + let (task_manager, client, backend, network_status_sinks) = new_light_base(config)?; - Ok((client, backend, network, task_manager)) + Ok((client, backend, network_status_sinks, task_manager)) }, ) } diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 43cdaf2be44b3..05c1f49791a1b 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -10,8 +10,7 @@ use sc_executor::native_executor_instance; pub use sc_executor::NativeExecutor; use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair}; use sc_finality_grandpa::{FinalityProofProvider as GrandpaFinalityProofProvider, SharedVoterState}; -use sp_runtime::traits::Block as BlockT; -use sc_network::NetworkService; +use sc_service::NetworkStatusSinks; // Our native executor instance. native_executor_instance!( @@ -226,7 +225,7 @@ pub fn new_full(config: Configuration) -> Result { pub(crate) fn new_light_base(config: Configuration) -> Result<( TaskManager, - Arc, Arc,Arc::Hash>>, + Arc, Arc, NetworkStatusSinks, ), ServiceError> { let (client, backend, keystore, mut task_manager, on_demand) = sc_service::new_light_parts::(&config)?; @@ -293,13 +292,13 @@ pub(crate) fn new_light_base(config: Configuration) -> Result<( keystore, backend: backend.clone(), network: network.clone(), - network_status_sinks, + network_status_sinks: network_status_sinks.clone(), system_rpc_tx, })?; network_starter.start_network(); - Ok((task_manager, client, backend, network)) + Ok((task_manager, client, backend, network_status_sinks)) } /// Builds a new service for a light client. diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index b73db4a585cf6..810e98b530cc1 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -103,9 +103,9 @@ pub fn run() -> Result<()> { Ok((client, backend, import_queue, task_manager)) }, |config| { - let (task_manager, client, backend, _, network, ..) = + let (task_manager, client, backend, _, network_status_sinks, ..) = new_light_base(config)?; - Ok((client, backend, network, task_manager)) + Ok((client, backend, network_status_sinks, task_manager)) }, ) } diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 5c60f5b7cac8d..3f77d93dac4c4 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -27,7 +27,7 @@ use node_primitives::Block; use node_runtime::RuntimeApi; use sc_service::{ config::{Role, Configuration}, error::{Error as ServiceError}, - RpcHandlers, TaskManager, + RpcHandlers, TaskManager, NetworkStatusSinks, }; use sp_inherents::InherentDataProviders; use sc_network::{Event, NetworkService}; @@ -344,7 +344,7 @@ pub fn new_full(config: Configuration) pub fn new_light_base(config: Configuration) -> Result<( TaskManager, Arc, Arc, - RpcHandlers, Arc::Hash>>, + RpcHandlers, NetworkStatusSinks, Arc>> ), ServiceError> { let (client, backend, keystore, mut task_manager, on_demand) = @@ -430,13 +430,14 @@ pub fn new_light_base(config: Configuration) -> Result<( client: client.clone(), backend: backend.clone(), transaction_pool: transaction_pool.clone(), - config, keystore, network_status_sinks, system_rpc_tx, + network_status_sinks: network_status_sinks.clone(), + config, keystore, system_rpc_tx, network: network.clone(), telemetry_connection_sinks: sc_service::TelemetryConnectionSinks::default(), task_manager: &mut task_manager, })?; - Ok((task_manager, client, backend, rpc_handlers, network, transaction_pool)) + Ok((task_manager, client, backend, rpc_handlers, network_status_sinks, transaction_pool)) } /// Builds a new service for a light client. diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index e29e5269ebb64..85a1eb0fe0a4c 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -36,7 +36,6 @@ sp-core = { version = "2.0.0-rc5", path = "../../primitives/core" } sc-service = { version = "0.8.0-rc5", default-features = false, path = "../service" } sp-state-machine = { version = "0.8.0-rc5", path = "../../primitives/state-machine" } sc-telemetry = { version = "2.0.0-rc5", path = "../telemetry" } -sp-consensus = { version = "0.8.0-rc5", path = "../../primitives/consensus/common" } substrate-prometheus-endpoint = { path = "../../utils/prometheus" , version = "0.8.0-rc5"} sp-keyring = { version = "2.0.0-rc5", path = "../../primitives/keyring" } names = "0.11.0" diff --git a/client/cli/src/commands/build_spec_cmd.rs b/client/cli/src/commands/build_spec_cmd.rs index 616c5139f64f0..509c4fbea8e16 100644 --- a/client/cli/src/commands/build_spec_cmd.rs +++ b/client/cli/src/commands/build_spec_cmd.rs @@ -17,14 +17,20 @@ // along with this program. If not, see . use crate::error; -use crate::params::NodeKeyParams; -use crate::params::SharedParams; +use crate::params::{SharedParams, NetworkParams}; use crate::CliConfiguration; +use crate::Role; use log::info; use sc_network::config::build_multiaddr; use sc_service::{config::{MultiaddrWithPeerId, NetworkConfiguration}, ChainSpec}; use structopt::StructOpt; use std::io::Write; +use std::sync::Arc; +use sp_runtime::traits::Block as BlockT; +use sc_service::chain_ops::{MaybeChtRootStorageProvider, build_light_sync_state}; +use sc_service::NetworkStatusSinks; +use futures::stream::StreamExt; +use futures::future::ready; /// The `build-spec` command used to build a specification. #[derive(Debug, StructOpt)] @@ -33,6 +39,11 @@ pub struct BuildSpecCmd { #[structopt(long = "raw")] pub raw: bool, + /// Sync the chain using a light client, and export the state in the chain spec so that other + /// light clients can sync faster. + #[structopt(long = "export-sync-state")] + pub export_sync_state: bool, + /// Disable adding the default bootnode to the specification. /// /// By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the @@ -46,7 +57,7 @@ pub struct BuildSpecCmd { #[allow(missing_docs)] #[structopt(flatten)] - pub node_key_params: NodeKeyParams, + pub network_params: NetworkParams, } impl BuildSpecCmd { @@ -75,6 +86,34 @@ impl BuildSpecCmd { } Ok(()) } + + /// Sync the light client, export the sync state and run the command as per normal. + pub async fn run_export_sync_state( + &self, + mut spec: Box, + network_config: NetworkConfiguration, + client: Arc, + backend: Arc, + network_status_sinks: NetworkStatusSinks, + ) -> error::Result<()> + where + B: BlockT, + CL: sp_blockchain::HeaderBackend, + BA: MaybeChtRootStorageProvider, + { + network_status_sinks.network_status(std::time::Duration::from_secs(1)).take_while(|(status, _)| { + ready(if status.sync_state == sc_network::SyncState::Idle && status.num_sync_peers > 0 { + false + } else { + true + }) + }).for_each(|_| ready(())).await; + + let light_sync_state = build_light_sync_state(client, backend)?; + spec.set_light_sync_state(light_sync_state.to_serializable()); + + self.run(spec, network_config) + } } impl CliConfiguration for BuildSpecCmd { @@ -82,7 +121,11 @@ impl CliConfiguration for BuildSpecCmd { &self.shared_params } - fn node_key_params(&self) -> Option<&NodeKeyParams> { - Some(&self.node_key_params) + fn network_params(&self) -> Option<&NetworkParams> { + Some(&self.network_params) + } + + fn role(&self, _is_dev: bool) -> error::Result { + Ok(Role::Light) } } diff --git a/client/cli/src/commands/export_sync_state_cmd.rs b/client/cli/src/commands/export_sync_state_cmd.rs deleted file mode 100644 index 6f6d149cc131e..0000000000000 --- a/client/cli/src/commands/export_sync_state_cmd.rs +++ /dev/null @@ -1,96 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2020 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -use crate::error; -use crate::params::{SharedParams, NetworkParams}; -use crate::CliConfiguration; -use crate::Role; -use log::info; -use sc_service::ChainSpec; -use sc_service::chain_ops::{MaybeChtRootStorageProvider, build_light_sync_state}; -use structopt::StructOpt; -use std::io::Write; -use std::sync::Arc; -use std::task::Poll; -use sp_runtime::traits::Block as BlockT; - -/// Export the chain spec containing a state to sync the light client. -#[derive(Debug, StructOpt)] -pub struct ExportSyncStateCmd { - /// Force raw genesis storage output in the chain spec. - #[structopt(long = "raw")] - pub raw: bool, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub shared_params: SharedParams, - - #[allow(missing_docs)] - #[structopt(flatten)] - pub network_params: NetworkParams, -} - -impl ExportSyncStateCmd { - /// Run the `export-sync-state` command - pub async fn run( - &self, - mut spec: Box, - client: Arc, - backend: Arc, - mut sync_oracle: SO, - ) -> error::Result<()> - where - B: BlockT, - CL: sp_blockchain::HeaderBackend, - BA: MaybeChtRootStorageProvider, - SO: sp_consensus::SyncOracle, - { - futures::future::poll_fn(|_| { - match sync_oracle.is_major_syncing() || sync_oracle.is_offline() { - true => Poll::Pending, - false => Poll::Ready(()) - } - }).await; - - info!("Building chain spec"); - - let light_sync_state = build_light_sync_state(client, backend)?; - - spec.set_light_sync_state(light_sync_state.to_serializable()); - - let json = sc_service::chain_ops::build_spec(&*spec, self.raw)?; - if std::io::stdout().write_all(json.as_bytes()).is_err() { - let _ = std::io::stderr().write_all(b"Error writing to stdout\n"); - } - Ok(()) - } -} - -impl CliConfiguration for ExportSyncStateCmd { - fn shared_params(&self) -> &SharedParams { - &self.shared_params - } - - fn network_params(&self) -> Option<&NetworkParams> { - Some(&self.network_params) - } - - fn role(&self, _is_dev: bool) -> error::Result { - Ok(Role::Light) - } -} diff --git a/client/cli/src/commands/mod.rs b/client/cli/src/commands/mod.rs index 227e17a7348d1..04cce66bef80d 100644 --- a/client/cli/src/commands/mod.rs +++ b/client/cli/src/commands/mod.rs @@ -23,7 +23,6 @@ mod import_blocks_cmd; mod purge_chain_cmd; mod revert_cmd; mod run_cmd; -mod export_sync_state_cmd; pub use self::build_spec_cmd::BuildSpecCmd; pub use self::check_block_cmd::CheckBlockCmd; @@ -33,7 +32,6 @@ pub use self::import_blocks_cmd::ImportBlocksCmd; pub use self::purge_chain_cmd::PurgeChainCmd; pub use self::revert_cmd::RevertCmd; pub use self::run_cmd::RunCmd; -pub use self::export_sync_state_cmd::ExportSyncStateCmd; use std::fmt::Debug; use structopt::StructOpt; @@ -64,9 +62,6 @@ pub enum Subcommand { /// Export state as raw chain spec. ExportState(ExportStateCmd), - - /// Export the chain spec containing a state to sync the light client. - ExportSyncState(ExportSyncStateCmd), } // TODO: move to config.rs? @@ -418,5 +413,5 @@ macro_rules! substrate_cli_subcommands { } substrate_cli_subcommands!( - Subcommand => BuildSpec, ExportBlocks, ImportBlocks, CheckBlock, Revert, PurgeChain, ExportState, ExportSyncState + Subcommand => BuildSpec, ExportBlocks, ImportBlocks, CheckBlock, Revert, PurgeChain, ExportState ); diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index 08735a0bdb806..6716a153b0131 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -31,6 +31,7 @@ use sp_utils::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL}; use std::{fmt::Debug, marker::PhantomData, str::FromStr, sync::Arc}; use sc_client_api::{UsageProvider, BlockBackend, StorageProvider}; use sp_blockchain::HeaderBackend; +use sc_service::NetworkStatusSinks; #[cfg(target_family = "unix")] async fn main(func: F) -> std::result::Result<(), Box> @@ -178,14 +179,14 @@ impl Runner { /// A helper function that runs a future with tokio and stops if the process receives the signal /// `SIGTERM` or `SIGINT`. - pub fn run_subcommand( + pub fn run_subcommand( self, subcommand: &Subcommand, full_builder: FB, light_builder: LB, ) -> Result<()> where FB: FnOnce(Configuration) -> sc_service::error::Result<(Arc, Arc, IQ, TaskManager)>, LB: FnOnce(Configuration) - -> sc_service::error::Result<(Arc, Arc, SO, TaskManager)>, + -> sc_service::error::Result<(Arc, Arc, NetworkStatusSinks, TaskManager)>, B: BlockT + for<'de> serde::Deserialize<'de>, FBA: sc_client_api::backend::Backend + 'static, IQ: sc_service::ImportQueue + 'static, @@ -196,14 +197,24 @@ impl Runner { 'static, LCL: HeaderBackend, LBA: sc_service::chain_ops::MaybeChtRootStorageProvider, - SO: sp_consensus::SyncOracle, { let chain_spec = self.config.chain_spec.cloned_box(); let network_config = self.config.network.clone(); let db_config = self.config.database.clone(); match subcommand { - Subcommand::BuildSpec(cmd) => cmd.run(chain_spec, network_config), + Subcommand::BuildSpec(cmd) => { + if cmd.export_sync_state { + let (client, backend, network_status_rx, task_manager) = light_builder(self.config)?; + run_until_exit( + self.tokio_runtime, + cmd.run_export_sync_state(chain_spec, network_config, client, backend, network_status_rx), + task_manager + ) + } else { + cmd.run(chain_spec, network_config) + } + }, Subcommand::ExportBlocks(cmd) => { let (client, _, _, task_manager) = full_builder(self.config)?; run_until_exit(self.tokio_runtime, cmd.run(client, db_config), task_manager) @@ -225,10 +236,6 @@ impl Runner { let (client, _, _, task_manager) = full_builder(self.config)?; run_until_exit(self.tokio_runtime, cmd.run(client, chain_spec), task_manager) }, - Subcommand::ExportSyncState(cmd) => { - let (client, backend, sync_oracle, task_manager) = light_builder(self.config)?; - run_until_exit(self.tokio_runtime, cmd.run(chain_spec, client, backend, sync_oracle), task_manager) - } } } diff --git a/client/service/src/chain_ops/export_sync_state.rs b/client/service/src/chain_ops/build_spec.rs similarity index 100% rename from client/service/src/chain_ops/export_sync_state.rs rename to client/service/src/chain_ops/build_spec.rs diff --git a/client/service/src/chain_ops/mod.rs b/client/service/src/chain_ops/mod.rs index 942ec82fae653..19f5e346820aa 100644 --- a/client/service/src/chain_ops/mod.rs +++ b/client/service/src/chain_ops/mod.rs @@ -21,11 +21,11 @@ mod export_blocks; mod export_raw_state; mod import_blocks; mod revert_chain; -mod export_sync_state; +mod build_spec; pub use check_block::*; pub use export_blocks::*; pub use export_raw_state::*; pub use import_blocks::*; pub use revert_chain::*; -pub use export_sync_state::*; +pub use build_spec::*; From 004a8a46cf1fd6c3bf30ef88e3475c0c1ff4e1e2 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Mon, 17 Aug 2020 18:27:43 +0200 Subject: [PATCH 15/19] Remove sc-network from node-template --- Cargo.lock | 1 - bin/node-template/node/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d521e48b5c797..59f4b34bfed76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3822,7 +3822,6 @@ dependencies = [ "sc-consensus-aura", "sc-executor", "sc-finality-grandpa", - "sc-network", "sc-rpc", "sc-rpc-api", "sc-service", diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index f4f524fdb7e35..0c988ebd1a22b 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -32,7 +32,6 @@ sc-consensus = { version = "0.8.0-rc5", path = "../../../client/consensus/common sc-finality-grandpa = { version = "0.8.0-rc5", path = "../../../client/finality-grandpa" } sp-finality-grandpa = { version = "2.0.0-rc5", path = "../../../primitives/finality-grandpa" } sc-client-api = { version = "2.0.0-rc5", path = "../../../client/api" } -sc-network = { version = "0.8.0-rc5", path = "../../../client/network" } sp-runtime = { version = "2.0.0-rc5", path = "../../../primitives/runtime" } # These dependencies are used for the node template's RPCs From ea62d80809bca32bc8f1bc7b87599fbbc586e6ab Mon Sep 17 00:00:00 2001 From: Ashley Date: Mon, 17 Aug 2020 23:04:14 +0200 Subject: [PATCH 16/19] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Köcher --- client/cli/src/commands/build_spec_cmd.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/client/cli/src/commands/build_spec_cmd.rs b/client/cli/src/commands/build_spec_cmd.rs index 509c4fbea8e16..060bbfae26237 100644 --- a/client/cli/src/commands/build_spec_cmd.rs +++ b/client/cli/src/commands/build_spec_cmd.rs @@ -41,7 +41,7 @@ pub struct BuildSpecCmd { /// Sync the chain using a light client, and export the state in the chain spec so that other /// light clients can sync faster. - #[structopt(long = "export-sync-state")] + #[structopt(long)] pub export_sync_state: bool, /// Disable adding the default bootnode to the specification. @@ -101,13 +101,9 @@ impl BuildSpecCmd { CL: sp_blockchain::HeaderBackend, BA: MaybeChtRootStorageProvider, { - network_status_sinks.network_status(std::time::Duration::from_secs(1)).take_while(|(status, _)| { - ready(if status.sync_state == sc_network::SyncState::Idle && status.num_sync_peers > 0 { - false - } else { - true - }) - }).for_each(|_| ready(())).await; + network_status_sinks.network_status(std::time::Duration::from_secs(1)).filter(|(status, _)| { + ready(status.sync_state == sc_network::SyncState::Idle && status.num_sync_peers > 0) + }).into_future().await; let light_sync_state = build_light_sync_state(client, backend)?; spec.set_light_sync_state(light_sync_state.to_serializable()); From 9b558eb5bd6afd714cf1b0a89e5e8e0921515069 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Tue, 18 Aug 2020 10:32:29 +0200 Subject: [PATCH 17/19] Various changes, split the flag up into 2 pieces to make testing easier. --- client/cli/src/commands/build_spec_cmd.rs | 64 ++++++++++++---------- client/cli/src/runner.rs | 20 +++---- client/service/src/chain_ops/build_spec.rs | 7 ++- 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/client/cli/src/commands/build_spec_cmd.rs b/client/cli/src/commands/build_spec_cmd.rs index 060bbfae26237..e6690e5bc6baf 100644 --- a/client/cli/src/commands/build_spec_cmd.rs +++ b/client/cli/src/commands/build_spec_cmd.rs @@ -27,9 +27,9 @@ use structopt::StructOpt; use std::io::Write; use std::sync::Arc; use sp_runtime::traits::Block as BlockT; -use sc_service::chain_ops::{MaybeChtRootStorageProvider, build_light_sync_state}; +use sc_service::chain_ops::{MaybeChtRootStorageProvider, build_light_sync_state, CHT_ROOT_ERROR}; use sc_service::NetworkStatusSinks; -use futures::stream::StreamExt; +use futures::{FutureExt, StreamExt}; use futures::future::ready; /// The `build-spec` command used to build a specification. @@ -39,11 +39,16 @@ pub struct BuildSpecCmd { #[structopt(long = "raw")] pub raw: bool, - /// Sync the chain using a light client, and export the state in the chain spec so that other - /// light clients can sync faster. + /// Export the light client state in the chain spec so that other light clients can sync faster. + /// Generally called together with `--sync-light-client`. #[structopt(long)] pub export_sync_state: bool, + /// Sync the chain using a light client first. Does nothing unless `--export-sync-state` is also + /// called. + #[structopt(long)] + pub sync_light_client: bool, + /// Disable adding the default bootnode to the specification. /// /// By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the @@ -62,11 +67,34 @@ pub struct BuildSpecCmd { impl BuildSpecCmd { /// Run the build-spec command - pub fn run( + pub async fn run( &self, mut spec: Box, network_config: NetworkConfiguration, - ) -> error::Result<()> { + client: Arc, + backend: Arc, + network_status_sinks: NetworkStatusSinks, + ) -> error::Result<()> + where + B: BlockT, + CL: sp_blockchain::HeaderBackend, + BA: MaybeChtRootStorageProvider, + { + if self.export_sync_state { + if backend.cht_root_storage().is_none() { + return Err(CHT_ROOT_ERROR.into()); + } + + if self.sync_light_client { + network_status_sinks.network_status(std::time::Duration::from_secs(1)).filter(|(status, _)| { + ready(status.sync_state == sc_network::SyncState::Idle && status.num_sync_peers > 0) + }).into_future().map(drop).await; + } + + let light_sync_state = build_light_sync_state(client, backend)?; + spec.set_light_sync_state(light_sync_state.to_serializable()); + } + info!("Building chain spec"); let raw_output = self.raw; @@ -86,30 +114,6 @@ impl BuildSpecCmd { } Ok(()) } - - /// Sync the light client, export the sync state and run the command as per normal. - pub async fn run_export_sync_state( - &self, - mut spec: Box, - network_config: NetworkConfiguration, - client: Arc, - backend: Arc, - network_status_sinks: NetworkStatusSinks, - ) -> error::Result<()> - where - B: BlockT, - CL: sp_blockchain::HeaderBackend, - BA: MaybeChtRootStorageProvider, - { - network_status_sinks.network_status(std::time::Duration::from_secs(1)).filter(|(status, _)| { - ready(status.sync_state == sc_network::SyncState::Idle && status.num_sync_peers > 0) - }).into_future().await; - - let light_sync_state = build_light_sync_state(client, backend)?; - spec.set_light_sync_state(light_sync_state.to_serializable()); - - self.run(spec, network_config) - } } impl CliConfiguration for BuildSpecCmd { diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index 6716a153b0131..59a55040f2577 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -98,7 +98,7 @@ pub fn build_runtime() -> std::result::Result( mut tokio_runtime: tokio::runtime::Runtime, future: FUT, - mut task_manager: TaskManager, + task_manager: TaskManager, ) -> Result<()> where FUT: Future> + future::Future, @@ -109,8 +109,7 @@ where tokio_runtime.block_on(main(f)).map_err(|e| e.to_string())?; - task_manager.terminate(); - drop(tokio_runtime); + task_manager.clean_shutdown(); Ok(()) } @@ -204,16 +203,11 @@ impl Runner { match subcommand { Subcommand::BuildSpec(cmd) => { - if cmd.export_sync_state { - let (client, backend, network_status_rx, task_manager) = light_builder(self.config)?; - run_until_exit( - self.tokio_runtime, - cmd.run_export_sync_state(chain_spec, network_config, client, backend, network_status_rx), - task_manager - ) - } else { - cmd.run(chain_spec, network_config) - } + let (client, backend, network_status_rx, task_manager) = light_builder(self.config)?; + run_until_exit(self.tokio_runtime, + cmd.run(chain_spec, network_config, client, backend, network_status_rx), + task_manager, + ) }, Subcommand::ExportBlocks(cmd) => { let (client, _, _, task_manager) = full_builder(self.config)?; diff --git a/client/service/src/chain_ops/build_spec.rs b/client/service/src/chain_ops/build_spec.rs index ae65861eb5fc5..c84c1c754adac 100644 --- a/client/service/src/chain_ops/build_spec.rs +++ b/client/service/src/chain_ops/build_spec.rs @@ -20,6 +20,10 @@ use crate::{TFullBackend, TLightBackend}; use std::sync::Arc; use sp_runtime::generic::BlockId; +/// An error for if this function is being called on a full node. +pub const CHT_ROOT_ERROR: &str = + "Backend doesn't store CHT roots. Make sure you're calling this on a light client."; + /// Something that might allow access to a `ChtRootStorage`. pub trait MaybeChtRootStorageProvider { /// Potentially get a reference to a `ChtRootStorage`. @@ -48,8 +52,7 @@ pub fn build_light_sync_state( TCl: HeaderBackend, TBackend: MaybeChtRootStorageProvider, { - let cht_root_error = "Backend doesn't store CHT roots. Make sure you're calling this on a light client."; - let storage = backend.cht_root_storage().ok_or(cht_root_error)?; + let storage = backend.cht_root_storage().ok_or(CHT_ROOT_ERROR)?; let finalized_hash = client.info().finalized_hash; let finalized_number = client.info().finalized_number; From fd1d2b1cbec75a36a696b574c5f6594031af7843 Mon Sep 17 00:00:00 2001 From: Ashley Date: Tue, 18 Aug 2020 11:10:14 +0200 Subject: [PATCH 18/19] Update client/cli/src/commands/build_spec_cmd.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Köcher --- client/cli/src/commands/build_spec_cmd.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/client/cli/src/commands/build_spec_cmd.rs b/client/cli/src/commands/build_spec_cmd.rs index e6690e5bc6baf..430e04101a662 100644 --- a/client/cli/src/commands/build_spec_cmd.rs +++ b/client/cli/src/commands/build_spec_cmd.rs @@ -44,9 +44,8 @@ pub struct BuildSpecCmd { #[structopt(long)] pub export_sync_state: bool, - /// Sync the chain using a light client first. Does nothing unless `--export-sync-state` is also - /// called. - #[structopt(long)] + /// Sync the chain using a light client first. + #[structopt(long, requires = "export-sync-state")] pub sync_light_client: bool, /// Disable adding the default bootnode to the specification. From fa5171722d77e06aa5f369e6f070a965fc54f630 Mon Sep 17 00:00:00 2001 From: Ashley Ruglys Date: Wed, 19 Aug 2020 13:42:13 +0200 Subject: [PATCH 19/19] Revert a lot of changes --- bin/node-template/node/src/command.rs | 20 +++----- bin/node-template/node/src/service.rs | 31 ++++--------- bin/node/cli/src/browser.rs | 2 +- bin/node/cli/src/chain_spec.rs | 2 +- bin/node/cli/src/command.rs | 20 +++----- bin/node/cli/src/service.rs | 19 ++++---- client/chain-spec/src/chain_spec.rs | 11 ----- client/chain-spec/src/lib.rs | 2 - client/cli/src/commands/build_spec_cmd.rs | 56 +++-------------------- client/cli/src/runner.rs | 42 ++++++----------- 10 files changed, 54 insertions(+), 151 deletions(-) diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index 4e5f45828cd8f..b3f1cfaf11f55 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -20,7 +20,7 @@ use crate::cli::Cli; use crate::service; use sc_cli::{SubstrateCli, RuntimeVersion, Role, ChainSpec}; use sc_service::PartialComponents; -use crate::service::{new_partial, new_light_base}; +use crate::service::new_partial; impl SubstrateCli for Cli { fn impl_name() -> String { @@ -69,19 +69,11 @@ pub fn run() -> sc_cli::Result<()> { match &cli.subcommand { Some(subcommand) => { let runner = cli.create_runner(subcommand)?; - runner.run_subcommand( - subcommand, - |config| { - let PartialComponents { client, backend, task_manager, import_queue, .. } - = new_partial(&config)?; - Ok((client, backend, import_queue, task_manager)) - }, - |config| { - let (task_manager, client, backend, network_status_sinks) - = new_light_base(config)?; - Ok((client, backend, network_status_sinks, task_manager)) - }, - ) + runner.run_subcommand(subcommand, |config| { + let PartialComponents { client, backend, task_manager, import_queue, .. } + = new_partial(&config)?; + Ok((client, backend, import_queue, task_manager)) + }) } None => { let runner = cli.create_runner(&cli.run)?; diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index 05c1f49791a1b..5984d67322333 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -10,7 +10,6 @@ use sc_executor::native_executor_instance; pub use sc_executor::NativeExecutor; use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair}; use sc_finality_grandpa::{FinalityProofProvider as GrandpaFinalityProofProvider, SharedVoterState}; -use sc_service::NetworkStatusSinks; // Our native executor instance. native_executor_instance!( @@ -22,8 +21,6 @@ native_executor_instance!( type FullClient = sc_service::TFullClient; type FullBackend = sc_service::TFullBackend; type FullSelectChain = sc_consensus::LongestChain; -type LightClient = sc_service::TLightClient; -type LightBackend = sc_service::TLightBackend; pub fn new_partial(config: &Configuration) -> Result Result { Ok(task_manager) } -pub(crate) fn new_light_base(config: Configuration) -> Result<( - TaskManager, - Arc, Arc, NetworkStatusSinks, -), ServiceError> { +/// Builds a new service for a light client. +pub fn new_light(config: Configuration) -> Result { let (client, backend, keystore, mut task_manager, on_demand) = - sc_service::new_light_parts::(&config)?; + sc_service::new_light_parts::(&config)?; let transaction_pool = Arc::new(sc_transaction_pool::BasicPool::new_light( config.transaction_pool.clone(), @@ -288,21 +283,15 @@ pub(crate) fn new_light_base(config: Configuration) -> Result<( rpc_extensions_builder: Box::new(|_, _| ()), telemetry_connection_sinks: sc_service::TelemetryConnectionSinks::default(), config, - client: client.clone(), + client, keystore, - backend: backend.clone(), - network: network.clone(), - network_status_sinks: network_status_sinks.clone(), + backend, + network, + network_status_sinks, system_rpc_tx, - })?; + })?; - network_starter.start_network(); + network_starter.start_network(); - Ok((task_manager, client, backend, network_status_sinks)) -} - -/// Builds a new service for a light client. -pub fn new_light(config: Configuration) -> Result { - let (task_manager, ..) = new_light_base(config)?; - Ok(task_manager) + Ok(task_manager) } diff --git a/bin/node/cli/src/browser.rs b/bin/node/cli/src/browser.rs index 4cd517a43236c..3aec486c10370 100644 --- a/bin/node/cli/src/browser.rs +++ b/bin/node/cli/src/browser.rs @@ -54,7 +54,7 @@ async fn start_inner(chain_spec: Option, log_level: String) -> Result String { @@ -95,19 +95,11 @@ pub fn run() -> Result<()> { } Some(Subcommand::Base(subcommand)) => { let runner = cli.create_runner(subcommand)?; - runner.run_subcommand( - subcommand, - |config| { - let PartialComponents { client, backend, task_manager, import_queue, ..} - = new_partial(&config)?; - Ok((client, backend, import_queue, task_manager)) - }, - |config| { - let (task_manager, client, backend, _, network_status_sinks, ..) = - new_light_base(config)?; - Ok((client, backend, network_status_sinks, task_manager)) - }, - ) + runner.run_subcommand(subcommand, |config| { + let PartialComponents { client, backend, task_manager, import_queue, ..} + = new_partial(&config)?; + Ok((client, backend, import_queue, task_manager)) + }) } } } diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 3f77d93dac4c4..d91696ab7d6bc 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -27,7 +27,7 @@ use node_primitives::Block; use node_runtime::RuntimeApi; use sc_service::{ config::{Role, Configuration}, error::{Error as ServiceError}, - RpcHandlers, TaskManager, NetworkStatusSinks, + RpcHandlers, TaskManager, }; use sp_inherents::InherentDataProviders; use sc_network::{Event, NetworkService}; @@ -43,7 +43,6 @@ type FullSelectChain = sc_consensus::LongestChain; type FullGrandpaBlockImport = grandpa::GrandpaBlockImport; type LightClient = sc_service::TLightClient; -type LightBackend = sc_service::TLightBackend; pub fn new_partial(config: &Configuration) -> Result Result<( - TaskManager, Arc, Arc, - RpcHandlers, NetworkStatusSinks, + TaskManager, RpcHandlers, Arc, + Arc::Hash>>, Arc>> ), ServiceError> { let (client, backend, keystore, mut task_manager, on_demand) = @@ -428,21 +427,19 @@ pub fn new_light_base(config: Configuration) -> Result<( remote_blockchain: Some(backend.remote_blockchain()), rpc_extensions_builder: Box::new(sc_service::NoopRpcExtensionBuilder(rpc_extensions)), client: client.clone(), - backend: backend.clone(), transaction_pool: transaction_pool.clone(), - network_status_sinks: network_status_sinks.clone(), - config, keystore, system_rpc_tx, + config, keystore, backend, network_status_sinks, system_rpc_tx, network: network.clone(), telemetry_connection_sinks: sc_service::TelemetryConnectionSinks::default(), task_manager: &mut task_manager, })?; - Ok((task_manager, client, backend, rpc_handlers, network_status_sinks, transaction_pool)) + Ok((task_manager, rpc_handlers, client, network, transaction_pool)) } /// Builds a new service for a light client. pub fn new_light(config: Configuration) -> Result { - new_light_base(config).map(|(task_manager, ..)| { + new_light_base(config).map(|(task_manager, _, _, _, _)| { task_manager }) } @@ -518,7 +515,7 @@ mod tests { Ok((node, (inherent_data_providers, setup_handles.unwrap()))) }, |config| { - let (keep_alive, client, _, _, network, transaction_pool) = new_light_base(config)?; + let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) }, |service, &mut (ref inherent_data_providers, (ref mut block_import, ref babe_link))| { @@ -668,7 +665,7 @@ mod tests { Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) }, |config| { - let (keep_alive, client, _, _, network, transaction_pool) = new_light_base(config)?; + let (keep_alive, _, client, network, transaction_pool) = new_light_base(config)?; Ok(sc_service_test::TestNetComponents::new(keep_alive, client, network, transaction_pool)) }, vec![ diff --git a/client/chain-spec/src/chain_spec.rs b/client/chain-spec/src/chain_spec.rs index b7e3e9572953c..20811394c56d7 100644 --- a/client/chain-spec/src/chain_spec.rs +++ b/client/chain-spec/src/chain_spec.rs @@ -333,13 +333,6 @@ impl ChainSpec { json::to_string_pretty(&container) .map_err(|e| format!("Error generating spec json: {}", e)) } - - /// Dump to json value - pub fn as_json_value(&self, raw: bool) -> Result { - let container = self.json_container(raw)?; - json::to_value(container) - .map_err(|e| format!("Error generating spec json: {}", e)) - } } impl crate::ChainSpec for ChainSpec @@ -387,10 +380,6 @@ where ChainSpec::as_json(self, raw) } - fn as_json_value(&self, raw: bool) -> Result { - ChainSpec::as_json_value(self, raw) - } - fn as_storage_builder(&self) -> &dyn BuildStorage { self } diff --git a/client/chain-spec/src/lib.rs b/client/chain-spec/src/lib.rs index a1802d5b523e3..f5afe496f1980 100644 --- a/client/chain-spec/src/lib.rs +++ b/client/chain-spec/src/lib.rs @@ -149,8 +149,6 @@ pub trait ChainSpec: BuildStorage + Send { fn add_boot_node(&mut self, addr: MultiaddrWithPeerId); /// Return spec as JSON. fn as_json(&self, raw: bool) -> Result; - /// Return spec as JSON value. - fn as_json_value(&self, raw: bool) -> Result; /// Return StorageBuilder for this spec. fn as_storage_builder(&self) -> &dyn BuildStorage; /// Returns a cloned `Box`. diff --git a/client/cli/src/commands/build_spec_cmd.rs b/client/cli/src/commands/build_spec_cmd.rs index 430e04101a662..616c5139f64f0 100644 --- a/client/cli/src/commands/build_spec_cmd.rs +++ b/client/cli/src/commands/build_spec_cmd.rs @@ -17,20 +17,14 @@ // along with this program. If not, see . use crate::error; -use crate::params::{SharedParams, NetworkParams}; +use crate::params::NodeKeyParams; +use crate::params::SharedParams; use crate::CliConfiguration; -use crate::Role; use log::info; use sc_network::config::build_multiaddr; use sc_service::{config::{MultiaddrWithPeerId, NetworkConfiguration}, ChainSpec}; use structopt::StructOpt; use std::io::Write; -use std::sync::Arc; -use sp_runtime::traits::Block as BlockT; -use sc_service::chain_ops::{MaybeChtRootStorageProvider, build_light_sync_state, CHT_ROOT_ERROR}; -use sc_service::NetworkStatusSinks; -use futures::{FutureExt, StreamExt}; -use futures::future::ready; /// The `build-spec` command used to build a specification. #[derive(Debug, StructOpt)] @@ -39,15 +33,6 @@ pub struct BuildSpecCmd { #[structopt(long = "raw")] pub raw: bool, - /// Export the light client state in the chain spec so that other light clients can sync faster. - /// Generally called together with `--sync-light-client`. - #[structopt(long)] - pub export_sync_state: bool, - - /// Sync the chain using a light client first. - #[structopt(long, requires = "export-sync-state")] - pub sync_light_client: bool, - /// Disable adding the default bootnode to the specification. /// /// By default the `/ip4/127.0.0.1/tcp/30333/p2p/NODE_PEER_ID` bootnode is added to the @@ -61,39 +46,16 @@ pub struct BuildSpecCmd { #[allow(missing_docs)] #[structopt(flatten)] - pub network_params: NetworkParams, + pub node_key_params: NodeKeyParams, } impl BuildSpecCmd { /// Run the build-spec command - pub async fn run( + pub fn run( &self, mut spec: Box, network_config: NetworkConfiguration, - client: Arc, - backend: Arc, - network_status_sinks: NetworkStatusSinks, - ) -> error::Result<()> - where - B: BlockT, - CL: sp_blockchain::HeaderBackend, - BA: MaybeChtRootStorageProvider, - { - if self.export_sync_state { - if backend.cht_root_storage().is_none() { - return Err(CHT_ROOT_ERROR.into()); - } - - if self.sync_light_client { - network_status_sinks.network_status(std::time::Duration::from_secs(1)).filter(|(status, _)| { - ready(status.sync_state == sc_network::SyncState::Idle && status.num_sync_peers > 0) - }).into_future().map(drop).await; - } - - let light_sync_state = build_light_sync_state(client, backend)?; - spec.set_light_sync_state(light_sync_state.to_serializable()); - } - + ) -> error::Result<()> { info!("Building chain spec"); let raw_output = self.raw; @@ -120,11 +82,7 @@ impl CliConfiguration for BuildSpecCmd { &self.shared_params } - fn network_params(&self) -> Option<&NetworkParams> { - Some(&self.network_params) - } - - fn role(&self, _is_dev: bool) -> error::Result { - Ok(Role::Light) + fn node_key_params(&self) -> Option<&NodeKeyParams> { + Some(&self.node_key_params) } } diff --git a/client/cli/src/runner.rs b/client/cli/src/runner.rs index 59a55040f2577..bdbf55eb8326a 100644 --- a/client/cli/src/runner.rs +++ b/client/cli/src/runner.rs @@ -30,8 +30,6 @@ use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; use sp_utils::metrics::{TOKIO_THREADS_ALIVE, TOKIO_THREADS_TOTAL}; use std::{fmt::Debug, marker::PhantomData, str::FromStr, sync::Arc}; use sc_client_api::{UsageProvider, BlockBackend, StorageProvider}; -use sp_blockchain::HeaderBackend; -use sc_service::NetworkStatusSinks; #[cfg(target_family = "unix")] async fn main(func: F) -> std::result::Result<(), Box> @@ -98,7 +96,7 @@ pub fn build_runtime() -> std::result::Result( mut tokio_runtime: tokio::runtime::Runtime, future: FUT, - task_manager: TaskManager, + mut task_manager: TaskManager, ) -> Result<()> where FUT: Future> + future::Future, @@ -109,7 +107,8 @@ where tokio_runtime.block_on(main(f)).map_err(|e| e.to_string())?; - task_manager.clean_shutdown(); + task_manager.terminate(); + drop(tokio_runtime); Ok(()) } @@ -178,56 +177,45 @@ impl Runner { /// A helper function that runs a future with tokio and stops if the process receives the signal /// `SIGTERM` or `SIGINT`. - pub fn run_subcommand( - self, subcommand: &Subcommand, full_builder: FB, light_builder: LB, - ) -> Result<()> + pub fn run_subcommand(self, subcommand: &Subcommand, builder: BU) + -> Result<()> where - FB: FnOnce(Configuration) - -> sc_service::error::Result<(Arc, Arc, IQ, TaskManager)>, - LB: FnOnce(Configuration) - -> sc_service::error::Result<(Arc, Arc, NetworkStatusSinks, TaskManager)>, + BU: FnOnce(Configuration) + -> sc_service::error::Result<(Arc, Arc, IQ, TaskManager)>, B: BlockT + for<'de> serde::Deserialize<'de>, - FBA: sc_client_api::backend::Backend + 'static, + BA: sc_client_api::backend::Backend + 'static, IQ: sc_service::ImportQueue + 'static, ::Hash: FromStr, <::Hash as FromStr>::Err: Debug, <<::Header as HeaderT>::Number as FromStr>::Err: Debug, - FCL: UsageProvider + BlockBackend + StorageProvider + Send + Sync + + CL: UsageProvider + BlockBackend + StorageProvider + Send + Sync + 'static, - LCL: HeaderBackend, - LBA: sc_service::chain_ops::MaybeChtRootStorageProvider, { let chain_spec = self.config.chain_spec.cloned_box(); let network_config = self.config.network.clone(); let db_config = self.config.database.clone(); match subcommand { - Subcommand::BuildSpec(cmd) => { - let (client, backend, network_status_rx, task_manager) = light_builder(self.config)?; - run_until_exit(self.tokio_runtime, - cmd.run(chain_spec, network_config, client, backend, network_status_rx), - task_manager, - ) - }, + Subcommand::BuildSpec(cmd) => cmd.run(chain_spec, network_config), Subcommand::ExportBlocks(cmd) => { - let (client, _, _, task_manager) = full_builder(self.config)?; + let (client, _, _, task_manager) = builder(self.config)?; run_until_exit(self.tokio_runtime, cmd.run(client, db_config), task_manager) } Subcommand::ImportBlocks(cmd) => { - let (client, _, import_queue, task_manager) = full_builder(self.config)?; + let (client, _, import_queue, task_manager) = builder(self.config)?; run_until_exit(self.tokio_runtime, cmd.run(client, import_queue), task_manager) } Subcommand::CheckBlock(cmd) => { - let (client, _, import_queue, task_manager) = full_builder(self.config)?; + let (client, _, import_queue, task_manager) = builder(self.config)?; run_until_exit(self.tokio_runtime, cmd.run(client, import_queue), task_manager) } Subcommand::Revert(cmd) => { - let (client, backend, _, task_manager) = full_builder(self.config)?; + let (client, backend, _, task_manager) = builder(self.config)?; run_until_exit(self.tokio_runtime, cmd.run(client, backend), task_manager) }, Subcommand::PurgeChain(cmd) => cmd.run(db_config), Subcommand::ExportState(cmd) => { - let (client, _, _, task_manager) = full_builder(self.config)?; + let (client, _, _, task_manager) = builder(self.config)?; run_until_exit(self.tokio_runtime, cmd.run(client, chain_spec), task_manager) }, }