From 9ecdc7454f9b91d35926a6c16b13492ef3346537 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 21 Jun 2021 16:48:05 -0600 Subject: [PATCH 01/99] Add initial migrations pallet sketch --- pallets/migrations/Cargo.toml | 22 +++++++ pallets/migrations/src/lib.rs | 104 ++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 pallets/migrations/Cargo.toml create mode 100644 pallets/migrations/src/lib.rs diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml new file mode 100644 index 0000000000..7838d72bf4 --- /dev/null +++ b/pallets/migrations/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "migrations" +version = "0.1.0" +authors = ["PureStake"] +edition = "2018" +description = "migrations management pallet" + +[dependencies] +frame-support = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } +frame-system = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } +log = "0.4" +sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } +sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } + +[features] +default = ["std"] +std = [ + "frame-support/std", + "frame-system/std", + "sp-std/std", + "sp-runtime/std", +] diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs new file mode 100644 index 0000000000..5da677ac4c --- /dev/null +++ b/pallets/migrations/src/lib.rs @@ -0,0 +1,104 @@ +// Copyright 2019-2020 PureStake Inc. +// This file is part of Moonbeam. + +// Moonbeam is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Moonbeam is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Moonbeam. If not, see . + +//! # Migrations + +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_support::pallet; + +#[pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use sp_std::prelude::*; + + #[pallet::pallet] + pub struct Pallet(PhantomData); + + #[derive(Copy, Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug)] + /// A Migration that must happen on-chain upon a runtime-upgrade + pub struct Migration { + // TODO: this would involve some metadata about the migration as well as a means of calling + // the actual migration function + } + + /// Our list of migrations. Any ordering considerations can be specified here (?). + // const MIGRATIONS: // TODO: this would be a constant vec (or similar) of individual migrations + + /// Configuration trait of this pallet. + #[pallet::config] + pub trait Config: frame_system::Config { + /// Overarching event type + type Event: From> + IsType<::Event>; + /// Weight information for extrinsics in this pallet. + type WeightInfo: WeightInfo; + } + + #[pallet::error] + pub enum Error { + // errors in this pallet would be quite bad... + } + + #[pallet::event] + #[pallet::generate_deposit(pub(crate) fn deposit_event)] + pub enum Event { + // e.g. runtime upgrade started, completed, etc. + } + + #[pallet::hooks] + impl Hooks> for Pallet { + + /// on_runtime_upgrade is expected to be called exactly once after a runtime upgrade. + /// We use this as a chance to flag that we are now in upgrade-mode and begin our + /// migrations. + /// + /// In the event that a migration is expected to take more than one block, ongoing migration + /// work could continue from block-to-block in this pallet's on_initialize function. + fn on_runtime_upgrade() -> Weight { + // start by flagging that we are not fully upgraded + >::put(false); + + let weight: u32 = 0; + + let info = process_runtime_upgrades(); + weight += info.actual_weight.expect("Weight not provided"); + + // now flag that we are done with our runtime upgrade + >::put(true); + + weight + } + } + + #[pallet::storage] + #[pallet::getter(fn is_fully_upgraded)] + /// True if all required migrations have completed + type FullyUpgraded = StorageValue<_, bool, ValueQuery>; + + #[pallet::call] + impl Pallet { + + // TODO: this isn't a call, but it should forward weight info + #[pallet::weight(0)] + pub fn process_runtime_upgrades(origin: OriginFor) -> DispatchResultWithPostInfo { + frame_system::ensure_root(origin)?; + // TODO: iterate over MIGRATIONS here and ensure that each one has been fully applied. + // additionally, write to storage about our progress if multi-block-update functionality + // is required. + } + } +} From e4f15ce2a29512d63353f423d8d46d3e63e3fccf Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 25 Jun 2021 10:23:43 -0600 Subject: [PATCH 02/99] Sketch out Migration impls --- pallets/migrations/src/lib.rs | 13 +++++++-- pallets/migrations/src/migrations.rs | 43 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 pallets/migrations/src/migrations.rs diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 5da677ac4c..356b8715df 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -14,11 +14,12 @@ // You should have received a copy of the GNU General Public License // along with Moonbeam. If not, see . -//! # Migrations +//! # Migration Pallet #![cfg_attr(not(feature = "std"), no_std)] use frame_support::pallet; +pub mod migrations; #[pallet] pub mod pallet { @@ -31,13 +32,19 @@ pub mod pallet { #[derive(Copy, Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug)] /// A Migration that must happen on-chain upon a runtime-upgrade - pub struct Migration { + pub trait Migration { // TODO: this would involve some metadata about the migration as well as a means of calling // the actual migration function + + fn friendly_name() -> str; } /// Our list of migrations. Any ordering considerations can be specified here (?). - // const MIGRATIONS: // TODO: this would be a constant vec (or similar) of individual migrations + static MIGRATIONS: [Migration] = [ + MM_001_AuthorMappingAddDeposit, + MM_002_StakingFixTotalBalance, + MM_003_StakingTransitionBoundedSet, + ]; /// Configuration trait of this pallet. #[pallet::config] diff --git a/pallets/migrations/src/migrations.rs b/pallets/migrations/src/migrations.rs new file mode 100644 index 0000000000..c552c44a0a --- /dev/null +++ b/pallets/migrations/src/migrations.rs @@ -0,0 +1,43 @@ +// Copyright 2019-2020 PureStake Inc. +// This file is part of Moonbeam. + +// Moonbeam is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Moonbeam is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Moonbeam. If not, see . + +//! # Migrations + +use crate::*; + +/// This module acts as a registry where each migration is defined. Each migration should implement +/// the "Migration" trait declared in this crate. + +struct MM_001_AuthorMappingAddDeposit; +impl Migration for MM_001_AuthorMappingAddDeposit { + fn friendly_name() -> str { + "AuthorMappingAddDeposit" + } +} + +struct MM_002_StakingFixTotalBalance; +impl Migration for StakingFixTotalBalance { + fn friendly_name() -> str { + "StakingFixTotalBalance" + } +} + +struct MM_003_StakingTransitionBoundedSet; // TODO: better name +impl Migration for MM_003_StakingTransitionBoundedSet { + fn friendly_name() -> str { + "StakingTransitionBoundedSet" + } +} From ca3620eba41286737738f7e563a53645f25a9b68 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 25 Jun 2021 11:22:06 -0600 Subject: [PATCH 03/99] Make it build --- Cargo.lock | 14 ++++++-- Cargo.toml | 1 + pallets/migrations/Cargo.toml | 1 + pallets/migrations/src/lib.rs | 51 ++++++++++++---------------- pallets/migrations/src/migrations.rs | 20 +++++++---- 5 files changed, 48 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 98f7981244..d4d55e568e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,7 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - [[package]] name = "Inflector" version = "0.11.4" @@ -4664,6 +4662,18 @@ dependencies = [ "thrift", ] +[[package]] +name = "migrations" +version = "0.1.0" +dependencies = [ + "frame-support", + "frame-system", + "log", + "parity-scale-codec", + "sp-runtime", + "sp-std", +] + [[package]] name = "minicbor" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index e20d54cb15..6896074f96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ 'node', 'node/cli', 'node/service', + 'pallets/migrations', 'bin/utils/moonkey' ] exclude = [ diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index 7838d72bf4..4a6cf5ba3f 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -11,6 +11,7 @@ frame-system = { git = "https://github.com/paritytech/substrate", branch = "polk log = "0.4" sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } +parity-scale-codec = { version = "2.0.0", default-features = false } [features] default = ["std"] diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 356b8715df..ef265655a0 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -21,29 +21,29 @@ use frame_support::pallet; pub mod migrations; +/// A Migration that must happen on-chain upon a runtime-upgrade +pub trait Migration { + // TODO: this would involve some metadata about the migration as well as a means of calling + // the actual migration function + + // fn friendly_name() -> &str; +} + #[pallet] pub mod pallet { use super::*; use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; use sp_std::prelude::*; #[pallet::pallet] pub struct Pallet(PhantomData); - #[derive(Copy, Clone, PartialEq, Eq, Encode, Decode, RuntimeDebug)] - /// A Migration that must happen on-chain upon a runtime-upgrade - pub trait Migration { - // TODO: this would involve some metadata about the migration as well as a means of calling - // the actual migration function - - fn friendly_name() -> str; - } - /// Our list of migrations. Any ordering considerations can be specified here (?). - static MIGRATIONS: [Migration] = [ - MM_001_AuthorMappingAddDeposit, - MM_002_StakingFixTotalBalance, - MM_003_StakingTransitionBoundedSet, + const MIGRATIONS: [&dyn Migration; 3] = [ + &migrations::MM_001_AuthorMappingAddDeposit {}, + &migrations::MM_002_StakingFixTotalBalance {}, + &migrations::MM_003_StakingTransitionBoundedSet {}, ]; /// Configuration trait of this pallet. @@ -51,8 +51,6 @@ pub mod pallet { pub trait Config: frame_system::Config { /// Overarching event type type Event: From> + IsType<::Event>; - /// Weight information for extrinsics in this pallet. - type WeightInfo: WeightInfo; } #[pallet::error] @@ -79,15 +77,14 @@ pub mod pallet { // start by flagging that we are not fully upgraded >::put(false); - let weight: u32 = 0; + let mut weight: Weight = 0u64.into(); - let info = process_runtime_upgrades(); - weight += info.actual_weight.expect("Weight not provided"); + weight += process_runtime_upgrades(); // now flag that we are done with our runtime upgrade >::put(true); - weight + weight.into() } } @@ -96,16 +93,10 @@ pub mod pallet { /// True if all required migrations have completed type FullyUpgraded = StorageValue<_, bool, ValueQuery>; - #[pallet::call] - impl Pallet { - - // TODO: this isn't a call, but it should forward weight info - #[pallet::weight(0)] - pub fn process_runtime_upgrades(origin: OriginFor) -> DispatchResultWithPostInfo { - frame_system::ensure_root(origin)?; - // TODO: iterate over MIGRATIONS here and ensure that each one has been fully applied. - // additionally, write to storage about our progress if multi-block-update functionality - // is required. - } + fn process_runtime_upgrades() -> Weight { + // TODO: iterate over MIGRATIONS here and ensure that each one has been fully applied. + // additionally, write to storage about our progress if multi-block-update functionality + // is required. + 0u64.into() } } diff --git a/pallets/migrations/src/migrations.rs b/pallets/migrations/src/migrations.rs index c552c44a0a..126c3434af 100644 --- a/pallets/migrations/src/migrations.rs +++ b/pallets/migrations/src/migrations.rs @@ -21,23 +21,29 @@ use crate::*; /// This module acts as a registry where each migration is defined. Each migration should implement /// the "Migration" trait declared in this crate. -struct MM_001_AuthorMappingAddDeposit; +pub struct MM_001_AuthorMappingAddDeposit; impl Migration for MM_001_AuthorMappingAddDeposit { - fn friendly_name() -> str { + /* + fn friendly_name() -> &str { "AuthorMappingAddDeposit" } + */ } -struct MM_002_StakingFixTotalBalance; -impl Migration for StakingFixTotalBalance { - fn friendly_name() -> str { +pub struct MM_002_StakingFixTotalBalance; +impl Migration for MM_002_StakingFixTotalBalance { + /* + fn friendly_name() -> &str { "StakingFixTotalBalance" } + */ } -struct MM_003_StakingTransitionBoundedSet; // TODO: better name +pub struct MM_003_StakingTransitionBoundedSet; // TODO: better name impl Migration for MM_003_StakingTransitionBoundedSet { - fn friendly_name() -> str { + /* + fn friendly_name() -> &str { "StakingTransitionBoundedSet" } + */ } From cdc042aef9d6d68d65df136dbb9357b69e91de13 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 25 Jun 2021 11:27:53 -0600 Subject: [PATCH 04/99] Squelch warnings --- pallets/migrations/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index ef265655a0..45c5b3aeb4 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -16,6 +16,8 @@ //! # Migration Pallet +#![allow(non_camel_case_types)] + #![cfg_attr(not(feature = "std"), no_std)] use frame_support::pallet; @@ -34,6 +36,7 @@ pub mod pallet { use super::*; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; + #[allow(unused_imports)] // TODO: why does it detect this as unused? use sp_std::prelude::*; #[pallet::pallet] @@ -59,7 +62,6 @@ pub mod pallet { } #[pallet::event] - #[pallet::generate_deposit(pub(crate) fn deposit_event)] pub enum Event { // e.g. runtime upgrade started, completed, etc. } From c4204b4fb2ab701e0613ff110540497a3a1058d6 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 25 Jun 2021 11:52:35 -0600 Subject: [PATCH 05/99] Sketch out process_runtime_upgrades --- pallets/migrations/src/lib.rs | 55 +++++++++++++++++++++++----- pallets/migrations/src/migrations.rs | 9 +++++ 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 45c5b3aeb4..55cd14021a 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -20,15 +20,18 @@ #![cfg_attr(not(feature = "std"), no_std)] -use frame_support::pallet; +use frame_support::{pallet, weights::Weight}; pub mod migrations; /// A Migration that must happen on-chain upon a runtime-upgrade pub trait Migration { - // TODO: this would involve some metadata about the migration as well as a means of calling - // the actual migration function + /// A human-readable name for this migration. Also used as storage key. + fn friendly_name() -> &str; - // fn friendly_name() -> &str; + /// Apply this migration. Will be called exactly once for this migration. + /// TODO: refactor to support multi-block migrations (or, alternatively, allow each Migration + /// to specify whether it requires this support and provide a different path) + fn apply() -> Weight; } #[pallet] @@ -76,6 +79,8 @@ pub mod pallet { /// In the event that a migration is expected to take more than one block, ongoing migration /// work could continue from block-to-block in this pallet's on_initialize function. fn on_runtime_upgrade() -> Weight { + log::warn!("Performing on_runtime_upgrade"); + // start by flagging that we are not fully upgraded >::put(false); @@ -95,10 +100,42 @@ pub mod pallet { /// True if all required migrations have completed type FullyUpgraded = StorageValue<_, bool, ValueQuery>; - fn process_runtime_upgrades() -> Weight { - // TODO: iterate over MIGRATIONS here and ensure that each one has been fully applied. - // additionally, write to storage about our progress if multi-block-update functionality - // is required. - 0u64.into() + #[pallet::storage] + #[pallet::getter(fn migration_state)] + /// MigrationState tracks the status of each migration + type MigrationState = StorageMap< + _, + Twox64Concat, + str, + bool, // whether it's been applied or not -- TODO: use struct or enum + OptionQuery, // TODO: what is this...? + >; + + fn process_runtime_upgrades() -> Weight { + log::info!("stepping runtime upgrade"); + + let weight: Weight = 0u64.into(); + + for migration in MIGRATIONS { + + // let migration_name = migration.friendly_name(); + let migration_name = "TODO"; // fix fn signature in trait... + log::trace!("evaluating migration {}", migration_name); + + let migration_state = >::get(migration_name); + if ! migration_state { + + + // Apply the migration. Here we assume that this can fit within our current block, + // but this could me modified to step through a migration across blocks until it + // is done. + weight += migration.apply(); + + >::insert(migration_name, true); + } + + } + + weight } } diff --git a/pallets/migrations/src/migrations.rs b/pallets/migrations/src/migrations.rs index 126c3434af..55ff486a23 100644 --- a/pallets/migrations/src/migrations.rs +++ b/pallets/migrations/src/migrations.rs @@ -28,6 +28,9 @@ impl Migration for MM_001_AuthorMappingAddDeposit { "AuthorMappingAddDeposit" } */ + fn apply() -> Weight { + 0u64.into() + } } pub struct MM_002_StakingFixTotalBalance; @@ -37,6 +40,9 @@ impl Migration for MM_002_StakingFixTotalBalance { "StakingFixTotalBalance" } */ + fn apply() -> Weight { + 0u64.into() + } } pub struct MM_003_StakingTransitionBoundedSet; // TODO: better name @@ -46,4 +52,7 @@ impl Migration for MM_003_StakingTransitionBoundedSet { "StakingTransitionBoundedSet" } */ + fn apply() -> Weight { + 0u64.into() + } } From 0cc885df4a35ed49ae8d428869bf2673daca3b0e Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 25 Jun 2021 11:55:06 -0600 Subject: [PATCH 06/99] Leave note for reviewers --- pallets/migrations/src/migrations.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pallets/migrations/src/migrations.rs b/pallets/migrations/src/migrations.rs index 55ff486a23..bca6d3ef61 100644 --- a/pallets/migrations/src/migrations.rs +++ b/pallets/migrations/src/migrations.rs @@ -29,6 +29,8 @@ impl Migration for MM_001_AuthorMappingAddDeposit { } */ fn apply() -> Weight { + /// reviewer note: this isn't meant to imply that migration code must live here. As noted + /// elsewhere, I would expect migration code to live close to the pallet it affects. 0u64.into() } } From 95308a15ca3fc2e162c82d92d0dc5f5c4acd7422 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 28 Jun 2021 15:23:48 -0600 Subject: [PATCH 07/99] Add &self to Migrations trait fns --- pallets/migrations/src/lib.rs | 18 +++++++++--------- pallets/migrations/src/migrations.rs | 18 ++++++------------ 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 55cd14021a..c8c681c4de 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -26,14 +26,21 @@ pub mod migrations; /// A Migration that must happen on-chain upon a runtime-upgrade pub trait Migration { /// A human-readable name for this migration. Also used as storage key. - fn friendly_name() -> &str; + fn friendly_name(&self) -> &str; /// Apply this migration. Will be called exactly once for this migration. /// TODO: refactor to support multi-block migrations (or, alternatively, allow each Migration /// to specify whether it requires this support and provide a different path) - fn apply() -> Weight; + fn apply(&self) -> Weight; } +/// Our list of migrations. Any ordering considerations can be specified here (?). +const MIGRATIONS: [&dyn Migration; 3] = [ + &migrations::MM_001_AuthorMappingAddDeposit {}, + &migrations::MM_002_StakingFixTotalBalance {}, + &migrations::MM_003_StakingTransitionBoundedSet {}, +]; + #[pallet] pub mod pallet { use super::*; @@ -45,13 +52,6 @@ pub mod pallet { #[pallet::pallet] pub struct Pallet(PhantomData); - /// Our list of migrations. Any ordering considerations can be specified here (?). - const MIGRATIONS: [&dyn Migration; 3] = [ - &migrations::MM_001_AuthorMappingAddDeposit {}, - &migrations::MM_002_StakingFixTotalBalance {}, - &migrations::MM_003_StakingTransitionBoundedSet {}, - ]; - /// Configuration trait of this pallet. #[pallet::config] pub trait Config: frame_system::Config { diff --git a/pallets/migrations/src/migrations.rs b/pallets/migrations/src/migrations.rs index bca6d3ef61..f2f7677386 100644 --- a/pallets/migrations/src/migrations.rs +++ b/pallets/migrations/src/migrations.rs @@ -23,12 +23,10 @@ use crate::*; pub struct MM_001_AuthorMappingAddDeposit; impl Migration for MM_001_AuthorMappingAddDeposit { - /* - fn friendly_name() -> &str { + fn friendly_name(&self) -> &str { "AuthorMappingAddDeposit" } - */ - fn apply() -> Weight { + fn apply(&self) -> Weight { /// reviewer note: this isn't meant to imply that migration code must live here. As noted /// elsewhere, I would expect migration code to live close to the pallet it affects. 0u64.into() @@ -37,24 +35,20 @@ impl Migration for MM_001_AuthorMappingAddDeposit { pub struct MM_002_StakingFixTotalBalance; impl Migration for MM_002_StakingFixTotalBalance { - /* - fn friendly_name() -> &str { + fn friendly_name(&self) -> &str { "StakingFixTotalBalance" } - */ - fn apply() -> Weight { + fn apply(&self) -> Weight { 0u64.into() } } pub struct MM_003_StakingTransitionBoundedSet; // TODO: better name impl Migration for MM_003_StakingTransitionBoundedSet { - /* - fn friendly_name() -> &str { + fn friendly_name(&self) -> &str { "StakingTransitionBoundedSet" } - */ - fn apply() -> Weight { + fn apply(&self) -> Weight { 0u64.into() } } From 93b96cecc71291c389d47be0d97b5de2c98989ef Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 28 Jun 2021 15:41:59 -0600 Subject: [PATCH 08/99] Make it compile --- pallets/migrations/src/lib.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index c8c681c4de..e184669f7b 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -86,7 +86,7 @@ pub mod pallet { let mut weight: Weight = 0u64.into(); - weight += process_runtime_upgrades(); + weight += process_runtime_upgrades::(); // now flag that we are done with our runtime upgrade >::put(true); @@ -106,7 +106,7 @@ pub mod pallet { type MigrationState = StorageMap< _, Twox64Concat, - str, + String, bool, // whether it's been applied or not -- TODO: use struct or enum OptionQuery, // TODO: what is this...? >; @@ -114,15 +114,16 @@ pub mod pallet { fn process_runtime_upgrades() -> Weight { log::info!("stepping runtime upgrade"); - let weight: Weight = 0u64.into(); + let mut weight: Weight = 0u64.into(); - for migration in MIGRATIONS { + for migration in &MIGRATIONS { // let migration_name = migration.friendly_name(); let migration_name = "TODO"; // fix fn signature in trait... log::trace!("evaluating migration {}", migration_name); - let migration_state = >::get(migration_name); + let migration_state = >::get(migration_name) + .unwrap_or(false); if ! migration_state { From 275531c0bbe81d14ad595a10633a1daba86fd642 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 28 Jun 2021 16:49:35 -0600 Subject: [PATCH 09/99] Refactor migrations design to use stepping instead of one-shot --- pallets/migrations/src/lib.rs | 51 ++++++++++++++++++++-------- pallets/migrations/src/migrations.rs | 12 +++---- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index e184669f7b..b734ab1b3c 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -21,6 +21,7 @@ #![cfg_attr(not(feature = "std"), no_std)] use frame_support::{pallet, weights::Weight}; +use sp_runtime::Perbill; pub mod migrations; /// A Migration that must happen on-chain upon a runtime-upgrade @@ -28,10 +29,20 @@ pub trait Migration { /// A human-readable name for this migration. Also used as storage key. fn friendly_name(&self) -> &str; - /// Apply this migration. Will be called exactly once for this migration. - /// TODO: refactor to support multi-block migrations (or, alternatively, allow each Migration - /// to specify whether it requires this support and provide a different path) - fn apply(&self) -> Weight; + /// Step through this migration, taking up to `available_weight` of execution time and providing + /// a status on the progress as well as the consumed weight. This allows a migration to perform + /// its logic in small batches across as many blocks as needed. + /// + /// Implementations should perform as much migration work as possible and then leave their + /// pallet in a valid state from which another 'step' of migration work can be performed. In no + /// case should a step consume more than `available_weight`. + /// + /// This should return a perbill indicating the aggregate progress of the migration. If + /// `Perbill::one()` is returned, the migration is considered complete and no further calls to + /// `step()` will be made. Any value less than `Perbill::one()` will result in another future + /// call to `step()`. Indeed, values < 1 are arbitrary, but the intent is to indicate progress + /// (so they should at least be monotonically increasing). + fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight); } /// Our list of migrations. Any ordering considerations can be specified here (?). @@ -102,37 +113,49 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn migration_state)] - /// MigrationState tracks the status of each migration + /// MigrationState tracks the progress of a migration. Migrations with progress < 1 type MigrationState = StorageMap< _, Twox64Concat, String, - bool, // whether it's been applied or not -- TODO: use struct or enum + Perbill, OptionQuery, // TODO: what is this...? >; fn process_runtime_upgrades() -> Weight { log::info!("stepping runtime upgrade"); + // TODO: query proper value or make configurable + let available_weight = 500_000_000_000u64.into(); let mut weight: Weight = 0u64.into(); for migration in &MIGRATIONS { // let migration_name = migration.friendly_name(); - let migration_name = "TODO"; // fix fn signature in trait... + let migration_name = migration.friendly_name(); log::trace!("evaluating migration {}", migration_name); let migration_state = >::get(migration_name) - .unwrap_or(false); - if ! migration_state { + .unwrap_or(Perbill::zero()); + if migration_state < Perbill::one() { - // Apply the migration. Here we assume that this can fit within our current block, - // but this could me modified to step through a migration across blocks until it - // is done. - weight += migration.apply(); + let available_for_step = available_weight - weight; - >::insert(migration_name, true); + // perform a step of this migration + let (updated_progress, consumed_weight) + = migration.step(migration_state, available_weight); + + weight += consumed_weight; + if weight > available_weight { + // TODO: the intent here is to complain obnoxiously so that this is caught + // during development. In production, this should probably be tolerated because + // failing is catastrophic. + log::error!("Migration {} consumed more weight than it was given! ({} > {})", + migration_name, consumed_weight, available_for_step); + } + + >::insert(migration_name, updated_progress); } } diff --git a/pallets/migrations/src/migrations.rs b/pallets/migrations/src/migrations.rs index f2f7677386..03bc69b1c1 100644 --- a/pallets/migrations/src/migrations.rs +++ b/pallets/migrations/src/migrations.rs @@ -26,10 +26,10 @@ impl Migration for MM_001_AuthorMappingAddDeposit { fn friendly_name(&self) -> &str { "AuthorMappingAddDeposit" } - fn apply(&self) -> Weight { + fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { /// reviewer note: this isn't meant to imply that migration code must live here. As noted /// elsewhere, I would expect migration code to live close to the pallet it affects. - 0u64.into() + (Perbill::one(), 0u64.into()) } } @@ -38,8 +38,8 @@ impl Migration for MM_002_StakingFixTotalBalance { fn friendly_name(&self) -> &str { "StakingFixTotalBalance" } - fn apply(&self) -> Weight { - 0u64.into() + fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { + (Perbill::one(), 0u64.into()) } } @@ -48,7 +48,7 @@ impl Migration for MM_003_StakingTransitionBoundedSet { fn friendly_name(&self) -> &str { "StakingTransitionBoundedSet" } - fn apply(&self) -> Weight { - 0u64.into() + fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { + (Perbill::one(), 0u64.into()) } } From d5abf1ffdc20ed840f599b71a0f00122d4e433ac Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 28 Jun 2021 16:58:07 -0600 Subject: [PATCH 10/99] Fix typo/bug --- pallets/migrations/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index b734ab1b3c..82775f00b1 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -141,10 +141,12 @@ pub mod pallet { if migration_state < Perbill::one() { let available_for_step = available_weight - weight; + log::trace!("stepping migration {}, prev: {}, avail weight: {}", + migration_name, migration_state, available_for_step); // perform a step of this migration let (updated_progress, consumed_weight) - = migration.step(migration_state, available_weight); + = migration.step(migration_state, available_for_step); weight += consumed_weight; if weight > available_weight { From d25ffc17d46f61337038041d65c71685af6db87d Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 28 Jun 2021 17:04:15 -0600 Subject: [PATCH 11/99] Track overall migration doneness --- pallets/migrations/src/lib.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 82775f00b1..b81ac45720 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -99,9 +99,6 @@ pub mod pallet { weight += process_runtime_upgrades::(); - // now flag that we are done with our runtime upgrade - >::put(true); - weight.into() } } @@ -113,7 +110,7 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn migration_state)] - /// MigrationState tracks the progress of a migration. Migrations with progress < 1 + /// MigrationState tracks the progress of a migration. type MigrationState = StorageMap< _, Twox64Concat, @@ -128,6 +125,7 @@ pub mod pallet { // TODO: query proper value or make configurable let available_weight = 500_000_000_000u64.into(); let mut weight: Weight = 0u64.into(); + let mut done: bool = true; for migration in &MIGRATIONS { @@ -141,7 +139,7 @@ pub mod pallet { if migration_state < Perbill::one() { let available_for_step = available_weight - weight; - log::trace!("stepping migration {}, prev: {}, avail weight: {}", + log::trace!("stepping migration {}, prev: {:?}, avail weight: {}", migration_name, migration_state, available_for_step); // perform a step of this migration @@ -157,11 +155,20 @@ pub mod pallet { migration_name, consumed_weight, available_for_step); } + // make note of any unfinished migrations + if updated_progress < Perbill::one() { + done = false; + } + >::insert(migration_name, updated_progress); } } + if done { + >::put(true); + } + weight } } From 0c9ce740d965c7056cb65b4906231f6289cdc634 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 28 Jun 2021 17:05:35 -0600 Subject: [PATCH 12/99] Optimize when progress remains unchanged --- pallets/migrations/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index b81ac45720..7112835faa 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -160,7 +160,9 @@ pub mod pallet { done = false; } - >::insert(migration_name, updated_progress); + if migration_state != updated_progress { + >::insert(migration_name, updated_progress); + } } } From 23837f8de399abc667de05b9426ab8385919f811 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 29 Jun 2021 08:52:38 -0600 Subject: [PATCH 13/99] Resolve compiler warnings --- pallets/migrations/src/migrations.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pallets/migrations/src/migrations.rs b/pallets/migrations/src/migrations.rs index 03bc69b1c1..07dc13f208 100644 --- a/pallets/migrations/src/migrations.rs +++ b/pallets/migrations/src/migrations.rs @@ -26,9 +26,9 @@ impl Migration for MM_001_AuthorMappingAddDeposit { fn friendly_name(&self) -> &str { "AuthorMappingAddDeposit" } - fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { - /// reviewer note: this isn't meant to imply that migration code must live here. As noted - /// elsewhere, I would expect migration code to live close to the pallet it affects. + fn step(&self, _previous_progress: Perbill, _available_weight: Weight) -> (Perbill, Weight) { + // reviewer note: this isn't meant to imply that migration code must live here. As noted + // elsewhere, I would expect migration code to live close to the pallet it affects. (Perbill::one(), 0u64.into()) } } @@ -38,7 +38,7 @@ impl Migration for MM_002_StakingFixTotalBalance { fn friendly_name(&self) -> &str { "StakingFixTotalBalance" } - fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { + fn step(&self, _previous_progress: Perbill, _available_weight: Weight) -> (Perbill, Weight) { (Perbill::one(), 0u64.into()) } } @@ -48,7 +48,7 @@ impl Migration for MM_003_StakingTransitionBoundedSet { fn friendly_name(&self) -> &str { "StakingTransitionBoundedSet" } - fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { + fn step(&self, _previous_progress: Perbill, _available_weight: Weight) -> (Perbill, Weight) { (Perbill::one(), 0u64.into()) } } From 85ecaa12eaae6e0afc58b08065b425dfded57e88 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 29 Jun 2021 10:26:30 -0600 Subject: [PATCH 14/99] Incremental progress on mock --- Cargo.lock | 1 + pallets/migrations/Cargo.toml | 5 ++ pallets/migrations/src/lib.rs | 44 +++++++++++-- pallets/migrations/src/mock.rs | 107 ++++++++++++++++++++++++++++++++ pallets/migrations/src/tests.rs | 32 ++++++++++ 5 files changed, 183 insertions(+), 6 deletions(-) create mode 100644 pallets/migrations/src/mock.rs create mode 100644 pallets/migrations/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index d4d55e568e..aaed615936 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4670,6 +4670,7 @@ dependencies = [ "frame-system", "log", "parity-scale-codec", + "sp-io", "sp-runtime", "sp-std", ] diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index 4a6cf5ba3f..96f5348ff4 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -13,6 +13,10 @@ sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } parity-scale-codec = { version = "2.0.0", default-features = false } +[dev-dependencies] +sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } + + [features] default = ["std"] std = [ @@ -20,4 +24,5 @@ std = [ "frame-system/std", "sp-std/std", "sp-runtime/std", + "sp-io/std", ] diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 7112835faa..a04a5b1410 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -20,10 +20,17 @@ #![cfg_attr(not(feature = "std"), no_std)] +#[cfg(test)] +mod mock; +#[cfg(test)] +mod tests; + use frame_support::{pallet, weights::Weight}; use sp_runtime::Perbill; pub mod migrations; +pub use pallet::*; + /// A Migration that must happen on-chain upon a runtime-upgrade pub trait Migration { /// A human-readable name for this migration. Also used as storage key. @@ -61,7 +68,7 @@ pub mod pallet { use sp_std::prelude::*; #[pallet::pallet] - pub struct Pallet(PhantomData); + pub struct Pallet(_); /// Configuration trait of this pallet. #[pallet::config] @@ -70,14 +77,14 @@ pub mod pallet { type Event: From> + IsType<::Event>; } - #[pallet::error] - pub enum Error { - // errors in this pallet would be quite bad... - } - #[pallet::event] pub enum Event { // e.g. runtime upgrade started, completed, etc. + RuntimeUpgradeStarted, + RuntimeUpgradeCompleted, + MigrationStarted(String), + MigrationProgress(String, Perbill), + MigrationCompleted(String), } #[pallet::hooks] @@ -119,6 +126,31 @@ pub mod pallet { OptionQuery, // TODO: what is this...? >; + #[pallet::genesis_config] + pub struct GenesisConfig { + pub completed_migrations: Vec, + pub dummy: PhantomData // TODO: + } + + #[cfg(feature = "std")] + impl Default for GenesisConfig { + fn default() -> Self { + Self { + completed_migrations: vec![], + dummy: PhantomData, + } + } + } + + #[pallet::genesis_build] + impl GenesisBuild for GenesisConfig { + fn build(&self) { + for migration_name in &self.completed_migrations { + >::insert(migration_name, Perbill::one()); + } + } + } + fn process_runtime_upgrades() -> Weight { log::info!("stepping runtime upgrade"); diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs new file mode 100644 index 0000000000..a97de84005 --- /dev/null +++ b/pallets/migrations/src/mock.rs @@ -0,0 +1,107 @@ +// Copyright 2019-2021 PureStake Inc. +// This file is part of Moonbeam. + +// Moonbeam is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Moonbeam is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Moonbeam. If not, see . + +//! A minimal runtime including the migrations pallet +use crate as pallet_migrations; +use frame_support::{construct_runtime, parameter_types, traits::GenesisBuild, weights::Weight}; +use sp_runtime::{Perbill, {traits::IdentityLookup}}; + +pub type AccountId = u64; +pub type Balance = u128; +pub type BlockNumber = u64; + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; + +// Configure a mock runtime to test the pallet. +construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, + } +); + +parameter_types! { + pub const BlockHashCount: u64 = 250; + pub const MaximumBlockWeight: Weight = 1024; + pub const MaximumBlockLength: u32 = 2 * 1024; + pub const AvailableBlockRatio: Perbill = Perbill::one(); +} +impl frame_system::Config for Test { + type BaseCallFilter = (); + type DbWeight = (); + type Origin = Origin; + type Index = u64; + type BlockNumber = BlockNumber; + type Call = Call; + type Hash = (); + type Hashing = (); + type AccountId = (); + type Lookup = IdentityLookup; + type Header = (); + type Event = Event; + type BlockHashCount = BlockHashCount; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type BlockWeights = (); + type BlockLength = (); + type SS58Prefix = (); + type OnSetCode = (); +} +impl pallet_migrations::Config for Test { +} + +/* +/// Externality builder for pallet migration's mock runtime +pub(crate) struct ExtBuilder { +} + +impl Default for ExtBuilder { + fn default() -> ExtBuilder { + ExtBuilder {} + } +} + +impl ExtBuilder { + + pub(crate) fn build(self) -> sp_io::TestExternalities { + let mut t = frame_system::GenesisConfig::default() + .build_storage::() + .expect("Frame system builds valid default genesis config"); + + pallet_migrations::GenesisConfig:: { + } + .assimilate_storage(&mut t) + .expect("Pallet migration's storage can be assimilated"); + + let mut ext = sp_io::TestExternalities::new(t); + ext.execute_with(|| System::set_block_number(1)); + ext + } +} + +pub(crate) fn last_event() -> Event { + System::events().pop().expect("Event expected").event +} +*/ diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs new file mode 100644 index 0000000000..bd9c95b5fe --- /dev/null +++ b/pallets/migrations/src/tests.rs @@ -0,0 +1,32 @@ +// Copyright 2019-2021 PureStake Inc. +// This file is part of Moonbeam. + +// Moonbeam is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Moonbeam is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Moonbeam. If not, see . + +/* +//! Unit testing +use crate::mock::{ + last_event, Migrations, Event as MetaEvent, ExtBuilder, Origin, System, Test, +}; + +#[test] +fn genesis_builder_works() { + ExtBuilder::default() + .build() + .execute_with(|| { + assert!(System::events().is_empty()); + }) +} +*/ + From f22ce8d354e84fff163798ab384d56e83be8af7c Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 29 Jun 2021 12:42:41 -0600 Subject: [PATCH 15/99] Mock is getting close --- Cargo.lock | 1 + pallets/migrations/Cargo.toml | 3 ++- pallets/migrations/src/lib.rs | 2 -- pallets/migrations/src/mock.rs | 20 +++++++++++++------- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aaed615936..e42ae9ebc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4670,6 +4670,7 @@ dependencies = [ "frame-system", "log", "parity-scale-codec", + "sp-core", "sp-io", "sp-runtime", "sp-std", diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index 96f5348ff4..2b27bad885 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -15,7 +15,7 @@ parity-scale-codec = { version = "2.0.0", default-features = false } [dev-dependencies] sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } - +sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } [features] default = ["std"] @@ -25,4 +25,5 @@ std = [ "sp-std/std", "sp-runtime/std", "sp-io/std", + "sp-core/std", ] diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index a04a5b1410..72f314917d 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -29,8 +29,6 @@ use frame_support::{pallet, weights::Weight}; use sp_runtime::Perbill; pub mod migrations; -pub use pallet::*; - /// A Migration that must happen on-chain upon a runtime-upgrade pub trait Migration { /// A human-readable name for this migration. Also used as storage key. diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index a97de84005..50bd7f89c0 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -16,8 +16,13 @@ //! A minimal runtime including the migrations pallet use crate as pallet_migrations; -use frame_support::{construct_runtime, parameter_types, traits::GenesisBuild, weights::Weight}; -use sp_runtime::{Perbill, {traits::IdentityLookup}}; +use frame_support::{construct_runtime, parameter_types, weights::Weight}; +use sp_core::H256; +use sp_runtime::{ + testing::Header, + traits::{BlakeTwo256, IdentityLookup}, + Perbill, +}; pub type AccountId = u64; pub type Balance = u128; @@ -51,11 +56,11 @@ impl frame_system::Config for Test { type Index = u64; type BlockNumber = BlockNumber; type Call = Call; - type Hash = (); - type Hashing = (); - type AccountId = (); + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = AccountId; type Lookup = IdentityLookup; - type Header = (); + type Header = Header; type Event = Event; type BlockHashCount = BlockHashCount; type Version = (); @@ -69,7 +74,8 @@ impl frame_system::Config for Test { type SS58Prefix = (); type OnSetCode = (); } -impl pallet_migrations::Config for Test { +impl Config for Test { + type Event: Event; } /* From d8d2c3cf5c79d2b9c8fe5e846443495c1d4ad680 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 29 Jun 2021 16:40:14 -0600 Subject: [PATCH 16/99] Make mock build --- pallets/migrations/src/lib.rs | 5 ++++- pallets/migrations/src/mock.rs | 15 ++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 72f314917d..8c0f70f393 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -29,6 +29,8 @@ use frame_support::{pallet, weights::Weight}; use sp_runtime::Perbill; pub mod migrations; +pub use pallet::*; + /// A Migration that must happen on-chain upon a runtime-upgrade pub trait Migration { /// A human-readable name for this migration. Also used as storage key. @@ -65,8 +67,9 @@ pub mod pallet { #[allow(unused_imports)] // TODO: why does it detect this as unused? use sp_std::prelude::*; + /// Pallet for migrations #[pallet::pallet] - pub struct Pallet(_); + pub struct Pallet(PhantomData); /// Configuration trait of this pallet. #[pallet::config] diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 50bd7f89c0..88462c4e5c 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -15,13 +15,17 @@ // along with Moonbeam. If not, see . //! A minimal runtime including the migrations pallet +use super::*; use crate as pallet_migrations; -use frame_support::{construct_runtime, parameter_types, weights::Weight}; +use frame_support::{ + construct_runtime, parameter_types, + weights::Weight, +}; use sp_core::H256; use sp_runtime::{ testing::Header, traits::{BlakeTwo256, IdentityLookup}, - Perbill, + Perbill, Percent, }; pub type AccountId = u64; @@ -39,7 +43,7 @@ construct_runtime!( UncheckedExtrinsic = UncheckedExtrinsic, { System: frame_system::{Pallet, Call, Config, Storage, Event}, - Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, + Migrations: pallet_migrations::{Pallet, Call, Storage, Config, Event}, } ); @@ -48,6 +52,7 @@ parameter_types! { pub const MaximumBlockWeight: Weight = 1024; pub const MaximumBlockLength: u32 = 2 * 1024; pub const AvailableBlockRatio: Perbill = Perbill::one(); + pub const SS58Prefix: u8 = 42; } impl frame_system::Config for Test { type BaseCallFilter = (); @@ -71,11 +76,11 @@ impl frame_system::Config for Test { type SystemWeightInfo = (); type BlockWeights = (); type BlockLength = (); - type SS58Prefix = (); + type SS58Prefix = SS58Prefix; type OnSetCode = (); } impl Config for Test { - type Event: Event; + type Event = Event; } /* From bfd24f60d2d3cd7665d13f4ff6a9e0d637cad8a3 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 29 Jun 2021 19:41:21 -0600 Subject: [PATCH 17/99] Plumb genesis building in mock --- pallets/migrations/src/mock.rs | 10 +++++++--- pallets/migrations/src/tests.rs | 2 -- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 88462c4e5c..d4dcf92f96 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -19,6 +19,7 @@ use super::*; use crate as pallet_migrations; use frame_support::{ construct_runtime, parameter_types, + traits::{GenesisBuild}, weights::Weight, }; use sp_core::H256; @@ -83,14 +84,16 @@ impl Config for Test { type Event = Event; } -/* /// Externality builder for pallet migration's mock runtime pub(crate) struct ExtBuilder { + completed_migrations: Vec, } impl Default for ExtBuilder { fn default() -> ExtBuilder { - ExtBuilder {} + ExtBuilder { + completed_migrations: vec![], + } } } @@ -102,6 +105,8 @@ impl ExtBuilder { .expect("Frame system builds valid default genesis config"); pallet_migrations::GenesisConfig:: { + completed_migrations: self.completed_migrations, + dummy: Default::default(), } .assimilate_storage(&mut t) .expect("Pallet migration's storage can be assimilated"); @@ -115,4 +120,3 @@ impl ExtBuilder { pub(crate) fn last_event() -> Event { System::events().pop().expect("Event expected").event } -*/ diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index bd9c95b5fe..107a8096b5 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -14,7 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Moonbeam. If not, see . -/* //! Unit testing use crate::mock::{ last_event, Migrations, Event as MetaEvent, ExtBuilder, Origin, System, Test, @@ -28,5 +27,4 @@ fn genesis_builder_works() { assert!(System::events().is_empty()); }) } -*/ From 9404fbed2c32ec8919ba5301153eaa557eebb7f2 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 29 Jun 2021 20:17:53 -0600 Subject: [PATCH 18/99] Baby's first tests --- pallets/migrations/src/mock.rs | 16 +++++++++++++++- pallets/migrations/src/tests.rs | 29 ++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index d4dcf92f96..db7097a9d1 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -19,7 +19,7 @@ use super::*; use crate as pallet_migrations; use frame_support::{ construct_runtime, parameter_types, - traits::{GenesisBuild}, + traits::{GenesisBuild, OnRuntimeUpgrade}, weights::Weight, }; use sp_core::H256; @@ -117,6 +117,20 @@ impl ExtBuilder { } } +pub(crate) fn events() -> Vec> { + System::events() + .into_iter() + .map(|r| r.event) + .filter_map(|e| { + if let Event::pallet_migrations(inner) = e { + Some(inner) + } else { + None + } + }) + .collect::>() +} + pub(crate) fn last_event() -> Event { System::events().pop().expect("Event expected").event } diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 107a8096b5..387ed8878c 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -16,7 +16,11 @@ //! Unit testing use crate::mock::{ - last_event, Migrations, Event as MetaEvent, ExtBuilder, Origin, System, Test, + last_event, events, Migrations, Event as MetaEvent, ExtBuilder, Origin, System, Test, +}; +use crate::Event; +use frame_support::{ + traits::{OnRuntimeUpgrade}, }; #[test] @@ -28,3 +32,26 @@ fn genesis_builder_works() { }) } +#[test] +fn on_runtime_upgrade_returns() { + ExtBuilder::default() + .build() + .execute_with(|| { + Migrations::on_runtime_upgrade(); + }) +} + +#[test] +fn on_runtime_upgrade_emits_events() { + ExtBuilder::default() + .build() + .execute_with(|| { + Migrations::on_runtime_upgrade(); + + let mut expected = vec![ + Event::RuntimeUpgradeStarted, + Event::RuntimeUpgradeCompleted, + ]; + assert_eq!(events(), expected); + }); +} From edee830e2ab0e8bd0c0316048ef3458bed80137f Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 29 Jun 2021 20:35:24 -0600 Subject: [PATCH 19/99] Fix events --- pallets/migrations/src/lib.rs | 7 +++++-- pallets/migrations/src/tests.rs | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 8c0f70f393..423544ba1c 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -79,10 +79,11 @@ pub mod pallet { } #[pallet::event] + #[pallet::generate_deposit(pub(crate) fn deposit_event)] pub enum Event { // e.g. runtime upgrade started, completed, etc. - RuntimeUpgradeStarted, - RuntimeUpgradeCompleted, + RuntimeUpgradeStarted(), + RuntimeUpgradeCompleted(), MigrationStarted(String), MigrationProgress(String, Perbill), MigrationCompleted(String), @@ -102,6 +103,7 @@ pub mod pallet { // start by flagging that we are not fully upgraded >::put(false); + Self::deposit_event(Event::RuntimeUpgradeStarted()); let mut weight: Weight = 0u64.into(); @@ -201,6 +203,7 @@ pub mod pallet { } if done { + >::deposit_event(Event::RuntimeUpgradeCompleted()); >::put(true); } diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 387ed8878c..e6e86514a3 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -49,8 +49,8 @@ fn on_runtime_upgrade_emits_events() { Migrations::on_runtime_upgrade(); let mut expected = vec![ - Event::RuntimeUpgradeStarted, - Event::RuntimeUpgradeCompleted, + Event::RuntimeUpgradeStarted(), + Event::RuntimeUpgradeCompleted(), ]; assert_eq!(events(), expected); }); From d13a471b21d5120feae4133c3fd84ac78e04bd79 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 30 Jun 2021 16:58:38 -0600 Subject: [PATCH 20/99] Use Vec instead of String --- pallets/migrations/src/lib.rs | 19 ++++++++++++------- pallets/migrations/src/mock.rs | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 423544ba1c..817ed12058 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -84,9 +84,9 @@ pub mod pallet { // e.g. runtime upgrade started, completed, etc. RuntimeUpgradeStarted(), RuntimeUpgradeCompleted(), - MigrationStarted(String), - MigrationProgress(String, Perbill), - MigrationCompleted(String), + MigrationStarted(Vec), + MigrationProgress(Vec, Perbill), + MigrationCompleted(Vec), } #[pallet::hooks] @@ -121,17 +121,18 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn migration_state)] /// MigrationState tracks the progress of a migration. + /// Maps name (Vec) -> migration progress (Perbill) type MigrationState = StorageMap< _, Twox64Concat, - String, + Vec, Perbill, OptionQuery, // TODO: what is this...? >; #[pallet::genesis_config] pub struct GenesisConfig { - pub completed_migrations: Vec, + pub completed_migrations: Vec>, pub dummy: PhantomData // TODO: } @@ -168,11 +169,15 @@ pub mod pallet { let migration_name = migration.friendly_name(); log::trace!("evaluating migration {}", migration_name); - let migration_state = >::get(migration_name) + let migration_state = >::get(migration_name.as_bytes()) .unwrap_or(Perbill::zero()); if migration_state < Perbill::one() { + // TODO: we don't currently have a reliable way to know "started" + // TODO: multiple calls to as_bytes() or to_vec() may be expensive + >::deposit_event(Event::MigrationStarted(migration_name.as_bytes().to_vec())); + let available_for_step = available_weight - weight; log::trace!("stepping migration {}, prev: {:?}, avail weight: {}", migration_name, migration_state, available_for_step); @@ -196,7 +201,7 @@ pub mod pallet { } if migration_state != updated_progress { - >::insert(migration_name, updated_progress); + >::insert(migration_name.as_bytes(), updated_progress); } } diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index db7097a9d1..0fba9d159f 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -86,7 +86,7 @@ impl Config for Test { /// Externality builder for pallet migration's mock runtime pub(crate) struct ExtBuilder { - completed_migrations: Vec, + completed_migrations: Vec>, } impl Default for ExtBuilder { From 986433e44680d8e6515c15fcd242e3f41f4afa92 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 1 Jul 2021 06:05:19 -0600 Subject: [PATCH 21/99] Make MigrationsList part of pallet config; plumb through Moonbase runtime --- node/service/src/chain_spec/moonbase.rs | 1 + pallets/migrations/Cargo.toml | 2 +- pallets/migrations/src/lib.rs | 13 ++++------ pallets/migrations/src/mock.rs | 15 ++++++++++++ runtime/common/Cargo.toml | 13 +++++++++- runtime/common/src/lib.rs | 1 + .../common}/src/migrations.rs | 24 +++++++++++++++++-- runtime/moonbase/Cargo.toml | 2 ++ runtime/moonbase/src/lib.rs | 7 ++++++ 9 files changed, 65 insertions(+), 13 deletions(-) rename {pallets/migrations => runtime/common}/src/migrations.rs (73%) diff --git a/node/service/src/chain_spec/moonbase.rs b/node/service/src/chain_spec/moonbase.rs index 8fced8c49c..53d71b004b 100644 --- a/node/service/src/chain_spec/moonbase.rs +++ b/node/service/src/chain_spec/moonbase.rs @@ -241,6 +241,7 @@ pub fn testnet_genesis( .collect(), }, pallet_treasury: Default::default(), + pallet_migrations: Default::default(), } } diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index 2b27bad885..fc926eef4f 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "migrations" +name = "pallet-migrations" version = "0.1.0" authors = ["PureStake"] edition = "2018" diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 817ed12058..c92620d3ef 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -27,7 +27,6 @@ mod tests; use frame_support::{pallet, weights::Weight}; use sp_runtime::Perbill; -pub mod migrations; pub use pallet::*; @@ -52,13 +51,6 @@ pub trait Migration { fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight); } -/// Our list of migrations. Any ordering considerations can be specified here (?). -const MIGRATIONS: [&dyn Migration; 3] = [ - &migrations::MM_001_AuthorMappingAddDeposit {}, - &migrations::MM_002_StakingFixTotalBalance {}, - &migrations::MM_003_StakingTransitionBoundedSet {}, -]; - #[pallet] pub mod pallet { use super::*; @@ -76,6 +68,8 @@ pub mod pallet { pub trait Config: frame_system::Config { /// Overarching event type type Event: From> + IsType<::Event>; + /// The list of migrations that will be performed + type MigrationsList: Get>>; } #[pallet::event] @@ -163,7 +157,7 @@ pub mod pallet { let mut weight: Weight = 0u64.into(); let mut done: bool = true; - for migration in &MIGRATIONS { + for migration in &T::MigrationsList::get() { // let migration_name = migration.friendly_name(); let migration_name = migration.friendly_name(); @@ -183,6 +177,7 @@ pub mod pallet { migration_name, migration_state, available_for_step); // perform a step of this migration + >::deposit_event(Event::MigrationStarted(migration_name.into())); let (updated_progress, consumed_weight) = migration.step(migration_state, available_for_step); diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 0fba9d159f..3f87288d83 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -17,7 +17,9 @@ //! A minimal runtime including the migrations pallet use super::*; use crate as pallet_migrations; +use crate::migrations; use frame_support::{ + pallet_prelude::*, construct_runtime, parameter_types, traits::{GenesisBuild, OnRuntimeUpgrade}, weights::Weight, @@ -80,8 +82,21 @@ impl frame_system::Config for Test { type SS58Prefix = SS58Prefix; type OnSetCode = (); } + +pub struct MockMigrations; +impl Get>> for MockMigrations { + fn get() -> Vec> { + vec![ + Box::new(migrations::MM_001_AuthorMappingAddDeposit), + Box::new(migrations::MM_002_StakingFixTotalBalance), + Box::new(migrations::MM_003_StakingTransitionBoundedSet), + ] + } +} + impl Config for Test { type Event = Event; + type MigrationsList = MockMigrations; } /// Externality builder for pallet migration's mock runtime diff --git a/runtime/common/Cargo.toml b/runtime/common/Cargo.toml index 7333a3bf37..82915bc4d8 100644 --- a/runtime/common/Cargo.toml +++ b/runtime/common/Cargo.toml @@ -7,5 +7,16 @@ version = '0.8.0-dev' authors = ["PureStake"] edition = '2018' +[dependencies] +pallet-migrations = { path = "../../pallets/migrations", default-features = false } +sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } +sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } +frame-support = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-v0.9.4" } + [features] -std = [] \ No newline at end of file +std = [ + "pallet-migrations/std", + "sp-runtime/std", + "sp-std/std", + "frame-support/std", +] diff --git a/runtime/common/src/lib.rs b/runtime/common/src/lib.rs index 3ae7fa674b..391f508452 100644 --- a/runtime/common/src/lib.rs +++ b/runtime/common/src/lib.rs @@ -17,3 +17,4 @@ #![cfg_attr(not(feature = "std"), no_std)] mod apis; +pub mod migrations; diff --git a/pallets/migrations/src/migrations.rs b/runtime/common/src/migrations.rs similarity index 73% rename from pallets/migrations/src/migrations.rs rename to runtime/common/src/migrations.rs index 07dc13f208..89a94eb5b0 100644 --- a/pallets/migrations/src/migrations.rs +++ b/runtime/common/src/migrations.rs @@ -16,10 +16,16 @@ //! # Migrations -use crate::*; +use frame_support::{ + weights::Weight, + pallet_prelude::Get, +}; +use pallet_migrations::Migration; +use sp_runtime::Perbill; +use sp_std::prelude::*; /// This module acts as a registry where each migration is defined. Each migration should implement -/// the "Migration" trait declared in this crate. +/// the "Migration" trait declared in the pallet-migrations crate. pub struct MM_001_AuthorMappingAddDeposit; impl Migration for MM_001_AuthorMappingAddDeposit { @@ -52,3 +58,17 @@ impl Migration for MM_003_StakingTransitionBoundedSet { (Perbill::one(), 0u64.into()) } } + +pub struct CommonMigrations; +impl Get>> for CommonMigrations { + fn get() -> Vec> { + // TODO: this is a lot of allocation to do upon every get() call. this *should* be avoided + // except when pallet_migrations undergoes a runtime upgrade -- but TODO: review + vec![ + Box::new(MM_001_AuthorMappingAddDeposit), + // Box::new(migrations::MM_002_StakingFixTotalBalance), + // Box::new(migrations::MM_003_StakingTransitionBoundedSet), + ] + } +} + diff --git a/runtime/moonbase/Cargo.toml b/runtime/moonbase/Cargo.toml index ce4ec44c11..c1c6c2a52d 100644 --- a/runtime/moonbase/Cargo.toml +++ b/runtime/moonbase/Cargo.toml @@ -23,6 +23,7 @@ pallet-ethereum-chain-id = { path = "../../pallets/ethereum-chain-id", default-f parachain-staking = { path = "../../pallets/parachain-staking", default-features = false } parachain-staking-precompiles = { path = "../../precompiles/parachain-staking", default-features = false } pallet-author-slot-filter = { git = "https://github.com/purestake/cumulus", branch = "nimbus-polkadot-v0.9.4", default-features = false } +pallet-migrations = { path = "../../pallets/migrations", default-features = false } nimbus-primitives = { git = "https://github.com/purestake/cumulus", branch = "nimbus-polkadot-v0.9.4", default-features = false } pallet-author-mapping = { path = "../../pallets/author-mapping", default-features = false } evm = { version="0.27.0", default-features=false, features=["with-codec"] } @@ -144,6 +145,7 @@ std = [ "parachain-staking/std", "parachain-staking-precompiles/std", "pallet-author-slot-filter/std", + "pallet-migrations/std", "pallet-crowdloan-rewards/std", "frame-benchmarking/std", "pallet-society/std", diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index a53f1a7376..7011bc954a 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -53,6 +53,7 @@ use pallet_evm::{ }; use pallet_transaction_payment::CurrencyAdapter; pub use parachain_staking::{InflationInfo, Range}; +use pallet_migrations::*; use parity_scale_codec::{Decode, Encode}; use sp_api::impl_runtime_apis; use sp_core::{u32_trait::*, OpaqueMetadata, H160, H256, U256}; @@ -708,6 +709,11 @@ impl pallet_proxy::Config for Runtime { type AnnouncementDepositFactor = AnnouncementDepositFactor; } +impl Config for Runtime { + type Event = Event; + type MigrationsList = runtime_common::migrations::CommonMigrations; +} + construct_runtime! { pub enum Runtime where Block = Block, @@ -739,6 +745,7 @@ construct_runtime! { CrowdloanRewards: pallet_crowdloan_rewards::{Pallet, Call, Config, Storage, Event}, AuthorMapping: pallet_author_mapping::{Pallet, Call, Config, Storage, Event}, Proxy: pallet_proxy::{Pallet, Call, Storage, Event}, + Migrations: pallet_migrations::{Pallet, Call, Storage, Config, Event}, } } From 92047ee1da4883dc2432a4e02121c421e6f6ea77 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 1 Jul 2021 06:15:13 -0600 Subject: [PATCH 22/99] Appease the compiler --- Cargo.lock | 35 +++++++++++++++++++------------- pallets/migrations/src/mock.rs | 12 ++--------- pallets/migrations/src/tests.rs | 4 ++-- runtime/common/src/migrations.rs | 3 +++ 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e42ae9ebc1..250f039553 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4662,20 +4662,6 @@ dependencies = [ "thrift", ] -[[package]] -name = "migrations" -version = "0.1.0" -dependencies = [ - "frame-support", - "frame-system", - "log", - "parity-scale-codec", - "sp-core", - "sp-io", - "sp-runtime", - "sp-std", -] - [[package]] name = "minicbor" version = "0.8.1" @@ -4822,6 +4808,7 @@ dependencies = [ "pallet-evm-precompile-modexp", "pallet-evm-precompile-sha3fips", "pallet-evm-precompile-simple", + "pallet-migrations", "pallet-proxy", "pallet-randomness-collective-flip", "pallet-scheduler", @@ -6274,6 +6261,20 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-migrations" +version = "0.1.0" +dependencies = [ + "frame-support", + "frame-system", + "log", + "parity-scale-codec", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-mmr" version = "3.0.0" @@ -9002,6 +9003,12 @@ dependencies = [ [[package]] name = "runtime-common" version = "0.8.0-dev" +dependencies = [ + "frame-support", + "pallet-migrations", + "sp-runtime", + "sp-std", +] [[package]] name = "rust-argon2" diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 3f87288d83..1a302da4c9 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -17,22 +17,20 @@ //! A minimal runtime including the migrations pallet use super::*; use crate as pallet_migrations; -use crate::migrations; use frame_support::{ pallet_prelude::*, construct_runtime, parameter_types, - traits::{GenesisBuild, OnRuntimeUpgrade}, + traits::GenesisBuild, weights::Weight, }; use sp_core::H256; use sp_runtime::{ testing::Header, traits::{BlakeTwo256, IdentityLookup}, - Perbill, Percent, + Perbill, }; pub type AccountId = u64; -pub type Balance = u128; pub type BlockNumber = u64; type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; @@ -87,9 +85,6 @@ pub struct MockMigrations; impl Get>> for MockMigrations { fn get() -> Vec> { vec![ - Box::new(migrations::MM_001_AuthorMappingAddDeposit), - Box::new(migrations::MM_002_StakingFixTotalBalance), - Box::new(migrations::MM_003_StakingTransitionBoundedSet), ] } } @@ -146,6 +141,3 @@ pub(crate) fn events() -> Vec> { .collect::>() } -pub(crate) fn last_event() -> Event { - System::events().pop().expect("Event expected").event -} diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index e6e86514a3..e84e80caba 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -16,7 +16,7 @@ //! Unit testing use crate::mock::{ - last_event, events, Migrations, Event as MetaEvent, ExtBuilder, Origin, System, Test, + events, Migrations, ExtBuilder, System, }; use crate::Event; use frame_support::{ @@ -48,7 +48,7 @@ fn on_runtime_upgrade_emits_events() { .execute_with(|| { Migrations::on_runtime_upgrade(); - let mut expected = vec![ + let expected = vec![ Event::RuntimeUpgradeStarted(), Event::RuntimeUpgradeCompleted(), ]; diff --git a/runtime/common/src/migrations.rs b/runtime/common/src/migrations.rs index 89a94eb5b0..bb1079d400 100644 --- a/runtime/common/src/migrations.rs +++ b/runtime/common/src/migrations.rs @@ -27,6 +27,7 @@ use sp_std::prelude::*; /// This module acts as a registry where each migration is defined. Each migration should implement /// the "Migration" trait declared in the pallet-migrations crate. +#[allow(non_camel_case_types)] pub struct MM_001_AuthorMappingAddDeposit; impl Migration for MM_001_AuthorMappingAddDeposit { fn friendly_name(&self) -> &str { @@ -39,6 +40,7 @@ impl Migration for MM_001_AuthorMappingAddDeposit { } } +#[allow(non_camel_case_types)] pub struct MM_002_StakingFixTotalBalance; impl Migration for MM_002_StakingFixTotalBalance { fn friendly_name(&self) -> &str { @@ -49,6 +51,7 @@ impl Migration for MM_002_StakingFixTotalBalance { } } +#[allow(non_camel_case_types)] pub struct MM_003_StakingTransitionBoundedSet; // TODO: better name impl Migration for MM_003_StakingTransitionBoundedSet { fn friendly_name(&self) -> &str { From b885fc5d0e419ae45e118a673587b7d571c9fcdd Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 1 Jul 2021 06:21:26 -0600 Subject: [PATCH 23/99] Fix up CommonMigrations list --- runtime/common/src/migrations.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/runtime/common/src/migrations.rs b/runtime/common/src/migrations.rs index bb1079d400..caab568e07 100644 --- a/runtime/common/src/migrations.rs +++ b/runtime/common/src/migrations.rs @@ -52,10 +52,10 @@ impl Migration for MM_002_StakingFixTotalBalance { } #[allow(non_camel_case_types)] -pub struct MM_003_StakingTransitionBoundedSet; // TODO: better name -impl Migration for MM_003_StakingTransitionBoundedSet { +pub struct MM_003_StakingUnboundedCollatorNominations; +impl Migration for MM_003_StakingUnboundedCollatorNominations { fn friendly_name(&self) -> &str { - "StakingTransitionBoundedSet" + "StakingUnboundedCollatorNominations" } fn step(&self, _previous_progress: Perbill, _available_weight: Weight) -> (Perbill, Weight) { (Perbill::one(), 0u64.into()) @@ -69,8 +69,8 @@ impl Get>> for CommonMigrations { // except when pallet_migrations undergoes a runtime upgrade -- but TODO: review vec![ Box::new(MM_001_AuthorMappingAddDeposit), - // Box::new(migrations::MM_002_StakingFixTotalBalance), - // Box::new(migrations::MM_003_StakingTransitionBoundedSet), + Box::new(MM_002_StakingFixTotalBalance), + Box::new(MM_003_StakingUnboundedCollatorNominations), ] } } From aefb302a3d595479a0eafbdc2c928d011e854c4e Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 1 Jul 2021 06:36:55 -0600 Subject: [PATCH 24/99] Remove comment --- pallets/migrations/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index c92620d3ef..ad34b3ac61 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -121,7 +121,7 @@ pub mod pallet { Twox64Concat, Vec, Perbill, - OptionQuery, // TODO: what is this...? + OptionQuery, >; #[pallet::genesis_config] From f9f8a2a1239b15a359a90680c9f7b5a0cc9c6234 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 1 Jul 2021 07:21:48 -0600 Subject: [PATCH 25/99] Cargo fmt --- pallets/migrations/src/lib.rs | 49 ++++++++++++++++---------------- pallets/migrations/src/mock.rs | 10 ++----- pallets/migrations/src/tests.rs | 44 +++++++++++----------------- runtime/common/src/migrations.rs | 6 +--- runtime/moonbase/src/lib.rs | 2 +- 5 files changed, 45 insertions(+), 66 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index ad34b3ac61..1150eee1bc 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -17,7 +17,6 @@ //! # Migration Pallet #![allow(non_camel_case_types)] - #![cfg_attr(not(feature = "std"), no_std)] #[cfg(test)] @@ -37,8 +36,8 @@ pub trait Migration { /// Step through this migration, taking up to `available_weight` of execution time and providing /// a status on the progress as well as the consumed weight. This allows a migration to perform - /// its logic in small batches across as many blocks as needed. - /// + /// its logic in small batches across as many blocks as needed. + /// /// Implementations should perform as much migration work as possible and then leave their /// pallet in a valid state from which another 'step' of migration work can be performed. In no /// case should a step consume more than `available_weight`. @@ -85,11 +84,10 @@ pub mod pallet { #[pallet::hooks] impl Hooks> for Pallet { - /// on_runtime_upgrade is expected to be called exactly once after a runtime upgrade. /// We use this as a chance to flag that we are now in upgrade-mode and begin our /// migrations. - /// + /// /// In the event that a migration is expected to take more than one block, ongoing migration /// work could continue from block-to-block in this pallet's on_initialize function. fn on_runtime_upgrade() -> Weight { @@ -116,18 +114,12 @@ pub mod pallet { #[pallet::getter(fn migration_state)] /// MigrationState tracks the progress of a migration. /// Maps name (Vec) -> migration progress (Perbill) - type MigrationState = StorageMap< - _, - Twox64Concat, - Vec, - Perbill, - OptionQuery, - >; + type MigrationState = StorageMap<_, Twox64Concat, Vec, Perbill, OptionQuery>; #[pallet::genesis_config] pub struct GenesisConfig { pub completed_migrations: Vec>, - pub dummy: PhantomData // TODO: + pub dummy: PhantomData, // TODO: } #[cfg(feature = "std")] @@ -158,36 +150,44 @@ pub mod pallet { let mut done: bool = true; for migration in &T::MigrationsList::get() { - // let migration_name = migration.friendly_name(); let migration_name = migration.friendly_name(); log::trace!("evaluating migration {}", migration_name); - let migration_state = >::get(migration_name.as_bytes()) - .unwrap_or(Perbill::zero()); + let migration_state = + >::get(migration_name.as_bytes()).unwrap_or(Perbill::zero()); if migration_state < Perbill::one() { - // TODO: we don't currently have a reliable way to know "started" // TODO: multiple calls to as_bytes() or to_vec() may be expensive - >::deposit_event(Event::MigrationStarted(migration_name.as_bytes().to_vec())); + >::deposit_event(Event::MigrationStarted( + migration_name.as_bytes().to_vec(), + )); let available_for_step = available_weight - weight; - log::trace!("stepping migration {}, prev: {:?}, avail weight: {}", - migration_name, migration_state, available_for_step); + log::trace!( + "stepping migration {}, prev: {:?}, avail weight: {}", + migration_name, + migration_state, + available_for_step + ); // perform a step of this migration >::deposit_event(Event::MigrationStarted(migration_name.into())); - let (updated_progress, consumed_weight) - = migration.step(migration_state, available_for_step); + let (updated_progress, consumed_weight) = + migration.step(migration_state, available_for_step); weight += consumed_weight; if weight > available_weight { // TODO: the intent here is to complain obnoxiously so that this is caught // during development. In production, this should probably be tolerated because // failing is catastrophic. - log::error!("Migration {} consumed more weight than it was given! ({} > {})", - migration_name, consumed_weight, available_for_step); + log::error!( + "Migration {} consumed more weight than it was given! ({} > {})", + migration_name, + consumed_weight, + available_for_step + ); } // make note of any unfinished migrations @@ -199,7 +199,6 @@ pub mod pallet { >::insert(migration_name.as_bytes(), updated_progress); } } - } if done { diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 1a302da4c9..845f7800e9 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -18,10 +18,7 @@ use super::*; use crate as pallet_migrations; use frame_support::{ - pallet_prelude::*, - construct_runtime, parameter_types, - traits::GenesisBuild, - weights::Weight, + construct_runtime, pallet_prelude::*, parameter_types, traits::GenesisBuild, weights::Weight, }; use sp_core::H256; use sp_runtime::{ @@ -84,8 +81,7 @@ impl frame_system::Config for Test { pub struct MockMigrations; impl Get>> for MockMigrations { fn get() -> Vec> { - vec![ - ] + vec![] } } @@ -108,7 +104,6 @@ impl Default for ExtBuilder { } impl ExtBuilder { - pub(crate) fn build(self) -> sp_io::TestExternalities { let mut t = frame_system::GenesisConfig::default() .build_storage::() @@ -140,4 +135,3 @@ pub(crate) fn events() -> Vec> { }) .collect::>() } - diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index e84e80caba..9bff3fa8e8 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -15,43 +15,33 @@ // along with Moonbeam. If not, see . //! Unit testing -use crate::mock::{ - events, Migrations, ExtBuilder, System, -}; +use crate::mock::{events, ExtBuilder, Migrations, System}; use crate::Event; -use frame_support::{ - traits::{OnRuntimeUpgrade}, -}; +use frame_support::traits::OnRuntimeUpgrade; #[test] fn genesis_builder_works() { - ExtBuilder::default() - .build() - .execute_with(|| { - assert!(System::events().is_empty()); - }) + ExtBuilder::default().build().execute_with(|| { + assert!(System::events().is_empty()); + }) } #[test] fn on_runtime_upgrade_returns() { - ExtBuilder::default() - .build() - .execute_with(|| { - Migrations::on_runtime_upgrade(); - }) + ExtBuilder::default().build().execute_with(|| { + Migrations::on_runtime_upgrade(); + }) } #[test] fn on_runtime_upgrade_emits_events() { - ExtBuilder::default() - .build() - .execute_with(|| { - Migrations::on_runtime_upgrade(); - - let expected = vec![ - Event::RuntimeUpgradeStarted(), - Event::RuntimeUpgradeCompleted(), - ]; - assert_eq!(events(), expected); - }); + ExtBuilder::default().build().execute_with(|| { + Migrations::on_runtime_upgrade(); + + let expected = vec![ + Event::RuntimeUpgradeStarted(), + Event::RuntimeUpgradeCompleted(), + ]; + assert_eq!(events(), expected); + }); } diff --git a/runtime/common/src/migrations.rs b/runtime/common/src/migrations.rs index caab568e07..abbc87a2ed 100644 --- a/runtime/common/src/migrations.rs +++ b/runtime/common/src/migrations.rs @@ -16,10 +16,7 @@ //! # Migrations -use frame_support::{ - weights::Weight, - pallet_prelude::Get, -}; +use frame_support::{pallet_prelude::Get, weights::Weight}; use pallet_migrations::Migration; use sp_runtime::Perbill; use sp_std::prelude::*; @@ -74,4 +71,3 @@ impl Get>> for CommonMigrations { ] } } - diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index 7011bc954a..5b2a64e5ef 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -51,9 +51,9 @@ use pallet_evm::{ Account as EVMAccount, EnsureAddressNever, EnsureAddressRoot, FeeCalculator, IdentityAddressMapping, Runner, }; +use pallet_migrations::*; use pallet_transaction_payment::CurrencyAdapter; pub use parachain_staking::{InflationInfo, Range}; -use pallet_migrations::*; use parity_scale_codec::{Decode, Encode}; use sp_api::impl_runtime_apis; use sp_core::{u32_trait::*, OpaqueMetadata, H160, H256, U256}; From bc2e5386977bdc4c55ca58ed849c35cec8f16549 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 1 Jul 2021 12:41:53 -0600 Subject: [PATCH 26/99] Per-test MigrationsList --- Cargo.lock | 29 +++++++++++++------------- pallets/migrations/Cargo.toml | 1 + pallets/migrations/src/mock.rs | 37 ++++++++++++++++++++++++++++++++- pallets/migrations/src/tests.rs | 31 +++++++++++++++++++++++++-- 4 files changed, 81 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 250f039553..38965c9aed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -245,7 +245,7 @@ dependencies = [ "concurrent-queue", "fastrand", "futures-lite", - "once_cell 1.7.2", + "once_cell 1.8.0", "vec-arena", ] @@ -262,7 +262,7 @@ dependencies = [ "blocking", "futures-lite", "num_cpus", - "once_cell 1.7.2", + "once_cell 1.8.0", ] [[package]] @@ -277,7 +277,7 @@ dependencies = [ "libc", "log", "nb-connect", - "once_cell 1.7.2", + "once_cell 1.8.0", "parking", "polling", "vec-arena", @@ -314,7 +314,7 @@ dependencies = [ "cfg-if 1.0.0", "event-listener", "futures-lite", - "once_cell 1.7.2", + "once_cell 1.8.0", "signal-hook", "winapi 0.3.9", ] @@ -341,7 +341,7 @@ dependencies = [ "log", "memchr", "num_cpus", - "once_cell 1.7.2", + "once_cell 1.8.0", "pin-project-lite 0.2.6", "pin-utils", "slab", @@ -729,7 +729,7 @@ dependencies = [ "atomic-waker", "fastrand", "futures-lite", - "once_cell 1.7.2", + "once_cell 1.8.0", ] [[package]] @@ -2012,7 +2012,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab01c2450bed354679e78bedbff1484e02910ef1be96755086a36cadd1247efa" dependencies = [ "log", - "once_cell 1.7.2", + "once_cell 1.8.0", "serde", "serde_json", ] @@ -2539,7 +2539,7 @@ dependencies = [ "impl-trait-for-tuples 0.2.1", "log", "max-encoded-len", - "once_cell 1.7.2", + "once_cell 1.8.0", "parity-scale-codec", "paste 1.0.5", "serde", @@ -5730,9 +5730,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" dependencies = [ "parking_lot 0.11.1", ] @@ -6268,6 +6268,7 @@ dependencies = [ "frame-support", "frame-system", "log", + "once_cell 1.8.0", "parity-scale-codec", "sp-core", "sp-io", @@ -8874,7 +8875,7 @@ checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" dependencies = [ "cc", "libc", - "once_cell 1.7.2", + "once_cell 1.8.0", "spin", "untrusted", "web-sys", @@ -10060,7 +10061,7 @@ dependencies = [ "erased-serde", "lazy_static", "log", - "once_cell 1.7.2", + "once_cell 1.8.0", "parking_lot 0.11.1", "regex", "rustc-hash", @@ -11715,7 +11716,7 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" dependencies = [ - "once_cell 1.7.2", + "once_cell 1.8.0", ] [[package]] @@ -11774,7 +11775,7 @@ checksum = "d9e44c4759bae7f1032e286a7ef990bd9ed23fe831b7eeba0beb97484c2e59b8" dependencies = [ "anyhow", "hmac 0.8.1", - "once_cell 1.7.2", + "once_cell 1.8.0", "pbkdf2 0.4.0", "rand 0.7.3", "rustc-hash", diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index fc926eef4f..d4f1a6e679 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -16,6 +16,7 @@ parity-scale-codec = { version = "2.0.0", default-features = false } [dev-dependencies] sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } +once_cell = "1.8.0" [features] default = ["std"] diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 845f7800e9..72c4bef415 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -17,6 +17,8 @@ //! A minimal runtime including the migrations pallet use super::*; use crate as pallet_migrations; +use std::sync::{Arc, Mutex}; +use once_cell::sync::Lazy; use frame_support::{ construct_runtime, pallet_prelude::*, parameter_types, traits::GenesisBuild, weights::Weight, }; @@ -78,10 +80,43 @@ impl frame_system::Config for Test { type OnSetCode = (); } +type MigrationStepFn = fn (Perbill, Weight) -> (Perbill, Weight); + +#[derive(Clone)] +pub struct MockMigration { + pub name: String, + pub callback: MigrationStepFn, +} + +impl Migration for MockMigration { + fn friendly_name(&self) -> &str { + &self.name[..] + } + fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { + let f = self.callback; + f(previous_progress, available_weight) + } +} + +pub static MOCK_MIGRATIONS_LIST: Lazy>> = Lazy::new(|| { + Mutex::new(vec![]) +}); +pub fn replace_mock_migrations_list(new_vec: &mut Vec) { + let mut list = MOCK_MIGRATIONS_LIST.lock().unwrap(); + list.clear(); + list.append(new_vec); +} + pub struct MockMigrations; impl Get>> for MockMigrations { fn get() -> Vec> { - vec![] + + let mut migrations_list: Vec> = Vec::new(); + for mock in &*MOCK_MIGRATIONS_LIST.lock().unwrap() { + migrations_list.push(Box::new(mock.clone())); + } + + migrations_list } } diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 9bff3fa8e8..740070aad2 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -15,9 +15,16 @@ // along with Moonbeam. If not, see . //! Unit testing -use crate::mock::{events, ExtBuilder, Migrations, System}; +use crate::mock::{ + events, ExtBuilder, Migrations, System, MockMigration, replace_mock_migrations_list +}; use crate::Event; -use frame_support::traits::OnRuntimeUpgrade; +use std::sync::{Arc, Mutex}; +use frame_support::{ + traits::OnRuntimeUpgrade, + weights::Weight, +}; +use sp_runtime::Perbill; #[test] fn genesis_builder_works() { @@ -26,6 +33,26 @@ fn genesis_builder_works() { }) } +#[test] +fn mock_migrations_static_hack_works() { + let mut flip_me: bool = false; + replace_mock_migrations_list(&mut vec![ + MockMigration { + name: "test".into(), + callback: |_: Perbill, _: Weight| -> (Perbill, Weight) { + flip_me = true; + (Perbill::one(), 0u64.into()) + } + }, + ]); + + ExtBuilder::default().build().execute_with(|| { + Migrations::on_runtime_upgrade(); + }); + + assert_eq!(flip_me, true, "mock migration callback should work with closure"); +} + #[test] fn on_runtime_upgrade_returns() { ExtBuilder::default().build().execute_with(|| { From 19a99123633e17d2bc917cca24224326e2193370 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 2 Jul 2021 19:37:33 -0600 Subject: [PATCH 27/99] Attempt at a glue --- pallets/migrations/src/mock.rs | 63 ++++++++++++++++++++++++++++++++- pallets/migrations/src/tests.rs | 4 ++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 72c4bef415..faa2a0d10f 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -80,8 +80,68 @@ impl frame_system::Config for Test { type OnSetCode = (); } -type MigrationStepFn = fn (Perbill, Weight) -> (Perbill, Weight); +type MigrationNameFn = dyn FnMut() -> &'static str + Send + Sync; +type MigrationStepFn = dyn FnMut(Perbill, Weight) -> (Perbill, Weight) + Send + Sync; +#[derive(Default)] +pub struct MockMigrationManager { + name_fn_callbacks: Vec>>, + step_fn_callbacks: Vec>>, +} + +impl MockMigrationManager { + fn registerCallback(&mut self, name_fn: &MigrationNameFn, step_fn: &MigrationStepFn) { + // self.name_fn_callbacks.push(Arc::new(name_fn)); + // self.step_fn_callbacks.push(Arc::new(step_fn)); + } + + fn invoke_name_fn(&mut self, index: usize) -> &'static str { + // MigrationNameFn returns a String, we need a &str + let arc = self.name_fn_callbacks[index].clone(); + let mut f = arc.lock().unwrap(); + f() + } + + fn invoke_step_fn(&mut self, index: usize, previous_progress: Perbill, available_weight: Weight) + -> (Perbill, Weight) + { + let arc = self.step_fn_callbacks[index].clone(); + let mut f = arc.lock().unwrap(); + f(previous_progress, available_weight) + } + + fn generate_migrations_list(&self) -> Vec> { + panic!("FIXME"); + } +} + +#[derive(Clone)] +pub struct MockMigration { + pub index: usize, +} + +impl Migration for MockMigration { + fn friendly_name(&self) -> &str { + MOCK_MIGRATIONS_LIST.lock().unwrap().invoke_name_fn(self.index) + } + fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { + MOCK_MIGRATIONS_LIST.lock().unwrap() + .invoke_step_fn(self.index, previous_progress, available_weight) + } +} + +pub static MOCK_MIGRATIONS_LIST: Lazy> = Lazy::new(|| { + Default::default() +}); + +pub struct MockMigrations; +impl Get>> for MockMigrations { + fn get() -> Vec> { + MOCK_MIGRATIONS_LIST.lock().unwrap().generate_migrations_list() + } +} + +/* #[derive(Clone)] pub struct MockMigration { pub name: String, @@ -119,6 +179,7 @@ impl Get>> for MockMigrations { migrations_list } } +*/ impl Config for Test { type Event = Event; diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 740070aad2..cc4b396153 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -16,7 +16,7 @@ //! Unit testing use crate::mock::{ - events, ExtBuilder, Migrations, System, MockMigration, replace_mock_migrations_list + events, ExtBuilder, Migrations, System, MockMigration }; use crate::Event; use std::sync::{Arc, Mutex}; @@ -33,6 +33,7 @@ fn genesis_builder_works() { }) } +/* #[test] fn mock_migrations_static_hack_works() { let mut flip_me: bool = false; @@ -52,6 +53,7 @@ fn mock_migrations_static_hack_works() { assert_eq!(flip_me, true, "mock migration callback should work with closure"); } +*/ #[test] fn on_runtime_upgrade_returns() { From 1f70d01867d7ccbb3de48462f3488dc6b92a430c Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 2 Jul 2021 19:41:40 -0600 Subject: [PATCH 28/99] Fix FIXME --- pallets/migrations/src/mock.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index faa2a0d10f..8b1fc19e8d 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -111,7 +111,11 @@ impl MockMigrationManager { } fn generate_migrations_list(&self) -> Vec> { - panic!("FIXME"); + let mut migrations: Vec> = Vec::new(); + for i in 0..self.name_fn_callbacks.len() { + migrations.push(Box::new(MockMigration{index: i})); + } + migrations } } From 6e581f3ff0a08dc77cdfeae866770cc67fe22266 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 2 Jul 2021 20:07:08 -0600 Subject: [PATCH 29/99] Getting close --- pallets/migrations/src/mock.rs | 52 ++++++--------------------------- pallets/migrations/src/tests.rs | 30 +++++++++++-------- 2 files changed, 27 insertions(+), 55 deletions(-) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 8b1fc19e8d..bc0c30d4b9 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -80,8 +80,10 @@ impl frame_system::Config for Test { type OnSetCode = (); } -type MigrationNameFn = dyn FnMut() -> &'static str + Send + Sync; -type MigrationStepFn = dyn FnMut(Perbill, Weight) -> (Perbill, Weight) + Send + Sync; +pub trait MigrationNameTrait: FnMut() -> &'static str + Send + Sync {} +pub trait MigrationStepTrait: FnMut(Perbill, Weight) -> (Perbill, Weight) + Send + Sync {} +type MigrationNameFn = dyn MigrationNameTrait; +type MigrationStepFn = dyn MigrationStepTrait; #[derive(Default)] pub struct MockMigrationManager { @@ -90,7 +92,11 @@ pub struct MockMigrationManager { } impl MockMigrationManager { - fn registerCallback(&mut self, name_fn: &MigrationNameFn, step_fn: &MigrationStepFn) { + pub fn registerCallback(&mut self, name_fn: &FN, step_fn: &FS) + where + FN: FnMut() -> &'static str + Send + Sync, + FS: FnMut(Perbill, Weight) -> (Perbill, Weight) + Send + Sync, + { // self.name_fn_callbacks.push(Arc::new(name_fn)); // self.step_fn_callbacks.push(Arc::new(step_fn)); } @@ -145,46 +151,6 @@ impl Get>> for MockMigrations { } } -/* -#[derive(Clone)] -pub struct MockMigration { - pub name: String, - pub callback: MigrationStepFn, -} - -impl Migration for MockMigration { - fn friendly_name(&self) -> &str { - &self.name[..] - } - fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { - let f = self.callback; - f(previous_progress, available_weight) - } -} - -pub static MOCK_MIGRATIONS_LIST: Lazy>> = Lazy::new(|| { - Mutex::new(vec![]) -}); -pub fn replace_mock_migrations_list(new_vec: &mut Vec) { - let mut list = MOCK_MIGRATIONS_LIST.lock().unwrap(); - list.clear(); - list.append(new_vec); -} - -pub struct MockMigrations; -impl Get>> for MockMigrations { - fn get() -> Vec> { - - let mut migrations_list: Vec> = Vec::new(); - for mock in &*MOCK_MIGRATIONS_LIST.lock().unwrap() { - migrations_list.push(Box::new(mock.clone())); - } - - migrations_list - } -} -*/ - impl Config for Test { type Event = Event; type MigrationsList = MockMigrations; diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index cc4b396153..4b1ef5a91f 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -33,27 +33,33 @@ fn genesis_builder_works() { }) } -/* #[test] fn mock_migrations_static_hack_works() { - let mut flip_me: bool = false; - replace_mock_migrations_list(&mut vec![ - MockMigration { - name: "test".into(), - callback: |_: Perbill, _: Weight| -> (Perbill, Weight) { - flip_me = true; - (Perbill::one(), 0u64.into()) + let mut name_fn_called: bool = false; + let mut step_fn_called: bool = false; + + // works: + // let name_fn: &(FnMut() -> &'static str + Send + Sync) = &|| { "hi" }; + + crate::mock::MOCK_MIGRATIONS_LIST.lock().unwrap() + .registerCallback( + &|| { + name_fn_called = true; + "hello, world" + }, + &|_, _| -> (Perbill, Weight) { + step_fn_called = true; + (Perbill::zero(), 0u64.into()) } - }, - ]); + ); ExtBuilder::default().build().execute_with(|| { Migrations::on_runtime_upgrade(); }); - assert_eq!(flip_me, true, "mock migration callback should work with closure"); + assert_eq!(name_fn_called, true, "mock migration should call friendly_name()"); + assert_eq!(step_fn_called, true, "mock migration should call step()"); } -*/ #[test] fn on_runtime_upgrade_returns() { From 2ea1e3b7ecdda42931e41c3b4b272ffddf3786f3 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 2 Jul 2021 21:52:48 -0600 Subject: [PATCH 30/99] Sort out lifetimes --- pallets/migrations/src/mock.rs | 24 +++++++++++------------- pallets/migrations/src/tests.rs | 11 +++++++---- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index bc0c30d4b9..0b8299dd9a 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -80,25 +80,23 @@ impl frame_system::Config for Test { type OnSetCode = (); } -pub trait MigrationNameTrait: FnMut() -> &'static str + Send + Sync {} -pub trait MigrationStepTrait: FnMut(Perbill, Weight) -> (Perbill, Weight) + Send + Sync {} -type MigrationNameFn = dyn MigrationNameTrait; -type MigrationStepFn = dyn MigrationStepTrait; +type MigrationNameFn<'test> = dyn FnMut() -> &'static str + Send + Sync + 'test; +type MigrationStepFn<'test> = dyn FnMut(Perbill, Weight) -> (Perbill, Weight) + Send + Sync + 'test; #[derive(Default)] -pub struct MockMigrationManager { - name_fn_callbacks: Vec>>, - step_fn_callbacks: Vec>>, +pub struct MockMigrationManager<'test> { + name_fn_callbacks: Vec>>>, + step_fn_callbacks: Vec>>>, } -impl MockMigrationManager { - pub fn registerCallback(&mut self, name_fn: &FN, step_fn: &FS) +impl<'test> MockMigrationManager<'test> { + pub fn registerCallback(&'test mut self, name_fn: &'test mut FN, step_fn: &'test mut FS) where - FN: FnMut() -> &'static str + Send + Sync, - FS: FnMut(Perbill, Weight) -> (Perbill, Weight) + Send + Sync, + FN: 'test + FnMut() -> &'static str + Send + Sync, + FS: 'test + FnMut(Perbill, Weight) -> (Perbill, Weight) + Send + Sync, { - // self.name_fn_callbacks.push(Arc::new(name_fn)); - // self.step_fn_callbacks.push(Arc::new(step_fn)); + self.name_fn_callbacks.push(Arc::new(Mutex::new(name_fn))); + self.step_fn_callbacks.push(Arc::new(Mutex::new(step_fn))); } fn invoke_name_fn(&mut self, index: usize) -> &'static str { diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 4b1ef5a91f..a53bad9aea 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -34,20 +34,23 @@ fn genesis_builder_works() { } #[test] -fn mock_migrations_static_hack_works() { +fn mock_migrations_static_hack_works<'test>() { let mut name_fn_called: bool = false; let mut step_fn_called: bool = false; + let mut mgr: crate::mock::MockMigrationManager = Default::default(); + // works: // let name_fn: &(FnMut() -> &'static str + Send + Sync) = &|| { "hi" }; - crate::mock::MOCK_MIGRATIONS_LIST.lock().unwrap() + // crate::mock::MOCK_MIGRATIONS_LIST.lock().unwrap() + mgr .registerCallback( - &|| { + &mut|| { name_fn_called = true; "hello, world" }, - &|_, _| -> (Perbill, Weight) { + &mut|_, _| -> (Perbill, Weight) { step_fn_called = true; (Perbill::zero(), 0u64.into()) } From 4afc532fcc4573bcf49190102e739b9ea4be779e Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 6 Jul 2021 16:01:28 -0600 Subject: [PATCH 31/99] Simplify FnMut arguments/storage --- Cargo.lock | 1 + pallets/migrations/Cargo.toml | 1 + pallets/migrations/src/lib.rs | 4 +++ pallets/migrations/src/mock.rs | 51 ++++++++++++++++----------------- pallets/migrations/src/tests.rs | 28 +++++++++--------- 5 files changed, 46 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38965c9aed..f2edb4b27f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6265,6 +6265,7 @@ dependencies = [ name = "pallet-migrations" version = "0.1.0" dependencies = [ + "environmental", "frame-support", "frame-system", "log", diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index d4f1a6e679..e01667624b 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -12,6 +12,7 @@ log = "0.4" sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } parity-scale-codec = { version = "2.0.0", default-features = false } +environmental = "1.1.0" [dev-dependencies] sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 1150eee1bc..1ab5ca6701 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -29,6 +29,10 @@ use sp_runtime::Perbill; pub use pallet::*; +// TODO: compile error if this is in mock.rs: +// "an `extern crate` loading macros must be at the crate root" +#[macro_use] extern crate environmental; + /// A Migration that must happen on-chain upon a runtime-upgrade pub trait Migration { /// A human-readable name for this migration. Also used as storage key. diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 0b8299dd9a..0dffdd2cd7 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -80,38 +80,35 @@ impl frame_system::Config for Test { type OnSetCode = (); } -type MigrationNameFn<'test> = dyn FnMut() -> &'static str + Send + Sync + 'test; -type MigrationStepFn<'test> = dyn FnMut(Perbill, Weight) -> (Perbill, Weight) + Send + Sync + 'test; +type MigrationNameFn = dyn FnMut() -> &'static str; +type MigrationStepFn = dyn FnMut(Perbill, Weight) -> (Perbill, Weight); #[derive(Default)] pub struct MockMigrationManager<'test> { - name_fn_callbacks: Vec>>>, - step_fn_callbacks: Vec>>>, + // name_fn_callbacks: Vec<&'test mut MigrationNameFn>, + // step_fn_callbacks: Vec<&'test mut MigrationStepFn>, + name_fn_callbacks: Vec &'static str>>, + step_fn_callbacks: Vec (Perbill, Weight)>>, } impl<'test> MockMigrationManager<'test> { - pub fn registerCallback(&'test mut self, name_fn: &'test mut FN, step_fn: &'test mut FS) + pub fn registerCallback(&mut self, name_fn: FN, step_fn: FS) where - FN: 'test + FnMut() -> &'static str + Send + Sync, - FS: 'test + FnMut(Perbill, Weight) -> (Perbill, Weight) + Send + Sync, + FN: 'test + FnMut() -> &'static str, + FS: 'test + FnMut(Perbill, Weight) -> (Perbill, Weight), { - self.name_fn_callbacks.push(Arc::new(Mutex::new(name_fn))); - self.step_fn_callbacks.push(Arc::new(Mutex::new(step_fn))); + self.name_fn_callbacks.push(Box::new(name_fn)); + self.step_fn_callbacks.push(Box::new(step_fn)); } - fn invoke_name_fn(&mut self, index: usize) -> &'static str { - // MigrationNameFn returns a String, we need a &str - let arc = self.name_fn_callbacks[index].clone(); - let mut f = arc.lock().unwrap(); - f() + pub(crate) fn invoke_name_fn(&mut self, index: usize) -> &'static str { + self.name_fn_callbacks[index]() } - fn invoke_step_fn(&mut self, index: usize, previous_progress: Perbill, available_weight: Weight) + pub(crate) fn invoke_step_fn(&mut self, index: usize, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { - let arc = self.step_fn_callbacks[index].clone(); - let mut f = arc.lock().unwrap(); - f(previous_progress, available_weight) + self.step_fn_callbacks[index](previous_progress, available_weight) } fn generate_migrations_list(&self) -> Vec> { @@ -122,6 +119,7 @@ impl<'test> MockMigrationManager<'test> { migrations } } +// environmental!(MOCK_MIGRATIONS_LIST: MockMigrationManager<'static>); #[derive(Clone)] pub struct MockMigration { @@ -130,22 +128,23 @@ pub struct MockMigration { impl Migration for MockMigration { fn friendly_name(&self) -> &str { - MOCK_MIGRATIONS_LIST.lock().unwrap().invoke_name_fn(self.index) + panic!("fixme"); + // MOCK_MIGRATIONS_LIST.lock().unwrap().invoke_name_fn(self.index) } fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { - MOCK_MIGRATIONS_LIST.lock().unwrap() - .invoke_step_fn(self.index, previous_progress, available_weight) + panic!("fixme"); + // MOCK_MIGRATIONS_LIST.lock().unwrap() + // .invoke_step_fn(self.index, previous_progress, available_weight) } } -pub static MOCK_MIGRATIONS_LIST: Lazy> = Lazy::new(|| { - Default::default() -}); - pub struct MockMigrations; impl Get>> for MockMigrations { fn get() -> Vec> { - MOCK_MIGRATIONS_LIST.lock().unwrap().generate_migrations_list() + let mut migrations: Vec> = Vec::new(); + // MOCK_MIGRATIONS_LIST::with(|m| { migrations = m.generate_migrations_list(); }); + panic!("fixme"); + migrations } } diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index a53bad9aea..005e3be2e0 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -34,31 +34,33 @@ fn genesis_builder_works() { } #[test] -fn mock_migrations_static_hack_works<'test>() { +fn mock_migrations_static_hack_works() { let mut name_fn_called: bool = false; let mut step_fn_called: bool = false; - let mut mgr: crate::mock::MockMigrationManager = Default::default(); + { + let mut mgr: crate::mock::MockMigrationManager = Default::default(); - // works: - // let name_fn: &(FnMut() -> &'static str + Send + Sync) = &|| { "hi" }; - - // crate::mock::MOCK_MIGRATIONS_LIST.lock().unwrap() - mgr - .registerCallback( - &mut|| { + mgr.registerCallback( + || { name_fn_called = true; "hello, world" }, - &mut|_, _| -> (Perbill, Weight) { + |_, _| -> (Perbill, Weight) { step_fn_called = true; (Perbill::zero(), 0u64.into()) } ); - ExtBuilder::default().build().execute_with(|| { - Migrations::on_runtime_upgrade(); - }); + mgr.invoke_name_fn(0); + mgr.invoke_step_fn(0, Perbill::zero(), 1u64.into()); + + /* + ExtBuilder::default().build().execute_with(|| { + Migrations::on_runtime_upgrade(); + }); + */ + } assert_eq!(name_fn_called, true, "mock migration should call friendly_name()"); assert_eq!(step_fn_called, true, "mock migration should call step()"); From 85ed484ff424eccfd3362ff8c32ec254a9369519 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 6 Jul 2021 16:09:33 -0600 Subject: [PATCH 32/99] Clean up, fix FIXMEs --- pallets/migrations/src/mock.rs | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 0dffdd2cd7..ee75d28407 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -80,13 +80,8 @@ impl frame_system::Config for Test { type OnSetCode = (); } -type MigrationNameFn = dyn FnMut() -> &'static str; -type MigrationStepFn = dyn FnMut(Perbill, Weight) -> (Perbill, Weight); - #[derive(Default)] pub struct MockMigrationManager<'test> { - // name_fn_callbacks: Vec<&'test mut MigrationNameFn>, - // step_fn_callbacks: Vec<&'test mut MigrationStepFn>, name_fn_callbacks: Vec &'static str>>, step_fn_callbacks: Vec (Perbill, Weight)>>, } @@ -119,7 +114,7 @@ impl<'test> MockMigrationManager<'test> { migrations } } -// environmental!(MOCK_MIGRATIONS_LIST: MockMigrationManager<'static>); +environmental!(MOCK_MIGRATIONS_LIST: MockMigrationManager<'static>); #[derive(Clone)] pub struct MockMigration { @@ -128,13 +123,18 @@ pub struct MockMigration { impl Migration for MockMigration { fn friendly_name(&self) -> &str { - panic!("fixme"); - // MOCK_MIGRATIONS_LIST.lock().unwrap().invoke_name_fn(self.index) + let mut result: &str = ""; + MOCK_MIGRATIONS_LIST::with(|mgr: &mut MockMigrationManager| { + result = mgr.invoke_name_fn(self.index); + }); + result } fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { - panic!("fixme"); - // MOCK_MIGRATIONS_LIST.lock().unwrap() - // .invoke_step_fn(self.index, previous_progress, available_weight) + let mut result: (Perbill, Weight) = (Perbill::zero(), 0u64.into()); + MOCK_MIGRATIONS_LIST::with(|mgr: &mut MockMigrationManager| { + result = mgr.invoke_step_fn(self.index, previous_progress, available_weight); + }); + result } } @@ -142,8 +142,9 @@ pub struct MockMigrations; impl Get>> for MockMigrations { fn get() -> Vec> { let mut migrations: Vec> = Vec::new(); - // MOCK_MIGRATIONS_LIST::with(|m| { migrations = m.generate_migrations_list(); }); - panic!("fixme"); + MOCK_MIGRATIONS_LIST::with(|mgr: &mut MockMigrationManager| { + migrations = mgr.generate_migrations_list(); + }); migrations } } From e377cafd2c5a26e35d7dda965d9cfe39205a4d9b Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 6 Jul 2021 20:33:55 -0600 Subject: [PATCH 33/99] It works --- Cargo.lock | 1 - pallets/migrations/Cargo.toml | 1 - pallets/migrations/src/mock.rs | 18 ++++++++++++-- pallets/migrations/src/tests.rs | 42 ++++++++++++++++----------------- 4 files changed, 37 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2edb4b27f..9707e4a396 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6269,7 +6269,6 @@ dependencies = [ "frame-support", "frame-system", "log", - "once_cell 1.8.0", "parity-scale-codec", "sp-core", "sp-io", diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index e01667624b..92792f2c8a 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -17,7 +17,6 @@ environmental = "1.1.0" [dev-dependencies] sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } -once_cell = "1.8.0" [features] default = ["std"] diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index ee75d28407..9d8c08eb91 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -17,8 +17,6 @@ //! A minimal runtime including the migrations pallet use super::*; use crate as pallet_migrations; -use std::sync::{Arc, Mutex}; -use once_cell::sync::Lazy; use frame_support::{ construct_runtime, pallet_prelude::*, parameter_types, traits::GenesisBuild, weights::Weight, }; @@ -116,6 +114,22 @@ impl<'test> MockMigrationManager<'test> { } environmental!(MOCK_MIGRATIONS_LIST: MockMigrationManager<'static>); +pub fn execute_with_mock_migrations(callback: &mut CB) +where + CB: FnMut(&mut MockMigrationManager) +{ + let mut original_mgr: MockMigrationManager = Default::default(); + MOCK_MIGRATIONS_LIST::using(&mut original_mgr, || { + MOCK_MIGRATIONS_LIST::with(|inner_mgr: &mut MockMigrationManager| { + callback(inner_mgr); + }); + + ExtBuilder::default().build().execute_with(|| { + Migrations::on_runtime_upgrade(); + }); + }); +} + #[derive(Clone)] pub struct MockMigration { pub index: usize, diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 005e3be2e0..37276b302d 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -16,7 +16,7 @@ //! Unit testing use crate::mock::{ - events, ExtBuilder, Migrations, System, MockMigration + events, ExtBuilder, Migrations, System, MockMigrationManager, }; use crate::Event; use std::sync::{Arc, Mutex}; @@ -35,35 +35,35 @@ fn genesis_builder_works() { #[test] fn mock_migrations_static_hack_works() { - let mut name_fn_called: bool = false; - let mut step_fn_called: bool = false; - { - let mut mgr: crate::mock::MockMigrationManager = Default::default(); + let name_fn_called = Arc::new(Mutex::new(false)); + let step_fn_called = Arc::new(Mutex::new(false)); + println!("Calling execute_with_mock_migrations..."); + crate::mock::execute_with_mock_migrations(&mut |mgr: &mut MockMigrationManager| { + println!("Inside execute_with_mock_migrations"); + let name_fn_called = Arc::clone(&name_fn_called); + let step_fn_called = Arc::clone(&step_fn_called); + + println!("Registering callbacks..."); mgr.registerCallback( - || { - name_fn_called = true; + move || { + println!("inside name_fn callback!"); + *name_fn_called.lock().unwrap() = true; "hello, world" }, - |_, _| -> (Perbill, Weight) { - step_fn_called = true; + move |_, _| -> (Perbill, Weight) { + println!("inside step_fn callback!"); + *step_fn_called.lock().unwrap() = true; (Perbill::zero(), 0u64.into()) } ); + println!("Done registering callbacks."); + }); + println!("Done with execute_with_mock_migrations"); - mgr.invoke_name_fn(0); - mgr.invoke_step_fn(0, Perbill::zero(), 1u64.into()); - - /* - ExtBuilder::default().build().execute_with(|| { - Migrations::on_runtime_upgrade(); - }); - */ - } - - assert_eq!(name_fn_called, true, "mock migration should call friendly_name()"); - assert_eq!(step_fn_called, true, "mock migration should call step()"); + assert_eq!(*name_fn_called.lock().unwrap(), true, "mock migration should call friendly_name()"); + assert_eq!(*step_fn_called.lock().unwrap(), true, "mock migration should call step()"); } #[test] From 95d6b956aa054cec2dccb8c2c5d76ce95d9ac22d Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 7 Jul 2021 10:56:56 -0600 Subject: [PATCH 34/99] Implement Migrations::on_initialize --- pallets/migrations/src/lib.rs | 17 ++++++++++++++ pallets/migrations/src/mock.rs | 18 +++++++++++++++ pallets/migrations/src/tests.rs | 39 ++++++++++++++++++++++++++------- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 1ab5ca6701..2cacea7e8c 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -80,6 +80,7 @@ pub mod pallet { pub enum Event { // e.g. runtime upgrade started, completed, etc. RuntimeUpgradeStarted(), + RuntimeUpgradeStepped(), RuntimeUpgradeCompleted(), MigrationStarted(Vec), MigrationProgress(Vec, Perbill), @@ -107,6 +108,22 @@ pub mod pallet { weight.into() } + + /// on_initialize implementation. Calls process_runtime_upgrades() if we are still in the + /// middle of a runtime upgrade. + /// TODO: use on_idle or some other hook? + fn on_initialize(_: T::BlockNumber) -> Weight { + + // TODO: should account for the minimum one DB read + let mut weight: Weight = 0u64.into(); + + if ! >::get() { + Self::deposit_event(Event::RuntimeUpgradeStepped()); + weight += process_runtime_upgrades::(); + } + + weight + } } #[pallet::storage] diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 9d8c08eb91..c4aeb4aaaa 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -124,8 +124,26 @@ where callback(inner_mgr); }); + // mimic the calls that would occur from the time a runtime upgrade starts until the + // Migrations pallet indicates that all upgrades are complete + ExtBuilder::default().build().execute_with(|| { + let mut block_number = 1u64; Migrations::on_runtime_upgrade(); + + while ! Migrations::is_fully_upgraded() { + System::set_block_number(block_number); + System::on_initialize(System::block_number()); + Migrations::on_initialize(System::block_number()); + Migrations::on_finalize(System::block_number()); + System::on_finalize(System::block_number()); + + block_number += 1; + + if block_number > 99999 { + panic!("Infinite loop?"); + } + } }); }); } diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 37276b302d..623edeb0b2 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -39,28 +39,21 @@ fn mock_migrations_static_hack_works() { let name_fn_called = Arc::new(Mutex::new(false)); let step_fn_called = Arc::new(Mutex::new(false)); - println!("Calling execute_with_mock_migrations..."); crate::mock::execute_with_mock_migrations(&mut |mgr: &mut MockMigrationManager| { - println!("Inside execute_with_mock_migrations"); let name_fn_called = Arc::clone(&name_fn_called); let step_fn_called = Arc::clone(&step_fn_called); - println!("Registering callbacks..."); mgr.registerCallback( move || { - println!("inside name_fn callback!"); *name_fn_called.lock().unwrap() = true; "hello, world" }, move |_, _| -> (Perbill, Weight) { - println!("inside step_fn callback!"); *step_fn_called.lock().unwrap() = true; - (Perbill::zero(), 0u64.into()) + (Perbill::one(), 0u64.into()) } ); - println!("Done registering callbacks."); }); - println!("Done with execute_with_mock_migrations"); assert_eq!(*name_fn_called.lock().unwrap(), true, "mock migration should call friendly_name()"); assert_eq!(*step_fn_called.lock().unwrap(), true, "mock migration should call step()"); @@ -85,3 +78,33 @@ fn on_runtime_upgrade_emits_events() { assert_eq!(events(), expected); }); } + +#[test] +fn step_called_until_done() { + + let num_step_calls = Arc::new(Mutex::new(0usize)); + + println!("step_called_until_done()..."); + + crate::mock::execute_with_mock_migrations(&mut |mgr: &mut MockMigrationManager| { + let num_step_calls = Arc::clone(&num_step_calls); + + mgr.registerCallback( + move || { + "migration1" + }, + move |_, _| -> (Perbill, Weight) { + let mut num_step_calls = num_step_calls.lock().unwrap(); + println!("step fn called, num times previously: {}", num_step_calls); + *num_step_calls += 1; + if *num_step_calls == 10 { + (Perbill::one(), 0u64.into()) + } else { + (Perbill::zero(), 0u64.into()) + } + } + ); + }); + + assert_eq!(*num_step_calls.lock().unwrap(), 10, "migration step should be called until done"); +} From d5da28af59c1c9a05956ce6782849269fb00ce92 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 7 Jul 2021 11:13:08 -0600 Subject: [PATCH 35/99] Resolve compilation warnings, add comments about how mock glue works --- pallets/migrations/src/lib.rs | 3 +-- pallets/migrations/src/mock.rs | 22 +++++++++++++++++++++- pallets/migrations/src/tests.rs | 4 ++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 2cacea7e8c..cdbb0cb613 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -29,8 +29,7 @@ use sp_runtime::Perbill; pub use pallet::*; -// TODO: compile error if this is in mock.rs: -// "an `extern crate` loading macros must be at the crate root" +#[cfg(test)] #[macro_use] extern crate environmental; /// A Migration that must happen on-chain upon a runtime-upgrade diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index c4aeb4aaaa..68e009de55 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -78,6 +78,15 @@ impl frame_system::Config for Test { type OnSetCode = (); } +/// MockMigrationManager stores the test-side callbacks/closures used in the Migrations list glue. +/// It is is expected to exist as a singleton, but only remain relevant within the scope of a test. +/// +/// Tests should use execute_with_mock_migrations(), which will create a MockMigrationManager and +/// provide it to the test. +/// +/// A pair of callbacks provided to register_callback() will map directly to a single instance of +/// Migration (done by the MockMigration glue below). Treat each pair of callbacks as though it were +/// a custom implementation of the Migration trait just as a normal Pallet would. #[derive(Default)] pub struct MockMigrationManager<'test> { name_fn_callbacks: Vec &'static str>>, @@ -85,7 +94,7 @@ pub struct MockMigrationManager<'test> { } impl<'test> MockMigrationManager<'test> { - pub fn registerCallback(&mut self, name_fn: FN, step_fn: FS) + pub fn register_callback(&mut self, name_fn: FN, step_fn: FS) where FN: 'test + FnMut() -> &'static str, FS: 'test + FnMut(Perbill, Weight) -> (Perbill, Weight), @@ -112,8 +121,14 @@ impl<'test> MockMigrationManager<'test> { migrations } } + +// Our global Migrations list. Necessary because the Get impl must be fulfilled with nothing but +// a static context. environmental!(MOCK_MIGRATIONS_LIST: MockMigrationManager<'static>); +/// Utility method for tests to implement their logic with a pre-generated MockMigrationManager. +/// This helps avoid lifetime issues between the implied 'static lifetime of MOCK_MIGRATIONS_LIST +/// and the function-scoped lifetimes of closures used in tests. pub fn execute_with_mock_migrations(callback: &mut CB) where CB: FnMut(&mut MockMigrationManager) @@ -153,6 +168,9 @@ pub struct MockMigration { pub index: usize, } +/// The implementation of Migration for our glue: MockMigration contains nothing more than an index +/// which is used inside of the callbacks at runtime to look up our global callbacks stored in +/// MOCK_MIGRATIONS_LIST and invoke those. impl Migration for MockMigration { fn friendly_name(&self) -> &str { let mut result: &str = ""; @@ -170,6 +188,8 @@ impl Migration for MockMigration { } } +/// Implementation of Migrations. Generates a Vec of MockMigrations on the fly based on the current +/// contents of MOCK_MIGRATIONS_LIST. pub struct MockMigrations; impl Get>> for MockMigrations { fn get() -> Vec> { diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 623edeb0b2..91232a8b1e 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -43,7 +43,7 @@ fn mock_migrations_static_hack_works() { let name_fn_called = Arc::clone(&name_fn_called); let step_fn_called = Arc::clone(&step_fn_called); - mgr.registerCallback( + mgr.register_callback( move || { *name_fn_called.lock().unwrap() = true; "hello, world" @@ -89,7 +89,7 @@ fn step_called_until_done() { crate::mock::execute_with_mock_migrations(&mut |mgr: &mut MockMigrationManager| { let num_step_calls = Arc::clone(&num_step_calls); - mgr.registerCallback( + mgr.register_callback( move || { "migration1" }, From 2c0bfe558b0b06df1899664d3ac011c9241c3f7f Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 7 Jul 2021 12:32:38 -0600 Subject: [PATCH 36/99] Move migration event impl --- pallets/migrations/src/lib.rs | 34 ++++++++++++------ pallets/migrations/src/mock.rs | 7 ++-- pallets/migrations/src/tests.rs | 63 ++++++++++++++++++++++++++++++++- 3 files changed, 90 insertions(+), 14 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index cdbb0cb613..8bdc51aa9b 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -79,10 +79,10 @@ pub mod pallet { pub enum Event { // e.g. runtime upgrade started, completed, etc. RuntimeUpgradeStarted(), - RuntimeUpgradeStepped(), + RuntimeUpgradeStepped(Weight), RuntimeUpgradeCompleted(), MigrationStarted(Vec), - MigrationProgress(Vec, Perbill), + MigrationStepped(Vec, Perbill, Weight), MigrationCompleted(Vec), } @@ -117,7 +117,6 @@ pub mod pallet { let mut weight: Weight = 0u64.into(); if ! >::get() { - Self::deposit_event(Event::RuntimeUpgradeStepped()); weight += process_runtime_upgrades::(); } @@ -165,24 +164,26 @@ pub mod pallet { log::info!("stepping runtime upgrade"); // TODO: query proper value or make configurable - let available_weight = 500_000_000_000u64.into(); + let available_weight: Weight = 500_000_000_000u64.into(); let mut weight: Weight = 0u64.into(); let mut done: bool = true; for migration in &T::MigrationsList::get() { // let migration_name = migration.friendly_name(); let migration_name = migration.friendly_name(); + let migration_name_as_bytes = migration_name.as_bytes(); log::trace!("evaluating migration {}", migration_name); let migration_state = - >::get(migration_name.as_bytes()).unwrap_or(Perbill::zero()); + >::get(migration_name_as_bytes).unwrap_or(Perbill::zero()); if migration_state < Perbill::one() { - // TODO: we don't currently have a reliable way to know "started" - // TODO: multiple calls to as_bytes() or to_vec() may be expensive - >::deposit_event(Event::MigrationStarted( - migration_name.as_bytes().to_vec(), - )); + + if migration_state.is_zero() { + >::deposit_event(Event::MigrationStarted( + migration_name_as_bytes.into() + )); + } let available_for_step = available_weight - weight; log::trace!( @@ -193,9 +194,14 @@ pub mod pallet { ); // perform a step of this migration - >::deposit_event(Event::MigrationStarted(migration_name.into())); let (updated_progress, consumed_weight) = migration.step(migration_state, available_for_step); + // TODO: error if progress == 0 still? + >::deposit_event(Event::MigrationStepped( + migration_name_as_bytes.into(), + updated_progress, + consumed_weight, + )); weight += consumed_weight; if weight > available_weight { @@ -213,6 +219,10 @@ pub mod pallet { // make note of any unfinished migrations if updated_progress < Perbill::one() { done = false; + } else { + >::deposit_event(Event::MigrationCompleted( + migration_name_as_bytes.into() + )); } if migration_state != updated_progress { @@ -221,6 +231,8 @@ pub mod pallet { } } + >::deposit_event(Event::RuntimeUpgradeStepped(weight)); + if done { >::deposit_event(Event::RuntimeUpgradeCompleted()); >::put(true); diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 68e009de55..5614530e3b 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -129,9 +129,10 @@ environmental!(MOCK_MIGRATIONS_LIST: MockMigrationManager<'static>); /// Utility method for tests to implement their logic with a pre-generated MockMigrationManager. /// This helps avoid lifetime issues between the implied 'static lifetime of MOCK_MIGRATIONS_LIST /// and the function-scoped lifetimes of closures used in tests. -pub fn execute_with_mock_migrations(callback: &mut CB) +pub fn execute_with_mock_migrations(callback: &mut CB, post_migration_callback: &mut ECB) where - CB: FnMut(&mut MockMigrationManager) + CB: FnMut(&mut MockMigrationManager), + ECB: FnMut(), { let mut original_mgr: MockMigrationManager = Default::default(); MOCK_MIGRATIONS_LIST::using(&mut original_mgr, || { @@ -159,6 +160,8 @@ where panic!("Infinite loop?"); } } + + post_migration_callback(); }); }); } diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 91232a8b1e..b335936771 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -38,6 +38,7 @@ fn mock_migrations_static_hack_works() { let name_fn_called = Arc::new(Mutex::new(false)); let step_fn_called = Arc::new(Mutex::new(false)); + let ecb_fn_called = Arc::new(Mutex::new(false)); crate::mock::execute_with_mock_migrations(&mut |mgr: &mut MockMigrationManager| { let name_fn_called = Arc::clone(&name_fn_called); @@ -53,10 +54,14 @@ fn mock_migrations_static_hack_works() { (Perbill::one(), 0u64.into()) } ); + }, + &mut || { + *ecb_fn_called.lock().unwrap() = true; }); assert_eq!(*name_fn_called.lock().unwrap(), true, "mock migration should call friendly_name()"); assert_eq!(*step_fn_called.lock().unwrap(), true, "mock migration should call step()"); + assert_eq!(*ecb_fn_called.lock().unwrap(), true, "mock migration should call ECB callback"); } #[test] @@ -73,6 +78,7 @@ fn on_runtime_upgrade_emits_events() { let expected = vec![ Event::RuntimeUpgradeStarted(), + Event::RuntimeUpgradeStepped(0u64.into()), Event::RuntimeUpgradeCompleted(), ]; assert_eq!(events(), expected); @@ -104,7 +110,62 @@ fn step_called_until_done() { } } ); - }); + }, + &mut || {} ); assert_eq!(*num_step_calls.lock().unwrap(), 10, "migration step should be called until done"); } + +#[test] +fn migration_progress_should_emit_events() { + + let num_steps = Arc::new(Mutex::new(0usize)); + + crate::mock::execute_with_mock_migrations(&mut |mgr: &mut MockMigrationManager| { + let num_steps = Arc::clone(&num_steps); + + mgr.register_callback( + move || { + "migration1" + }, + move |_, _| -> (Perbill, Weight) { + let mut num_steps = num_steps.lock().unwrap(); + + let result: (Perbill, Weight) = match *num_steps { + 0 => (Perbill::from_percent(50), 50), + 1 => (Perbill::from_percent(60), 51), + 2 => (Perbill::from_percent(70), 52), + 3 => (Perbill::from_percent(80), 53), + 4 => (Perbill::from_percent(100), 1), + _ => { unreachable!(); } + }; + + *num_steps += 1; + result + } + ); + }, + &mut || { + + let expected = vec![ + Event::RuntimeUpgradeStarted(), + Event::MigrationStarted("migration1".into()), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(50), 50), + Event::RuntimeUpgradeStepped(50), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(60), 51), + Event::RuntimeUpgradeStepped(51), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(70), 52), + Event::RuntimeUpgradeStepped(52), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(80), 53), + Event::RuntimeUpgradeStepped(53), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(100), 1), + Event::MigrationCompleted("migration1".into()), + Event::RuntimeUpgradeStepped(1), + Event::RuntimeUpgradeCompleted(), + ]; + assert_eq!(events(), expected); + }); + + assert_eq!(*num_steps.lock().unwrap(), 5, "migration step should be called until done"); + +} From 67bede2f8622a805e250af9155c26aa3b33926a8 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 8 Jul 2021 15:56:17 -0600 Subject: [PATCH 37/99] Let tests manage ExtBuilder ... execute_with() --- pallets/migrations/src/mock.rs | 59 +++++++++++++++++++-------------- pallets/migrations/src/tests.rs | 51 ++++++++++++++++------------ 2 files changed, 64 insertions(+), 46 deletions(-) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 5614530e3b..3007f51cda 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -139,30 +139,7 @@ where MOCK_MIGRATIONS_LIST::with(|inner_mgr: &mut MockMigrationManager| { callback(inner_mgr); }); - - // mimic the calls that would occur from the time a runtime upgrade starts until the - // Migrations pallet indicates that all upgrades are complete - - ExtBuilder::default().build().execute_with(|| { - let mut block_number = 1u64; - Migrations::on_runtime_upgrade(); - - while ! Migrations::is_fully_upgraded() { - System::set_block_number(block_number); - System::on_initialize(System::block_number()); - Migrations::on_initialize(System::block_number()); - Migrations::on_finalize(System::block_number()); - System::on_finalize(System::block_number()); - - block_number += 1; - - if block_number > 99999 { - panic!("Infinite loop?"); - } - } - - post_migration_callback(); - }); + post_migration_callback(); }); } @@ -254,3 +231,37 @@ pub(crate) fn events() -> Vec> { }) .collect::>() } + +pub(crate) fn roll_to(block_number: u64, invoke_on_runtime_upgrade_first: bool) { + + if invoke_on_runtime_upgrade_first { + Migrations::on_runtime_upgrade(); + } + + while System::block_number() < block_number { + System::set_block_number(System::block_number() + 1); + System::on_initialize(System::block_number()); + Migrations::on_initialize(System::block_number()); + Migrations::on_finalize(System::block_number()); + System::on_finalize(System::block_number()); + } +} + +pub(crate) fn roll_until_upgraded(invoke_on_runtime_upgrade_first: bool) { + + if invoke_on_runtime_upgrade_first { + Migrations::on_runtime_upgrade(); + } + + while ! Migrations::is_fully_upgraded() { + System::set_block_number(System::block_number() + 1); + System::on_initialize(System::block_number()); + Migrations::on_initialize(System::block_number()); + Migrations::on_finalize(System::block_number()); + System::on_finalize(System::block_number()); + + if System::block_number() > 99999 { + panic!("Infinite loop?"); + } + } +} diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index b335936771..8403e2e354 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -56,6 +56,9 @@ fn mock_migrations_static_hack_works() { ); }, &mut || { + ExtBuilder::default().build().execute_with(|| { + crate::mock::roll_until_upgraded(true); + }); *ecb_fn_called.lock().unwrap() = true; }); @@ -90,8 +93,6 @@ fn step_called_until_done() { let num_step_calls = Arc::new(Mutex::new(0usize)); - println!("step_called_until_done()..."); - crate::mock::execute_with_mock_migrations(&mut |mgr: &mut MockMigrationManager| { let num_step_calls = Arc::clone(&num_step_calls); @@ -101,7 +102,6 @@ fn step_called_until_done() { }, move |_, _| -> (Perbill, Weight) { let mut num_step_calls = num_step_calls.lock().unwrap(); - println!("step fn called, num times previously: {}", num_step_calls); *num_step_calls += 1; if *num_step_calls == 10 { (Perbill::one(), 0u64.into()) @@ -111,7 +111,11 @@ fn step_called_until_done() { } ); }, - &mut || {} ); + &mut || { + ExtBuilder::default().build().execute_with(|| { + crate::mock::roll_until_upgraded(true); + }); + }); assert_eq!(*num_step_calls.lock().unwrap(), 10, "migration step should be called until done"); } @@ -146,24 +150,27 @@ fn migration_progress_should_emit_events() { ); }, &mut || { - - let expected = vec![ - Event::RuntimeUpgradeStarted(), - Event::MigrationStarted("migration1".into()), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(50), 50), - Event::RuntimeUpgradeStepped(50), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(60), 51), - Event::RuntimeUpgradeStepped(51), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(70), 52), - Event::RuntimeUpgradeStepped(52), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(80), 53), - Event::RuntimeUpgradeStepped(53), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(100), 1), - Event::MigrationCompleted("migration1".into()), - Event::RuntimeUpgradeStepped(1), - Event::RuntimeUpgradeCompleted(), - ]; - assert_eq!(events(), expected); + ExtBuilder::default().build().execute_with(|| { + crate::mock::roll_until_upgraded(true); + + let expected = vec![ + Event::RuntimeUpgradeStarted(), + Event::MigrationStarted("migration1".into()), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(50), 50), + Event::RuntimeUpgradeStepped(50), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(60), 51), + Event::RuntimeUpgradeStepped(51), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(70), 52), + Event::RuntimeUpgradeStepped(52), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(80), 53), + Event::RuntimeUpgradeStepped(53), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(100), 1), + Event::MigrationCompleted("migration1".into()), + Event::RuntimeUpgradeStepped(1), + Event::RuntimeUpgradeCompleted(), + ]; + assert_eq!(events(), expected); + }); }); assert_eq!(*num_steps.lock().unwrap(), 5, "migration step should be called until done"); From f92411241bef09796e79066d534114f41d6e28d7 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 8 Jul 2021 16:22:40 -0600 Subject: [PATCH 38/99] Test that migrations are only run once --- pallets/migrations/src/tests.rs | 68 +++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 8403e2e354..45d590eceb 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -176,3 +176,71 @@ fn migration_progress_should_emit_events() { assert_eq!(*num_steps.lock().unwrap(), 5, "migration step should be called until done"); } + +#[test] +fn migration_should_only_be_invoked_once() { + + let num_name_fn_calls = Arc::new(Mutex::new(0usize)); + let num_step_fn_calls = Arc::new(Mutex::new(0usize)); + + crate::mock::execute_with_mock_migrations(&mut |mgr: &mut MockMigrationManager| { + let num_name_fn_calls = Arc::clone(&num_name_fn_calls); + let num_step_fn_calls = Arc::clone(&num_step_fn_calls); + + mgr.register_callback( + move || { + let mut num_name_fn_calls = num_name_fn_calls.lock().unwrap(); + *num_name_fn_calls += 1; + "migration1" + }, + move |_, _| -> (Perbill, Weight) { + let mut num_step_fn_calls = num_step_fn_calls.lock().unwrap(); + *num_step_fn_calls += 1; + (Perbill::one(), 1) // immediately done + } + ); + }, + &mut || { + ExtBuilder::default().build().execute_with(|| { + // roll forward until upgraded, should happen before block even increments + crate::mock::roll_until_upgraded(true); + + assert_eq!(System::block_number(), 1); + assert_eq!(*num_name_fn_calls.lock().unwrap(), 1, "migration name needed once"); + assert_eq!(*num_step_fn_calls.lock().unwrap(), 1, "migration step needed once"); + let mut expected = vec![ + Event::RuntimeUpgradeStarted(), + Event::MigrationStarted("migration1".into()), + Event::MigrationStepped("migration1".into(), Perbill::one(), 1), + Event::MigrationCompleted("migration1".into()), + Event::RuntimeUpgradeStepped(1), + Event::RuntimeUpgradeCompleted(), + ]; + assert_eq!(events(), expected); + + // attempt to roll forward again, block should still not increment, and migration + // name fn should be called but pallet_migrations should immediately recognize that + // no work needs to be done (and not call step) + crate::mock::roll_until_upgraded(true); + + assert_eq!(System::block_number(), 1); + assert_eq!(*num_name_fn_calls.lock().unwrap(), 2, "migration name needed twice"); + assert_eq!(*num_step_fn_calls.lock().unwrap(), 1, "migration step not needed again"); + expected.append(&mut vec![ + Event::RuntimeUpgradeStarted(), + // TODO: it might be nice to see an event here about a migration being skipped + // is there much overhead in emitting events? + Event::RuntimeUpgradeStepped(0), + Event::RuntimeUpgradeCompleted(), + ]); + assert_eq!(events(), expected); + + // roll forward a few blocks + crate::mock::roll_to(3, false); + assert_eq!(*num_name_fn_calls.lock().unwrap(), 2, "migration name not needed again"); + assert_eq!(*num_step_fn_calls.lock().unwrap(), 1, "migration step not needed again"); + // assert that no new events have been emitted + assert_eq!(events(), expected); + }); + }); +} From 5022dcc83a032a4d5d94ef454423848bca30e9fb Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 8 Jul 2021 16:39:32 -0600 Subject: [PATCH 39/99] Remove TODO/comment; events are not cheap and should be used conservatively --- pallets/migrations/src/tests.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 45d590eceb..036fe9a01c 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -228,8 +228,6 @@ fn migration_should_only_be_invoked_once() { assert_eq!(*num_step_fn_calls.lock().unwrap(), 1, "migration step not needed again"); expected.append(&mut vec![ Event::RuntimeUpgradeStarted(), - // TODO: it might be nice to see an event here about a migration being skipped - // is there much overhead in emitting events? Event::RuntimeUpgradeStepped(0), Event::RuntimeUpgradeCompleted(), ]); From a2177dd430b7415bb5ed6104df5ed598e8c19940 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 19 Jul 2021 10:57:14 -0600 Subject: [PATCH 40/99] Post merge-master fixes --- Cargo.lock | 2 -- node/service/src/chain_spec/moonbase.rs | 2 +- pallets/migrations/Cargo.toml | 15 ++++++++------- pallets/migrations/src/mock.rs | 4 ++-- runtime/common/Cargo.toml | 6 +++--- runtime/moonbase/src/lib.rs | 2 +- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd725b2a09..01e8632654 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -248,7 +248,6 @@ dependencies = [ "fastrand", "futures-lite", "once_cell 1.8.0", - "vec-arena", "slab", ] @@ -278,7 +277,6 @@ dependencies = [ "futures-lite", "libc", "log", - "nb-connect", "once_cell 1.8.0", "parking", "polling", diff --git a/node/service/src/chain_spec/moonbase.rs b/node/service/src/chain_spec/moonbase.rs index bff8c4a945..cd971ef85e 100644 --- a/node/service/src/chain_spec/moonbase.rs +++ b/node/service/src/chain_spec/moonbase.rs @@ -260,7 +260,7 @@ pub fn testnet_genesis( .collect(), }, treasury: Default::default(), - pallet_migrations: Default::default(), + migrations: Default::default(), } } diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index 92792f2c8a..8ba3532dd3 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -6,17 +6,17 @@ edition = "2018" description = "migrations management pallet" [dependencies] -frame-support = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } -frame-system = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } +frame-support = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } +frame-system = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } log = "0.4" -sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } -sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } +sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } +sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } parity-scale-codec = { version = "2.0.0", default-features = false } -environmental = "1.1.0" +environmental = { version = "1.1.2", default-features = false } +sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } +sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } [dev-dependencies] -sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } -sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } [features] default = ["std"] @@ -27,4 +27,5 @@ std = [ "sp-runtime/std", "sp-io/std", "sp-core/std", + "environmental/std", ] diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 3007f51cda..8102338884 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -41,7 +41,7 @@ construct_runtime!( UncheckedExtrinsic = UncheckedExtrinsic, { System: frame_system::{Pallet, Call, Config, Storage, Event}, - Migrations: pallet_migrations::{Pallet, Call, Storage, Config, Event}, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, } ); @@ -223,7 +223,7 @@ pub(crate) fn events() -> Vec> { .into_iter() .map(|r| r.event) .filter_map(|e| { - if let Event::pallet_migrations(inner) = e { + if let Event::Migrations(inner) = e { Some(inner) } else { None diff --git a/runtime/common/Cargo.toml b/runtime/common/Cargo.toml index 82915bc4d8..e3bb6cc812 100644 --- a/runtime/common/Cargo.toml +++ b/runtime/common/Cargo.toml @@ -9,9 +9,9 @@ edition = '2018' [dependencies] pallet-migrations = { path = "../../pallets/migrations", default-features = false } -sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } -sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false } -frame-support = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-v0.9.4" } +sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } +sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } +frame-support = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-v0.9.8" } [features] std = [ diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index 1b9da88600..afe07161d1 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -789,7 +789,7 @@ construct_runtime! { CrowdloanRewards: pallet_crowdloan_rewards::{Pallet, Call, Config, Storage, Event}, AuthorMapping: pallet_author_mapping::{Pallet, Call, Config, Storage, Event}, Proxy: pallet_proxy::{Pallet, Call, Storage, Event}, - Migrations: pallet_migrations::{Pallet, Call, Storage, Config, Event}, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, } } From 01727f8f2bbfab7cbb8853bb7eef512468b1318d Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 19 Jul 2021 11:11:51 -0600 Subject: [PATCH 41/99] Remove cruft --- pallets/migrations/Cargo.toml | 2 -- pallets/migrations/src/lib.rs | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index 8ba3532dd3..5f0491616b 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -16,8 +16,6 @@ environmental = { version = "1.1.2", default-features = false } sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } -[dev-dependencies] - [features] default = ["std"] std = [ diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 8bdc51aa9b..5f96e01c3e 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -77,7 +77,6 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(crate) fn deposit_event)] pub enum Event { - // e.g. runtime upgrade started, completed, etc. RuntimeUpgradeStarted(), RuntimeUpgradeStepped(Weight), RuntimeUpgradeCompleted(), @@ -110,7 +109,6 @@ pub mod pallet { /// on_initialize implementation. Calls process_runtime_upgrades() if we are still in the /// middle of a runtime upgrade. - /// TODO: use on_idle or some other hook? fn on_initialize(_: T::BlockNumber) -> Weight { // TODO: should account for the minimum one DB read @@ -138,7 +136,7 @@ pub mod pallet { #[pallet::genesis_config] pub struct GenesisConfig { pub completed_migrations: Vec>, - pub dummy: PhantomData, // TODO: + pub dummy: PhantomData, } #[cfg(feature = "std")] From 8ed861f7c7f48f706d5cbf6843126a1ade14fcdd Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 19 Jul 2021 12:29:27 -0600 Subject: [PATCH 42/99] Track some db reads and writes and charge accordingly --- pallets/migrations/src/lib.rs | 11 ++++++----- pallets/migrations/src/mock.rs | 5 +++-- pallets/migrations/src/tests.rs | 31 +++++++++++++++++++++++++++++-- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 5f96e01c3e..deb5ceb230 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -96,23 +96,23 @@ pub mod pallet { fn on_runtime_upgrade() -> Weight { log::warn!("Performing on_runtime_upgrade"); + let mut weight: Weight = 0u64.into(); + // start by flagging that we are not fully upgraded >::put(false); + weight += T::DbWeight::get().writes(1); Self::deposit_event(Event::RuntimeUpgradeStarted()); - let mut weight: Weight = 0u64.into(); - weight += process_runtime_upgrades::(); - weight.into() + weight } /// on_initialize implementation. Calls process_runtime_upgrades() if we are still in the /// middle of a runtime upgrade. fn on_initialize(_: T::BlockNumber) -> Weight { - // TODO: should account for the minimum one DB read - let mut weight: Weight = 0u64.into(); + let mut weight: Weight = T::DbWeight::get().reads(1 as Weight); if ! >::get() { weight += process_runtime_upgrades::(); @@ -234,6 +234,7 @@ pub mod pallet { if done { >::deposit_event(Event::RuntimeUpgradeCompleted()); >::put(true); + weight += T::DbWeight::get().writes(1); } weight diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 8102338884..066a24158a 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -18,7 +18,8 @@ use super::*; use crate as pallet_migrations; use frame_support::{ - construct_runtime, pallet_prelude::*, parameter_types, traits::GenesisBuild, weights::Weight, + construct_runtime, pallet_prelude::*, parameter_types, traits::GenesisBuild, + weights::{constants::RocksDbWeight, Weight}, }; use sp_core::H256; use sp_runtime::{ @@ -54,7 +55,7 @@ parameter_types! { } impl frame_system::Config for Test { type BaseCallFilter = (); - type DbWeight = (); + type DbWeight = RocksDbWeight; type Origin = Origin; type Index = u64; type BlockNumber = BlockNumber; diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 036fe9a01c..a5f65d9cf5 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -21,8 +21,8 @@ use crate::mock::{ use crate::Event; use std::sync::{Arc, Mutex}; use frame_support::{ - traits::OnRuntimeUpgrade, - weights::Weight, + traits::{OnRuntimeUpgrade, OnInitialize}, + weights::{constants::RocksDbWeight, Weight}, }; use sp_runtime::Perbill; @@ -242,3 +242,30 @@ fn migration_should_only_be_invoked_once() { }); }); } + +#[test] +fn on_initialize_should_charge_at_least_one_db_read() { + + ExtBuilder::default().build().execute_with(|| { + // first call to on_runtime_upgrade should flag FullyUpgraded as true + Migrations::on_runtime_upgrade(); + assert_eq!(Migrations::is_fully_upgraded(), true); + + // and subsequent call to on_initialize should do nothing but read this value and return + let weight = >::on_initialize(1); + assert_eq!(weight, RocksDbWeight::get().reads(1)); + }) +} + +#[test] +fn on_runtime_upgrade_charges_minimum_two_db_writes() { + + ExtBuilder::default().build().execute_with(|| { + let mut weight = Migrations::on_runtime_upgrade(); + + // substrate seems to add a write to this call, so substract one for our logic + weight -= RocksDbWeight::get().writes(1); + + assert_eq!(weight, RocksDbWeight::get().writes(2)); + }) +} From 0785e4891fb2804ae67d392d42ddce05b6b67f9b Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 20 Jul 2021 13:41:51 -0600 Subject: [PATCH 43/99] cargo fmt --- pallets/migrations/src/lib.rs | 11 +- pallets/migrations/src/mock.rs | 26 ++- pallets/migrations/src/tests.rs | 379 ++++++++++++++++++-------------- 3 files changed, 230 insertions(+), 186 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index deb5ceb230..fe0b380d63 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -30,7 +30,8 @@ use sp_runtime::Perbill; pub use pallet::*; #[cfg(test)] -#[macro_use] extern crate environmental; +#[macro_use] +extern crate environmental; /// A Migration that must happen on-chain upon a runtime-upgrade pub trait Migration { @@ -111,10 +112,9 @@ pub mod pallet { /// on_initialize implementation. Calls process_runtime_upgrades() if we are still in the /// middle of a runtime upgrade. fn on_initialize(_: T::BlockNumber) -> Weight { - let mut weight: Weight = T::DbWeight::get().reads(1 as Weight); - if ! >::get() { + if !>::get() { weight += process_runtime_upgrades::(); } @@ -176,10 +176,9 @@ pub mod pallet { >::get(migration_name_as_bytes).unwrap_or(Perbill::zero()); if migration_state < Perbill::one() { - if migration_state.is_zero() { >::deposit_event(Event::MigrationStarted( - migration_name_as_bytes.into() + migration_name_as_bytes.into(), )); } @@ -219,7 +218,7 @@ pub mod pallet { done = false; } else { >::deposit_event(Event::MigrationCompleted( - migration_name_as_bytes.into() + migration_name_as_bytes.into(), )); } diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 066a24158a..e674e383d2 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -18,7 +18,10 @@ use super::*; use crate as pallet_migrations; use frame_support::{ - construct_runtime, pallet_prelude::*, parameter_types, traits::GenesisBuild, + construct_runtime, + pallet_prelude::*, + parameter_types, + traits::GenesisBuild, weights::{constants::RocksDbWeight, Weight}, }; use sp_core::H256; @@ -81,10 +84,10 @@ impl frame_system::Config for Test { /// MockMigrationManager stores the test-side callbacks/closures used in the Migrations list glue. /// It is is expected to exist as a singleton, but only remain relevant within the scope of a test. -/// +/// /// Tests should use execute_with_mock_migrations(), which will create a MockMigrationManager and /// provide it to the test. -/// +/// /// A pair of callbacks provided to register_callback() will map directly to a single instance of /// Migration (done by the MockMigration glue below). Treat each pair of callbacks as though it were /// a custom implementation of the Migration trait just as a normal Pallet would. @@ -108,16 +111,19 @@ impl<'test> MockMigrationManager<'test> { self.name_fn_callbacks[index]() } - pub(crate) fn invoke_step_fn(&mut self, index: usize, previous_progress: Perbill, available_weight: Weight) - -> (Perbill, Weight) - { + pub(crate) fn invoke_step_fn( + &mut self, + index: usize, + previous_progress: Perbill, + available_weight: Weight, + ) -> (Perbill, Weight) { self.step_fn_callbacks[index](previous_progress, available_weight) } fn generate_migrations_list(&self) -> Vec> { let mut migrations: Vec> = Vec::new(); for i in 0..self.name_fn_callbacks.len() { - migrations.push(Box::new(MockMigration{index: i})); + migrations.push(Box::new(MockMigration { index: i })); } migrations } @@ -170,7 +176,7 @@ impl Migration for MockMigration { } /// Implementation of Migrations. Generates a Vec of MockMigrations on the fly based on the current -/// contents of MOCK_MIGRATIONS_LIST. +/// contents of MOCK_MIGRATIONS_LIST. pub struct MockMigrations; impl Get>> for MockMigrations { fn get() -> Vec> { @@ -234,7 +240,6 @@ pub(crate) fn events() -> Vec> { } pub(crate) fn roll_to(block_number: u64, invoke_on_runtime_upgrade_first: bool) { - if invoke_on_runtime_upgrade_first { Migrations::on_runtime_upgrade(); } @@ -249,12 +254,11 @@ pub(crate) fn roll_to(block_number: u64, invoke_on_runtime_upgrade_first: bool) } pub(crate) fn roll_until_upgraded(invoke_on_runtime_upgrade_first: bool) { - if invoke_on_runtime_upgrade_first { Migrations::on_runtime_upgrade(); } - while ! Migrations::is_fully_upgraded() { + while !Migrations::is_fully_upgraded() { System::set_block_number(System::block_number() + 1); System::on_initialize(System::block_number()); Migrations::on_initialize(System::block_number()); diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index a5f65d9cf5..7b88e176f4 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -15,16 +15,14 @@ // along with Moonbeam. If not, see . //! Unit testing -use crate::mock::{ - events, ExtBuilder, Migrations, System, MockMigrationManager, -}; +use crate::mock::{events, ExtBuilder, Migrations, MockMigrationManager, System}; use crate::Event; -use std::sync::{Arc, Mutex}; use frame_support::{ - traits::{OnRuntimeUpgrade, OnInitialize}, + traits::{OnInitialize, OnRuntimeUpgrade}, weights::{constants::RocksDbWeight, Weight}, }; use sp_runtime::Perbill; +use std::sync::{Arc, Mutex}; #[test] fn genesis_builder_works() { @@ -35,36 +33,49 @@ fn genesis_builder_works() { #[test] fn mock_migrations_static_hack_works() { - let name_fn_called = Arc::new(Mutex::new(false)); let step_fn_called = Arc::new(Mutex::new(false)); let ecb_fn_called = Arc::new(Mutex::new(false)); - crate::mock::execute_with_mock_migrations(&mut |mgr: &mut MockMigrationManager| { - let name_fn_called = Arc::clone(&name_fn_called); - let step_fn_called = Arc::clone(&step_fn_called); - - mgr.register_callback( - move || { - *name_fn_called.lock().unwrap() = true; - "hello, world" - }, - move |_, _| -> (Perbill, Weight) { - *step_fn_called.lock().unwrap() = true; - (Perbill::one(), 0u64.into()) - } - ); - }, - &mut || { - ExtBuilder::default().build().execute_with(|| { - crate::mock::roll_until_upgraded(true); - }); - *ecb_fn_called.lock().unwrap() = true; - }); - - assert_eq!(*name_fn_called.lock().unwrap(), true, "mock migration should call friendly_name()"); - assert_eq!(*step_fn_called.lock().unwrap(), true, "mock migration should call step()"); - assert_eq!(*ecb_fn_called.lock().unwrap(), true, "mock migration should call ECB callback"); + crate::mock::execute_with_mock_migrations( + &mut |mgr: &mut MockMigrationManager| { + let name_fn_called = Arc::clone(&name_fn_called); + let step_fn_called = Arc::clone(&step_fn_called); + + mgr.register_callback( + move || { + *name_fn_called.lock().unwrap() = true; + "hello, world" + }, + move |_, _| -> (Perbill, Weight) { + *step_fn_called.lock().unwrap() = true; + (Perbill::one(), 0u64.into()) + }, + ); + }, + &mut || { + ExtBuilder::default().build().execute_with(|| { + crate::mock::roll_until_upgraded(true); + }); + *ecb_fn_called.lock().unwrap() = true; + }, + ); + + assert_eq!( + *name_fn_called.lock().unwrap(), + true, + "mock migration should call friendly_name()" + ); + assert_eq!( + *step_fn_called.lock().unwrap(), + true, + "mock migration should call step()" + ); + assert_eq!( + *ecb_fn_called.lock().unwrap(), + true, + "mock migration should call ECB callback" + ); } #[test] @@ -90,162 +101,193 @@ fn on_runtime_upgrade_emits_events() { #[test] fn step_called_until_done() { - let num_step_calls = Arc::new(Mutex::new(0usize)); - crate::mock::execute_with_mock_migrations(&mut |mgr: &mut MockMigrationManager| { - let num_step_calls = Arc::clone(&num_step_calls); - - mgr.register_callback( - move || { - "migration1" - }, - move |_, _| -> (Perbill, Weight) { - let mut num_step_calls = num_step_calls.lock().unwrap(); - *num_step_calls += 1; - if *num_step_calls == 10 { - (Perbill::one(), 0u64.into()) - } else { - (Perbill::zero(), 0u64.into()) - } - } - ); - }, - &mut || { - ExtBuilder::default().build().execute_with(|| { - crate::mock::roll_until_upgraded(true); - }); - }); - - assert_eq!(*num_step_calls.lock().unwrap(), 10, "migration step should be called until done"); + crate::mock::execute_with_mock_migrations( + &mut |mgr: &mut MockMigrationManager| { + let num_step_calls = Arc::clone(&num_step_calls); + + mgr.register_callback( + move || "migration1", + move |_, _| -> (Perbill, Weight) { + let mut num_step_calls = num_step_calls.lock().unwrap(); + *num_step_calls += 1; + if *num_step_calls == 10 { + (Perbill::one(), 0u64.into()) + } else { + (Perbill::zero(), 0u64.into()) + } + }, + ); + }, + &mut || { + ExtBuilder::default().build().execute_with(|| { + crate::mock::roll_until_upgraded(true); + }); + }, + ); + + assert_eq!( + *num_step_calls.lock().unwrap(), + 10, + "migration step should be called until done" + ); } #[test] fn migration_progress_should_emit_events() { - let num_steps = Arc::new(Mutex::new(0usize)); - crate::mock::execute_with_mock_migrations(&mut |mgr: &mut MockMigrationManager| { - let num_steps = Arc::clone(&num_steps); - - mgr.register_callback( - move || { - "migration1" - }, - move |_, _| -> (Perbill, Weight) { - let mut num_steps = num_steps.lock().unwrap(); - - let result: (Perbill, Weight) = match *num_steps { - 0 => (Perbill::from_percent(50), 50), - 1 => (Perbill::from_percent(60), 51), - 2 => (Perbill::from_percent(70), 52), - 3 => (Perbill::from_percent(80), 53), - 4 => (Perbill::from_percent(100), 1), - _ => { unreachable!(); } - }; - - *num_steps += 1; - result - } - ); - }, - &mut || { - ExtBuilder::default().build().execute_with(|| { - crate::mock::roll_until_upgraded(true); - - let expected = vec![ - Event::RuntimeUpgradeStarted(), - Event::MigrationStarted("migration1".into()), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(50), 50), - Event::RuntimeUpgradeStepped(50), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(60), 51), - Event::RuntimeUpgradeStepped(51), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(70), 52), - Event::RuntimeUpgradeStepped(52), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(80), 53), - Event::RuntimeUpgradeStepped(53), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(100), 1), - Event::MigrationCompleted("migration1".into()), - Event::RuntimeUpgradeStepped(1), - Event::RuntimeUpgradeCompleted(), - ]; - assert_eq!(events(), expected); - }); - }); - - assert_eq!(*num_steps.lock().unwrap(), 5, "migration step should be called until done"); - + crate::mock::execute_with_mock_migrations( + &mut |mgr: &mut MockMigrationManager| { + let num_steps = Arc::clone(&num_steps); + + mgr.register_callback( + move || "migration1", + move |_, _| -> (Perbill, Weight) { + let mut num_steps = num_steps.lock().unwrap(); + + let result: (Perbill, Weight) = match *num_steps { + 0 => (Perbill::from_percent(50), 50), + 1 => (Perbill::from_percent(60), 51), + 2 => (Perbill::from_percent(70), 52), + 3 => (Perbill::from_percent(80), 53), + 4 => (Perbill::from_percent(100), 1), + _ => { + unreachable!(); + } + }; + + *num_steps += 1; + result + }, + ); + }, + &mut || { + ExtBuilder::default().build().execute_with(|| { + crate::mock::roll_until_upgraded(true); + + let expected = vec![ + Event::RuntimeUpgradeStarted(), + Event::MigrationStarted("migration1".into()), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(50), 50), + Event::RuntimeUpgradeStepped(50), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(60), 51), + Event::RuntimeUpgradeStepped(51), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(70), 52), + Event::RuntimeUpgradeStepped(52), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(80), 53), + Event::RuntimeUpgradeStepped(53), + Event::MigrationStepped("migration1".into(), Perbill::from_percent(100), 1), + Event::MigrationCompleted("migration1".into()), + Event::RuntimeUpgradeStepped(1), + Event::RuntimeUpgradeCompleted(), + ]; + assert_eq!(events(), expected); + }); + }, + ); + + assert_eq!( + *num_steps.lock().unwrap(), + 5, + "migration step should be called until done" + ); } #[test] fn migration_should_only_be_invoked_once() { - let num_name_fn_calls = Arc::new(Mutex::new(0usize)); let num_step_fn_calls = Arc::new(Mutex::new(0usize)); - crate::mock::execute_with_mock_migrations(&mut |mgr: &mut MockMigrationManager| { - let num_name_fn_calls = Arc::clone(&num_name_fn_calls); - let num_step_fn_calls = Arc::clone(&num_step_fn_calls); - - mgr.register_callback( - move || { - let mut num_name_fn_calls = num_name_fn_calls.lock().unwrap(); - *num_name_fn_calls += 1; - "migration1" - }, - move |_, _| -> (Perbill, Weight) { - let mut num_step_fn_calls = num_step_fn_calls.lock().unwrap(); - *num_step_fn_calls += 1; - (Perbill::one(), 1) // immediately done - } - ); - }, - &mut || { - ExtBuilder::default().build().execute_with(|| { - // roll forward until upgraded, should happen before block even increments - crate::mock::roll_until_upgraded(true); - - assert_eq!(System::block_number(), 1); - assert_eq!(*num_name_fn_calls.lock().unwrap(), 1, "migration name needed once"); - assert_eq!(*num_step_fn_calls.lock().unwrap(), 1, "migration step needed once"); - let mut expected = vec![ - Event::RuntimeUpgradeStarted(), - Event::MigrationStarted("migration1".into()), - Event::MigrationStepped("migration1".into(), Perbill::one(), 1), - Event::MigrationCompleted("migration1".into()), - Event::RuntimeUpgradeStepped(1), - Event::RuntimeUpgradeCompleted(), - ]; - assert_eq!(events(), expected); - - // attempt to roll forward again, block should still not increment, and migration - // name fn should be called but pallet_migrations should immediately recognize that - // no work needs to be done (and not call step) - crate::mock::roll_until_upgraded(true); - - assert_eq!(System::block_number(), 1); - assert_eq!(*num_name_fn_calls.lock().unwrap(), 2, "migration name needed twice"); - assert_eq!(*num_step_fn_calls.lock().unwrap(), 1, "migration step not needed again"); - expected.append(&mut vec![ - Event::RuntimeUpgradeStarted(), - Event::RuntimeUpgradeStepped(0), - Event::RuntimeUpgradeCompleted(), - ]); - assert_eq!(events(), expected); - - // roll forward a few blocks - crate::mock::roll_to(3, false); - assert_eq!(*num_name_fn_calls.lock().unwrap(), 2, "migration name not needed again"); - assert_eq!(*num_step_fn_calls.lock().unwrap(), 1, "migration step not needed again"); - // assert that no new events have been emitted - assert_eq!(events(), expected); - }); - }); + crate::mock::execute_with_mock_migrations( + &mut |mgr: &mut MockMigrationManager| { + let num_name_fn_calls = Arc::clone(&num_name_fn_calls); + let num_step_fn_calls = Arc::clone(&num_step_fn_calls); + + mgr.register_callback( + move || { + let mut num_name_fn_calls = num_name_fn_calls.lock().unwrap(); + *num_name_fn_calls += 1; + "migration1" + }, + move |_, _| -> (Perbill, Weight) { + let mut num_step_fn_calls = num_step_fn_calls.lock().unwrap(); + *num_step_fn_calls += 1; + (Perbill::one(), 1) // immediately done + }, + ); + }, + &mut || { + ExtBuilder::default().build().execute_with(|| { + // roll forward until upgraded, should happen before block even increments + crate::mock::roll_until_upgraded(true); + + assert_eq!(System::block_number(), 1); + assert_eq!( + *num_name_fn_calls.lock().unwrap(), + 1, + "migration name needed once" + ); + assert_eq!( + *num_step_fn_calls.lock().unwrap(), + 1, + "migration step needed once" + ); + let mut expected = vec![ + Event::RuntimeUpgradeStarted(), + Event::MigrationStarted("migration1".into()), + Event::MigrationStepped("migration1".into(), Perbill::one(), 1), + Event::MigrationCompleted("migration1".into()), + Event::RuntimeUpgradeStepped(1), + Event::RuntimeUpgradeCompleted(), + ]; + assert_eq!(events(), expected); + + // attempt to roll forward again, block should still not increment, and migration + // name fn should be called but pallet_migrations should immediately recognize that + // no work needs to be done (and not call step) + crate::mock::roll_until_upgraded(true); + + assert_eq!(System::block_number(), 1); + assert_eq!( + *num_name_fn_calls.lock().unwrap(), + 2, + "migration name needed twice" + ); + assert_eq!( + *num_step_fn_calls.lock().unwrap(), + 1, + "migration step not needed again" + ); + expected.append(&mut vec![ + Event::RuntimeUpgradeStarted(), + Event::RuntimeUpgradeStepped(0), + Event::RuntimeUpgradeCompleted(), + ]); + assert_eq!(events(), expected); + + // roll forward a few blocks + crate::mock::roll_to(3, false); + assert_eq!( + *num_name_fn_calls.lock().unwrap(), + 2, + "migration name not needed again" + ); + assert_eq!( + *num_step_fn_calls.lock().unwrap(), + 1, + "migration step not needed again" + ); + // assert that no new events have been emitted + assert_eq!(events(), expected); + }); + }, + ); } #[test] fn on_initialize_should_charge_at_least_one_db_read() { - ExtBuilder::default().build().execute_with(|| { // first call to on_runtime_upgrade should flag FullyUpgraded as true Migrations::on_runtime_upgrade(); @@ -259,7 +301,6 @@ fn on_initialize_should_charge_at_least_one_db_read() { #[test] fn on_runtime_upgrade_charges_minimum_two_db_writes() { - ExtBuilder::default().build().execute_with(|| { let mut weight = Migrations::on_runtime_upgrade(); From 8c444e77a3f7ad58efc6caa1b874e50cc5a2e9d9 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 21 Jul 2021 10:03:29 -0600 Subject: [PATCH 44/99] Add failing test about one-migration-at-a-time --- pallets/migrations/src/tests.rs | 51 +++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 7b88e176f4..b677e6a87c 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -310,3 +310,54 @@ fn on_runtime_upgrade_charges_minimum_two_db_writes() { assert_eq!(weight, RocksDbWeight::get().writes(2)); }) } + +#[test] +fn only_one_outstanding_test_at_a_time() { + let num_migration1_calls = Arc::new(Mutex::new(0usize)); + let num_migration2_calls = Arc::new(Mutex::new(0usize)); + + // create two migrations. the first will return < Perbill::one(), which should cause + // pallet-migrations to not call the second in the first step. We only step once. + + crate::mock::execute_with_mock_migrations( + &mut |mgr: &mut MockMigrationManager| { + let num_migration1_calls = Arc::clone(&num_migration1_calls); + let num_migration2_calls = Arc::clone(&num_migration2_calls); + + mgr.register_callback( + move || "migration1", + move |_, _| -> (Perbill, Weight) { + let mut num_migration1_calls = num_migration1_calls.lock().unwrap(); + *num_migration1_calls += 1; + (Perbill::zero(), 0u64.into()) + }, + ); + + mgr.register_callback( + move || "migration2", + move |_, _| -> (Perbill, Weight) { + let mut num_migration2_calls = num_migration2_calls.lock().unwrap(); + *num_migration2_calls += 1; + (Perbill::zero(), 0u64.into()) + }, + ); + }, + &mut || { + ExtBuilder::default().build().execute_with(|| { + Migrations::on_runtime_upgrade(); + }); + }, + ); + + assert_eq!( + *num_migration1_calls.lock().unwrap(), + 1, + "mock migration should call friendly_name()" + ); + + assert_eq!( + *num_migration2_calls.lock().unwrap(), + 0, + "mock migration should call friendly_name()" + ); +} From 2e12fd2eea27adbfa15cd193aea9e68869107884 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 21 Jul 2021 10:19:59 -0600 Subject: [PATCH 45/99] Don't start next migration until current is done --- pallets/migrations/src/lib.rs | 9 ++++--- pallets/migrations/src/tests.rs | 43 +++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index fe0b380d63..76b9976169 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -213,18 +213,19 @@ pub mod pallet { ); } + if migration_state != updated_progress { + >::insert(migration_name.as_bytes(), updated_progress); + } + // make note of any unfinished migrations if updated_progress < Perbill::one() { done = false; + break; } else { >::deposit_event(Event::MigrationCompleted( migration_name_as_bytes.into(), )); } - - if migration_state != updated_progress { - >::insert(migration_name.as_bytes(), updated_progress); - } } } diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index b677e6a87c..2bf42f3295 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -316,8 +316,8 @@ fn only_one_outstanding_test_at_a_time() { let num_migration1_calls = Arc::new(Mutex::new(0usize)); let num_migration2_calls = Arc::new(Mutex::new(0usize)); - // create two migrations. the first will return < Perbill::one(), which should cause - // pallet-migrations to not call the second in the first step. We only step once. + // create two migrations. the first will return < Perbill::one() until its 3rd step, which + // should prevent the second from running. Once it s done, the second should execute. crate::mock::execute_with_mock_migrations( &mut |mgr: &mut MockMigrationManager| { @@ -329,7 +329,13 @@ fn only_one_outstanding_test_at_a_time() { move |_, _| -> (Perbill, Weight) { let mut num_migration1_calls = num_migration1_calls.lock().unwrap(); *num_migration1_calls += 1; - (Perbill::zero(), 0u64.into()) + + // this migration is done on its 3rd step + if *num_migration1_calls < 3 { + (Perbill::zero(), 0u64.into()) + } else { + (Perbill::one(), 0u64.into()) + } }, ); @@ -338,26 +344,31 @@ fn only_one_outstanding_test_at_a_time() { move |_, _| -> (Perbill, Weight) { let mut num_migration2_calls = num_migration2_calls.lock().unwrap(); *num_migration2_calls += 1; - (Perbill::zero(), 0u64.into()) + (Perbill::one(), 0u64.into()) }, ); }, &mut || { ExtBuilder::default().build().execute_with(|| { + + // first pass should invoke migration1 once and not move on to migration2 Migrations::on_runtime_upgrade(); - }); - }, - ); + assert_eq!(*num_migration1_calls.lock().unwrap(), 1); + assert_eq!(*num_migration2_calls.lock().unwrap(), 0); - assert_eq!( - *num_migration1_calls.lock().unwrap(), - 1, - "mock migration should call friendly_name()" - ); + // second pass should do the same + crate::mock::roll_to(2, false); + assert_eq!(*num_migration1_calls.lock().unwrap(), 2); + assert_eq!(*num_migration2_calls.lock().unwrap(), 0); - assert_eq!( - *num_migration2_calls.lock().unwrap(), - 0, - "mock migration should call friendly_name()" + // third pass should invoke both + crate::mock::roll_to(3, false); + assert_eq!(*num_migration1_calls.lock().unwrap(), 3); + assert_eq!(*num_migration2_calls.lock().unwrap(), 1); + + // and both should be done now + assert_eq!(Migrations::is_fully_upgraded(), true); + }); + }, ); } From 9a9bfd4d2256ca0127574b6e55100bae65b8f3d1 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 22 Jul 2021 09:46:04 -0600 Subject: [PATCH 46/99] Add notes from meeting --- pallets/migrations/src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 76b9976169..d580f24d93 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -100,6 +100,10 @@ pub mod pallet { let mut weight: Weight = 0u64.into(); // start by flagging that we are not fully upgraded + // TODO: case where this is already false (e.g. we started an upgrade while one was + // already in progress) + // notably, we might have started an individual migration and the list of + // migrations might change on our next on_runtime_upgrade() >::put(false); weight += T::DbWeight::get().writes(1); Self::deposit_event(Event::RuntimeUpgradeStarted()); @@ -114,6 +118,7 @@ pub mod pallet { fn on_initialize(_: T::BlockNumber) -> Weight { let mut weight: Weight = T::DbWeight::get().reads(1 as Weight); + // TODO: don't support this now, it's dangerous! if !>::get() { weight += process_runtime_upgrades::(); } From 274457150809b9506ca785326fc592bffcd3e159 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 22 Jul 2021 11:05:53 -0600 Subject: [PATCH 47/99] Allow multi-block migrations to be disabled --- pallets/migrations/src/lib.rs | 17 ++++++++++++++--- pallets/migrations/src/mock.rs | 25 +++++++++++++++++++++++- pallets/migrations/src/tests.rs | 34 +++++++++++++++++++++++++++++++++ runtime/moonbase/src/lib.rs | 6 ++++++ 4 files changed, 78 insertions(+), 4 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index d580f24d93..055b193838 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -73,6 +73,8 @@ pub mod pallet { type Event: From> + IsType<::Event>; /// The list of migrations that will be performed type MigrationsList: Get>>; + /// Whether or not multi-block migrations are supported + type MultiBlockMigrationsSupported: Get; } #[pallet::event] @@ -110,6 +112,14 @@ pub mod pallet { weight += process_runtime_upgrades::(); + // at least emit a warning if we aren't going to end up finishing our migrations... + if ! T::MultiBlockMigrationsSupported::get() { + if ! >::get() { + log::warn!("migrations weren't completed in on_runtime_upgrade(), but we're not + configured for multi-block migrations; state is potentially inconsistent!"); + } + } + weight } @@ -118,9 +128,10 @@ pub mod pallet { fn on_initialize(_: T::BlockNumber) -> Weight { let mut weight: Weight = T::DbWeight::get().reads(1 as Weight); - // TODO: don't support this now, it's dangerous! - if !>::get() { - weight += process_runtime_upgrades::(); + if T::MultiBlockMigrationsSupported::get() { + if !>::get() { + weight += process_runtime_upgrades::(); + } } weight diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index e674e383d2..84954450ef 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -91,10 +91,20 @@ impl frame_system::Config for Test { /// A pair of callbacks provided to register_callback() will map directly to a single instance of /// Migration (done by the MockMigration glue below). Treat each pair of callbacks as though it were /// a custom implementation of the Migration trait just as a normal Pallet would. -#[derive(Default)] pub struct MockMigrationManager<'test> { name_fn_callbacks: Vec &'static str>>, step_fn_callbacks: Vec (Perbill, Weight)>>, + pub is_multi_block: bool, // corresponds to MultiBlockMigrationsSupported. defaults to true. +} + +impl Default for MockMigrationManager<'_> { + fn default() -> Self { + Self { + name_fn_callbacks: Default::default(), + step_fn_callbacks: Default::default(), + is_multi_block: true, + } + } } impl<'test> MockMigrationManager<'test> { @@ -188,9 +198,22 @@ impl Get>> for MockMigrations { } } +/// Similar to the impl for Get>> but we return a value suitable for +/// MultiBlockMigrationsSupported +impl Get for MockMigrations { + fn get() -> bool { + let mut supported = false; + MOCK_MIGRATIONS_LIST::with(|mgr: &mut MockMigrationManager| { + supported = mgr.is_multi_block; + }); + supported + } +} + impl Config for Test { type Event = Event; type MigrationsList = MockMigrations; + type MultiBlockMigrationsSupported = MockMigrations; } /// Externality builder for pallet migration's mock runtime diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 2bf42f3295..9e6d333f3a 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -372,3 +372,37 @@ fn only_one_outstanding_test_at_a_time() { }, ); } + +#[test] +fn multi_block_migration_flag_works() { + let num_migration_calls = Arc::new(Mutex::new(0u32)); + + // we create a single migration that wants to take more than one block and ensure that it's only + // called once + + crate::mock::execute_with_mock_migrations( + &mut |mgr: &mut MockMigrationManager| { + let num_migration_calls = Arc::clone(&num_migration_calls); + + mgr.is_multi_block = false; + + mgr.register_callback( + move || { + "migration1" + }, + move |_, _| -> (Perbill, Weight) { + *num_migration_calls.lock().unwrap() += 1; + (Perbill::zero(), 0u64.into()) + }, + ); + }, + &mut || { + ExtBuilder::default().build().execute_with(|| { + crate::mock::roll_to(5, true); + + assert_eq!(*num_migration_calls.lock().unwrap(), 1); + assert_eq!(Migrations::is_fully_upgraded(), false); + }); + }, + ); +} diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index afe07161d1..de374e71eb 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -753,9 +753,15 @@ impl pallet_proxy::Config for Runtime { type AnnouncementDepositFactor = AnnouncementDepositFactor; } +parameter_types! { + // Our runtime dosen't support multi-block migrations currently + pub const MultiBlockMigrationsNotSupported: bool = false; +} + impl Config for Runtime { type Event = Event; type MigrationsList = runtime_common::migrations::CommonMigrations; + type MultiBlockMigrationsSupported = MultiBlockMigrationsNotSupported; } construct_runtime! { From e1f49c6ddc23f3231baa0123f53f6e8b803f6db5 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 22 Jul 2021 11:30:44 -0600 Subject: [PATCH 48/99] Add failing test about overweight migrations --- pallets/migrations/src/tests.rs | 65 +++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 9e6d333f3a..cd6e6fbf34 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -406,3 +406,68 @@ fn multi_block_migration_flag_works() { }, ); } + +#[test] +fn overweight_migrations_tolerated() { + + // pallet-migrations currently tolerates a migration going over-weight. not only does it + // tolerate it, but it continues on to the next migration even if it's already overweight. + // + // Now that the pallet can be configured to not support multi-block migrations, this is sort of + // a feature and not really a bug -- this test case exists to explicitly acknowledge/protect + // that. + // + // The logic behind this is that we would rather go over-weight and risk a block taking too long + // (which *might* be "catastrophic") than outright prevent migrations from proceeding (which is + // certainly "catastrophic"). + // + // maybe_catastrophic > certainly_catastrophic + + let num_migration1_calls = Arc::new(Mutex::new(0u32)); + let num_migration2_calls = Arc::new(Mutex::new(0u32)); + let num_migration3_calls = Arc::new(Mutex::new(0u32)); + + crate::mock::execute_with_mock_migrations( + &mut |mgr: &mut MockMigrationManager| { + let num_migration1_calls = Arc::clone(&num_migration1_calls); + let num_migration2_calls = Arc::clone(&num_migration2_calls); + let num_migration3_calls = Arc::clone(&num_migration3_calls); + + mgr.is_multi_block = false; + + mgr.register_callback( + move || { "migration1" }, + move |_, _| -> (Perbill, Weight) { + *num_migration1_calls.lock().unwrap() += 1; + (Perbill::one(), 1_000_000_000_000u64.into()) + }, + ); + + mgr.register_callback( + move || { "migration2" }, + move |_, _| -> (Perbill, Weight) { + *num_migration2_calls.lock().unwrap() += 1; + (Perbill::one(), 1_000_000_000_000u64.into()) + }, + ); + + mgr.register_callback( + move || { "migration3" }, + move |_, _| -> (Perbill, Weight) { + *num_migration3_calls.lock().unwrap() += 1; + (Perbill::one(), 1_000_000_000_000u64.into()) + }, + ); + }, + &mut || { + ExtBuilder::default().build().execute_with(|| { + Migrations::on_runtime_upgrade(); + + assert_eq!(*num_migration1_calls.lock().unwrap(), 1); + assert_eq!(*num_migration2_calls.lock().unwrap(), 1); + assert_eq!(*num_migration3_calls.lock().unwrap(), 1); + assert_eq!(Migrations::is_fully_upgraded(), true); + }); + }, + ); +} From 0684874a864b3ea547aad09ffc3dd9c4851c0f06 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 22 Jul 2021 11:44:42 -0600 Subject: [PATCH 49/99] Explicitly embrace allowing overweight migrations --- pallets/migrations/src/lib.rs | 20 +++++++++++++++++++- pallets/migrations/src/tests.rs | 2 ++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 055b193838..eea437b21d 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -198,7 +198,25 @@ pub mod pallet { )); } - let available_for_step = available_weight - weight; + // TODO: here we allow migrations to go overweight because that's "safer" than not + // doing so in the case of MultiBlockMigrationsSupported == false. In this case, any + // migrations not handled in this pass will not occur at all, which will leave + // pallets unmigrated. + // + // Options for handling this more gracefully: + // 1) make this particular logic (whether or not we tolerate this) configurable + // 2) only follow this logic when MultiBlockMigrationsSupported == false + let available_for_step = if available_weight > weight { + available_weight - weight + } else { + log::warn!("previous migration went overweight; + ignoring and providing migration {} 0 weight.", + migration_name, + ); + + 0u64.into() + }; + log::trace!( "stepping migration {}, prev: {:?}, avail weight: {}", migration_name, diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index cd6e6fbf34..e017c8cd25 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -439,6 +439,8 @@ fn overweight_migrations_tolerated() { move || { "migration1" }, move |_, _| -> (Perbill, Weight) { *num_migration1_calls.lock().unwrap() += 1; + // TODO: this is brittle because it assumes it is larger than the value used at + // the top of process_runtime_upgrades() (Perbill::one(), 1_000_000_000_000u64.into()) }, ); From c19a4a1ebf2e020fed98065d8fd05f7cbf3a22da Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 22 Jul 2021 11:50:15 -0600 Subject: [PATCH 50/99] cargo fmt --- pallets/migrations/src/lib.rs | 13 ++++++++----- pallets/migrations/src/tests.rs | 12 ++++-------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index eea437b21d..b5c0593e43 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -113,10 +113,12 @@ pub mod pallet { weight += process_runtime_upgrades::(); // at least emit a warning if we aren't going to end up finishing our migrations... - if ! T::MultiBlockMigrationsSupported::get() { - if ! >::get() { - log::warn!("migrations weren't completed in on_runtime_upgrade(), but we're not - configured for multi-block migrations; state is potentially inconsistent!"); + if !T::MultiBlockMigrationsSupported::get() { + if !>::get() { + log::warn!( + "migrations weren't completed in on_runtime_upgrade(), but we're not + configured for multi-block migrations; state is potentially inconsistent!" + ); } } @@ -209,7 +211,8 @@ pub mod pallet { let available_for_step = if available_weight > weight { available_weight - weight } else { - log::warn!("previous migration went overweight; + log::warn!( + "previous migration went overweight; ignoring and providing migration {} 0 weight.", migration_name, ); diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index e017c8cd25..40ef4847c3 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -350,7 +350,6 @@ fn only_one_outstanding_test_at_a_time() { }, &mut || { ExtBuilder::default().build().execute_with(|| { - // first pass should invoke migration1 once and not move on to migration2 Migrations::on_runtime_upgrade(); assert_eq!(*num_migration1_calls.lock().unwrap(), 1); @@ -387,9 +386,7 @@ fn multi_block_migration_flag_works() { mgr.is_multi_block = false; mgr.register_callback( - move || { - "migration1" - }, + move || "migration1", move |_, _| -> (Perbill, Weight) { *num_migration_calls.lock().unwrap() += 1; (Perbill::zero(), 0u64.into()) @@ -409,7 +406,6 @@ fn multi_block_migration_flag_works() { #[test] fn overweight_migrations_tolerated() { - // pallet-migrations currently tolerates a migration going over-weight. not only does it // tolerate it, but it continues on to the next migration even if it's already overweight. // @@ -436,7 +432,7 @@ fn overweight_migrations_tolerated() { mgr.is_multi_block = false; mgr.register_callback( - move || { "migration1" }, + move || "migration1", move |_, _| -> (Perbill, Weight) { *num_migration1_calls.lock().unwrap() += 1; // TODO: this is brittle because it assumes it is larger than the value used at @@ -446,7 +442,7 @@ fn overweight_migrations_tolerated() { ); mgr.register_callback( - move || { "migration2" }, + move || "migration2", move |_, _| -> (Perbill, Weight) { *num_migration2_calls.lock().unwrap() += 1; (Perbill::one(), 1_000_000_000_000u64.into()) @@ -454,7 +450,7 @@ fn overweight_migrations_tolerated() { ); mgr.register_callback( - move || { "migration3" }, + move || "migration3", move |_, _| -> (Perbill, Weight) { *num_migration3_calls.lock().unwrap() += 1; (Perbill::one(), 1_000_000_000_000u64.into()) From 7d57a4ebb5180cc7356e77fea1f3191e34a4897c Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 26 Jul 2021 08:14:55 -0600 Subject: [PATCH 51/99] Clean up / add comments --- pallets/migrations/src/lib.rs | 12 ++++-------- pallets/migrations/src/mock.rs | 3 ++- pallets/migrations/src/tests.rs | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index b5c0593e43..f578ac9374 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -43,8 +43,9 @@ pub trait Migration { /// its logic in small batches across as many blocks as needed. /// /// Implementations should perform as much migration work as possible and then leave their - /// pallet in a valid state from which another 'step' of migration work can be performed. In no - /// case should a step consume more than `available_weight`. + /// pallet in a valid state from which another 'step' of migration work can be performed. + /// Consuming more weight than `available_weight` is dangerous and can lead to invalid blocks + /// for parachains. /// /// This should return a perbill indicating the aggregate progress of the migration. If /// `Perbill::one()` is returned, the migration is considered complete and no further calls to @@ -59,7 +60,6 @@ pub mod pallet { use super::*; use frame_support::pallet_prelude::*; use frame_system::pallet_prelude::*; - #[allow(unused_imports)] // TODO: why does it detect this as unused? use sp_std::prelude::*; /// Pallet for migrations @@ -230,7 +230,6 @@ pub mod pallet { // perform a step of this migration let (updated_progress, consumed_weight) = migration.step(migration_state, available_for_step); - // TODO: error if progress == 0 still? >::deposit_event(Event::MigrationStepped( migration_name_as_bytes.into(), updated_progress, @@ -239,9 +238,6 @@ pub mod pallet { weight += consumed_weight; if weight > available_weight { - // TODO: the intent here is to complain obnoxiously so that this is caught - // during development. In production, this should probably be tolerated because - // failing is catastrophic. log::error!( "Migration {} consumed more weight than it was given! ({} > {})", migration_name, @@ -254,7 +250,7 @@ pub mod pallet { >::insert(migration_name.as_bytes(), updated_progress); } - // make note of any unfinished migrations + // if we encounter any migrations which are incomplete, we're done for this block if updated_progress < Perbill::one() { done = false; break; diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 84954450ef..6de409b578 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -86,7 +86,8 @@ impl frame_system::Config for Test { /// It is is expected to exist as a singleton, but only remain relevant within the scope of a test. /// /// Tests should use execute_with_mock_migrations(), which will create a MockMigrationManager and -/// provide it to the test. +/// provide it to the test, ensuring that the MockMigrationManager remains valid for the duration of +/// the callback. /// /// A pair of callbacks provided to register_callback() will map directly to a single instance of /// Migration (done by the MockMigration glue below). Treat each pair of callbacks as though it were diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 40ef4847c3..27139c0c8b 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -31,28 +31,45 @@ fn genesis_builder_works() { }) } +// This test ensures that the mock migration mess works, but also serves as a minimal[-ish] example +// of how to use it. See comments within the fn itself for details. #[test] fn mock_migrations_static_hack_works() { let name_fn_called = Arc::new(Mutex::new(false)); let step_fn_called = Arc::new(Mutex::new(false)); let ecb_fn_called = Arc::new(Mutex::new(false)); + // invoke execute_with_mock_migrations(), which will set up the MockMigrationManager properly + // and provide a valid reference to it in the callbacks we create. crate::mock::execute_with_mock_migrations( + + // This callback receives a mutable ref to the mock which we can use to set up the + // migrations we wish to mock. &mut |mgr: &mut MockMigrationManager| { let name_fn_called = Arc::clone(&name_fn_called); let step_fn_called = Arc::clone(&step_fn_called); + // For each migration we wish to mock, we should call register_callback(). The + // callbacks we provide map to pallet-migration's Migration trait functions. mgr.register_callback( + + // mock Migration::friendly_name() move || { *name_fn_called.lock().unwrap() = true; "hello, world" }, + + // mock Migration::step() move |_, _| -> (Perbill, Weight) { *step_fn_called.lock().unwrap() = true; (Perbill::one(), 0u64.into()) }, ); }, + + // This callback provides no parameters, but ensures that the MockMigrationManager + // "singleton" is still valid. Interactions with the pallet should occur here since they + // will implicitly require MockMigrationManager to be in a valid state. &mut || { ExtBuilder::default().build().execute_with(|| { crate::mock::roll_until_upgraded(true); From 3f87f79857a5f411757ac675ae1461b620a737fa Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 26 Jul 2021 08:48:27 -0600 Subject: [PATCH 52/99] Derive block weight from Config (still needs improvement) --- pallets/migrations/src/lib.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index f578ac9374..210b1ba224 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -100,6 +100,8 @@ pub mod pallet { log::warn!("Performing on_runtime_upgrade"); let mut weight: Weight = 0u64.into(); + // TODO: derive a suitable value here, which is probably something < max_block + let available_weight: Weight = T::BlockWeights::get().max_block; // start by flagging that we are not fully upgraded // TODO: case where this is already false (e.g. we started an upgrade while one was @@ -110,7 +112,7 @@ pub mod pallet { weight += T::DbWeight::get().writes(1); Self::deposit_event(Event::RuntimeUpgradeStarted()); - weight += process_runtime_upgrades::(); + weight += process_runtime_upgrades::(available_weight - weight); // at least emit a warning if we aren't going to end up finishing our migrations... if !T::MultiBlockMigrationsSupported::get() { @@ -130,9 +132,12 @@ pub mod pallet { fn on_initialize(_: T::BlockNumber) -> Weight { let mut weight: Weight = T::DbWeight::get().reads(1 as Weight); + // TODO: derive a suitable value here, which is probably something < max_block + let available_weight: Weight = T::BlockWeights::get().max_block; + if T::MultiBlockMigrationsSupported::get() { if !>::get() { - weight += process_runtime_upgrades::(); + weight += process_runtime_upgrades::(available_weight); } } @@ -176,11 +181,9 @@ pub mod pallet { } } - fn process_runtime_upgrades() -> Weight { + fn process_runtime_upgrades(available_weight: Weight) -> Weight { log::info!("stepping runtime upgrade"); - // TODO: query proper value or make configurable - let available_weight: Weight = 500_000_000_000u64.into(); let mut weight: Weight = 0u64.into(); let mut done: bool = true; From 8f6e9e58b375dbf3670153b02b52017fa74da18d Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 26 Jul 2021 09:43:23 -0600 Subject: [PATCH 53/99] cargo fmt --- pallets/migrations/src/tests.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 27139c0c8b..d4df1c6b20 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -42,9 +42,8 @@ fn mock_migrations_static_hack_works() { // invoke execute_with_mock_migrations(), which will set up the MockMigrationManager properly // and provide a valid reference to it in the callbacks we create. crate::mock::execute_with_mock_migrations( - // This callback receives a mutable ref to the mock which we can use to set up the - // migrations we wish to mock. + // migrations we wish to mock. &mut |mgr: &mut MockMigrationManager| { let name_fn_called = Arc::clone(&name_fn_called); let step_fn_called = Arc::clone(&step_fn_called); @@ -52,13 +51,11 @@ fn mock_migrations_static_hack_works() { // For each migration we wish to mock, we should call register_callback(). The // callbacks we provide map to pallet-migration's Migration trait functions. mgr.register_callback( - // mock Migration::friendly_name() move || { *name_fn_called.lock().unwrap() = true; "hello, world" }, - // mock Migration::step() move |_, _| -> (Perbill, Weight) { *step_fn_called.lock().unwrap() = true; @@ -66,7 +63,6 @@ fn mock_migrations_static_hack_works() { }, ); }, - // This callback provides no parameters, but ensures that the MockMigrationManager // "singleton" is still valid. Interactions with the pallet should occur here since they // will implicitly require MockMigrationManager to be in a valid state. From a4e2acc39f5795401aa4a4e1861f7e42ce6fe125 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 26 Jul 2021 10:50:39 -0600 Subject: [PATCH 54/99] Configure all runtimes to include pallet-migrations --- Cargo.lock | 3 +++ node/service/src/chain_spec/moonbeam.rs | 1 + node/service/src/chain_spec/moonriver.rs | 1 + node/service/src/chain_spec/moonshadow.rs | 1 + runtime/moonbeam/Cargo.toml | 2 ++ runtime/moonbeam/src/lib.rs | 13 +++++++++++++ runtime/moonriver/Cargo.toml | 2 ++ runtime/moonriver/src/lib.rs | 13 +++++++++++++ runtime/moonshadow/Cargo.toml | 2 ++ runtime/moonshadow/src/lib.rs | 13 +++++++++++++ 10 files changed, 51 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 91c54befa6..ac2dac1fbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5206,6 +5206,7 @@ dependencies = [ "pallet-evm-precompile-modexp", "pallet-evm-precompile-sha3fips", "pallet-evm-precompile-simple", + "pallet-migrations", "pallet-proxy", "pallet-randomness-collective-flip", "pallet-scheduler", @@ -5398,6 +5399,7 @@ dependencies = [ "pallet-evm-precompile-modexp", "pallet-evm-precompile-sha3fips", "pallet-evm-precompile-simple", + "pallet-migrations", "pallet-proxy", "pallet-randomness-collective-flip", "pallet-scheduler", @@ -5472,6 +5474,7 @@ dependencies = [ "pallet-evm-precompile-modexp", "pallet-evm-precompile-sha3fips", "pallet-evm-precompile-simple", + "pallet-migrations", "pallet-proxy", "pallet-randomness-collective-flip", "pallet-scheduler", diff --git a/node/service/src/chain_spec/moonbeam.rs b/node/service/src/chain_spec/moonbeam.rs index d3cebce922..8b727ec853 100644 --- a/node/service/src/chain_spec/moonbeam.rs +++ b/node/service/src/chain_spec/moonbeam.rs @@ -250,6 +250,7 @@ pub fn testnet_genesis( .collect(), }, treasury: Default::default(), + migrations: Default::default(), } } diff --git a/node/service/src/chain_spec/moonriver.rs b/node/service/src/chain_spec/moonriver.rs index 16f5a3d4b4..4894c2872c 100644 --- a/node/service/src/chain_spec/moonriver.rs +++ b/node/service/src/chain_spec/moonriver.rs @@ -250,6 +250,7 @@ pub fn testnet_genesis( .collect(), }, treasury: Default::default(), + migrations: Default::default(), } } diff --git a/node/service/src/chain_spec/moonshadow.rs b/node/service/src/chain_spec/moonshadow.rs index d4235dd159..eb64d255f6 100644 --- a/node/service/src/chain_spec/moonshadow.rs +++ b/node/service/src/chain_spec/moonshadow.rs @@ -250,6 +250,7 @@ pub fn testnet_genesis( .collect(), }, treasury: Default::default(), + migrations: Default::default(), } } diff --git a/runtime/moonbeam/Cargo.toml b/runtime/moonbeam/Cargo.toml index b1801f40d9..8c47d5bf5a 100644 --- a/runtime/moonbeam/Cargo.toml +++ b/runtime/moonbeam/Cargo.toml @@ -23,6 +23,7 @@ parachain-staking = { path = "../../pallets/parachain-staking", default-features parachain-staking-precompiles = { path = "../../precompiles/parachain-staking", default-features = false } pallet-author-slot-filter = { git = "https://github.com/purestake/cumulus", branch = "joshy-np098", default-features = false } nimbus-primitives = { git = "https://github.com/purestake/cumulus", branch = "joshy-np098", default-features = false } +pallet-migrations = { path = "../../pallets/migrations", default-features = false } pallet-author-mapping = { path = "../../pallets/author-mapping", default-features = false } evm = { version="0.27.0", default-features=false, features=["with-codec"] } pallet-evm-precompile-bn128 = { git="https://github.com/purestake/frontier", default-features=false, branch="moonbeam-polkadot-v0.9.8" } @@ -146,6 +147,7 @@ std = [ "parachain-staking/std", "parachain-staking-precompiles/std", "pallet-author-slot-filter/std", + "pallet-migrations/std", "pallet-crowdloan-rewards/std", "frame-benchmarking/std", "pallet-society/std", diff --git a/runtime/moonbeam/src/lib.rs b/runtime/moonbeam/src/lib.rs index 7abc91221d..b2b429a7b2 100644 --- a/runtime/moonbeam/src/lib.rs +++ b/runtime/moonbeam/src/lib.rs @@ -52,6 +52,7 @@ use pallet_evm::{ Account as EVMAccount, EnsureAddressNever, EnsureAddressRoot, FeeCalculator, IdentityAddressMapping, Runner, }; +use pallet_migrations::*; use pallet_transaction_payment::{CurrencyAdapter, Multiplier, TargetedFeeAdjustment}; pub use parachain_staking::{InflationInfo, Range}; use parity_scale_codec::{Decode, Encode}; @@ -766,6 +767,17 @@ impl pallet_proxy::Config for Runtime { type AnnouncementDepositFactor = AnnouncementDepositFactor; } +parameter_types! { + // Our runtime dosen't support multi-block migrations currently + pub const MultiBlockMigrationsNotSupported: bool = false; +} + +impl Config for Runtime { + type Event = Event; + type MigrationsList = runtime_common::migrations::CommonMigrations; + type MultiBlockMigrationsSupported = MultiBlockMigrationsNotSupported; +} + construct_runtime! { pub enum Runtime where Block = Block, @@ -797,6 +809,7 @@ construct_runtime! { CrowdloanRewards: pallet_crowdloan_rewards::{Pallet, Call, Config, Storage, Event}, AuthorMapping: pallet_author_mapping::{Pallet, Call, Config, Storage, Event}, Proxy: pallet_proxy::{Pallet, Call, Storage, Event}, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, } } diff --git a/runtime/moonriver/Cargo.toml b/runtime/moonriver/Cargo.toml index 6f40348807..f6e0bb8c35 100644 --- a/runtime/moonriver/Cargo.toml +++ b/runtime/moonriver/Cargo.toml @@ -23,6 +23,7 @@ parachain-staking = { path = "../../pallets/parachain-staking", default-features parachain-staking-precompiles = { path = "../../precompiles/parachain-staking", default-features = false } pallet-author-slot-filter = { git = "https://github.com/purestake/cumulus", branch = "joshy-np098", default-features = false } nimbus-primitives = { git = "https://github.com/purestake/cumulus", branch = "joshy-np098", default-features = false } +pallet-migrations = { path = "../../pallets/migrations", default-features = false } pallet-author-mapping = { path = "../../pallets/author-mapping", default-features = false } evm = { version="0.27.0", default-features=false, features=["with-codec"] } pallet-evm-precompile-bn128 = { git="https://github.com/purestake/frontier", default-features=false, branch="moonbeam-polkadot-v0.9.8" } @@ -146,6 +147,7 @@ std = [ "parachain-staking/std", "parachain-staking-precompiles/std", "pallet-author-slot-filter/std", + "pallet-migrations/std", "pallet-crowdloan-rewards/std", "frame-benchmarking/std", "pallet-society/std", diff --git a/runtime/moonriver/src/lib.rs b/runtime/moonriver/src/lib.rs index a83dcbd26c..6c53ba76a9 100644 --- a/runtime/moonriver/src/lib.rs +++ b/runtime/moonriver/src/lib.rs @@ -52,6 +52,7 @@ use pallet_evm::{ Account as EVMAccount, EnsureAddressNever, EnsureAddressRoot, FeeCalculator, IdentityAddressMapping, Runner, }; +use pallet_migrations::*; use pallet_transaction_payment::{CurrencyAdapter, Multiplier, TargetedFeeAdjustment}; pub use parachain_staking::{InflationInfo, Range}; use parity_scale_codec::{Decode, Encode}; @@ -778,6 +779,17 @@ impl pallet_proxy::Config for Runtime { type AnnouncementDepositFactor = AnnouncementDepositFactor; } +parameter_types! { + // Our runtime dosen't support multi-block migrations currently + pub const MultiBlockMigrationsNotSupported: bool = false; +} + +impl Config for Runtime { + type Event = Event; + type MigrationsList = runtime_common::migrations::CommonMigrations; + type MultiBlockMigrationsSupported = MultiBlockMigrationsNotSupported; +} + construct_runtime! { pub enum Runtime where Block = Block, @@ -828,6 +840,7 @@ construct_runtime! { // Crowdloan stuff. CrowdloanRewards: pallet_crowdloan_rewards::{Pallet, Call, Config, Storage, Event} = 90, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, } } diff --git a/runtime/moonshadow/Cargo.toml b/runtime/moonshadow/Cargo.toml index 2d59942ca3..c1f36f01bc 100644 --- a/runtime/moonshadow/Cargo.toml +++ b/runtime/moonshadow/Cargo.toml @@ -23,6 +23,7 @@ parachain-staking = { path = "../../pallets/parachain-staking", default-features parachain-staking-precompiles = { path = "../../precompiles/parachain-staking", default-features = false } pallet-author-slot-filter = { git = "https://github.com/purestake/cumulus", branch = "joshy-np098", default-features = false } nimbus-primitives = { git = "https://github.com/purestake/cumulus", branch = "joshy-np098", default-features = false } +pallet-migrations = { path = "../../pallets/migrations", default-features = false } pallet-author-mapping = { path = "../../pallets/author-mapping", default-features = false } evm = { version="0.27.0", default-features=false, features=["with-codec"] } pallet-evm-precompile-bn128 = { git="https://github.com/purestake/frontier", default-features=false, branch="moonbeam-polkadot-v0.9.8" } @@ -146,6 +147,7 @@ std = [ "parachain-staking/std", "parachain-staking-precompiles/std", "pallet-author-slot-filter/std", + "pallet-migrations/std", "pallet-crowdloan-rewards/std", "frame-benchmarking/std", "pallet-society/std", diff --git a/runtime/moonshadow/src/lib.rs b/runtime/moonshadow/src/lib.rs index c3e1423374..430f3f1328 100644 --- a/runtime/moonshadow/src/lib.rs +++ b/runtime/moonshadow/src/lib.rs @@ -52,6 +52,7 @@ use pallet_evm::{ Account as EVMAccount, EnsureAddressNever, EnsureAddressRoot, FeeCalculator, IdentityAddressMapping, Runner, }; +use pallet_migrations::*; use pallet_transaction_payment::{CurrencyAdapter, Multiplier, TargetedFeeAdjustment}; pub use parachain_staking::{InflationInfo, Range}; use parity_scale_codec::{Decode, Encode}; @@ -765,6 +766,17 @@ impl pallet_proxy::Config for Runtime { type AnnouncementDepositFactor = AnnouncementDepositFactor; } +parameter_types! { + // Our runtime dosen't support multi-block migrations currently + pub const MultiBlockMigrationsNotSupported: bool = false; +} + +impl Config for Runtime { + type Event = Event; + type MigrationsList = runtime_common::migrations::CommonMigrations; + type MultiBlockMigrationsSupported = MultiBlockMigrationsNotSupported; +} + construct_runtime! { pub enum Runtime where Block = Block, @@ -796,6 +808,7 @@ construct_runtime! { CrowdloanRewards: pallet_crowdloan_rewards::{Pallet, Call, Config, Storage, Event}, AuthorMapping: pallet_author_mapping::{Pallet, Call, Config, Storage, Event}, Proxy: pallet_proxy::{Pallet, Call, Storage, Event}, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, } } From 41e9d0bc40a584f173a410529f11ace64df79971 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Mon, 26 Jul 2021 12:19:00 -0600 Subject: [PATCH 55/99] Add pallet-migrations genesis to specs --- pallets/migrations/src/lib.rs | 4 ++-- pallets/migrations/src/mock.rs | 2 +- specs/alphanet/parachain-specs-template.json | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 210b1ba224..59b474c1b7 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -159,7 +159,7 @@ pub mod pallet { #[pallet::genesis_config] pub struct GenesisConfig { pub completed_migrations: Vec>, - pub dummy: PhantomData, + pub phantom: PhantomData, } #[cfg(feature = "std")] @@ -167,7 +167,7 @@ pub mod pallet { fn default() -> Self { Self { completed_migrations: vec![], - dummy: PhantomData, + phantom: PhantomData, } } } diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 6de409b578..bd65255777 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -238,7 +238,7 @@ impl ExtBuilder { pallet_migrations::GenesisConfig:: { completed_migrations: self.completed_migrations, - dummy: Default::default(), + phantom: Default::default(), } .assimilate_storage(&mut t) .expect("Pallet migration's storage can be assimilated"); diff --git a/specs/alphanet/parachain-specs-template.json b/specs/alphanet/parachain-specs-template.json index f70884be21..03b8ed63d4 100644 --- a/specs/alphanet/parachain-specs-template.json +++ b/specs/alphanet/parachain-specs-template.json @@ -199,7 +199,11 @@ ] }, "treasury": {}, - "scheduler": null + "scheduler": null, + "migrations": { + "phantom": null, + "completedMigrations": [] + } } } } From 0a3ae7ea2b3b2ae2da590d756f2b1e99ebd39122 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 29 Jul 2021 12:13:11 -0600 Subject: [PATCH 56/99] Update pallets/migrations/src/lib.rs Co-authored-by: Alexander Popiak --- pallets/migrations/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 59b474c1b7..cc438e265d 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -117,7 +117,7 @@ pub mod pallet { // at least emit a warning if we aren't going to end up finishing our migrations... if !T::MultiBlockMigrationsSupported::get() { if !>::get() { - log::warn!( + log::error!( "migrations weren't completed in on_runtime_upgrade(), but we're not configured for multi-block migrations; state is potentially inconsistent!" ); From 26ef7dbddb0864980889ba96593d641095c883ee Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 29 Jul 2021 12:13:20 -0600 Subject: [PATCH 57/99] Update pallets/migrations/src/lib.rs Co-authored-by: Alexander Popiak --- pallets/migrations/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index cc438e265d..0e5390e5b3 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -112,7 +112,7 @@ pub mod pallet { weight += T::DbWeight::get().writes(1); Self::deposit_event(Event::RuntimeUpgradeStarted()); - weight += process_runtime_upgrades::(available_weight - weight); + weight += process_runtime_upgrades::(available_weight.saturating_sub(weight)); // at least emit a warning if we aren't going to end up finishing our migrations... if !T::MultiBlockMigrationsSupported::get() { From e6543e61c5034697132dba20bbf88426d55f60ba Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 13 Aug 2021 14:26:11 -0600 Subject: [PATCH 58/99] First pass at ripping out multi-block migration support --- pallets/migrations/src/lib.rs | 140 ++++++++-------------------------- 1 file changed, 33 insertions(+), 107 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 0e5390e5b3..7858197bfe 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -25,7 +25,6 @@ mod mock; mod tests; use frame_support::{pallet, weights::Weight}; -use sp_runtime::Perbill; pub use pallet::*; @@ -38,21 +37,13 @@ pub trait Migration { /// A human-readable name for this migration. Also used as storage key. fn friendly_name(&self) -> &str; - /// Step through this migration, taking up to `available_weight` of execution time and providing - /// a status on the progress as well as the consumed weight. This allows a migration to perform - /// its logic in small batches across as many blocks as needed. - /// - /// Implementations should perform as much migration work as possible and then leave their - /// pallet in a valid state from which another 'step' of migration work can be performed. - /// Consuming more weight than `available_weight` is dangerous and can lead to invalid blocks - /// for parachains. - /// - /// This should return a perbill indicating the aggregate progress of the migration. If - /// `Perbill::one()` is returned, the migration is considered complete and no further calls to - /// `step()` will be made. Any value less than `Perbill::one()` will result in another future - /// call to `step()`. Indeed, values < 1 are arbitrary, but the intent is to indicate progress - /// (so they should at least be monotonically increasing). - fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight); + /// Perform the required migration and return the weight consumed. + /// + /// Currently there is no way to migrate across blocks, so this method must (1) perform its full + /// migration and (2) not produce a block that has gone over-weight. Not meeting these strict + /// constraints will lead to a bricked chain upon a runtime upgrade because the parachain will + /// not be able to produce a block that the relay chain will accept. + fn migrate(&self, available_weight: Weight) -> Weight; } #[pallet] @@ -73,19 +64,15 @@ pub mod pallet { type Event: From> + IsType<::Event>; /// The list of migrations that will be performed type MigrationsList: Get>>; - /// Whether or not multi-block migrations are supported - type MultiBlockMigrationsSupported: Get; } #[pallet::event] #[pallet::generate_deposit(pub(crate) fn deposit_event)] pub enum Event { RuntimeUpgradeStarted(), - RuntimeUpgradeStepped(Weight), RuntimeUpgradeCompleted(), MigrationStarted(Vec), - MigrationStepped(Vec, Perbill, Weight), - MigrationCompleted(Vec), + MigrationCompleted(Vec, Weight), } #[pallet::hooks] @@ -93,9 +80,6 @@ pub mod pallet { /// on_runtime_upgrade is expected to be called exactly once after a runtime upgrade. /// We use this as a chance to flag that we are now in upgrade-mode and begin our /// migrations. - /// - /// In the event that a migration is expected to take more than one block, ongoing migration - /// work could continue from block-to-block in this pallet's on_initialize function. fn on_runtime_upgrade() -> Weight { log::warn!("Performing on_runtime_upgrade"); @@ -104,41 +88,17 @@ pub mod pallet { let available_weight: Weight = T::BlockWeights::get().max_block; // start by flagging that we are not fully upgraded - // TODO: case where this is already false (e.g. we started an upgrade while one was - // already in progress) - // notably, we might have started an individual migration and the list of - // migrations might change on our next on_runtime_upgrade() >::put(false); weight += T::DbWeight::get().writes(1); Self::deposit_event(Event::RuntimeUpgradeStarted()); - weight += process_runtime_upgrades::(available_weight.saturating_sub(weight)); + weight += perform_runtime_upgrades::(available_weight.saturating_sub(weight)); - // at least emit a warning if we aren't going to end up finishing our migrations... - if !T::MultiBlockMigrationsSupported::get() { - if !>::get() { - log::error!( - "migrations weren't completed in on_runtime_upgrade(), but we're not - configured for multi-block migrations; state is potentially inconsistent!" - ); - } - } - - weight - } - - /// on_initialize implementation. Calls process_runtime_upgrades() if we are still in the - /// middle of a runtime upgrade. - fn on_initialize(_: T::BlockNumber) -> Weight { - let mut weight: Weight = T::DbWeight::get().reads(1 as Weight); - - // TODO: derive a suitable value here, which is probably something < max_block - let available_weight: Weight = T::BlockWeights::get().max_block; - - if T::MultiBlockMigrationsSupported::get() { - if !>::get() { - weight += process_runtime_upgrades::(available_weight); - } + if !>::get() { + log::error!( + "migrations weren't completed in on_runtime_upgrade(), but we're not + configured for multi-block migrations; state is potentially inconsistent!" + ); } weight @@ -153,8 +113,8 @@ pub mod pallet { #[pallet::storage] #[pallet::getter(fn migration_state)] /// MigrationState tracks the progress of a migration. - /// Maps name (Vec) -> migration progress (Perbill) - type MigrationState = StorageMap<_, Twox64Concat, Vec, Perbill, OptionQuery>; + /// Maps name (Vec) -> whether or not migration has been completed (bool) + type MigrationState = StorageMap<_, Twox64Concat, Vec, bool, OptionQuery>; #[pallet::genesis_config] pub struct GenesisConfig { @@ -176,45 +136,33 @@ pub mod pallet { impl GenesisBuild for GenesisConfig { fn build(&self) { for migration_name in &self.completed_migrations { - >::insert(migration_name, Perbill::one()); + >::insert(migration_name, true); } } } - fn process_runtime_upgrades(available_weight: Weight) -> Weight { - log::info!("stepping runtime upgrade"); - + fn perform_runtime_upgrades(available_weight: Weight) -> Weight { let mut weight: Weight = 0u64.into(); - let mut done: bool = true; for migration in &T::MigrationsList::get() { - // let migration_name = migration.friendly_name(); let migration_name = migration.friendly_name(); let migration_name_as_bytes = migration_name.as_bytes(); log::trace!("evaluating migration {}", migration_name); - let migration_state = - >::get(migration_name_as_bytes).unwrap_or(Perbill::zero()); + let migration_done = + >::get(migration_name_as_bytes).unwrap_or(false); - if migration_state < Perbill::one() { - if migration_state.is_zero() { - >::deposit_event(Event::MigrationStarted( - migration_name_as_bytes.into(), - )); - } + if ! migration_done { + >::deposit_event(Event::MigrationStarted( + migration_name_as_bytes.into(), + )); - // TODO: here we allow migrations to go overweight because that's "safer" than not - // doing so in the case of MultiBlockMigrationsSupported == false. In this case, any - // migrations not handled in this pass will not occur at all, which will leave - // pallets unmigrated. - // - // Options for handling this more gracefully: - // 1) make this particular logic (whether or not we tolerate this) configurable - // 2) only follow this logic when MultiBlockMigrationsSupported == false + // when we go overweight, leave a warning... there's nothing we can really do about + // this scenario other than hope that the block is actually accepted. let available_for_step = if available_weight > weight { available_weight - weight } else { - log::warn!( + log::error!( "previous migration went overweight; ignoring and providing migration {} 0 weight.", migration_name, @@ -224,18 +172,14 @@ pub mod pallet { }; log::trace!( - "stepping migration {}, prev: {:?}, avail weight: {}", + "performing migration {}, avail weight: {}", migration_name, - migration_state, available_for_step ); - // perform a step of this migration - let (updated_progress, consumed_weight) = - migration.step(migration_state, available_for_step); - >::deposit_event(Event::MigrationStepped( + let consumed_weight = migration.migrate(available_for_step); + >::deposit_event(Event::MigrationCompleted( migration_name_as_bytes.into(), - updated_progress, consumed_weight, )); @@ -248,30 +192,12 @@ pub mod pallet { available_for_step ); } - - if migration_state != updated_progress { - >::insert(migration_name.as_bytes(), updated_progress); - } - - // if we encounter any migrations which are incomplete, we're done for this block - if updated_progress < Perbill::one() { - done = false; - break; - } else { - >::deposit_event(Event::MigrationCompleted( - migration_name_as_bytes.into(), - )); - } } } - >::deposit_event(Event::RuntimeUpgradeStepped(weight)); - - if done { - >::deposit_event(Event::RuntimeUpgradeCompleted()); - >::put(true); - weight += T::DbWeight::get().writes(1); - } + >::deposit_event(Event::RuntimeUpgradeCompleted()); + >::put(true); + weight += T::DbWeight::get().writes(1); weight } From 01bc09f4ee7258ff39df7a1b01b3aa63d699a716 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 13 Aug 2021 15:33:48 -0600 Subject: [PATCH 59/99] Incremental work @ removing multi-block migration support --- pallets/migrations/src/mock.rs | 38 +++------ pallets/migrations/src/tests.rs | 132 +++++--------------------------- 2 files changed, 31 insertions(+), 139 deletions(-) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index bd65255777..ed65286ff6 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -94,41 +94,38 @@ impl frame_system::Config for Test { /// a custom implementation of the Migration trait just as a normal Pallet would. pub struct MockMigrationManager<'test> { name_fn_callbacks: Vec &'static str>>, - step_fn_callbacks: Vec (Perbill, Weight)>>, - pub is_multi_block: bool, // corresponds to MultiBlockMigrationsSupported. defaults to true. + migrate_fn_callbacks: Vec Weight>>, } impl Default for MockMigrationManager<'_> { fn default() -> Self { Self { name_fn_callbacks: Default::default(), - step_fn_callbacks: Default::default(), - is_multi_block: true, + migrate_fn_callbacks: Default::default(), } } } impl<'test> MockMigrationManager<'test> { - pub fn register_callback(&mut self, name_fn: FN, step_fn: FS) + pub fn register_callback(&mut self, name_fn: FN, migrate_fn: FM) where FN: 'test + FnMut() -> &'static str, - FS: 'test + FnMut(Perbill, Weight) -> (Perbill, Weight), + FM: 'test + FnMut(Weight) -> Weight, { self.name_fn_callbacks.push(Box::new(name_fn)); - self.step_fn_callbacks.push(Box::new(step_fn)); + self.migrate_fn_callbacks.push(Box::new(migrate_fn)); } pub(crate) fn invoke_name_fn(&mut self, index: usize) -> &'static str { self.name_fn_callbacks[index]() } - pub(crate) fn invoke_step_fn( + pub(crate) fn invoke_migrate_fn( &mut self, index: usize, - previous_progress: Perbill, available_weight: Weight, - ) -> (Perbill, Weight) { - self.step_fn_callbacks[index](previous_progress, available_weight) + ) -> Weight { + self.migrate_fn_callbacks[index](available_weight) } fn generate_migrations_list(&self) -> Vec> { @@ -177,10 +174,10 @@ impl Migration for MockMigration { }); result } - fn step(&self, previous_progress: Perbill, available_weight: Weight) -> (Perbill, Weight) { - let mut result: (Perbill, Weight) = (Perbill::zero(), 0u64.into()); + fn migrate(&self, available_weight: Weight) -> Weight { + let mut result: Weight = 0u64.into(); MOCK_MIGRATIONS_LIST::with(|mgr: &mut MockMigrationManager| { - result = mgr.invoke_step_fn(self.index, previous_progress, available_weight); + result = mgr.invoke_migrate_fn(self.index, available_weight); }); result } @@ -199,22 +196,9 @@ impl Get>> for MockMigrations { } } -/// Similar to the impl for Get>> but we return a value suitable for -/// MultiBlockMigrationsSupported -impl Get for MockMigrations { - fn get() -> bool { - let mut supported = false; - MOCK_MIGRATIONS_LIST::with(|mgr: &mut MockMigrationManager| { - supported = mgr.is_multi_block; - }); - supported - } -} - impl Config for Test { type Event = Event; type MigrationsList = MockMigrations; - type MultiBlockMigrationsSupported = MockMigrations; } /// Externality builder for pallet migration's mock runtime diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index d4df1c6b20..86896fb07d 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -57,9 +57,9 @@ fn mock_migrations_static_hack_works() { "hello, world" }, // mock Migration::step() - move |_, _| -> (Perbill, Weight) { + move |_| -> Weight { *step_fn_called.lock().unwrap() = true; - (Perbill::one(), 0u64.into()) + 0u64.into() }, ); }, @@ -122,13 +122,13 @@ fn step_called_until_done() { mgr.register_callback( move || "migration1", - move |_, _| -> (Perbill, Weight) { + move |_| -> Weight { let mut num_step_calls = num_step_calls.lock().unwrap(); *num_step_calls += 1; if *num_step_calls == 10 { - (Perbill::one(), 0u64.into()) + 0u64.into() } else { - (Perbill::zero(), 0u64.into()) + 0u64.into() } }, ); @@ -147,67 +147,6 @@ fn step_called_until_done() { ); } -#[test] -fn migration_progress_should_emit_events() { - let num_steps = Arc::new(Mutex::new(0usize)); - - crate::mock::execute_with_mock_migrations( - &mut |mgr: &mut MockMigrationManager| { - let num_steps = Arc::clone(&num_steps); - - mgr.register_callback( - move || "migration1", - move |_, _| -> (Perbill, Weight) { - let mut num_steps = num_steps.lock().unwrap(); - - let result: (Perbill, Weight) = match *num_steps { - 0 => (Perbill::from_percent(50), 50), - 1 => (Perbill::from_percent(60), 51), - 2 => (Perbill::from_percent(70), 52), - 3 => (Perbill::from_percent(80), 53), - 4 => (Perbill::from_percent(100), 1), - _ => { - unreachable!(); - } - }; - - *num_steps += 1; - result - }, - ); - }, - &mut || { - ExtBuilder::default().build().execute_with(|| { - crate::mock::roll_until_upgraded(true); - - let expected = vec![ - Event::RuntimeUpgradeStarted(), - Event::MigrationStarted("migration1".into()), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(50), 50), - Event::RuntimeUpgradeStepped(50), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(60), 51), - Event::RuntimeUpgradeStepped(51), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(70), 52), - Event::RuntimeUpgradeStepped(52), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(80), 53), - Event::RuntimeUpgradeStepped(53), - Event::MigrationStepped("migration1".into(), Perbill::from_percent(100), 1), - Event::MigrationCompleted("migration1".into()), - Event::RuntimeUpgradeStepped(1), - Event::RuntimeUpgradeCompleted(), - ]; - assert_eq!(events(), expected); - }); - }, - ); - - assert_eq!( - *num_steps.lock().unwrap(), - 5, - "migration step should be called until done" - ); -} - #[test] fn migration_should_only_be_invoked_once() { let num_name_fn_calls = Arc::new(Mutex::new(0usize)); @@ -224,10 +163,10 @@ fn migration_should_only_be_invoked_once() { *num_name_fn_calls += 1; "migration1" }, - move |_, _| -> (Perbill, Weight) { + move |_| -> Weight { let mut num_step_fn_calls = num_step_fn_calls.lock().unwrap(); *num_step_fn_calls += 1; - (Perbill::one(), 1) // immediately done + 1u32.into() }, ); }, @@ -339,25 +278,25 @@ fn only_one_outstanding_test_at_a_time() { mgr.register_callback( move || "migration1", - move |_, _| -> (Perbill, Weight) { + move |_| -> Weight { let mut num_migration1_calls = num_migration1_calls.lock().unwrap(); *num_migration1_calls += 1; // this migration is done on its 3rd step if *num_migration1_calls < 3 { - (Perbill::zero(), 0u64.into()) + 0u64.into() } else { - (Perbill::one(), 0u64.into()) + 0u64.into() } }, ); mgr.register_callback( move || "migration2", - move |_, _| -> (Perbill, Weight) { + move |_| -> Weight { let mut num_migration2_calls = num_migration2_calls.lock().unwrap(); *num_migration2_calls += 1; - (Perbill::one(), 0u64.into()) + 0u64.into() }, ); }, @@ -385,38 +324,6 @@ fn only_one_outstanding_test_at_a_time() { ); } -#[test] -fn multi_block_migration_flag_works() { - let num_migration_calls = Arc::new(Mutex::new(0u32)); - - // we create a single migration that wants to take more than one block and ensure that it's only - // called once - - crate::mock::execute_with_mock_migrations( - &mut |mgr: &mut MockMigrationManager| { - let num_migration_calls = Arc::clone(&num_migration_calls); - - mgr.is_multi_block = false; - - mgr.register_callback( - move || "migration1", - move |_, _| -> (Perbill, Weight) { - *num_migration_calls.lock().unwrap() += 1; - (Perbill::zero(), 0u64.into()) - }, - ); - }, - &mut || { - ExtBuilder::default().build().execute_with(|| { - crate::mock::roll_to(5, true); - - assert_eq!(*num_migration_calls.lock().unwrap(), 1); - assert_eq!(Migrations::is_fully_upgraded(), false); - }); - }, - ); -} - #[test] fn overweight_migrations_tolerated() { // pallet-migrations currently tolerates a migration going over-weight. not only does it @@ -442,31 +349,32 @@ fn overweight_migrations_tolerated() { let num_migration2_calls = Arc::clone(&num_migration2_calls); let num_migration3_calls = Arc::clone(&num_migration3_calls); - mgr.is_multi_block = false; + panic!("fix me"); // this is a valid test but needs adapting to no-multi-block-support + // mgr.is_multi_block = false; mgr.register_callback( move || "migration1", - move |_, _| -> (Perbill, Weight) { + move |_| -> Weight { *num_migration1_calls.lock().unwrap() += 1; // TODO: this is brittle because it assumes it is larger than the value used at // the top of process_runtime_upgrades() - (Perbill::one(), 1_000_000_000_000u64.into()) + 1_000_000_000_000u64.into() }, ); mgr.register_callback( move || "migration2", - move |_, _| -> (Perbill, Weight) { + move |_| -> Weight { *num_migration2_calls.lock().unwrap() += 1; - (Perbill::one(), 1_000_000_000_000u64.into()) + 1_000_000_000_000u64.into() }, ); mgr.register_callback( move || "migration3", - move |_, _| -> (Perbill, Weight) { + move |_| -> Weight { *num_migration3_calls.lock().unwrap() += 1; - (Perbill::one(), 1_000_000_000_000u64.into()) + 1_000_000_000_000u64.into() }, ); }, From d2e5759c25f586b2099fc3329db71bd30b3ed7fa Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 13 Aug 2021 16:01:52 -0600 Subject: [PATCH 60/99] Make migration tests compile (not passing yet) --- pallets/migrations/src/lib.rs | 4 ++-- pallets/migrations/src/tests.rs | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 7858197bfe..c70720bb52 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -70,7 +70,7 @@ pub mod pallet { #[pallet::generate_deposit(pub(crate) fn deposit_event)] pub enum Event { RuntimeUpgradeStarted(), - RuntimeUpgradeCompleted(), + RuntimeUpgradeCompleted(Weight), MigrationStarted(Vec), MigrationCompleted(Vec, Weight), } @@ -195,9 +195,9 @@ pub mod pallet { } } - >::deposit_event(Event::RuntimeUpgradeCompleted()); >::put(true); weight += T::DbWeight::get().writes(1); + >::deposit_event(Event::RuntimeUpgradeCompleted(weight)); weight } diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 86896fb07d..b64b096afa 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -105,8 +105,7 @@ fn on_runtime_upgrade_emits_events() { let expected = vec![ Event::RuntimeUpgradeStarted(), - Event::RuntimeUpgradeStepped(0u64.into()), - Event::RuntimeUpgradeCompleted(), + Event::RuntimeUpgradeCompleted(0u64.into()), ]; assert_eq!(events(), expected); }); @@ -189,10 +188,8 @@ fn migration_should_only_be_invoked_once() { let mut expected = vec![ Event::RuntimeUpgradeStarted(), Event::MigrationStarted("migration1".into()), - Event::MigrationStepped("migration1".into(), Perbill::one(), 1), - Event::MigrationCompleted("migration1".into()), - Event::RuntimeUpgradeStepped(1), - Event::RuntimeUpgradeCompleted(), + Event::MigrationCompleted("migration1".into(), 1u32.into()), + Event::RuntimeUpgradeCompleted(1u32.into()), ]; assert_eq!(events(), expected); @@ -214,8 +211,7 @@ fn migration_should_only_be_invoked_once() { ); expected.append(&mut vec![ Event::RuntimeUpgradeStarted(), - Event::RuntimeUpgradeStepped(0), - Event::RuntimeUpgradeCompleted(), + Event::RuntimeUpgradeCompleted(0u32.into()), ]); assert_eq!(events(), expected); From 38ee3a4adc4d0a7db1cd8a2876eed6ebbfe58642 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 17 Aug 2021 13:47:31 -0600 Subject: [PATCH 61/99] Clean up runtime to reflect removal of multi-block migrations --- runtime/common/src/migrations.rs | 22 ++++++++++++---------- runtime/moonbase/src/lib.rs | 6 ------ runtime/moonbeam/src/lib.rs | 6 ------ runtime/moonriver/src/lib.rs | 6 ------ runtime/moonshadow/src/lib.rs | 6 ------ 5 files changed, 12 insertions(+), 34 deletions(-) diff --git a/runtime/common/src/migrations.rs b/runtime/common/src/migrations.rs index abbc87a2ed..e4c38cf21c 100644 --- a/runtime/common/src/migrations.rs +++ b/runtime/common/src/migrations.rs @@ -16,24 +16,23 @@ //! # Migrations -use frame_support::{pallet_prelude::Get, weights::Weight}; +use frame_support::pallet_prelude::Get; use pallet_migrations::Migration; -use sp_runtime::Perbill; use sp_std::prelude::*; /// This module acts as a registry where each migration is defined. Each migration should implement /// the "Migration" trait declared in the pallet-migrations crate. +/* + * These are left as examples. #[allow(non_camel_case_types)] pub struct MM_001_AuthorMappingAddDeposit; impl Migration for MM_001_AuthorMappingAddDeposit { fn friendly_name(&self) -> &str { "AuthorMappingAddDeposit" } - fn step(&self, _previous_progress: Perbill, _available_weight: Weight) -> (Perbill, Weight) { - // reviewer note: this isn't meant to imply that migration code must live here. As noted - // elsewhere, I would expect migration code to live close to the pallet it affects. - (Perbill::one(), 0u64.into()) + fn migrate(&self, _available_weight: Weight) -> Weight { + 0u64.into() } } @@ -43,8 +42,8 @@ impl Migration for MM_002_StakingFixTotalBalance { fn friendly_name(&self) -> &str { "StakingFixTotalBalance" } - fn step(&self, _previous_progress: Perbill, _available_weight: Weight) -> (Perbill, Weight) { - (Perbill::one(), 0u64.into()) + fn migrate(&self, _available_weight: Weight) -> Weight { + 0u64.into() } } @@ -54,10 +53,11 @@ impl Migration for MM_003_StakingUnboundedCollatorNominations { fn friendly_name(&self) -> &str { "StakingUnboundedCollatorNominations" } - fn step(&self, _previous_progress: Perbill, _available_weight: Weight) -> (Perbill, Weight) { - (Perbill::one(), 0u64.into()) + fn migrate(&self, _available_weight: Weight) -> Weight { + 0u64.into() } } +*/ pub struct CommonMigrations; impl Get>> for CommonMigrations { @@ -65,9 +65,11 @@ impl Get>> for CommonMigrations { // TODO: this is a lot of allocation to do upon every get() call. this *should* be avoided // except when pallet_migrations undergoes a runtime upgrade -- but TODO: review vec![ + /* Box::new(MM_001_AuthorMappingAddDeposit), Box::new(MM_002_StakingFixTotalBalance), Box::new(MM_003_StakingUnboundedCollatorNominations), + */ ] } } diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index e14303ce34..76fbc6f1b7 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -763,15 +763,9 @@ impl pallet_proxy::Config for Runtime { type AnnouncementDepositFactor = AnnouncementDepositFactor; } -parameter_types! { - // Our runtime dosen't support multi-block migrations currently - pub const MultiBlockMigrationsNotSupported: bool = false; -} - impl Config for Runtime { type Event = Event; type MigrationsList = runtime_common::migrations::CommonMigrations; - type MultiBlockMigrationsSupported = MultiBlockMigrationsNotSupported; } construct_runtime! { diff --git a/runtime/moonbeam/src/lib.rs b/runtime/moonbeam/src/lib.rs index b2b429a7b2..4f6185376c 100644 --- a/runtime/moonbeam/src/lib.rs +++ b/runtime/moonbeam/src/lib.rs @@ -767,15 +767,9 @@ impl pallet_proxy::Config for Runtime { type AnnouncementDepositFactor = AnnouncementDepositFactor; } -parameter_types! { - // Our runtime dosen't support multi-block migrations currently - pub const MultiBlockMigrationsNotSupported: bool = false; -} - impl Config for Runtime { type Event = Event; type MigrationsList = runtime_common::migrations::CommonMigrations; - type MultiBlockMigrationsSupported = MultiBlockMigrationsNotSupported; } construct_runtime! { diff --git a/runtime/moonriver/src/lib.rs b/runtime/moonriver/src/lib.rs index 6c53ba76a9..bf6e379f43 100644 --- a/runtime/moonriver/src/lib.rs +++ b/runtime/moonriver/src/lib.rs @@ -779,15 +779,9 @@ impl pallet_proxy::Config for Runtime { type AnnouncementDepositFactor = AnnouncementDepositFactor; } -parameter_types! { - // Our runtime dosen't support multi-block migrations currently - pub const MultiBlockMigrationsNotSupported: bool = false; -} - impl Config for Runtime { type Event = Event; type MigrationsList = runtime_common::migrations::CommonMigrations; - type MultiBlockMigrationsSupported = MultiBlockMigrationsNotSupported; } construct_runtime! { diff --git a/runtime/moonshadow/src/lib.rs b/runtime/moonshadow/src/lib.rs index 430f3f1328..cd2751aec2 100644 --- a/runtime/moonshadow/src/lib.rs +++ b/runtime/moonshadow/src/lib.rs @@ -766,15 +766,9 @@ impl pallet_proxy::Config for Runtime { type AnnouncementDepositFactor = AnnouncementDepositFactor; } -parameter_types! { - // Our runtime dosen't support multi-block migrations currently - pub const MultiBlockMigrationsNotSupported: bool = false; -} - impl Config for Runtime { type Event = Event; type MigrationsList = runtime_common::migrations::CommonMigrations; - type MultiBlockMigrationsSupported = MultiBlockMigrationsNotSupported; } construct_runtime! { From 30033ee7a6a5fa52f287cd221dab2d31ef91ab1f Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 17 Aug 2021 14:02:22 -0600 Subject: [PATCH 62/99] You know your tests are good when they catch a critical refactor mistake --- pallets/migrations/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index c70720bb52..f536320962 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -182,6 +182,7 @@ pub mod pallet { migration_name_as_bytes.into(), consumed_weight, )); + >::insert(migration_name_as_bytes, true); weight += consumed_weight; if weight > available_weight { From dfb6f42920cce08df0ee576a0fe885435fe181ea Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 17 Aug 2021 14:02:41 -0600 Subject: [PATCH 63/99] Fix test logic to reflect no multi-block migrations --- pallets/migrations/src/tests.rs | 125 +------------------------------- 1 file changed, 4 insertions(+), 121 deletions(-) diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index b64b096afa..92a7139099 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -18,10 +18,9 @@ use crate::mock::{events, ExtBuilder, Migrations, MockMigrationManager, System}; use crate::Event; use frame_support::{ - traits::{OnInitialize, OnRuntimeUpgrade}, + traits::OnRuntimeUpgrade, weights::{constants::RocksDbWeight, Weight}, }; -use sp_runtime::Perbill; use std::sync::{Arc, Mutex}; #[test] @@ -105,47 +104,12 @@ fn on_runtime_upgrade_emits_events() { let expected = vec![ Event::RuntimeUpgradeStarted(), - Event::RuntimeUpgradeCompleted(0u64.into()), + Event::RuntimeUpgradeCompleted(100000000u64.into()), ]; assert_eq!(events(), expected); }); } -#[test] -fn step_called_until_done() { - let num_step_calls = Arc::new(Mutex::new(0usize)); - - crate::mock::execute_with_mock_migrations( - &mut |mgr: &mut MockMigrationManager| { - let num_step_calls = Arc::clone(&num_step_calls); - - mgr.register_callback( - move || "migration1", - move |_| -> Weight { - let mut num_step_calls = num_step_calls.lock().unwrap(); - *num_step_calls += 1; - if *num_step_calls == 10 { - 0u64.into() - } else { - 0u64.into() - } - }, - ); - }, - &mut || { - ExtBuilder::default().build().execute_with(|| { - crate::mock::roll_until_upgraded(true); - }); - }, - ); - - assert_eq!( - *num_step_calls.lock().unwrap(), - 10, - "migration step should be called until done" - ); -} - #[test] fn migration_should_only_be_invoked_once() { let num_name_fn_calls = Arc::new(Mutex::new(0usize)); @@ -189,7 +153,7 @@ fn migration_should_only_be_invoked_once() { Event::RuntimeUpgradeStarted(), Event::MigrationStarted("migration1".into()), Event::MigrationCompleted("migration1".into(), 1u32.into()), - Event::RuntimeUpgradeCompleted(1u32.into()), + Event::RuntimeUpgradeCompleted(100000001u32.into()), // includes reads/writes ]; assert_eq!(events(), expected); @@ -211,7 +175,7 @@ fn migration_should_only_be_invoked_once() { ); expected.append(&mut vec![ Event::RuntimeUpgradeStarted(), - Event::RuntimeUpgradeCompleted(0u32.into()), + Event::RuntimeUpgradeCompleted(100000000u32.into()), ]); assert_eq!(events(), expected); @@ -234,19 +198,6 @@ fn migration_should_only_be_invoked_once() { ); } -#[test] -fn on_initialize_should_charge_at_least_one_db_read() { - ExtBuilder::default().build().execute_with(|| { - // first call to on_runtime_upgrade should flag FullyUpgraded as true - Migrations::on_runtime_upgrade(); - assert_eq!(Migrations::is_fully_upgraded(), true); - - // and subsequent call to on_initialize should do nothing but read this value and return - let weight = >::on_initialize(1); - assert_eq!(weight, RocksDbWeight::get().reads(1)); - }) -} - #[test] fn on_runtime_upgrade_charges_minimum_two_db_writes() { ExtBuilder::default().build().execute_with(|| { @@ -259,76 +210,11 @@ fn on_runtime_upgrade_charges_minimum_two_db_writes() { }) } -#[test] -fn only_one_outstanding_test_at_a_time() { - let num_migration1_calls = Arc::new(Mutex::new(0usize)); - let num_migration2_calls = Arc::new(Mutex::new(0usize)); - - // create two migrations. the first will return < Perbill::one() until its 3rd step, which - // should prevent the second from running. Once it s done, the second should execute. - - crate::mock::execute_with_mock_migrations( - &mut |mgr: &mut MockMigrationManager| { - let num_migration1_calls = Arc::clone(&num_migration1_calls); - let num_migration2_calls = Arc::clone(&num_migration2_calls); - - mgr.register_callback( - move || "migration1", - move |_| -> Weight { - let mut num_migration1_calls = num_migration1_calls.lock().unwrap(); - *num_migration1_calls += 1; - - // this migration is done on its 3rd step - if *num_migration1_calls < 3 { - 0u64.into() - } else { - 0u64.into() - } - }, - ); - - mgr.register_callback( - move || "migration2", - move |_| -> Weight { - let mut num_migration2_calls = num_migration2_calls.lock().unwrap(); - *num_migration2_calls += 1; - 0u64.into() - }, - ); - }, - &mut || { - ExtBuilder::default().build().execute_with(|| { - // first pass should invoke migration1 once and not move on to migration2 - Migrations::on_runtime_upgrade(); - assert_eq!(*num_migration1_calls.lock().unwrap(), 1); - assert_eq!(*num_migration2_calls.lock().unwrap(), 0); - - // second pass should do the same - crate::mock::roll_to(2, false); - assert_eq!(*num_migration1_calls.lock().unwrap(), 2); - assert_eq!(*num_migration2_calls.lock().unwrap(), 0); - - // third pass should invoke both - crate::mock::roll_to(3, false); - assert_eq!(*num_migration1_calls.lock().unwrap(), 3); - assert_eq!(*num_migration2_calls.lock().unwrap(), 1); - - // and both should be done now - assert_eq!(Migrations::is_fully_upgraded(), true); - }); - }, - ); -} - #[test] fn overweight_migrations_tolerated() { // pallet-migrations currently tolerates a migration going over-weight. not only does it // tolerate it, but it continues on to the next migration even if it's already overweight. // - // Now that the pallet can be configured to not support multi-block migrations, this is sort of - // a feature and not really a bug -- this test case exists to explicitly acknowledge/protect - // that. - // // The logic behind this is that we would rather go over-weight and risk a block taking too long // (which *might* be "catastrophic") than outright prevent migrations from proceeding (which is // certainly "catastrophic"). @@ -345,9 +231,6 @@ fn overweight_migrations_tolerated() { let num_migration2_calls = Arc::clone(&num_migration2_calls); let num_migration3_calls = Arc::clone(&num_migration3_calls); - panic!("fix me"); // this is a valid test but needs adapting to no-multi-block-support - // mgr.is_multi_block = false; - mgr.register_callback( move || "migration1", move |_| -> Weight { From f6919773996ddcd7fb8f09d3782493238466c188 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 17 Aug 2021 14:04:49 -0600 Subject: [PATCH 64/99] cargo fmt --- pallets/migrations/src/lib.rs | 11 ++++------- pallets/migrations/src/mock.rs | 6 +----- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index f536320962..15f0df89cb 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -38,7 +38,7 @@ pub trait Migration { fn friendly_name(&self) -> &str; /// Perform the required migration and return the weight consumed. - /// + /// /// Currently there is no way to migrate across blocks, so this method must (1) perform its full /// migration and (2) not produce a block that has gone over-weight. Not meeting these strict /// constraints will lead to a bricked chain upon a runtime upgrade because the parachain will @@ -149,13 +149,10 @@ pub mod pallet { let migration_name_as_bytes = migration_name.as_bytes(); log::trace!("evaluating migration {}", migration_name); - let migration_done = - >::get(migration_name_as_bytes).unwrap_or(false); + let migration_done = >::get(migration_name_as_bytes).unwrap_or(false); - if ! migration_done { - >::deposit_event(Event::MigrationStarted( - migration_name_as_bytes.into(), - )); + if !migration_done { + >::deposit_event(Event::MigrationStarted(migration_name_as_bytes.into())); // when we go overweight, leave a warning... there's nothing we can really do about // this scenario other than hope that the block is actually accepted. diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index ed65286ff6..c1bd8f6018 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -120,11 +120,7 @@ impl<'test> MockMigrationManager<'test> { self.name_fn_callbacks[index]() } - pub(crate) fn invoke_migrate_fn( - &mut self, - index: usize, - available_weight: Weight, - ) -> Weight { + pub(crate) fn invoke_migrate_fn(&mut self, index: usize, available_weight: Weight) -> Weight { self.migrate_fn_callbacks[index](available_weight) } From dc4281903a07cf9898355d93b8ad0a2032eec7f3 Mon Sep 17 00:00:00 2001 From: Amar Singh Date: Wed, 18 Aug 2021 12:12:33 -0400 Subject: [PATCH 65/99] Remove phantomdata field from pallet_migrations::GenesisConfig (#701) * remove phantomdata from pallet migrations genesis config struct * skip migration if no weight available for step * revert --- pallets/migrations/src/lib.rs | 16 +++------------- pallets/migrations/src/mock.rs | 13 +++++++------ runtime/moonbase/src/lib.rs | 2 +- runtime/moonbeam/src/lib.rs | 2 +- runtime/moonriver/src/lib.rs | 2 +- runtime/moonshadow/src/lib.rs | 2 +- specs/alphanet/parachain-specs-template.json | 1 - 7 files changed, 14 insertions(+), 24 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 15f0df89cb..61bc6d948d 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -117,23 +117,13 @@ pub mod pallet { type MigrationState = StorageMap<_, Twox64Concat, Vec, bool, OptionQuery>; #[pallet::genesis_config] - pub struct GenesisConfig { + #[derive(Default)] + pub struct GenesisConfig { pub completed_migrations: Vec>, - pub phantom: PhantomData, - } - - #[cfg(feature = "std")] - impl Default for GenesisConfig { - fn default() -> Self { - Self { - completed_migrations: vec![], - phantom: PhantomData, - } - } } #[pallet::genesis_build] - impl GenesisBuild for GenesisConfig { + impl GenesisBuild for GenesisConfig { fn build(&self) { for migration_name in &self.completed_migrations { >::insert(migration_name, true); diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index c1bd8f6018..47814ca4c0 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -45,7 +45,7 @@ construct_runtime!( UncheckedExtrinsic = UncheckedExtrinsic, { System: frame_system::{Pallet, Call, Config, Storage, Event}, - Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, } ); @@ -216,11 +216,12 @@ impl ExtBuilder { .build_storage::() .expect("Frame system builds valid default genesis config"); - pallet_migrations::GenesisConfig:: { - completed_migrations: self.completed_migrations, - phantom: Default::default(), - } - .assimilate_storage(&mut t) + GenesisBuild::::assimilate_storage( + &pallet_migrations::GenesisConfig { + completed_migrations: self.completed_migrations, + }, + &mut t, + ) .expect("Pallet migration's storage can be assimilated"); let mut ext = sp_io::TestExternalities::new(t); diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index 76fbc6f1b7..817771ea4d 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -799,7 +799,7 @@ construct_runtime! { CrowdloanRewards: pallet_crowdloan_rewards::{Pallet, Call, Config, Storage, Event}, AuthorMapping: pallet_author_mapping::{Pallet, Call, Config, Storage, Event}, Proxy: pallet_proxy::{Pallet, Call, Storage, Event}, - Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, } } diff --git a/runtime/moonbeam/src/lib.rs b/runtime/moonbeam/src/lib.rs index 4f6185376c..5883e0bdf8 100644 --- a/runtime/moonbeam/src/lib.rs +++ b/runtime/moonbeam/src/lib.rs @@ -803,7 +803,7 @@ construct_runtime! { CrowdloanRewards: pallet_crowdloan_rewards::{Pallet, Call, Config, Storage, Event}, AuthorMapping: pallet_author_mapping::{Pallet, Call, Config, Storage, Event}, Proxy: pallet_proxy::{Pallet, Call, Storage, Event}, - Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, } } diff --git a/runtime/moonriver/src/lib.rs b/runtime/moonriver/src/lib.rs index bf6e379f43..8302129e2e 100644 --- a/runtime/moonriver/src/lib.rs +++ b/runtime/moonriver/src/lib.rs @@ -834,7 +834,7 @@ construct_runtime! { // Crowdloan stuff. CrowdloanRewards: pallet_crowdloan_rewards::{Pallet, Call, Config, Storage, Event} = 90, - Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, } } diff --git a/runtime/moonshadow/src/lib.rs b/runtime/moonshadow/src/lib.rs index cd2751aec2..7f23b7506b 100644 --- a/runtime/moonshadow/src/lib.rs +++ b/runtime/moonshadow/src/lib.rs @@ -802,7 +802,7 @@ construct_runtime! { CrowdloanRewards: pallet_crowdloan_rewards::{Pallet, Call, Config, Storage, Event}, AuthorMapping: pallet_author_mapping::{Pallet, Call, Config, Storage, Event}, Proxy: pallet_proxy::{Pallet, Call, Storage, Event}, - Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, } } diff --git a/specs/alphanet/parachain-specs-template.json b/specs/alphanet/parachain-specs-template.json index 03b8ed63d4..0ba567276f 100644 --- a/specs/alphanet/parachain-specs-template.json +++ b/specs/alphanet/parachain-specs-template.json @@ -201,7 +201,6 @@ "treasury": {}, "scheduler": null, "migrations": { - "phantom": null, "completedMigrations": [] } } From f31807a44bf9bcb0cdfd6512c050e9348cd56ac6 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 18 Aug 2021 11:42:18 -0600 Subject: [PATCH 66/99] Better log statement Co-authored-by: Amar Singh --- pallets/migrations/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 61bc6d948d..6a97b0377a 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -159,7 +159,7 @@ pub mod pallet { }; log::trace!( - "performing migration {}, avail weight: {}", + "performing migration {}, available weight: {}", migration_name, available_for_step ); From 607e0ee38eb6bdeb2a7cbf0b444776f79e57490c Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 18 Aug 2021 12:16:25 -0600 Subject: [PATCH 67/99] Use ValueQuery instead of OptionQuery --- pallets/migrations/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 6a97b0377a..505694394d 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -114,7 +114,7 @@ pub mod pallet { #[pallet::getter(fn migration_state)] /// MigrationState tracks the progress of a migration. /// Maps name (Vec) -> whether or not migration has been completed (bool) - type MigrationState = StorageMap<_, Twox64Concat, Vec, bool, OptionQuery>; + type MigrationState = StorageMap<_, Twox64Concat, Vec, bool, ValueQuery>; #[pallet::genesis_config] #[derive(Default)] @@ -139,7 +139,7 @@ pub mod pallet { let migration_name_as_bytes = migration_name.as_bytes(); log::trace!("evaluating migration {}", migration_name); - let migration_done = >::get(migration_name_as_bytes).unwrap_or(false); + let migration_done = >::get(migration_name_as_bytes); if !migration_done { >::deposit_event(Event::MigrationStarted(migration_name_as_bytes.into())); From 91f92f7f5fe5172e390fb634bf697aa571795524 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 18 Aug 2021 12:40:56 -0600 Subject: [PATCH 68/99] Update Cargo.lock --- Cargo.lock | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 458e84d186..d5628c3236 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5402,6 +5402,81 @@ dependencies = [ "structopt", ] +[[package]] +name = "moonriver-runtime" +version = "0.8.4" +dependencies = [ + "account", + "crowdloan-rewards-precompiles", + "cumulus-pallet-parachain-system", + "cumulus-primitives-core", + "cumulus-primitives-parachain-inherent", + "cumulus-primitives-timestamp", + "cumulus-test-relay-sproof-builder", + "evm", + "fp-rpc", + "frame-benchmarking", + "frame-executive", + "frame-support", + "frame-system", + "frame-system-benchmarking", + "frame-system-rpc-runtime-api", + "hex", + "log", + "max-encoded-len", + "moonbeam-core-primitives", + "moonbeam-evm-tracer", + "moonbeam-rpc-primitives-debug", + "moonbeam-rpc-primitives-txpool", + "nimbus-primitives", + "pallet-author-inherent", + "pallet-author-mapping", + "pallet-author-slot-filter", + "pallet-balances", + "pallet-collective", + "pallet-crowdloan-rewards", + "pallet-democracy", + "pallet-ethereum", + "pallet-ethereum-chain-id", + "pallet-evm", + "pallet-evm-precompile-bn128", + "pallet-evm-precompile-dispatch", + "pallet-evm-precompile-modexp", + "pallet-evm-precompile-sha3fips", + "pallet-evm-precompile-simple", + "pallet-migrations", + "pallet-proxy", + "pallet-randomness-collective-flip", + "pallet-scheduler", + "pallet-society", + "pallet-timestamp", + "pallet-transaction-payment", + "pallet-transaction-payment-rpc-runtime-api", + "pallet-treasury", + "pallet-utility", + "parachain-info", + "parachain-staking", + "parachain-staking-precompiles", + "parity-scale-codec", + "precompile-utils", + "rlp", + "runtime-common", + "serde", + "sha3 0.8.2", + "sp-api", + "sp-block-builder", + "sp-core", + "sp-inherents", + "sp-io", + "sp-offchain", + "sp-runtime", + "sp-session", + "sp-std", + "sp-transaction-pool", + "sp-version", + "substrate-wasm-builder", +] + [[package]] name = "more-asserts" version = "0.2.1" From c0e6167886377b9f4b3cbc09814cfe583fb5f871 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 18 Aug 2021 16:13:49 -0600 Subject: [PATCH 69/99] Manually add back version = 3 --- Cargo.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index d5628c3236..2976a5ddcb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "Inflector" version = "0.11.4" From d9e561b74c36091d6f129a8333626b6e515e8ce3 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 18 Aug 2021 16:25:04 -0600 Subject: [PATCH 70/99] Make some deps dev-dependencies --- pallets/migrations/Cargo.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index 5f0491616b..2458095740 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -12,9 +12,11 @@ log = "0.4" sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } parity-scale-codec = { version = "2.0.0", default-features = false } -environmental = { version = "1.1.2", default-features = false } -sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } -sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } + +[dev-dependencies] +environmental = { version = "1.1.2" } +sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8" } +sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8" } [features] default = ["std"] From a585e3c46921dbbc3e3f4a4a61db20e506c64487 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 8 Sep 2021 14:25:33 -0600 Subject: [PATCH 71/99] Fix branch --- runtime/common/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/common/Cargo.toml b/runtime/common/Cargo.toml index b11774992b..161505421c 100644 --- a/runtime/common/Cargo.toml +++ b/runtime/common/Cargo.toml @@ -9,9 +9,9 @@ edition = '2018' [dependencies] pallet-migrations = { path = "../../pallets/migrations", default-features = false } -sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "notlesh-nimbus-v0.9.9-block-response-length", default-features = false } -sp-std = { git = "https://github.com/paritytech/substrate", branch = "notlesh-nimbus-v0.9.9-block-response-length", default-features = false } -frame-support = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "notlesh-nimbus-v0.9.9-block-response-length" } +sp-runtime = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length", default-features = false } +sp-std = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length", default-features = false } +frame-support = { git = "https://github.com/purestake/substrate", default-features = false, branch = "crystalin-v0.9.9-block-response-length" } log = "0.4" [features] From bdafb76177fdc063eb2378164e8b8676b8406171 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 8 Sep 2021 14:30:56 -0600 Subject: [PATCH 72/99] Use hotfix branch in Migrations --- pallets/migrations/Cargo.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index 2458095740..200048235b 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -6,17 +6,17 @@ edition = "2018" description = "migrations management pallet" [dependencies] -frame-support = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } -frame-system = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } +frame-support = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length", default-features = false } +frame-system = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length", default-features = false } log = "0.4" -sp-std = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } -sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8", default-features = false } +sp-std = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length", default-features = false } +sp-runtime = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length", default-features = false } parity-scale-codec = { version = "2.0.0", default-features = false } [dev-dependencies] environmental = { version = "1.1.2" } -sp-io = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8" } -sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.8" } +sp-io = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length" } +sp-core = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length" } [features] default = ["std"] From 9d27292bfa2419477049001ec901099ec8f918c3 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 8 Sep 2021 14:36:54 -0600 Subject: [PATCH 73/99] Clean up from merge --- Cargo.lock | 10 ++++------ runtime/moonbase/src/lib.rs | 7 ++----- runtime/moonbeam/src/lib.rs | 4 ++-- runtime/moonriver/src/lib.rs | 4 ++-- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0a5570d13b..8cd1f710d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,7 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - [[package]] name = "Inflector" version = "0.11.4" @@ -5019,9 +5017,9 @@ dependencies = [ "pallet-evm-precompile-modexp", "pallet-evm-precompile-sha3fips", "pallet-evm-precompile-simple", - "pallet-migrations", "pallet-identity", "pallet-maintenance-mode", + "pallet-migrations", "pallet-proxy", "pallet-randomness-collective-flip", "pallet-scheduler", @@ -5349,9 +5347,9 @@ dependencies = [ "pallet-evm-precompile-modexp", "pallet-evm-precompile-sha3fips", "pallet-evm-precompile-simple", - "pallet-migrations", "pallet-identity", "pallet-maintenance-mode", + "pallet-migrations", "pallet-proxy", "pallet-randomness-collective-flip", "pallet-scheduler", @@ -5544,9 +5542,9 @@ dependencies = [ "pallet-evm-precompile-modexp", "pallet-evm-precompile-sha3fips", "pallet-evm-precompile-simple", - "pallet-migrations", "pallet-identity", "pallet-maintenance-mode", + "pallet-migrations", "pallet-proxy", "pallet-randomness-collective-flip", "pallet-scheduler", @@ -9495,10 +9493,10 @@ name = "runtime-common" version = "0.8.0-dev" dependencies = [ "frame-support", + "log", "pallet-migrations", "sp-runtime", "sp-std", - "log", ] [[package]] diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index f76a24a8d3..3d3da9f9f8 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -52,7 +52,7 @@ use pallet_evm::{ Account as EVMAccount, EnsureAddressNever, EnsureAddressRoot, FeeCalculator, IdentityAddressMapping, Runner, }; -use pallet_migrations::{Pallet, Storage, Config, Event}; +use pallet_migrations::{Pallet, Config}; use pallet_transaction_payment::{CurrencyAdapter, Multiplier, TargetedFeeAdjustment}; pub use parachain_staking::{InflationInfo, Range}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; @@ -862,12 +862,9 @@ construct_runtime! { CrowdloanRewards: pallet_crowdloan_rewards::{Pallet, Call, Config, Storage, Event} = 20, AuthorMapping: pallet_author_mapping::{Pallet, Call, Config, Storage, Event} = 21, Proxy: pallet_proxy::{Pallet, Call, Storage, Event} = 22, -<<<<<<< HEAD - Migrations: pallet_migrations::{Pallet, Storage, Config, Event} = 23, -======= MaintenanceMode: pallet_maintenance_mode::{Pallet, Call, Config, Storage, Event} = 23, Identity: pallet_identity::{Pallet, Call, Storage, Event} = 24, ->>>>>>> master + Migrations: pallet_migrations::{Pallet, Storage, Config, Event} = 25, } } diff --git a/runtime/moonbeam/src/lib.rs b/runtime/moonbeam/src/lib.rs index 8ccf74bf52..2f927ad953 100644 --- a/runtime/moonbeam/src/lib.rs +++ b/runtime/moonbeam/src/lib.rs @@ -52,7 +52,7 @@ use pallet_evm::{ Account as EVMAccount, EnsureAddressNever, EnsureAddressRoot, FeeCalculator, IdentityAddressMapping, Runner, }; -use pallet_migrations::{Pallet, Storage, Config, Event}; +use pallet_migrations::{Pallet, Config}; use pallet_transaction_payment::{CurrencyAdapter, Multiplier, TargetedFeeAdjustment}; pub use parachain_staking::{InflationInfo, Range}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; @@ -856,9 +856,9 @@ construct_runtime! { CrowdloanRewards: pallet_crowdloan_rewards::{Pallet, Call, Config, Storage, Event}, AuthorMapping: pallet_author_mapping::{Pallet, Call, Config, Storage, Event}, Proxy: pallet_proxy::{Pallet, Call, Storage, Event}, - Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, MaintenanceMode: pallet_maintenance_mode::{Pallet, Call, Config, Storage, Event}, Identity: pallet_identity::{Pallet, Call, Storage, Event}, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, } } diff --git a/runtime/moonriver/src/lib.rs b/runtime/moonriver/src/lib.rs index d22cb101bf..f87b1d03d0 100644 --- a/runtime/moonriver/src/lib.rs +++ b/runtime/moonriver/src/lib.rs @@ -52,7 +52,7 @@ use pallet_evm::{ Account as EVMAccount, EnsureAddressNever, EnsureAddressRoot, FeeCalculator, IdentityAddressMapping, Runner, }; -use pallet_migrations::{Pallet, Storage, Config, Event}; +use pallet_migrations::{Pallet, Config}; use pallet_transaction_payment::{CurrencyAdapter, Multiplier, TargetedFeeAdjustment}; pub use parachain_staking::{InflationInfo, Range}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; @@ -843,6 +843,7 @@ construct_runtime! { Proxy: pallet_proxy::{Pallet, Call, Storage, Event} = 31, MaintenanceMode: pallet_maintenance_mode::{Pallet, Call, Config, Storage, Event} = 32, Identity: pallet_identity::{Pallet, Call, Storage, Event} = 33, + Migrations: pallet_migrations::{Pallet, Storage, Config, Event} = 34, // Sudo was previously index 40 @@ -866,7 +867,6 @@ construct_runtime! { // Crowdloan stuff. CrowdloanRewards: pallet_crowdloan_rewards::{Pallet, Call, Config, Storage, Event} = 90, - Migrations: pallet_migrations::{Pallet, Storage, Config, Event}, } } From e346883a56682771d72946898d2093c367257291 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 8 Sep 2021 14:40:24 -0600 Subject: [PATCH 74/99] cargo fmt --- runtime/moonbase/src/lib.rs | 2 +- runtime/moonbeam/src/lib.rs | 2 +- runtime/moonriver/src/lib.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index 3d3da9f9f8..881d8be9ac 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -52,7 +52,7 @@ use pallet_evm::{ Account as EVMAccount, EnsureAddressNever, EnsureAddressRoot, FeeCalculator, IdentityAddressMapping, Runner, }; -use pallet_migrations::{Pallet, Config}; +use pallet_migrations::{Config, Pallet}; use pallet_transaction_payment::{CurrencyAdapter, Multiplier, TargetedFeeAdjustment}; pub use parachain_staking::{InflationInfo, Range}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; diff --git a/runtime/moonbeam/src/lib.rs b/runtime/moonbeam/src/lib.rs index 2f927ad953..ac5c1ce72e 100644 --- a/runtime/moonbeam/src/lib.rs +++ b/runtime/moonbeam/src/lib.rs @@ -52,7 +52,7 @@ use pallet_evm::{ Account as EVMAccount, EnsureAddressNever, EnsureAddressRoot, FeeCalculator, IdentityAddressMapping, Runner, }; -use pallet_migrations::{Pallet, Config}; +use pallet_migrations::{Config, Pallet}; use pallet_transaction_payment::{CurrencyAdapter, Multiplier, TargetedFeeAdjustment}; pub use parachain_staking::{InflationInfo, Range}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; diff --git a/runtime/moonriver/src/lib.rs b/runtime/moonriver/src/lib.rs index f87b1d03d0..201e00839d 100644 --- a/runtime/moonriver/src/lib.rs +++ b/runtime/moonriver/src/lib.rs @@ -52,7 +52,7 @@ use pallet_evm::{ Account as EVMAccount, EnsureAddressNever, EnsureAddressRoot, FeeCalculator, IdentityAddressMapping, Runner, }; -use pallet_migrations::{Pallet, Config}; +use pallet_migrations::{Config, Pallet}; use pallet_transaction_payment::{CurrencyAdapter, Multiplier, TargetedFeeAdjustment}; pub use parachain_staking::{InflationInfo, Range}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; From 3cb8bb4c26dc065562e5f9acea7ab064ec492e85 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 8 Sep 2021 15:37:07 -0600 Subject: [PATCH 75/99] Remove prior hack in test --- pallets/migrations/src/tests.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 92a7139099..491535ea72 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -202,10 +202,6 @@ fn migration_should_only_be_invoked_once() { fn on_runtime_upgrade_charges_minimum_two_db_writes() { ExtBuilder::default().build().execute_with(|| { let mut weight = Migrations::on_runtime_upgrade(); - - // substrate seems to add a write to this call, so substract one for our logic - weight -= RocksDbWeight::get().writes(1); - assert_eq!(weight, RocksDbWeight::get().writes(2)); }) } From 1d86ea1a193d9be44f55b4c2773ee337746bed6d Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 9 Sep 2021 16:22:40 -0600 Subject: [PATCH 76/99] Initial concept of try-runtime support for pallet-migrations --- pallets/migrations/Cargo.toml | 1 + pallets/migrations/src/lib.rs | 50 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index 200048235b..0b0e623ce8 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -29,3 +29,4 @@ std = [ "sp-core/std", "environmental/std", ] +try-runtime = ["frame-support/try-runtime"] diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 505694394d..4e686d9f29 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -44,6 +44,14 @@ pub trait Migration { /// constraints will lead to a bricked chain upon a runtime upgrade because the parachain will /// not be able to produce a block that the relay chain will accept. fn migrate(&self, available_weight: Weight) -> Weight; + + /// Run a standard pre-runtime test. This works the same way as in a normal runtime upgrade. + #[cfg(feature = "try-runtime")] + fn pre_upgrade(&self) -> Result<(), &'static str> { Ok(()) } + + /// Run a standard post-runtime test. This works the same way as in a normal runtime upgrade. + #[cfg(feature = "try-runtime")] + fn post_upgrade(&self) -> Result<(), &'static str> { Ok(()) } } #[pallet] @@ -103,6 +111,48 @@ pub mod pallet { weight } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result<(), &'static str> { + let mut failed = false; + for migration in &T::MigrationsList::get() { + let migration_name = migration.friendly_name(); + let migration_name_as_bytes = migration_name.as_bytes(); + + let migration_done = >::get(migration_name_as_bytes); + if migration_done { + continue; + } + log::trace!("invoking pre_upgrade() on migration {}", migration_name); + + // TODO: use temp storage to make note of any migration that hasn't been performed. + // this is required because we won't know in post_upgrade() which migrations + // should have their own post_upgrade() hooks called since all upgrades will + // appear to be done by that point. + let result = migration.pre_upgrade(); + match result { + Ok(()) => { + log::info!("migration {} pre_upgrade() => Ok()", migration_name); + }, + Err(msg) => { + log::error!("migration {} pre_upgrade() => Err({})", migration_name, msg); + failed = true; + }, + } + } + + if failed { + Err("One or more pre_upgrade tests failed; see output above.") + } else { + Ok(()) + } + } + + /// Run a standard post-runtime test. This works the same way as in a normal runtime upgrade. + #[cfg(feature = "try-runtime")] + fn post_upgrade() -> Result<(), &'static str> { + todo!(); + } } #[pallet::storage] From d48140ca7fa7868d311565956fdc4b67f98d67c9 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 9 Sep 2021 16:44:17 -0600 Subject: [PATCH 77/99] Impl for post_upgrade() --- pallets/migrations/src/lib.rs | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 4e686d9f29..dc83e76860 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -151,7 +151,40 @@ pub mod pallet { /// Run a standard post-runtime test. This works the same way as in a normal runtime upgrade. #[cfg(feature = "try-runtime")] fn post_upgrade() -> Result<(), &'static str> { - todo!(); + // TODO: my desire to DRY all the things feels like this code is very repetitive... + + let mut failed = false; + for migration in &T::MigrationsList::get() { + let migration_name = migration.friendly_name(); + // let migration_name_as_bytes = migration_name.as_bytes(); + + // TODO: filter these migrations by the ones that weren't done prior to + // pre_upgrade() -- see the related comment above. + /* + let migration_done = >::get(migration_name_as_bytes); + if migration_done { + continue; + } + */ + log::trace!("invoking post_upgrade() on migration {}", migration_name); + + let result = migration.post_upgrade(); + match result { + Ok(()) => { + log::info!("migration {} post_upgrade() => Ok()", migration_name); + }, + Err(msg) => { + log::error!("migration {} post_upgrade() => Err({})", migration_name, msg); + failed = true; + }, + } + } + + if failed { + Err("One or more post_upgrade tests failed; see output above.") + } else { + Ok(()) + } } } From 06ee01fe6a4c5e8c7f22940ac3da1f2b9441e590 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 9 Sep 2021 20:27:33 -0600 Subject: [PATCH 78/99] Extend tests to work with pre_ and post_upgrade hooks --- pallets/migrations/src/mock.rs | 43 ++++++++++++++++++++++++++++++ pallets/migrations/src/tests.rs | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 47814ca4c0..f89e147182 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -95,6 +95,8 @@ impl frame_system::Config for Test { pub struct MockMigrationManager<'test> { name_fn_callbacks: Vec &'static str>>, migrate_fn_callbacks: Vec Weight>>, + pre_upgrade_fn_callbacks: Vec Result<(), &'static str>>>, + post_upgrade_fn_callbacks: Vec Result<(), &'static str>>>, } impl Default for MockMigrationManager<'_> { @@ -102,6 +104,8 @@ impl Default for MockMigrationManager<'_> { Self { name_fn_callbacks: Default::default(), migrate_fn_callbacks: Default::default(), + pre_upgrade_fn_callbacks: Default::default(), + post_upgrade_fn_callbacks: Default::default(), } } } @@ -114,6 +118,21 @@ impl<'test> MockMigrationManager<'test> { { self.name_fn_callbacks.push(Box::new(name_fn)); self.migrate_fn_callbacks.push(Box::new(migrate_fn)); + self.pre_upgrade_fn_callbacks.push(Box::new(|| Ok(()))); + self.post_upgrade_fn_callbacks.push(Box::new(|| Ok(()))); + } + pub fn register_callback_with_try_fns(&mut self, name_fn: FN, migrate_fn: FM, pre_upgrade_fn: FT1, post_upgrade_fn: FT2) + where + FN: 'test + FnMut() -> &'static str, + FM: 'test + FnMut(Weight) -> Weight, + FT1: 'test + FnMut() -> Result<(), &'static str>, + // no two closures, even if identical, have the same type + FT2: 'test + FnMut() -> Result<(), &'static str>, + { + self.name_fn_callbacks.push(Box::new(name_fn)); + self.migrate_fn_callbacks.push(Box::new(migrate_fn)); + self.pre_upgrade_fn_callbacks.push(Box::new(pre_upgrade_fn)); + self.post_upgrade_fn_callbacks.push(Box::new(post_upgrade_fn)); } pub(crate) fn invoke_name_fn(&mut self, index: usize) -> &'static str { @@ -124,6 +143,14 @@ impl<'test> MockMigrationManager<'test> { self.migrate_fn_callbacks[index](available_weight) } + pub(crate) fn invoke_pre_upgrade(&mut self, index: usize) -> Result<(), &'static str> { + self.pre_upgrade_fn_callbacks[index]() + } + + pub(crate) fn invoke_post_upgrade(&mut self, index: usize) -> Result<(), &'static str> { + self.post_upgrade_fn_callbacks[index]() + } + fn generate_migrations_list(&self) -> Vec> { let mut migrations: Vec> = Vec::new(); for i in 0..self.name_fn_callbacks.len() { @@ -177,6 +204,22 @@ impl Migration for MockMigration { }); result } + #[cfg(feature = "try-runtime")] + fn pre_upgrade(&self) -> Result<(), &'static str> { + let mut result: Result<(), &'static str> = Err("closure didn't set result"); + MOCK_MIGRATIONS_LIST::with(|mgr: &mut MockMigrationManager| { + result = mgr.invoke_pre_upgrade(self.index); + }); + result + } + #[cfg(feature = "try-runtime")] + fn post_upgrade(&self) -> Result<(), &'static str> { + let mut result: Result<(), &'static str> = Err("closure didn't set result"); + MOCK_MIGRATIONS_LIST::with(|mgr: &mut MockMigrationManager| { + result = mgr.invoke_post_upgrade(self.index); + }); + result + } } /// Implementation of Migrations. Generates a Vec of MockMigrations on the fly based on the current diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 491535ea72..90a4fd5c30 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -265,3 +265,49 @@ fn overweight_migrations_tolerated() { }, ); } + +#[test] +fn try_runtime_functions_work() { + let pre_fn_called = Arc::new(Mutex::new(false)); + let post_fn_called = Arc::new(Mutex::new(false)); + + crate::mock::execute_with_mock_migrations( + &mut |mgr: &mut MockMigrationManager| { + let pre_fn_called = Arc::clone(&pre_fn_called); + let post_fn_called = Arc::clone(&post_fn_called); + mgr.register_callback_with_try_fns( + move || { + "dummy_step" + }, + move |_| -> Weight { + 0u64.into() + }, + move || -> Result<(), &'static str> { + *pre_fn_called.lock().unwrap() = true; + Ok(()) + }, + move || -> Result<(), &'static str> { + *post_fn_called.lock().unwrap() = true; + Ok(()) + }, + ); + }, + &mut || { + ExtBuilder::default().build().execute_with(|| { + crate::mock::roll_until_upgraded(true); + }); + }, + ); + + assert_eq!( + *pre_fn_called.lock().unwrap(), + true, + "mock migration should call pre_upgrade()" + ); + + assert_eq!( + *post_fn_called.lock().unwrap(), + true, + "mock migration should call post_upgrade()" + ); +} From 5bf5b120fac3057ee4403d13326e6dbc8425feea Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 9 Sep 2021 20:39:23 -0600 Subject: [PATCH 79/99] Explicitly invoke try-runtime hooks in tests --- pallets/migrations/src/mock.rs | 9 +++++++++ pallets/migrations/src/tests.rs | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index f89e147182..4e38bda6c3 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -287,6 +287,15 @@ pub(crate) fn events() -> Vec> { .collect::>() } +pub(crate) fn invoke_all_upgrade_hooks() -> Weight { + let mut weight: Weight = 0u64.into(); + Migrations::pre_upgrade(); + weight = Migrations::on_runtime_upgrade(); + Migrations::post_upgrade(); + + weight +} + pub(crate) fn roll_to(block_number: u64, invoke_on_runtime_upgrade_first: bool) { if invoke_on_runtime_upgrade_first { Migrations::on_runtime_upgrade(); diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 90a4fd5c30..846aa313a2 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -294,7 +294,7 @@ fn try_runtime_functions_work() { }, &mut || { ExtBuilder::default().build().execute_with(|| { - crate::mock::roll_until_upgraded(true); + crate::mock::invoke_all_upgrade_hooks(); }); }, ); From 31d52fd7ebd6d00c26abe494b6d170dab2ebf6e2 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 9 Sep 2021 21:02:21 -0600 Subject: [PATCH 80/99] Leave hint about which migrations are performed for post_upgrade --- pallets/migrations/src/lib.rs | 27 +++++++++++++++------------ pallets/migrations/src/tests.rs | 2 ++ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index dc83e76860..9ec80b31d2 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -114,6 +114,8 @@ pub mod pallet { #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result<(), &'static str> { + use frame_support::traits::OnRuntimeUpgradeHelpersExt; + let mut failed = false; for migration in &T::MigrationsList::get() { let migration_name = migration.friendly_name(); @@ -125,10 +127,10 @@ pub mod pallet { } log::trace!("invoking pre_upgrade() on migration {}", migration_name); - // TODO: use temp storage to make note of any migration that hasn't been performed. - // this is required because we won't know in post_upgrade() which migrations - // should have their own post_upgrade() hooks called since all upgrades will - // appear to be done by that point. + // dump the migration name to temp storage so post_upgrade will know which + // migrations were performed (as opposed to skipped) + Self::set_temp_storage(true, migration_name); + let result = migration.pre_upgrade(); match result { Ok(()) => { @@ -151,21 +153,22 @@ pub mod pallet { /// Run a standard post-runtime test. This works the same way as in a normal runtime upgrade. #[cfg(feature = "try-runtime")] fn post_upgrade() -> Result<(), &'static str> { + use frame_support::traits::OnRuntimeUpgradeHelpersExt; + // TODO: my desire to DRY all the things feels like this code is very repetitive... let mut failed = false; for migration in &T::MigrationsList::get() { let migration_name = migration.friendly_name(); - // let migration_name_as_bytes = migration_name.as_bytes(); - // TODO: filter these migrations by the ones that weren't done prior to - // pre_upgrade() -- see the related comment above. - /* - let migration_done = >::get(migration_name_as_bytes); - if migration_done { - continue; + // we can't query MigrationState because on_runtime_upgrade() would have + // unconditionally set it to true, so we read a hint from temp storage which was + // left for us by pre_upgrade() + match Self::get_temp_storage::(migration_name) { + Some(value) => assert!(true == value, "our dummy value might as well be true"), + None => continue, } - */ + log::trace!("invoking post_upgrade() on migration {}", migration_name); let result = migration.post_upgrade(); diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 846aa313a2..d04dbf3cfc 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -311,3 +311,5 @@ fn try_runtime_functions_work() { "mock migration should call post_upgrade()" ); } +// TODO: a test to ensure that post_upgrade invokes the same set of migrations that pre_upgrade +// does would be useful From 64605c8f02b2fd4ddf641807302e255278ab064d Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Thu, 9 Sep 2021 21:29:12 -0600 Subject: [PATCH 81/99] cargo fmt --- pallets/migrations/src/lib.rs | 22 +++++++++++++++------- pallets/migrations/src/mock.rs | 12 +++++++++--- pallets/migrations/src/tests.rs | 8 ++------ 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 9ec80b31d2..7a2dc03c13 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -47,11 +47,15 @@ pub trait Migration { /// Run a standard pre-runtime test. This works the same way as in a normal runtime upgrade. #[cfg(feature = "try-runtime")] - fn pre_upgrade(&self) -> Result<(), &'static str> { Ok(()) } + fn pre_upgrade(&self) -> Result<(), &'static str> { + Ok(()) + } /// Run a standard post-runtime test. This works the same way as in a normal runtime upgrade. #[cfg(feature = "try-runtime")] - fn post_upgrade(&self) -> Result<(), &'static str> { Ok(()) } + fn post_upgrade(&self) -> Result<(), &'static str> { + Ok(()) + } } #[pallet] @@ -135,11 +139,11 @@ pub mod pallet { match result { Ok(()) => { log::info!("migration {} pre_upgrade() => Ok()", migration_name); - }, + } Err(msg) => { log::error!("migration {} pre_upgrade() => Err({})", migration_name, msg); failed = true; - }, + } } } @@ -175,11 +179,15 @@ pub mod pallet { match result { Ok(()) => { log::info!("migration {} post_upgrade() => Ok()", migration_name); - }, + } Err(msg) => { - log::error!("migration {} post_upgrade() => Err({})", migration_name, msg); + log::error!( + "migration {} post_upgrade() => Err({})", + migration_name, + msg + ); failed = true; - }, + } } } diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 4e38bda6c3..60f27749b3 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -121,8 +121,13 @@ impl<'test> MockMigrationManager<'test> { self.pre_upgrade_fn_callbacks.push(Box::new(|| Ok(()))); self.post_upgrade_fn_callbacks.push(Box::new(|| Ok(()))); } - pub fn register_callback_with_try_fns(&mut self, name_fn: FN, migrate_fn: FM, pre_upgrade_fn: FT1, post_upgrade_fn: FT2) - where + pub fn register_callback_with_try_fns( + &mut self, + name_fn: FN, + migrate_fn: FM, + pre_upgrade_fn: FT1, + post_upgrade_fn: FT2, + ) where FN: 'test + FnMut() -> &'static str, FM: 'test + FnMut(Weight) -> Weight, FT1: 'test + FnMut() -> Result<(), &'static str>, @@ -132,7 +137,8 @@ impl<'test> MockMigrationManager<'test> { self.name_fn_callbacks.push(Box::new(name_fn)); self.migrate_fn_callbacks.push(Box::new(migrate_fn)); self.pre_upgrade_fn_callbacks.push(Box::new(pre_upgrade_fn)); - self.post_upgrade_fn_callbacks.push(Box::new(post_upgrade_fn)); + self.post_upgrade_fn_callbacks + .push(Box::new(post_upgrade_fn)); } pub(crate) fn invoke_name_fn(&mut self, index: usize) -> &'static str { diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index d04dbf3cfc..f2810f44f6 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -276,12 +276,8 @@ fn try_runtime_functions_work() { let pre_fn_called = Arc::clone(&pre_fn_called); let post_fn_called = Arc::clone(&post_fn_called); mgr.register_callback_with_try_fns( - move || { - "dummy_step" - }, - move |_| -> Weight { - 0u64.into() - }, + move || "dummy_step", + move |_| -> Weight { 0u64.into() }, move || -> Result<(), &'static str> { *pre_fn_called.lock().unwrap() = true; Ok(()) From e58599f5ea73d19673c7e4ebbfdce8925792974f Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Fri, 10 Sep 2021 10:14:19 -0600 Subject: [PATCH 82/99] Add pallet-migration's try-runtime feature to all runtimes --- runtime/moonbase/Cargo.toml | 1 + runtime/moonbeam/Cargo.toml | 1 + runtime/moonriver/Cargo.toml | 1 + 3 files changed, 3 insertions(+) diff --git a/runtime/moonbase/Cargo.toml b/runtime/moonbase/Cargo.toml index 3370c391d3..0f9a438c91 100644 --- a/runtime/moonbase/Cargo.toml +++ b/runtime/moonbase/Cargo.toml @@ -198,4 +198,5 @@ try-runtime = [ "pallet-society/try-runtime", "pallet-author-mapping/try-runtime", #"pallet-crowdloan-rewards/try-runtime", + "pallet-migrations/try-runtime", ] diff --git a/runtime/moonbeam/Cargo.toml b/runtime/moonbeam/Cargo.toml index 86ae4a7fbb..c355083665 100644 --- a/runtime/moonbeam/Cargo.toml +++ b/runtime/moonbeam/Cargo.toml @@ -183,4 +183,5 @@ runtime-benchmarks = [ "pallet-society/runtime-benchmarks", "pallet-author-mapping/runtime-benchmarks", "pallet-crowdloan-rewards/runtime-benchmarks", + "pallet-migrations/try-runtime", ] diff --git a/runtime/moonriver/Cargo.toml b/runtime/moonriver/Cargo.toml index 26a8b919c5..5ab2f09e2e 100644 --- a/runtime/moonriver/Cargo.toml +++ b/runtime/moonriver/Cargo.toml @@ -195,4 +195,5 @@ try-runtime = [ "pallet-society/try-runtime", "pallet-author-mapping/try-runtime", #"pallet-crowdloan-rewards/try-runtime", + "pallet-migrations/try-runtime", ] From 3e3383693df23e0fdb8f8d3a4182d7771319520a Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Mon, 20 Sep 2021 16:36:16 -0400 Subject: [PATCH 83/99] fix warnings --- pallets/migrations/src/mock.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index 60f27749b3..b635b86c5b 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -294,10 +294,9 @@ pub(crate) fn events() -> Vec> { } pub(crate) fn invoke_all_upgrade_hooks() -> Weight { - let mut weight: Weight = 0u64.into(); - Migrations::pre_upgrade(); - weight = Migrations::on_runtime_upgrade(); - Migrations::post_upgrade(); + Migrations::pre_upgrade().expect("pre-upgrade hook succeeds"); + let weight = Migrations::on_runtime_upgrade(); + Migrations::post_upgrade().expect("post-upgrade hook succeeds"); weight } From 9370840daf8c9f950b90fe8b2a801089a2815509 Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Mon, 20 Sep 2021 16:50:05 -0400 Subject: [PATCH 84/99] Move autho mapping migration to dedicated type --- pallets/author-mapping/src/lib.rs | 119 +------------------- pallets/author-mapping/src/migrations.rs | 134 +++++++++++++++++++++++ 2 files changed, 137 insertions(+), 116 deletions(-) create mode 100644 pallets/author-mapping/src/migrations.rs diff --git a/pallets/author-mapping/src/lib.rs b/pallets/author-mapping/src/lib.rs index ce0d3d2995..a0d3ab99eb 100644 --- a/pallets/author-mapping/src/lib.rs +++ b/pallets/author-mapping/src/lib.rs @@ -37,6 +37,8 @@ mod mock; #[cfg(test)] mod tests; +mod migrations; + #[pallet] pub mod pallet { use crate::WeightInfo; @@ -237,7 +239,7 @@ pub mod pallet { #[pallet::getter(fn account_and_deposit_of)] /// We maintain a mapping from the AuthorIds used in the consensus layer /// to the AccountIds runtime (including this staking pallet). - type MappingWithDeposit = StorageMap< + pub type MappingWithDeposit = StorageMap< _, Blake2_128Concat, T::AuthorId, @@ -283,119 +285,4 @@ pub mod pallet { Self::account_and_deposit_of(author_id).map(|info| info.account) } } - - #[pallet::hooks] - impl Hooks> for Pallet { - fn on_runtime_upgrade() -> Weight { - use frame_support::storage::migration::{remove_storage_prefix, storage_key_iter}; - use sp_std::{convert::TryInto, vec::Vec}; - - let pallet_prefix: &[u8] = b"AuthorMapping"; - let storage_item_prefix: &[u8] = b"MappingWithDeposit"; - - // Read all the data into memory. - // https://crates.parity.io/frame_support/storage/migration/fn.storage_key_iter.html - let stored_data: Vec<_> = storage_key_iter::< - T::AuthorId, - RegistrationInfo>, - Twox64Concat, - >(pallet_prefix, storage_item_prefix) - .collect(); - - let migrated_count: Weight = stored_data - .len() - .try_into() - .expect("There are between 0 and 2**64 mappings stored."); - - // Now remove the old storage - // https://crates.parity.io/frame_support/storage/migration/fn.remove_storage_prefix.html - remove_storage_prefix(pallet_prefix, storage_item_prefix, &[]); - - // Assert that old storage is empty - assert!(storage_key_iter::< - T::AuthorId, - RegistrationInfo>, - Twox64Concat, - >(pallet_prefix, storage_item_prefix) - .next() - .is_none()); - - // Write the mappings back to storage with the new secure hasher - for (author_id, account_id) in stored_data { - MappingWithDeposit::::insert(author_id, account_id); - } - - // Return the weight used. For each migrated mapping there is a red to get it into - // memory, a write to clear the old stored value, and a write to re-store it. - let db_weights = T::DbWeight::get(); - migrated_count.saturating_mul(2 * db_weights.write + db_weights.read) - } - - #[cfg(feature = "try-runtime")] - fn pre_upgrade() -> Result<(), &'static str> { - use frame_support::storage::migration::{storage_iter, storage_key_iter}; - use frame_support::traits::OnRuntimeUpgradeHelpersExt; - - let pallet_prefix: &[u8] = b"AuthorMapping"; - let storage_item_prefix: &[u8] = b"MappingWithDeposit"; - - // We want to test that: - // There are no entries in the new storage beforehand - // The same number of mappings exist before and after - // As long as there are some mappings stored, one representative key maps to the - // same value after the migration. - // There are no entries in the old storage afterward - - // Assert new storage is empty - // Because the pallet and item prefixes are the same, the old storage is still at this - // key. However, the values can't be decoded so the assertion passes. - assert!(MappingWithDeposit::::iter().next().is_none()); - - // Check number of entries, and set it aside in temp storage - let mapping_count = storage_iter::>>( - pallet_prefix, - storage_item_prefix, - ) - .count() as u64; - Self::set_temp_storage(mapping_count, "mapping_count"); - - // Read an example pair from old storage and set it aside in temp storage - if mapping_count > 0 { - let example_pair = storage_key_iter::< - T::AuthorId, - RegistrationInfo>, - Twox64Concat, - >(pallet_prefix, storage_item_prefix) - .next() - .expect("We already confirmed that there was at least one item stored"); - - Self::set_temp_storage(example_pair, "example_pair"); - } - - Ok(()) - } - - #[cfg(feature = "try-runtime")] - fn post_upgrade() -> Result<(), &'static str> { - use frame_support::traits::OnRuntimeUpgradeHelpersExt; - - // Check number of entries matches what was set aside in pre_upgrade - let old_mapping_count: u64 = Self::get_temp_storage("mapping_count") - .expect("We stored a mapping count; it should be there; qed"); - let new_mapping_count = MappingWithDeposit::::iter().count() as u64; - assert_eq!(old_mapping_count, new_mapping_count); - - // Check that our example pair is still well-mapped after the migration - if new_mapping_count > 0 { - let (account, original_info): ( - T::AuthorId, - RegistrationInfo>, - ) = Self::get_temp_storage("example_pair").expect("qed"); - let migrated_info = MappingWithDeposit::::get(account).expect("qed"); - assert_eq!(original_info, migrated_info); - } - - Ok(()) - } - } } diff --git a/pallets/author-mapping/src/migrations.rs b/pallets/author-mapping/src/migrations.rs new file mode 100644 index 0000000000..7bdbdbfd72 --- /dev/null +++ b/pallets/author-mapping/src/migrations.rs @@ -0,0 +1,134 @@ +// Copyright 2019-2021 PureStake Inc. +// This file is part of Moonbeam. + +// Moonbeam is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Moonbeam is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Moonbeam. If not, see . + +use crate::{BalanceOf, Config, MappingWithDeposit, RegistrationInfo}; +use frame_support::{ + pallet_prelude::PhantomData, + storage::migration::{remove_storage_prefix, storage_key_iter}, + traits::{Get, OnRuntimeUpgrade}, + weights::Weight, + Twox64Concat, +}; +use sp_std::convert::TryInto; + +/// Migrates the AuthorMapping's storage map fro mthe insecure Twox64 hasher to the secure +/// BlakeTwo hasher. +pub struct TwoXToBlake(PhantomData); +impl OnRuntimeUpgrade for TwoXToBlake { + fn on_runtime_upgrade() -> Weight { + let pallet_prefix: &[u8] = b"AuthorMapping"; + let storage_item_prefix: &[u8] = b"MappingWithDeposit"; + + // Read all the data into memory. + // https://crates.parity.io/frame_support/storage/migration/fn.storage_key_iter.html + let stored_data: Vec<_> = storage_key_iter::< + T::AuthorId, + RegistrationInfo>, + Twox64Concat, + >(pallet_prefix, storage_item_prefix) + .collect(); + + let migrated_count: Weight = stored_data + .len() + .try_into() + .expect("There are between 0 and 2**64 mappings stored."); + + // Now remove the old storage + // https://crates.parity.io/frame_support/storage/migration/fn.remove_storage_prefix.html + remove_storage_prefix(pallet_prefix, storage_item_prefix, &[]); + + // Assert that old storage is empty + assert!(storage_key_iter::< + T::AuthorId, + RegistrationInfo>, + Twox64Concat, + >(pallet_prefix, storage_item_prefix) + .next() + .is_none()); + + // Write the mappings back to storage with the new secure hasher + for (author_id, account_id) in stored_data { + MappingWithDeposit::::insert(author_id, account_id); + } + + // Return the weight used. For each migrated mapping there is a red to get it into + // memory, a write to clear the old stored value, and a write to re-store it. + let db_weights = T::DbWeight::get(); + migrated_count.saturating_mul(2 * db_weights.write + db_weights.read) + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result<(), &'static str> { + let pallet_prefix: &[u8] = b"AuthorMapping"; + let storage_item_prefix: &[u8] = b"MappingWithDeposit"; + + // We want to test that: + // There are no entries in the new storage beforehand + // The same number of mappings exist before and after + // As long as there are some mappings stored, one representative key maps to the + // same value after the migration. + // There are no entries in the old storage afterward + + // Assert new storage is empty + // Because the pallet and item prefixes are the same, the old storage is still at this + // key. However, the values can't be decoded so the assertion passes. + assert!(MappingWithDeposit::::iter().next().is_none()); + + // Check number of entries, and set it aside in temp storage + let mapping_count = storage_iter::>>( + pallet_prefix, + storage_item_prefix, + ) + .count() as u64; + Self::set_temp_storage(mapping_count, "mapping_count"); + + // Read an example pair from old storage and set it aside in temp storage + if mapping_count > 0 { + let example_pair = storage_key_iter::< + T::AuthorId, + RegistrationInfo>, + Twox64Concat, + >(pallet_prefix, storage_item_prefix) + .next() + .expect("We already confirmed that there was at least one item stored"); + + Self::set_temp_storage(example_pair, "example_pair"); + } + + Ok(()) + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade() -> Result<(), &'static str> { + // Check number of entries matches what was set aside in pre_upgrade + let old_mapping_count: u64 = Self::get_temp_storage("mapping_count") + .expect("We stored a mapping count; it should be there; qed"); + let new_mapping_count = MappingWithDeposit::::iter().count() as u64; + assert_eq!(old_mapping_count, new_mapping_count); + + // Check that our example pair is still well-mapped after the migration + if new_mapping_count > 0 { + let (account, original_info): ( + T::AuthorId, + RegistrationInfo>, + ) = Self::get_temp_storage("example_pair").expect("qed"); + let migrated_info = MappingWithDeposit::::get(account).expect("qed"); + assert_eq!(original_info, migrated_info); + } + + Ok(()) + } +} From b90b7264ab5ced1989ba4e711f27e09eaa9cc6f6 Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Mon, 20 Sep 2021 16:53:10 -0400 Subject: [PATCH 85/99] brain dump --- pallets/migrations/Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index 68819464c9..dd7a372e8d 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -13,6 +13,11 @@ sp-std = { git = "https://github.com/purestake/substrate", branch = "crystalin-v sp-runtime = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length", default-features = false } parity-scale-codec = { version = "2.0.0", default-features = false } +# One weakness is that every pallet has to depend on the migrations trait. +# We should really only have to impl frame support traits. +# But what about the unique identifier... +syntax error as a reminer of where to pick back up + [dev-dependencies] environmental = "1.1.2" sp-io = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length" } From ac5f99f3818d81091fe4e0cc33105a6624baa29c Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Tue, 21 Sep 2021 11:26:05 -0400 Subject: [PATCH 86/99] Make author hasher migration compatible with pallet migrations. --- pallets/author-mapping/src/lib.rs | 2 +- pallets/author-mapping/src/migrations.rs | 2 ++ pallets/migrations/Cargo.toml | 5 ---- runtime/moonbase/src/lib.rs | 3 ++- runtime/moonbase/src/migrations.rs | 32 ++++++++++++++++++++++++ 5 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 runtime/moonbase/src/migrations.rs diff --git a/pallets/author-mapping/src/lib.rs b/pallets/author-mapping/src/lib.rs index a0d3ab99eb..c66ecf24e3 100644 --- a/pallets/author-mapping/src/lib.rs +++ b/pallets/author-mapping/src/lib.rs @@ -37,7 +37,7 @@ mod mock; #[cfg(test)] mod tests; -mod migrations; +pub mod migrations; #[pallet] pub mod pallet { diff --git a/pallets/author-mapping/src/migrations.rs b/pallets/author-mapping/src/migrations.rs index 7bdbdbfd72..00b076d1bc 100644 --- a/pallets/author-mapping/src/migrations.rs +++ b/pallets/author-mapping/src/migrations.rs @@ -23,6 +23,8 @@ use frame_support::{ Twox64Concat, }; use sp_std::convert::TryInto; +//TODO sometimes this is unused, sometimes its necessary +use sp_std::vec::Vec; /// Migrates the AuthorMapping's storage map fro mthe insecure Twox64 hasher to the secure /// BlakeTwo hasher. diff --git a/pallets/migrations/Cargo.toml b/pallets/migrations/Cargo.toml index dd7a372e8d..68819464c9 100644 --- a/pallets/migrations/Cargo.toml +++ b/pallets/migrations/Cargo.toml @@ -13,11 +13,6 @@ sp-std = { git = "https://github.com/purestake/substrate", branch = "crystalin-v sp-runtime = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length", default-features = false } parity-scale-codec = { version = "2.0.0", default-features = false } -# One weakness is that every pallet has to depend on the migrations trait. -# We should really only have to impl frame support traits. -# But what about the unique identifier... -syntax error as a reminer of where to pick back up - [dev-dependencies] environmental = "1.1.2" sp-io = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length" } diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index cd9b76825a..221f0d9122 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -69,7 +69,6 @@ use pallet_evm::{ Account as EVMAccount, EnsureAddressNever, EnsureAddressRoot, FeeCalculator, GasWeightMapping, IdentityAddressMapping, Runner, }; -use pallet_migrations::{Config, Pallet}; use pallet_transaction_payment::{CurrencyAdapter, Multiplier, TargetedFeeAdjustment}; pub use parachain_staking::{InflationInfo, Range}; use parity_scale_codec::{Decode, Encode, MaxEncodedLen}; @@ -100,6 +99,7 @@ use xcm::v0::{ use nimbus_primitives::{CanAuthor, NimbusId}; +mod migrations; mod precompiles; use precompiles::MoonbasePrecompiles; @@ -846,6 +846,7 @@ impl pallet_proxy::Config for Runtime { impl pallet_migrations::Config for Runtime { type Event = Event; + //TODO wire up our correct list of migrations here. Maybe this shouldn't be in `runtime_common`. type MigrationsList = runtime_common::migrations::CommonMigrations; } diff --git a/runtime/moonbase/src/migrations.rs b/runtime/moonbase/src/migrations.rs new file mode 100644 index 0000000000..9891833d79 --- /dev/null +++ b/runtime/moonbase/src/migrations.rs @@ -0,0 +1,32 @@ +//TODO license + +use frame_support::{traits::OnRuntimeUpgrade, weights::Weight}; +use pallet_author_mapping::{migrations::TwoXToBlake, Config as AuthorMappingConfig}; +use pallet_migrations::{Config as MigrationsConfig, Migration}; +use sp_std::marker::PhantomData; + +/// A moonbeam migration wrapping the similarly named migration in pallet-author-mapping +pub struct AuthorMappingTwoXToBlake(PhantomData); + +impl Migration for AuthorMappingTwoXToBlake { + fn friendly_name(&self) -> &str { + "MM_Author_Mapping_TwoXToBlake" + } + + fn migrate(&self, available_weight: Weight) -> Weight { + //TODO @notlesh, what should I be doing with the available_weight here + TwoXToBlake::::on_runtime_upgrade() + } + + /// Run a standard pre-runtime test. This works the same way as in a normal runtime upgrade. + #[cfg(feature = "try-runtime")] + fn pre_upgrade(&self) -> Result<(), &'static str> { + TwoXToBlake::::pre_upgrade() + } + + /// Run a standard post-runtime test. This works the same way as in a normal runtime upgrade. + #[cfg(feature = "try-runtime")] + fn post_upgrade(&self) -> Result<(), &'static str> { + TwoXToBlake::::post_upgrade() + } +} From 003a52c91890f4548e781475eae233045d70db97 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 21 Sep 2021 09:52:06 -0600 Subject: [PATCH 87/99] Condense code Co-authored-by: Joshy Orndorff --- pallets/migrations/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 7a2dc03c13..f51709626a 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -135,8 +135,7 @@ pub mod pallet { // migrations were performed (as opposed to skipped) Self::set_temp_storage(true, migration_name); - let result = migration.pre_upgrade(); - match result { + match migration.pre_upgrade() { Ok(()) => { log::info!("migration {} pre_upgrade() => Ok()", migration_name); } From 9a41ed320f2a389d8278f3d9a7e637d0500a0544 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 21 Sep 2021 10:40:05 -0600 Subject: [PATCH 88/99] Only run try-runtime tests with --features=try-runtime --- pallets/migrations/src/mock.rs | 1 + pallets/migrations/src/tests.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pallets/migrations/src/mock.rs b/pallets/migrations/src/mock.rs index b635b86c5b..d4c35a494c 100644 --- a/pallets/migrations/src/mock.rs +++ b/pallets/migrations/src/mock.rs @@ -293,6 +293,7 @@ pub(crate) fn events() -> Vec> { .collect::>() } +#[cfg(feature = "try-runtime")] pub(crate) fn invoke_all_upgrade_hooks() -> Weight { Migrations::pre_upgrade().expect("pre-upgrade hook succeeds"); let weight = Migrations::on_runtime_upgrade(); diff --git a/pallets/migrations/src/tests.rs b/pallets/migrations/src/tests.rs index 7f6b6ebe65..d7026bbd0f 100644 --- a/pallets/migrations/src/tests.rs +++ b/pallets/migrations/src/tests.rs @@ -266,7 +266,7 @@ fn overweight_migrations_tolerated() { ); } -#[test] +#[cfg(all(test, feature = "try-runtime"))] fn try_runtime_functions_work() { let pre_fn_called = Arc::new(Mutex::new(false)); let post_fn_called = Arc::new(Mutex::new(false)); From 9b476e28adc4072cc881474ee2b1be690eee783f Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 21 Sep 2021 10:48:58 -0600 Subject: [PATCH 89/99] Add missing import --- pallets/author-mapping/src/migrations.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pallets/author-mapping/src/migrations.rs b/pallets/author-mapping/src/migrations.rs index 00b076d1bc..7d9fb1d607 100644 --- a/pallets/author-mapping/src/migrations.rs +++ b/pallets/author-mapping/src/migrations.rs @@ -17,7 +17,7 @@ use crate::{BalanceOf, Config, MappingWithDeposit, RegistrationInfo}; use frame_support::{ pallet_prelude::PhantomData, - storage::migration::{remove_storage_prefix, storage_key_iter}, + storage::migration::{storage_iter, remove_storage_prefix, storage_key_iter}, traits::{Get, OnRuntimeUpgrade}, weights::Weight, Twox64Concat, From 4a53747d8f7045e25ea2ac9b3f39e6be69601cc3 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 21 Sep 2021 11:39:23 -0600 Subject: [PATCH 90/99] Add try-runtime-only imports --- pallets/author-mapping/src/migrations.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pallets/author-mapping/src/migrations.rs b/pallets/author-mapping/src/migrations.rs index 7d9fb1d607..24f6f4a85d 100644 --- a/pallets/author-mapping/src/migrations.rs +++ b/pallets/author-mapping/src/migrations.rs @@ -22,6 +22,7 @@ use frame_support::{ weights::Weight, Twox64Concat, }; + use sp_std::convert::TryInto; //TODO sometimes this is unused, sometimes its necessary use sp_std::vec::Vec; @@ -74,6 +75,8 @@ impl OnRuntimeUpgrade for TwoXToBlake { #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result<(), &'static str> { + use frame_support::traits::OnRuntimeUpgradeHelpersExt; + let pallet_prefix: &[u8] = b"AuthorMapping"; let storage_item_prefix: &[u8] = b"MappingWithDeposit"; @@ -115,6 +118,8 @@ impl OnRuntimeUpgrade for TwoXToBlake { #[cfg(feature = "try-runtime")] fn post_upgrade() -> Result<(), &'static str> { + use frame_support::traits::OnRuntimeUpgradeHelpersExt; + // Check number of entries matches what was set aside in pre_upgrade let old_mapping_count: u64 = Self::get_temp_storage("mapping_count") .expect("We stored a mapping count; it should be there; qed"); From 66762659b347bb17ccd728516d6a3b1916a5feca Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 21 Sep 2021 14:44:08 -0600 Subject: [PATCH 91/99] Hook AuthorMapping migration up to common migrations --- .cargo/config.toml | 2 ++ Cargo.lock | 1 + runtime/common/Cargo.toml | 1 + runtime/common/src/migrations.rs | 50 +++++++++++++----------------- runtime/moonbase/src/lib.rs | 1 - runtime/moonbase/src/migrations.rs | 32 ------------------- 6 files changed, 25 insertions(+), 62 deletions(-) delete mode 100644 runtime/moonbase/src/migrations.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 0d550ca9bb..ab9556237a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,5 @@ +paths = ["../substrate"] + [alias] moonbase = " build --release -p moonbeam --no-default-features --features moonbase-native" moonbase-rococo = " build --release -p moonbeam --no-default-features --features moonbase-native,rococo-native" diff --git a/Cargo.lock b/Cargo.lock index dd210cdef7..4c59e8508e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9637,6 +9637,7 @@ version = "0.8.0-dev" dependencies = [ "frame-support", "log", + "pallet-author-mapping", "pallet-migrations", "sp-runtime", "sp-std", diff --git a/runtime/common/Cargo.toml b/runtime/common/Cargo.toml index 161505421c..0e0ad76fdc 100644 --- a/runtime/common/Cargo.toml +++ b/runtime/common/Cargo.toml @@ -12,6 +12,7 @@ pallet-migrations = { path = "../../pallets/migrations", default-features = fals sp-runtime = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length", default-features = false } sp-std = { git = "https://github.com/purestake/substrate", branch = "crystalin-v0.9.9-block-response-length", default-features = false } frame-support = { git = "https://github.com/purestake/substrate", default-features = false, branch = "crystalin-v0.9.9-block-response-length" } +pallet-author-mapping = { path = "../../pallets/author-mapping", default-features = false } log = "0.4" [features] diff --git a/runtime/common/src/migrations.rs b/runtime/common/src/migrations.rs index e4c38cf21c..aa247d4e01 100644 --- a/runtime/common/src/migrations.rs +++ b/runtime/common/src/migrations.rs @@ -16,48 +16,40 @@ //! # Migrations -use frame_support::pallet_prelude::Get; -use pallet_migrations::Migration; -use sp_std::prelude::*; +use frame_support::{traits::OnRuntimeUpgrade, weights::Weight, pallet_prelude::Get}; +use pallet_migrations::{Config as MigrationsConfig, Migration}; +use sp_std::{ + prelude::*, + marker::PhantomData, +}; +use pallet_author_mapping::{migrations::TwoXToBlake, Config as AuthorMappingConfig}; /// This module acts as a registry where each migration is defined. Each migration should implement /// the "Migration" trait declared in the pallet-migrations crate. -/* - * These are left as examples. -#[allow(non_camel_case_types)] -pub struct MM_001_AuthorMappingAddDeposit; -impl Migration for MM_001_AuthorMappingAddDeposit { +/// A moonbeam migration wrapping the similarly named migration in pallet-author-mapping +pub struct AuthorMappingTwoXToBlake(PhantomData); +impl Migration for AuthorMappingTwoXToBlake { fn friendly_name(&self) -> &str { - "AuthorMappingAddDeposit" + "MM_Author_Mapping_TwoXToBlake" } - fn migrate(&self, _available_weight: Weight) -> Weight { - 0u64.into() - } -} -#[allow(non_camel_case_types)] -pub struct MM_002_StakingFixTotalBalance; -impl Migration for MM_002_StakingFixTotalBalance { - fn friendly_name(&self) -> &str { - "StakingFixTotalBalance" - } fn migrate(&self, _available_weight: Weight) -> Weight { - 0u64.into() + TwoXToBlake::::on_runtime_upgrade() } -} -#[allow(non_camel_case_types)] -pub struct MM_003_StakingUnboundedCollatorNominations; -impl Migration for MM_003_StakingUnboundedCollatorNominations { - fn friendly_name(&self) -> &str { - "StakingUnboundedCollatorNominations" + /// Run a standard pre-runtime test. This works the same way as in a normal runtime upgrade. + #[cfg(feature = "try-runtime")] + fn pre_upgrade(&self) -> Result<(), &'static str> { + TwoXToBlake::::pre_upgrade() } - fn migrate(&self, _available_weight: Weight) -> Weight { - 0u64.into() + + /// Run a standard post-runtime test. This works the same way as in a normal runtime upgrade. + #[cfg(feature = "try-runtime")] + fn post_upgrade(&self) -> Result<(), &'static str> { + TwoXToBlake::::post_upgrade() } } -*/ pub struct CommonMigrations; impl Get>> for CommonMigrations { diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index 221f0d9122..6d7510340e 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -99,7 +99,6 @@ use xcm::v0::{ use nimbus_primitives::{CanAuthor, NimbusId}; -mod migrations; mod precompiles; use precompiles::MoonbasePrecompiles; diff --git a/runtime/moonbase/src/migrations.rs b/runtime/moonbase/src/migrations.rs deleted file mode 100644 index 9891833d79..0000000000 --- a/runtime/moonbase/src/migrations.rs +++ /dev/null @@ -1,32 +0,0 @@ -//TODO license - -use frame_support::{traits::OnRuntimeUpgrade, weights::Weight}; -use pallet_author_mapping::{migrations::TwoXToBlake, Config as AuthorMappingConfig}; -use pallet_migrations::{Config as MigrationsConfig, Migration}; -use sp_std::marker::PhantomData; - -/// A moonbeam migration wrapping the similarly named migration in pallet-author-mapping -pub struct AuthorMappingTwoXToBlake(PhantomData); - -impl Migration for AuthorMappingTwoXToBlake { - fn friendly_name(&self) -> &str { - "MM_Author_Mapping_TwoXToBlake" - } - - fn migrate(&self, available_weight: Weight) -> Weight { - //TODO @notlesh, what should I be doing with the available_weight here - TwoXToBlake::::on_runtime_upgrade() - } - - /// Run a standard pre-runtime test. This works the same way as in a normal runtime upgrade. - #[cfg(feature = "try-runtime")] - fn pre_upgrade(&self) -> Result<(), &'static str> { - TwoXToBlake::::pre_upgrade() - } - - /// Run a standard post-runtime test. This works the same way as in a normal runtime upgrade. - #[cfg(feature = "try-runtime")] - fn post_upgrade(&self) -> Result<(), &'static str> { - TwoXToBlake::::post_upgrade() - } -} From 935d0fde7a293ba8c6ea08f4ee01fe84b517d065 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 21 Sep 2021 15:34:00 -0600 Subject: [PATCH 92/99] Actually hook it up... also, trait bounds --- runtime/common/src/migrations.rs | 20 ++++++++++++-------- runtime/moonbase/src/lib.rs | 2 +- runtime/moonbeam/src/lib.rs | 2 +- runtime/moonriver/src/lib.rs | 2 +- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/runtime/common/src/migrations.rs b/runtime/common/src/migrations.rs index aa247d4e01..535da762b4 100644 --- a/runtime/common/src/migrations.rs +++ b/runtime/common/src/migrations.rs @@ -29,7 +29,7 @@ use pallet_author_mapping::{migrations::TwoXToBlake, Config as AuthorMappingConf /// A moonbeam migration wrapping the similarly named migration in pallet-author-mapping pub struct AuthorMappingTwoXToBlake(PhantomData); -impl Migration for AuthorMappingTwoXToBlake { +impl Migration for AuthorMappingTwoXToBlake { fn friendly_name(&self) -> &str { "MM_Author_Mapping_TwoXToBlake" } @@ -51,17 +51,21 @@ impl Migration for AuthorMappingTwoXT } } -pub struct CommonMigrations; -impl Get>> for CommonMigrations { +pub struct CommonMigrations(PhantomData); +impl Get>> for CommonMigrations +where + Runtime: pallet_author_mapping::Config, +{ fn get() -> Vec> { + let migration_author_mapping_twox_to_blake = AuthorMappingTwoXToBlake:: { + 0: Default::default(), + }; + // TODO: this is a lot of allocation to do upon every get() call. this *should* be avoided // except when pallet_migrations undergoes a runtime upgrade -- but TODO: review + vec![ - /* - Box::new(MM_001_AuthorMappingAddDeposit), - Box::new(MM_002_StakingFixTotalBalance), - Box::new(MM_003_StakingUnboundedCollatorNominations), - */ + Box::new(migration_author_mapping_twox_to_blake), ] } } diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index 6d7510340e..c5dba4584a 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -846,7 +846,7 @@ impl pallet_proxy::Config for Runtime { impl pallet_migrations::Config for Runtime { type Event = Event; //TODO wire up our correct list of migrations here. Maybe this shouldn't be in `runtime_common`. - type MigrationsList = runtime_common::migrations::CommonMigrations; + type MigrationsList = runtime_common::migrations::CommonMigrations; } parameter_types! { diff --git a/runtime/moonbeam/src/lib.rs b/runtime/moonbeam/src/lib.rs index 1731cf9365..01919056c0 100644 --- a/runtime/moonbeam/src/lib.rs +++ b/runtime/moonbeam/src/lib.rs @@ -812,7 +812,7 @@ impl pallet_proxy::Config for Runtime { impl pallet_migrations::Config for Runtime { type Event = Event; - type MigrationsList = runtime_common::migrations::CommonMigrations; + type MigrationsList = runtime_common::migrations::CommonMigrations; } /// Call filter expected to be used during Phase 3 of the Moonbeam rollout diff --git a/runtime/moonriver/src/lib.rs b/runtime/moonriver/src/lib.rs index a398691d5e..cc26ae94c9 100644 --- a/runtime/moonriver/src/lib.rs +++ b/runtime/moonriver/src/lib.rs @@ -803,7 +803,7 @@ impl pallet_proxy::Config for Runtime { impl pallet_migrations::Config for Runtime { type Event = Event; - type MigrationsList = runtime_common::migrations::CommonMigrations; + type MigrationsList = runtime_common::migrations::CommonMigrations; } /// Call filter used during Phase 3 of the Moonriver rollout From b9e6a61b2e63ff71107b4fbb73fd153e26fb6b10 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 21 Sep 2021 15:57:53 -0600 Subject: [PATCH 93/99] cargo fmt --- pallets/author-mapping/src/migrations.rs | 2 +- runtime/common/src/migrations.rs | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/pallets/author-mapping/src/migrations.rs b/pallets/author-mapping/src/migrations.rs index 24f6f4a85d..9240cdf5ec 100644 --- a/pallets/author-mapping/src/migrations.rs +++ b/pallets/author-mapping/src/migrations.rs @@ -17,7 +17,7 @@ use crate::{BalanceOf, Config, MappingWithDeposit, RegistrationInfo}; use frame_support::{ pallet_prelude::PhantomData, - storage::migration::{storage_iter, remove_storage_prefix, storage_key_iter}, + storage::migration::{remove_storage_prefix, storage_iter, storage_key_iter}, traits::{Get, OnRuntimeUpgrade}, weights::Weight, Twox64Concat, diff --git a/runtime/common/src/migrations.rs b/runtime/common/src/migrations.rs index 535da762b4..dbe1714a0a 100644 --- a/runtime/common/src/migrations.rs +++ b/runtime/common/src/migrations.rs @@ -16,13 +16,10 @@ //! # Migrations -use frame_support::{traits::OnRuntimeUpgrade, weights::Weight, pallet_prelude::Get}; -use pallet_migrations::{Config as MigrationsConfig, Migration}; -use sp_std::{ - prelude::*, - marker::PhantomData, -}; +use frame_support::{pallet_prelude::Get, traits::OnRuntimeUpgrade, weights::Weight}; use pallet_author_mapping::{migrations::TwoXToBlake, Config as AuthorMappingConfig}; +use pallet_migrations::{Config as MigrationsConfig, Migration}; +use sp_std::{marker::PhantomData, prelude::*}; /// This module acts as a registry where each migration is defined. Each migration should implement /// the "Migration" trait declared in the pallet-migrations crate. @@ -64,8 +61,6 @@ where // TODO: this is a lot of allocation to do upon every get() call. this *should* be avoided // except when pallet_migrations undergoes a runtime upgrade -- but TODO: review - vec![ - Box::new(migration_author_mapping_twox_to_blake), - ] + vec![Box::new(migration_author_mapping_twox_to_blake)] } } From 85d126b42ff5c2292cdb3db8aacdf3d029c06957 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Tue, 21 Sep 2021 15:58:20 -0600 Subject: [PATCH 94/99] Remove accidental .cargo/config.toml addition --- .cargo/config.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index ab9556237a..0d550ca9bb 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,5 +1,3 @@ -paths = ["../substrate"] - [alias] moonbase = " build --release -p moonbeam --no-default-features --features moonbase-native" moonbase-rococo = " build --release -p moonbeam --no-default-features --features moonbase-native,rococo-native" From 4e459c43ff44b67a86ac399d78f2f87d1bea1bc7 Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Fri, 24 Sep 2021 10:03:48 -0400 Subject: [PATCH 95/99] better logging --- pallets/author-mapping/src/migrations.rs | 3 +++ pallets/migrations/src/lib.rs | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pallets/author-mapping/src/migrations.rs b/pallets/author-mapping/src/migrations.rs index 9240cdf5ec..f14bbe44d2 100644 --- a/pallets/author-mapping/src/migrations.rs +++ b/pallets/author-mapping/src/migrations.rs @@ -32,6 +32,7 @@ use sp_std::vec::Vec; pub struct TwoXToBlake(PhantomData); impl OnRuntimeUpgrade for TwoXToBlake { fn on_runtime_upgrade() -> Weight { + log::info!(target: "TwoXToBlake", "actually running it"); let pallet_prefix: &[u8] = b"AuthorMapping"; let storage_item_prefix: &[u8] = b"MappingWithDeposit"; @@ -67,6 +68,8 @@ impl OnRuntimeUpgrade for TwoXToBlake { MappingWithDeposit::::insert(author_id, account_id); } + log::info!(target: "TwoXToBlake", "almost done"); + // Return the weight used. For each migrated mapping there is a red to get it into // memory, a write to clear the old stored value, and a write to re-store it. let db_weights = T::DbWeight::get(); diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index f51709626a..b68b8518f7 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -129,7 +129,7 @@ pub mod pallet { if migration_done { continue; } - log::trace!("invoking pre_upgrade() on migration {}", migration_name); + log::debug!( target: "pallet-migrations", "invoking pre_upgrade() on migration {}", migration_name); // dump the migration name to temp storage so post_upgrade will know which // migrations were performed (as opposed to skipped) @@ -172,7 +172,7 @@ pub mod pallet { None => continue, } - log::trace!("invoking post_upgrade() on migration {}", migration_name); + log::debug!( target: "pallet-migrations", "invoking post_upgrade() on migration {}", migration_name); let result = migration.post_upgrade(); match result { @@ -230,7 +230,7 @@ pub mod pallet { for migration in &T::MigrationsList::get() { let migration_name = migration.friendly_name(); let migration_name_as_bytes = migration_name.as_bytes(); - log::trace!("evaluating migration {}", migration_name); + log::debug!( target: "pallet-migrations", "evaluating migration {}", migration_name); let migration_done = >::get(migration_name_as_bytes); @@ -251,7 +251,7 @@ pub mod pallet { 0u64.into() }; - log::trace!( + log::info!( target: "pallet-migrations", "performing migration {}, available weight: {}", migration_name, available_for_step From 7f546b761467c04487ca5328045a6847b10a776f Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Fri, 24 Sep 2021 10:04:29 -0400 Subject: [PATCH 96/99] bump spec version for testing purposes --- runtime/moonbase/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/moonbase/src/lib.rs b/runtime/moonbase/src/lib.rs index c5dba4584a..ec72f93b3b 100644 --- a/runtime/moonbase/src/lib.rs +++ b/runtime/moonbase/src/lib.rs @@ -162,7 +162,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("moonbase"), impl_name: create_runtime_str!("moonbase"), authoring_version: 3, - spec_version: 0700, + spec_version: 0900, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 2, From a9da31da9dfc7160451d32b20257c526472fe14b Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Fri, 1 Oct 2021 15:13:23 -0400 Subject: [PATCH 97/99] cleanups --- pallets/author-mapping/src/migrations.rs | 4 ++-- pallets/migrations/src/lib.rs | 2 +- runtime/common/src/migrations.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pallets/author-mapping/src/migrations.rs b/pallets/author-mapping/src/migrations.rs index f14bbe44d2..cf2d5df47b 100644 --- a/pallets/author-mapping/src/migrations.rs +++ b/pallets/author-mapping/src/migrations.rs @@ -17,7 +17,7 @@ use crate::{BalanceOf, Config, MappingWithDeposit, RegistrationInfo}; use frame_support::{ pallet_prelude::PhantomData, - storage::migration::{remove_storage_prefix, storage_iter, storage_key_iter}, + storage::migration::{remove_storage_prefix, storage_key_iter}, traits::{Get, OnRuntimeUpgrade}, weights::Weight, Twox64Concat, @@ -78,7 +78,7 @@ impl OnRuntimeUpgrade for TwoXToBlake { #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result<(), &'static str> { - use frame_support::traits::OnRuntimeUpgradeHelpersExt; + use frame_support::{storage::migration::storage_iter, traits::OnRuntimeUpgradeHelpersExt}; let pallet_prefix: &[u8] = b"AuthorMapping"; let storage_item_prefix: &[u8] = b"MappingWithDeposit"; diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index b68b8518f7..2dcdacbd48 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -251,7 +251,7 @@ pub mod pallet { 0u64.into() }; - log::info!( target: "pallet-migrations", + log::info!( target: "pallet-migrations", "performing migration {}, available weight: {}", migration_name, available_for_step diff --git a/runtime/common/src/migrations.rs b/runtime/common/src/migrations.rs index dbe1714a0a..08e916ddc1 100644 --- a/runtime/common/src/migrations.rs +++ b/runtime/common/src/migrations.rs @@ -18,7 +18,7 @@ use frame_support::{pallet_prelude::Get, traits::OnRuntimeUpgrade, weights::Weight}; use pallet_author_mapping::{migrations::TwoXToBlake, Config as AuthorMappingConfig}; -use pallet_migrations::{Config as MigrationsConfig, Migration}; +use pallet_migrations::Migration; use sp_std::{marker::PhantomData, prelude::*}; /// This module acts as a registry where each migration is defined. Each migration should implement From 95d69c50ff9fe0f635083d59e439e6986b5ab6d5 Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Fri, 1 Oct 2021 15:15:23 -0400 Subject: [PATCH 98/99] line length --- pallets/migrations/src/lib.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pallets/migrations/src/lib.rs b/pallets/migrations/src/lib.rs index 2dcdacbd48..56486ef915 100644 --- a/pallets/migrations/src/lib.rs +++ b/pallets/migrations/src/lib.rs @@ -129,7 +129,10 @@ pub mod pallet { if migration_done { continue; } - log::debug!( target: "pallet-migrations", "invoking pre_upgrade() on migration {}", migration_name); + log::debug!( + target: "pallet-migrations", + "invoking pre_upgrade() on migration {}", migration_name + ); // dump the migration name to temp storage so post_upgrade will know which // migrations were performed (as opposed to skipped) @@ -172,7 +175,10 @@ pub mod pallet { None => continue, } - log::debug!( target: "pallet-migrations", "invoking post_upgrade() on migration {}", migration_name); + log::debug!( + target: "pallet-migrations", + "invoking post_upgrade() on migration {}", migration_name + ); let result = migration.post_upgrade(); match result { From fe808355f1f9080867ca1418eff2ae509d6ac0ec Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Sat, 2 Oct 2021 13:31:34 -0400 Subject: [PATCH 99/99] std feature --- runtime/common/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/common/Cargo.toml b/runtime/common/Cargo.toml index d4600f82c7..7a51bdfd46 100644 --- a/runtime/common/Cargo.toml +++ b/runtime/common/Cargo.toml @@ -21,4 +21,5 @@ std = [ "sp-runtime/std", "sp-std/std", "frame-support/std", + "pallet-author-mapping/std", ]