Generate People chain specs with Identity off-chain migration

cumulus/scripts/create_people_rococo_spec.sh

@@ -41,24 +41,18 @@ cat chain-spec-plain.json | jq --rawfile code rt-hex.txt '.genesis.runtimeGenesi
     | jq '.id = "people-rococo"' \
     | jq '.chainType = "Live"' \
     | jq '.bootNodes = [
-		"/dns/rococo-people-collator-node-0.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWDZg5jMYhKXTu6RU491V5sxsFnP4oaEmZJEUfcRkYzps5",
-		"/dns/rococo-people-collator-node-0.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWDZg5jMYhKXTu6RU491V5sxsFnP4oaEmZJEUfcRkYzps5",
-		"/dns/rococo-people-collator-node-1.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWGGR5i6qQqfo7iDNp7vjDRKPWuDk53idGV6nFLwS12X5H",
-		"/dns/rococo-people-collator-node-1.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWGGR5i6qQqfo7iDNp7vjDRKPWuDk53idGV6nFLwS12X5H",
-		"/dns/rococo-people-collator-node-2.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWBvA9BmBfrsVMcAcqVXGYFCpMTvkSk2igNXpmoareYbeT",
-		"/dns/rococo-people-collator-node-2.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWBvA9BmBfrsVMcAcqVXGYFCpMTvkSk2igNXpmoareYbeT",
-		"/dns/rococo-people-collator-node-3.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWQ7Q9jLcJTPXy7KEp5hSZ8YMY9pHx9CnQVz3T8TKQ81UG",
-		"/dns/rococo-people-collator-node-3.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWQ7Q9jLcJTPXy7KEp5hSZ8YMY9pHx9CnQVz3T8TKQ81UG"
+        "/dns/rococo-people-collator-node-0.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWDZg5jMYhKXTu6RU491V5sxsFnP4oaEmZJEUfcRkYzps5",
+        "/dns/rococo-people-collator-node-0.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWDZg5jMYhKXTu6RU491V5sxsFnP4oaEmZJEUfcRkYzps5",
+        "/dns/rococo-people-collator-node-1.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWGGR5i6qQqfo7iDNp7vjDRKPWuDk53idGV6nFLwS12X5H",
+        "/dns/rococo-people-collator-node-1.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWGGR5i6qQqfo7iDNp7vjDRKPWuDk53idGV6nFLwS12X5H"
 	]' \
     | jq '.relay_chain = "rococo"' \
     | jq --argjson para_id $para_id '.para_id = $para_id' \
     | jq --argjson para_id $para_id '.genesis.runtimeGenesis.patch.parachainInfo.parachainId = $para_id' \
     | jq '.genesis.runtimeGenesis.patch.balances.balances = []' \
     | jq '.genesis.runtimeGenesis.patch.collatorSelection.invulnerables = [
 		"5Gnjmw1iuF2kV4PecFgetJed7B8quBKfLiRM99ELcXvFH9Vn",
-		"5FLZRxyeRPhG69zo4ZPqCJSYboSKaRBUjBvQc1nkuWoBpZ5P",
-		"5DNnmPH2MT6SXpfqbJZbTz4eERmuZegssfxc4ysL8PWrHaNN",
-		"5DkKcSP5MboNMpXScW1CyRqaktKMXH8QLP4Mn49TwS5vhL6k"
+		"5FLZRxyeRPhG69zo4ZPqCJSYboSKaRBUjBvQc1nkuWoBpZ5P"
 	]' \
     | jq '.genesis.runtimeGenesis.patch.session.keys = [
             [
@@ -74,20 +68,6 @@ cat chain-spec-plain.json | jq --rawfile code rt-hex.txt '.genesis.runtimeGenesi
                     {
                         "aura": "5FLZRxyeRPhG69zo4ZPqCJSYboSKaRBUjBvQc1nkuWoBpZ5P"
                     }
-            ],
-            [
-                "5DNnmPH2MT6SXpfqbJZbTz4eERmuZegssfxc4ysL8PWrHaNN",
-                "5DNnmPH2MT6SXpfqbJZbTz4eERmuZegssfxc4ysL8PWrHaNN",
-                    {
-                        "aura": "5DNnmPH2MT6SXpfqbJZbTz4eERmuZegssfxc4ysL8PWrHaNN"
-                    }
-            ],
-            [
-                "5DkKcSP5MboNMpXScW1CyRqaktKMXH8QLP4Mn49TwS5vhL6k",
-                "5DkKcSP5MboNMpXScW1CyRqaktKMXH8QLP4Mn49TwS5vhL6k",
-                    {
-                        "aura": "5DkKcSP5MboNMpXScW1CyRqaktKMXH8QLP4Mn49TwS5vhL6k"
-                    }
             ]
         ]' \
     > edited-chain-spec-plain.json

cumulus/scripts/create_people_westend_spec.sh

@@ -44,21 +44,15 @@ cat chain-spec-plain.json | jq --rawfile code rt-hex.txt '.genesis.runtimeGenesi
         "/dns/westend-people-collator-node-0.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWDcLjDLTu9fNhmas9DTWtqdv8eUbFMWQzVwvXRK7QcjHD",
         "/dns/westend-people-collator-node-0.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWDcLjDLTu9fNhmas9DTWtqdv8eUbFMWQzVwvXRK7QcjHD",
         "/dns/westend-people-collator-node-1.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWM56JbKWAXsDyWh313z73aKYVMp1Hj2nSnAKY3q6MnoC9",
-        "/dns/westend-people-collator-node-1.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWM56JbKWAXsDyWh313z73aKYVMp1Hj2nSnAKY3q6MnoC9",
-        "/dns/westend-people-collator-node-2.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWGVYTVKW7tYe51JvetvGvVLDPXzqQX1mueJgz14FgkmHG",
-        "/dns/westend-people-collator-node-2.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWGVYTVKW7tYe51JvetvGvVLDPXzqQX1mueJgz14FgkmHG",
-        "/dns/westend-people-collator-node-3.parity-testnet.parity.io/tcp/30333/p2p/12D3KooWCF1eA2Gap69zgXD7Df3e9DqDUsGoByocggTGejoHjK23",
-        "/dns/westend-people-collator-node-3.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWCF1eA2Gap69zgXD7Df3e9DqDUsGoByocggTGejoHjK23"
+        "/dns/westend-people-collator-node-1.parity-testnet.parity.io/tcp/443/wss/p2p/12D3KooWM56JbKWAXsDyWh313z73aKYVMp1Hj2nSnAKY3q6MnoC9"
     ]' \
     | jq '.relay_chain = "westend"' \
     | jq --argjson para_id $para_id '.para_id = $para_id' \
     | jq --argjson para_id $para_id '.genesis.runtimeGenesis.patch.parachainInfo.parachainId = $para_id' \
     | jq '.genesis.runtimeGenesis.patch.balances.balances = []' \
     | jq '.genesis.runtimeGenesis.patch.collatorSelection.invulnerables = [
         "5CFYvshLff1dHmT33jUcBc7mEKbVRJKbA9HzPqmLfjksHah6",
-        "5HgEdsYyVGVsyNmbE1sUxeDLrxTLJXnAKCNa2HJ9QXXEir1B",
-        "5EZmD6eA9wm1Y2Dy2wefLCsFJJcC7o8bVfWm7Mfbuanc8JYo",
-        "5EkJFfUtbo258dCaqgYSvajN1tNtXhT3SrybW8ZhygoMP3kE"
+        "5HgEdsYyVGVsyNmbE1sUxeDLrxTLJXnAKCNa2HJ9QXXEir1B"
     ]' \
     | jq '.genesis.runtimeGenesis.patch.session.keys = [
             [
@@ -74,22 +68,9 @@ cat chain-spec-plain.json | jq --rawfile code rt-hex.txt '.genesis.runtimeGenesi
                     {
                         "aura": "5HgEdsYyVGVsyNmbE1sUxeDLrxTLJXnAKCNa2HJ9QXXEir1B"
                     }
-            ],
-            [
-                "5EZmD6eA9wm1Y2Dy2wefLCsFJJcC7o8bVfWm7Mfbuanc8JYo",
-                "5EZmD6eA9wm1Y2Dy2wefLCsFJJcC7o8bVfWm7Mfbuanc8JYo",
-                    {
-                        "aura": "5EZmD6eA9wm1Y2Dy2wefLCsFJJcC7o8bVfWm7Mfbuanc8JYo"
-                    }
-            ],
-            [
-                "5EkJFfUtbo258dCaqgYSvajN1tNtXhT3SrybW8ZhygoMP3kE",
-                "5EkJFfUtbo258dCaqgYSvajN1tNtXhT3SrybW8ZhygoMP3kE",
-                    {
-                        "aura": "5EkJFfUtbo258dCaqgYSvajN1tNtXhT3SrybW8ZhygoMP3kE"
-                    }
             ]
         ]' \
+    | jq '.genesis.runtimeGenesis.patch.polkadotXcm.safeXcmVersion = 3' \
     > edited-chain-spec-plain.json
 
 # build a raw spec

cumulus/scripts/migrate_storage_to_genesis/README.md

@@ -0,0 +1,13 @@
+# Migrate storage to genesis state
+Script to query storage under a particular key and add it to a raw chain_spec json file as genesis state.
+
+# How to use
+The storage to be quired can be selected by either providing a pallet name or a key in hex format.
+```
+yarn migrate -c <wss_chain_enpoint> -f <chain_spec_raw_to_edit.json> -p <pallet_name> -k <storage_key> -m <js_migration_function_file>
+```
+
+Example for migrating `Identity` pallet from Polkadot:
+```
+yarn migrate -c wss://rpc.polkadot.io -f new_genesis_raw.json -p Identity -m ./identityMigration.js
+```

cumulus/scripts/migrate_storage_to_genesis/identityMigration.js

@@ -0,0 +1,108 @@
+const { u8aToHex, hexToString } = require('@polkadot/util');
+const { xxhashAsHex } = require('@polkadot/util-crypto');
+
+const migrateIdentity = async (key, data, api) => {
+	// Register the new `Registration` format type
+	api.registry.register({
+		IdentityInfoNew: {
+		_fallback: 'IdentityInfoTo198',
+		display: 'Data',
+		legal: 'Data',
+		web: 'Data',
+		matrix: 'Data',
+		email: 'Data',
+		pgpFingerprint: 'Option<H160>',
+		image: 'Data',
+		twitter: 'Data',
+		github: 'Data',
+		discord: 'Data',
+		},
+		RegistrationNew: {
+		_fallback: 'RegistrationTo198',
+		judgements: 'Vec<RegistrationJudgement>',
+		deposit: 'Balance',
+		info: 'IdentityInfoNew'
+		},
+		Username: 'Vec<u8>',
+	});
+
+	// We want to migrate `IdentityOf` storage item
+	let IdentityOfkeyToMigrate = "IdentityOf";
+	let IdentityOfHexkeyToMigrate = xxhashAsHex(IdentityOfkeyToMigrate, 128);
+
+	// We want to migrate `SubsOf` storage item
+	let SubsOfkeyToMigrate = "SubsOf";
+	let SubsOfHexkeyToMigrate = xxhashAsHex(SubsOfkeyToMigrate, 128);
+
+	// We take the second half of the key, which is the storage item identifier
+	let storageItem = u8aToHex(key.toU8a().slice(18, 34));
+
+	// Migrate `IdentityOf` data to its new format
+	if (IdentityOfHexkeyToMigrate === storageItem) {
+		let decoded = api.createType('(Registration, Option<Username>)', data.toU8a(true));
+
+		// Default value for `discord` and `github` fields.
+		let discord = { none: null };
+		let github = { none: null };
+
+		// Get the `Registration` part from the `IdentityInfo` tuple
+		let decodedJson = decoded.toJSON()[0];
+
+		// Look for `Discord` and `Github` keys in `additional` field
+		decodedJson.info.additional.forEach(([key, data]) => {
+			let keyString = hexToString(key.raw)
+
+			if (keyString.toLowerCase() === "discord") {
+				discord = { raw: data.raw };
+			} else if (keyString.toLowerCase() === "github") {
+				github = { raw: data.raw };
+			}
+		});
+
+		// Migrate `IdentityInfo` data to the new format:
+		// - remove `additional` field
+		// - add `discord` field
+		// - add `github` field
+		// - set `deposit` to 0
+		let decodedNew = api.createType(
+			'(RegistrationNew, Option<Username>)',
+			[
+				{
+					judgements: decodedJson.judgements,
+					deposit: 0,
+					info: {
+						display: decodedJson.info.display,
+						legal: decodedJson.info.legal,
+						web: decodedJson.info.web,
+						matrix: decodedJson.info.riot,
+						email: decodedJson.info.email,
+						pgpFingerprint: decodedJson.info.pgpFingerprint,
+						image: decodedJson.info.image,
+						twitter: decodedJson.info.twitter,
+						github: github,
+						discord: discord,
+					}
+				},
+				null
+			]
+		);
+
+		data = decodedNew.toHex();
+
+	} else if (SubsOfHexkeyToMigrate === storageItem) {
+		let decoded = api.createType('(Balance, BoundedVec<AccountId, MaxApprovals>)', data.toU8a(true));
+
+		// Migrate `SubsOf` data:
+		// - set Deposit to 0
+		let decodedNew = api.createType(
+			'(Balance, BoundedVec<AccountId, MaxApprovals>)',
+			[0, decoded.toJSON()[1]]
+		);
+
+		data = decodedNew.toHex();
+	}
+
+	return data;
+};
+
+module.exports = migrateIdentity;

cumulus/scripts/migrate_storage_to_genesis/index.js

@@ -0,0 +1,137 @@
+const fs = require('fs-extra');
+const yargs = require('yargs');
+const { resolve } = require('path');
+const { ApiPromise, WsProvider } = require('@polkadot/api');
+const { xxhashAsHex } = require('@polkadot/util-crypto');
+
+
+// Open chain_spec raw file
+const openFile = (filePath) => {
+	try {
+		const jsonData = fs.readJSONSync(filePath);
+		return jsonData;
+	} catch(e) {
+		console.log(`${filePath} file could not be opened. Make sure it exists`);
+		process.exit(1);
+	}
+}
+
+// Connect to chain
+const connect = async (endpoint) => {
+	console.log(`Conecting to ${endpoint}...`);
+	const wsProvider = new WsProvider(endpoint);
+	return await ApiPromise.create({ provider: wsProvider });
+}
+
+// Migrate data using the migration function
+const migrateData = async (path, key, data, api) => {
+	let absolutePath = resolve('.', path);
+	let migrationFunction;
+
+	try {
+		migrationFunction = await import(absolutePath);
+	} catch (e) {
+		console.log(`No data migration file can be found in ${path}`, e);
+		process.exit(1);
+	}
+
+	if (typeof migrationFunction.default === 'function') {
+		let migratedValue = await migrationFunction.default(key, data, api);
+		return migratedValue;
+	} else {
+		console.log(
+			`Data migration function must be default exported from the file ${path}`
+		);
+		process.exit(1);
+	}
+}
+
+// Query all storage under a certain key
+const queryStorage = async (api, argv) => {
+	let rootKey = argv.key ? argv.key : xxhashAsHex(argv.pallet);
+
+	let storageKeyValues = new Object();
+
+	let keys = await api.rpc.state.getKeys(rootKey);
+
+	console.log("Querying pallet state...")
+
+	for (let key of keys) {
+		let data = await api.rpc.state.getStorage(key);
+		data = await migrateData(argv.migration, key, data, api)
+		storageKeyValues[key] = data
+	}
+
+	return storageKeyValues;
+}
+
+// Edit chain_spec raw adding the queried storage
+const editChainSpec = (filePath, jsonData, storageKeyValues) => {
+	console.log("Editing json chain_spec...")
+
+	if (jsonData.genesis?.raw?.top) {
+		for (const key in storageKeyValues) {
+			jsonData.genesis.raw.top[key] = storageKeyValues[key];
+		}
+	} else {
+		console.log(`${filePath} - invalid raw chain_spec format json file`);
+		process.exit(1);
+	}
+
+	fs.writeJSONSync(filePath, jsonData, { spaces: 2 });
+}
+
+// Run the script
+const run = async () => {
+	const argv = yargs
+	.option('chain', {
+		alias: 'c',
+		describe: 'Endpoint to the chain to query sorage',
+		demandOption: true, // Requires the --chain option
+		type: 'string',
+	})
+	.option('file', {
+		alias: 'f',
+		describe: 'Path to the JSON file',
+		demandOption: true, // Requires the --file option
+		type: 'string',
+	})
+	.option('pallet', {
+		alias: 'p',
+		describe: 'Pallet name',
+		type: 'string',
+	})
+	.option('key', {
+		alias: 'k',
+		describe: 'Storage key',
+		type: 'string',
+	})
+	.option('migration', {
+		alias: 'm',
+		describe: 'Path to the migration file function',
+		demandOption: true, // Requires the --migration option
+		type: 'string',
+	})
+	.check((args) => {
+		if (!args.pallet && !args.key) {
+			throw new Error('Please provide either --pallet or --key option.');
+		}
+		return true;
+	})
+	.help()
+	.alias('help', 'h').argv;
+
+	const filePath = argv.file;
+
+	let jsonData = openFile(filePath);
+
+	let api = await connect(argv.chain);
+
+	let storageKeyValues = await queryStorage(api, argv);
+
+	editChainSpec(filePath, jsonData, storageKeyValues)
+
+	process.exit(0);
+}
+
+run()

cumulus/scripts/migrate_storage_to_genesis/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "migrate_storage_to_genesis",
+  "version": "1.0.0",
+  "description": "query storage from a pallet and add it to a new chain_spec_raw.json",
+  "main": "index.js",
+  "scripts": {
+    "migrate": "node index.js"
+  },
+  "author": "Parity Technologies <admin@parity.io>",
+  "license": "ISC",
+  "dependencies": {
+    "@polkadot/api": "^10.9.1",
+    "@polkadot/util-crypto": "^12.5.1",
+    "fs-extra": "^11.1.1",
+    "yargs": "^17.7.2"
+  }
+}