Off-chain migration to genesis Script

cumulus/scripts/migrate_storage_to_genesis/README.md

@@ -0,0 +1,13 @@
+# Migrate storage to genesis state
+Script to query storage under a particular key and add it to a raw chain_spec json file as genesis state.
+
+# How to use
+The storage to be quired can be selected by either providing a pallet name or a key in hex format.
+```
+yarn migrate -c <wss_chain_enpoint> -f <chain_spec_raw_to_edit.json> -p <pallet_name> -k <storage_key> -m <js_migration_function_file>
+```
+
+Example for migrating `Identity` pallet from Polkadot:
+```
+yarn migrate -c wss://rpc.polkadot.io -f new_genesis_raw.json -p Identity -m ./identityMigration.js
+```

cumulus/scripts/migrate_storage_to_genesis/identityMigration.js

@@ -0,0 +1,108 @@
+const { u8aToHex, hexToString } = require('@polkadot/util');
+const { xxhashAsHex } = require('@polkadot/util-crypto');
+
+const migrateIdentity = async (key, data, api) => {
+	// Register the new `Registration` format type
+	api.registry.register({
+		IdentityInfoNew: {
+		_fallback: 'IdentityInfoTo198',
+		display: 'Data',
+		legal: 'Data',
+		web: 'Data',
+		matrix: 'Data',
+		email: 'Data',
+		pgpFingerprint: 'Option<H160>',
+		image: 'Data',
+		twitter: 'Data',
+		github: 'Data',
+		discord: 'Data',
+		},
+		RegistrationNew: {
+		_fallback: 'RegistrationTo198',
+		judgements: 'Vec<RegistrationJudgement>',
+		deposit: 'Balance',
+		info: 'IdentityInfoNew'
+		},
+		Username: 'Vec<u8>',
+	});
+
+	// We want to migrate `IdentityOf` storage item
+	let IdentityOfkeyToMigrate = "IdentityOf";
+	let IdentityOfHexkeyToMigrate = xxhashAsHex(IdentityOfkeyToMigrate, 128);
+
+	// We want to migrate `SubsOf` storage item
+	let SubsOfkeyToMigrate = "SubsOf";
+	let SubsOfHexkeyToMigrate = xxhashAsHex(SubsOfkeyToMigrate, 128);
+
+	// We take the second half of the key, which is the storage item identifier
+	let storageItem = u8aToHex(key.toU8a().slice(18, 34));
+
+	// Migrate `IdentityOf` data to its new format
+	if (IdentityOfHexkeyToMigrate === storageItem) {
+		let decoded = api.createType('Registration', data.toU8a(true));
+
+		// Default value for `discord` and `github` fields.
+		let discord = { none: null };
+		let github = { none: null };
+
+		let decodedJson = decoded.toJSON();
+
+		// Look for `Discord` and `Github` keys in `additional` field
+		decodedJson.info.additional.forEach(([key, data]) => {
+			let keyString = hexToString(key.raw)
+
+			if (keyString.toLowerCase() === "discord") {
+				discord = { raw: data.raw };
+			} else if (keyString.toLowerCase() === "github") {
+				github = { raw: data.raw };
+			}
+		});
+
+		// Migrate `IdentityInfo` data to the new format:
+		// - remove `additional` field
+		// - add `discord` field
+		// - add `github` field
+		// - set `deposit` to 0
+		let decodedNew = api.createType(
+			'(RegistrationNew, Option<Username>)',
+			[
+				{
+					judgements: decodedJson.judgements,
+					deposit: 0,
+					info: {
+						display: decodedJson.info.display,
+						legal: decodedJson.info.legal,
+						web: decodedJson.info.web,
+						matrix: decodedJson.info.riot,
+						email: decodedJson.info.email,
+						pgpFingerprint: decodedJson.info.pgpFingerprint,
+						image: decodedJson.info.image,
+						twitter: decodedJson.info.twitter,
+						github: github,
+						discord: discord,
+					}
+				},
+				null
+			]
+
+		);
+
+		data = decodedNew.toHex();
+
+	} else if (SubsOfHexkeyToMigrate === storageItem) {
+		let decoded = api.createType('(Balance, BoundedVec<AccountId, MaxApprovals>)', data.toU8a(true));
+
+		// Migrate `SubsOf` data:
+		// - set Deposit to 0
+		let decodedNew = api.createType(
+			'(Balance, BoundedVec<AccountId, MaxApprovals>)',
+			[0, decoded.toJSON()[1]]
+		);
+
+		data = decodedNew.toHex();
+	}
+
+	return data;
+};
+
+module.exports = migrateIdentity;

cumulus/scripts/migrate_storage_to_genesis/index.js

@@ -0,0 +1,137 @@
+const fs = require('fs-extra');
+const yargs = require('yargs');
+const { resolve } = require('path');
+const { ApiPromise, WsProvider } = require('@polkadot/api');
+const { xxhashAsHex } = require('@polkadot/util-crypto');
+
+
+// Open chain_spec raw file
+const openFile = (filePath) => {
+	try {
+		const jsonData = fs.readJSONSync(filePath);
+		return jsonData;
+	} catch(e) {
+		console.log(`${filePath} file could not be opened. Make sure it exists`);
+		process.exit(1);
+	}
+}
+
+// Connect to chain
+const connect = async (endpoint) => {
+	console.log(`Conecting to ${endpoint}...`);
+	const wsProvider = new WsProvider(endpoint);
+	return await ApiPromise.create({ provider: wsProvider });
+}
+
+// Migrate data using the migration function
+const migrateData = async (path, key, data, api) => {
+	let absolutePath = resolve('.', path);
+	let migrationFunction;
+
+	try {
+		migrationFunction = await import(absolutePath);
+	} catch (e) {
+		console.log(`No data migration file can be found in ${path}`, e);
+		process.exit(1);
+	}
+
+	if (typeof migrationFunction.default === 'function') {
+		let migratedValue = await migrationFunction.default(key, data, api);
+		return migratedValue;
+	} else {
+		console.log(
+			`Data migration function must be default exported from the file ${path}`
+		);
+		process.exit(1);
+	}
+}
+
+// Query all storage under a certain key
+const queryStorage = async (api, argv) => {
+	let rootKey = argv.key ? argv.key : xxhashAsHex(argv.pallet);
+
+	let storageKeyValues = new Object();
+
+	let keys = await api.rpc.state.getKeys(rootKey);
+
+	console.log("Querying pallet state...")
+
+	for (let key of keys) {
+		let data = await api.rpc.state.getStorage(key);
+		data = await migrateData(argv.migration, key, data, api)
+		storageKeyValues[key] = data
+	}
+
+	return storageKeyValues;
+}
+
+// Edit chain_spec raw adding the queried storage
+const editChainSpec = (filePath, jsonData, storageKeyValues) => {
+	console.log("Editing json chain_spec...")
+
+	if (jsonData.genesis?.raw?.top) {
+		for (const key in storageKeyValues) {
+			jsonData.genesis.raw.top[key] = storageKeyValues[key];
+		}
+	} else {
+		console.log(`${filePath} - invalid raw chain_spec format json file`);
+		process.exit(1);
+	}
+
+	fs.writeJSONSync(filePath, jsonData, { spaces: 2 });
+}
+
+// Run the script
+const run = async () => {
+	const argv = yargs
+	.option('chain', {
+		alias: 'c',
+		describe: 'Endpoint to the chain to query sorage',
+		demandOption: true, // Requires the --chain option
+		type: 'string',
+	})
+	.option('file', {
+		alias: 'f',
+		describe: 'Path to the JSON file',
+		demandOption: true, // Requires the --file option
+		type: 'string',
+	})
+	.option('pallet', {
+		alias: 'p',
+		describe: 'Pallet name',
+		type: 'string',
+	})
+	.option('key', {
+		alias: 'k',
+		describe: 'Storage key',
+		type: 'string',
+	})
+	.option('migration', {
+		alias: 'm',
+		describe: 'Path to the migration file function',
+		demandOption: true, // Requires the --migration option
+		type: 'string',
+	})
+	.check((args) => {
+		if (!args.pallet && !args.key) {
+			throw new Error('Please provide either --pallet or --key option.');
+		}
+		return true;
+	})
+	.help()
+	.alias('help', 'h').argv;
+
+	const filePath = argv.file;
+
+	let jsonData = openFile(filePath);
+
+	let api = await connect(argv.chain);
+
+	let storageKeyValues = await queryStorage(api, argv);
+
+	editChainSpec(filePath, jsonData, storageKeyValues)
+
+	process.exit(0);
+}
+
+run()

cumulus/scripts/migrate_storage_to_genesis/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "migrate_storage_to_genesis",
+  "version": "1.0.0",
+  "description": "query storage from a pallet and add it to a new chain_spec_raw.json",
+  "main": "index.js",
+  "scripts": {
+    "migrate": "node index.js"
+  },
+  "author": "Parity Technologies <admin@parity.io>",
+  "license": "ISC",
+  "dependencies": {
+    "@polkadot/api": "^10.9.1",
+    "@polkadot/util-crypto": "^12.5.1",
+    "fs-extra": "^11.1.1",
+    "yargs": "^17.7.2"
+  }
+}