diff --git a/javascript/packages/orchestrator/src/cmdGenerator.ts b/javascript/packages/orchestrator/src/cmdGenerator.ts index a9c6701b4..3d445a94f 100644 --- a/javascript/packages/orchestrator/src/cmdGenerator.ts +++ b/javascript/packages/orchestrator/src/cmdGenerator.ts @@ -6,7 +6,7 @@ import { RPC_HTTP_PORT, RPC_WS_PORT, } from "./constants"; -import { Node, ZombieRole } from "./types"; +import { Node, SubstrateCliArgsVersion, ZombieRole } from "./types"; const debug = require("debug")("zombie::cmdGenerator"); @@ -18,11 +18,6 @@ interface PortsInterface { [key: string]: number; } -// Commented out as "Never used" -// interface ParachainCollatorsInterface { -// [key: number]: number; -// } - function parseCmdWithArguments( commandWithArgs: string, useWrapper = true, @@ -69,6 +64,7 @@ export async function genCumulusCollatorCmd( "--base-path": true, "--port": true, "--ws-port": true, + "--rpc-port": true, "--chain": true, "--prometheus-port": true, }; @@ -85,22 +81,20 @@ export async function genCumulusCollatorCmd( dataPath, "--listen-addr", `/ip4/0.0.0.0/tcp/${nodeSetup.p2pPort ? nodeSetup.p2pPort : P2P_PORT}/ws`, - "--rpc-port", - (nodeSetup.rpcPort ? nodeSetup.rpcPort : RPC_HTTP_PORT).toString(), - "--ws-port", - (nodeSetup.wsPort ? nodeSetup.wsPort : RPC_WS_PORT).toString(), "--prometheus-external", - "--prometheus-port", - (nodeSetup.prometheusPort - ? nodeSetup.prometheusPort - : PROMETHEUS_PORT - ).toString(), "--rpc-cors all", "--unsafe-rpc-external", "--rpc-methods unsafe", - "--unsafe-ws-external", ]; + if (nodeSetup.substrateCliArgsVersion === SubstrateCliArgsVersion.V0) + fullCmd.push("--unsafe-ws-external"); + const portFlags = getPortFlagsByCliArgsVersion(nodeSetup); + + for (const [k, v] of Object.entries(portFlags)) { + fullCmd.push(...[k, v.toString()]); + } + const chainParts = chain.split("_"); const relayChain = chainParts.length > 1 ? chainParts[chainParts.length - 1] : chainParts[0]; @@ -109,7 +103,6 @@ export async function genCumulusCollatorCmd( const collatorPorts: PortsInterface = { "--port": 0, - "--ws-port": 0, "--rpc-port": 0, }; @@ -284,12 +277,7 @@ export async function genCmd( if (bootnodes && bootnodes.length) args.push("--bootnodes", bootnodes.join(" ")); - // port flags logic - const portFlags = { - "--prometheus-port": nodeSetup.prometheusPort, - "--rpc-port": nodeSetup.rpcPort, - "--ws-port": nodeSetup.wsPort, - }; + const portFlags = getPortFlagsByCliArgsVersion(nodeSetup); for (const [k, v] of Object.entries(portFlags)) { args.push(...[k, v.toString()]); @@ -311,6 +299,9 @@ export async function genCmd( if (basePathFlagIndex >= 0) args.splice(basePathFlagIndex, 2); args.push(...["--base-path", dataPath]); + if (nodeSetup.substrateCliArgsVersion === SubstrateCliArgsVersion.V0) + args.push("--unsafe-ws-external"); + const finalArgs: string[] = [ command, "--chain", @@ -322,7 +313,6 @@ export async function genCmd( "--unsafe-rpc-external", "--rpc-methods", "unsafe", - "--unsafe-ws-external", ...args, ]; @@ -331,16 +321,24 @@ export async function genCmd( return resolvedCmd; } -// Commented out as "Never used" -// helper -// const parachainCollators: ParachainCollatorsInterface = -// {} as ParachainCollatorsInterface; +const getPortFlagsByCliArgsVersion = (nodeSetup: Node) => { + // port flags logic + const portFlags: { [key: string]: string } = { + "--prometheus-port": ( + nodeSetup.prometheusPort || PROMETHEUS_PORT + ).toString(), + }; -// Commented out as "Never used" -// function getCollatorIndex(paraId: number): number { -// if (parachainCollators[paraId] >= 0) -// parachainCollators[paraId] = parachainCollators[paraId] + 1; -// else parachainCollators[paraId] = 0; + if (nodeSetup.substrateCliArgsVersion === SubstrateCliArgsVersion.V0) { + portFlags["--rpc-port"] = (nodeSetup.rpcPort || RPC_HTTP_PORT).toString(); + portFlags["--ws-port"] = (nodeSetup.wsPort || RPC_WS_PORT).toString(); + } else { + // use ws port as default + const portToUse = nodeSetup.wsPort + ? nodeSetup.wsPort + : nodeSetup.rpcPort || RPC_HTTP_PORT; + portFlags["--rpc-port"] = portToUse.toString(); + } -// return parachainCollators[paraId]; -// } + return portFlags; +}; diff --git a/javascript/packages/orchestrator/src/orchestrator.ts b/javascript/packages/orchestrator/src/orchestrator.ts index d8a8ed1c9..da39ef187 100644 --- a/javascript/packages/orchestrator/src/orchestrator.ts +++ b/javascript/packages/orchestrator/src/orchestrator.ts @@ -100,9 +100,12 @@ export async function start( debug(JSON.stringify(networkSpec, null, 4)); - const { initClient, setupChainSpec, getChainSpecRaw } = getProvider( - networkSpec.settings.provider, - ); + const { + initClient, + setupChainSpec, + getChainSpecRaw, + setSubstrateCliArdsVersion, + } = getProvider(networkSpec.settings.provider); // global timeout to spin the network const timeoutTimer = setTimeout(() => { @@ -222,6 +225,10 @@ export async function start( await client.staticSetup(networkSpec.settings); await client.createPodMonitor("pod-monitor.yaml", chainName); + // Set substrate client argument version, needed from breaking change. + // see https://github.com/paritytech/substrate/pull/13384 + await setSubstrateCliArdsVersion(networkSpec); + // create or copy relay chain spec await setupChainSpec( namespace, diff --git a/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts b/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts index 21734d13d..1cfb746fd 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts @@ -46,6 +46,7 @@ export async function createTempNodeDef( image: string, chain: string, fullCommand: string, + useCommandSuffix = true, ) { const nodeName = getUniqueName("temp"); const node: Node = { @@ -53,7 +54,8 @@ export async function createTempNodeDef( key: getSha256(nodeName), image, fullCommand: - fullCommand + " && " + TMP_DONE + " && " + WAIT_UNTIL_SCRIPT_SUFIX, // leave the pod runnig until we finish transfer files + fullCommand + + (useCommandSuffix ? ` && ${TMP_DONE} && ${WAIT_UNTIL_SCRIPT_SUFIX}` : ""), // leave the pod runnig until we finish transfer files chain, validator: false, invulnerable: false, diff --git a/javascript/packages/orchestrator/src/providers/k8s/index.ts b/javascript/packages/orchestrator/src/providers/k8s/index.ts index 666583e08..9fabb98e2 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/index.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/index.ts @@ -5,6 +5,7 @@ import { replaceNetworkRef, } from "./dynResourceDefinition"; import { KubeClient, initClient } from "./kubeClient"; +import { setSubstrateCliArdsVersion } from "./substrateCliArgsHelper"; export const provider = { KubeClient, @@ -14,4 +15,5 @@ export const provider = { setupChainSpec, getChainSpecRaw, replaceNetworkRef, + setSubstrateCliArdsVersion, }; diff --git a/javascript/packages/orchestrator/src/providers/k8s/substrateCliArgsHelper.ts b/javascript/packages/orchestrator/src/providers/k8s/substrateCliArgsHelper.ts new file mode 100644 index 000000000..e6e13e6b1 --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/k8s/substrateCliArgsHelper.ts @@ -0,0 +1,82 @@ +import { series } from "@zombienet/utils"; +import { ComputedNetwork, SubstrateCliArgsVersion } from "../../types"; +import { getClient } from "../client"; +import { createTempNodeDef, genNodeDef } from "./dynResourceDefinition"; +import { KubeClient } from "./kubeClient"; + +const getVersion = async ( + image: string, + command: string, +): Promise => { + const client = getClient() as KubeClient; + const fullCmd = `${command} --help | grep ws-port`; + const node = await createTempNodeDef( + "temp", + image, + "", // don't used + fullCmd, + false, + ); + + const podDef = await genNodeDef(client.namespace, node); + const podName = podDef.metadata.name; + await client.spawnFromDef(podDef); + const logs = await client.getNodeLogs(podName); + + return logs.includes("--ws-port ") + ? SubstrateCliArgsVersion.V0 + : SubstrateCliArgsVersion.V1; +}; + +export const setSubstrateCliArdsVersion = async (network: ComputedNetwork) => { + // Calculate substrate cli version for each node + // and set in the node to use later when we build the cmd. + const imgCmdMap = new Map(); + network.relaychain.nodes.reduce((memo, node) => { + const uniq_image_cmd = `${node.image}_${node.command}`; + if (!memo.has(uniq_image_cmd)) + memo.set(uniq_image_cmd, { image: node.image, command: node.command }); + return memo; + }, imgCmdMap); + + network.parachains.reduce((memo, parachain) => { + for (const collator of parachain.collators) { + const uniq_image_cmd = `${collator.image}_${collator.command}`; + if (!memo.has(uniq_image_cmd)) + memo.set(uniq_image_cmd, { + image: collator.image, + command: collator.command, + }); + } + return memo; + }, imgCmdMap); + + // check versions in series + const promiseGenerators = []; + for (const [, v] of imgCmdMap) { + const getVersionPromise = async () => { + const version = await getVersion(v.image, v.command); + v.version = version; + return version; + }; + promiseGenerators.push(getVersionPromise); + } + + await series(promiseGenerators, 4); + + // now we need to iterate and set in each node the version + // IFF is not set + for (const node of network.relaychain.nodes) { + if (node.substrateCliArgsVersion) continue; + const uniq_image_cmd = `${node.image}_${node.command}`; + node.substrateCliArgsVersion = imgCmdMap.get(uniq_image_cmd).version; + } + + for (const parachain of network.parachains) { + for (const collator of parachain.collators) { + if (collator.substrateCliArgsVersion) continue; + const uniq_image_cmd = `${collator.image}_${collator.command}`; + collator.substrateCliArgsVersion = imgCmdMap.get(uniq_image_cmd).version; + } + } +}; diff --git a/javascript/packages/orchestrator/src/providers/native/index.ts b/javascript/packages/orchestrator/src/providers/native/index.ts index 54599f54a..7bc312c47 100644 --- a/javascript/packages/orchestrator/src/providers/native/index.ts +++ b/javascript/packages/orchestrator/src/providers/native/index.ts @@ -5,6 +5,7 @@ import { replaceNetworkRef, } from "./dynResourceDefinition"; import { NativeClient, initClient } from "./nativeClient"; +import { setSubstrateCliArdsVersion } from "./substrateCliArgsHelper"; export const provider = { NativeClient, @@ -14,4 +15,5 @@ export const provider = { setupChainSpec, getChainSpecRaw, replaceNetworkRef, + setSubstrateCliArdsVersion, }; diff --git a/javascript/packages/orchestrator/src/providers/native/substrateCliArgsHelper.ts b/javascript/packages/orchestrator/src/providers/native/substrateCliArgsHelper.ts new file mode 100644 index 000000000..c62083711 --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/native/substrateCliArgsHelper.ts @@ -0,0 +1,71 @@ +import { series } from "@zombienet/utils"; +import { ComputedNetwork, SubstrateCliArgsVersion } from "../../types"; +import { getClient } from "../client"; + +const getVersion = async ( + image: string, + command: string, +): Promise => { + const client = getClient(); + const fullCmd = `${command} --help | grep ws-port`; + const logs = (await client.runCommand(["-c", fullCmd], { allowFail: true })) + .stdout; + + return logs.includes("--ws-port ") + ? SubstrateCliArgsVersion.V0 + : SubstrateCliArgsVersion.V1; +}; + +export const setSubstrateCliArdsVersion = async (network: ComputedNetwork) => { + // Calculate substrate cli version for each node + // and set in the node to use later when we build the cmd. + const imgCmdMap = new Map(); + network.relaychain.nodes.reduce((memo, node) => { + const uniq_image_cmd = `${node.image}_${node.command}`; + if (!memo.has(uniq_image_cmd)) + memo.set(uniq_image_cmd, { image: node.image, command: node.command }); + return memo; + }, imgCmdMap); + + network.parachains.reduce((memo, parachain) => { + for (const collator of parachain.collators) { + const uniq_image_cmd = `${collator.image}_${collator.command}`; + if (!memo.has(uniq_image_cmd)) + memo.set(uniq_image_cmd, { + image: collator.image, + command: collator.command, + }); + } + return memo; + }, imgCmdMap); + + // check versions in series + const promiseGenerators = []; + + for (const [, v] of imgCmdMap) { + const getVersionPromise = async () => { + const version = await getVersion(v.image, v.command); + v.version = version; + return version; + }; + promiseGenerators.push(getVersionPromise); + } + + await series(promiseGenerators, 4); + + // now we need to iterate and set in each node the version + // IFF is not set + for (const node of network.relaychain.nodes) { + if (node.substrateCliArgsVersion) continue; + const uniq_image_cmd = `${node.image}_${node.command}`; + node.substrateCliArgsVersion = imgCmdMap.get(uniq_image_cmd).version; + } + + for (const parachain of network.parachains) { + for (const collator of parachain.collators) { + if (collator.substrateCliArgsVersion) continue; + const uniq_image_cmd = `${collator.image}_${collator.command}`; + collator.substrateCliArgsVersion = imgCmdMap.get(uniq_image_cmd).version; + } + } +}; diff --git a/javascript/packages/orchestrator/src/providers/podman/index.ts b/javascript/packages/orchestrator/src/providers/podman/index.ts index 6cd38ba0f..d35b2c2b6 100644 --- a/javascript/packages/orchestrator/src/providers/podman/index.ts +++ b/javascript/packages/orchestrator/src/providers/podman/index.ts @@ -5,6 +5,7 @@ import { replaceNetworkRef, } from "./dynResourceDefinition"; import { PodmanClient, initClient } from "./podmanClient"; +import { setSubstrateCliArdsVersion } from "./substrateCliArgsHelper"; export const provider = { PodmanClient, @@ -14,4 +15,5 @@ export const provider = { setupChainSpec, getChainSpecRaw, replaceNetworkRef, + setSubstrateCliArdsVersion, }; diff --git a/javascript/packages/orchestrator/src/providers/podman/podmanClient.ts b/javascript/packages/orchestrator/src/providers/podman/podmanClient.ts index 6d11872e4..9633ab31d 100644 --- a/javascript/packages/orchestrator/src/providers/podman/podmanClient.ts +++ b/javascript/packages/orchestrator/src/providers/podman/podmanClient.ts @@ -346,8 +346,8 @@ export class PodmanClient extends Client { async spawnFromDef( podDef: any, filesToCopy: fileMap[] = [], - keystore: string, - chainSpecId: string, + keystore?: string, + chainSpecId?: string, dbSnapshot?: string, ): Promise { const name = podDef.metadata.name; @@ -383,7 +383,7 @@ export class PodmanClient extends Client { ]); } - if (keystore) { + if (keystore && chainSpecId) { const keystoreRemoteDir = `${dataPath.hostPath.path}/chains/${chainSpecId}/keystore`; await makeDir(keystoreRemoteDir, true); const keystoreIsEmpty = diff --git a/javascript/packages/orchestrator/src/providers/podman/substrateCliArgsHelper.ts b/javascript/packages/orchestrator/src/providers/podman/substrateCliArgsHelper.ts new file mode 100644 index 000000000..f685db947 --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/podman/substrateCliArgsHelper.ts @@ -0,0 +1,81 @@ +import { series } from "@zombienet/utils"; +import { ComputedNetwork, SubstrateCliArgsVersion } from "../../types"; +import { getClient } from "../client"; +import { createTempNodeDef, genNodeDef } from "./dynResourceDefinition"; + +const getVersion = async ( + image: string, + command: string, +): Promise => { + const client = getClient(); + const fullCmd = `${command} --help | grep ws-port`; + const node = await createTempNodeDef( + "temp", + image, + "", // don't used + fullCmd, + ); + + const podDef = await genNodeDef(client.namespace, node); + const podName = podDef.metadata.name; + await client.spawnFromDef(podDef); + const logs = await client.getNodeLogs(podName); + + return logs.includes("--ws-port ") + ? SubstrateCliArgsVersion.V0 + : SubstrateCliArgsVersion.V1; +}; + +export const setSubstrateCliArdsVersion = async (network: ComputedNetwork) => { + // Calculate substrate cli version for each node + // and set in the node to use later when we build the cmd. + const imgCmdMap = new Map(); + network.relaychain.nodes.reduce((memo, node) => { + const uniq_image_cmd = `${node.image}_${node.command}`; + if (!memo.has(uniq_image_cmd)) + memo.set(uniq_image_cmd, { image: node.image, command: node.command }); + return memo; + }, imgCmdMap); + + network.parachains.reduce((memo, parachain) => { + for (const collator of parachain.collators) { + const uniq_image_cmd = `${collator.image}_${collator.command}`; + if (!memo.has(uniq_image_cmd)) + memo.set(uniq_image_cmd, { + image: collator.image, + command: collator.command, + }); + } + return memo; + }, imgCmdMap); + + // check versions in series + const promiseGenerators = []; + + for (const [, v] of imgCmdMap) { + const getVersionPromise = async () => { + const version = await getVersion(v.image, v.command); + v.version = version; + return version; + }; + promiseGenerators.push(getVersionPromise); + } + + await series(promiseGenerators, 4); + + // now we need to iterate and set in each node the version + // IFF is not set + for (const node of network.relaychain.nodes) { + if (node.substrateCliArgsVersion) continue; + const uniq_image_cmd = `${node.image}_${node.command}`; + node.substrateCliArgsVersion = imgCmdMap.get(uniq_image_cmd).version; + } + + for (const parachain of network.parachains) { + for (const collator of parachain.collators) { + if (collator.substrateCliArgsVersion) continue; + const uniq_image_cmd = `${collator.image}_${collator.command}`; + collator.substrateCliArgsVersion = imgCmdMap.get(uniq_image_cmd).version; + } + } +}; diff --git a/javascript/packages/orchestrator/src/types.ts b/javascript/packages/orchestrator/src/types.ts index c772c963f..0ae80ea5e 100644 --- a/javascript/packages/orchestrator/src/types.ts +++ b/javascript/packages/orchestrator/src/types.ts @@ -183,6 +183,7 @@ export interface Node { prometheusPort: number; p2pPort: number; }; + substrateCliArgsVersion?: SubstrateCliArgsVersion; } export interface Collator { @@ -339,3 +340,8 @@ export enum ZombieRole { } export type ZombieRoleLabel = ZombieRole | "authority" | "full-node"; + +export enum SubstrateCliArgsVersion { + V0 = 0, + V1 = 1, +}