diff --git a/javascript/package-lock.json b/javascript/package-lock.json index a650a7d76..ef2ecdb22 100644 --- a/javascript/package-lock.json +++ b/javascript/package-lock.json @@ -6524,11 +6524,11 @@ }, "packages/cli": { "name": "@zombienet/cli", - "version": "1.3.43", + "version": "1.3.44", "license": "GPL-3.0-or-later", "dependencies": { "@zombienet/dsl-parser-wrapper": "^0.1.7", - "@zombienet/orchestrator": "^0.0.34", + "@zombienet/orchestrator": "^0.0.35", "@zombienet/utils": "^0.0.19", "cli-progress": "^3.12.0", "commander": "^10.0.1", @@ -6550,7 +6550,7 @@ }, "packages/orchestrator": { "name": "@zombienet/orchestrator", - "version": "0.0.34", + "version": "0.0.35", "license": "GPL-3.0-or-later", "dependencies": { "@polkadot/api": "^10.4.1", diff --git a/javascript/packages/orchestrator/src/constants.ts b/javascript/packages/orchestrator/src/constants.ts index 3d0007724..88015763c 100644 --- a/javascript/packages/orchestrator/src/constants.ts +++ b/javascript/packages/orchestrator/src/constants.ts @@ -18,6 +18,13 @@ const DEFAULT_PORTS = { prometheusPort: PROMETHEUS_PORT, }; +// Jaeger agent container exposed port from: +// https://www.jaegertracing.io/docs/1.6/getting-started/#all-in-one-docker-image +const JAEGER_AGENT_ZIPKIN_COMPACT_PORT = 5775; +const JAEGER_AGENT_SERVE_CONFIGS_PORT = 5778; +const JAEGER_AGENT_THRIFT_COMPACT_PORT = 6831; +const JAEGER_AGENT_THRIFT_BINARY_PORT = 6832; + const DEFAULT_GLOBAL_TIMEOUT = 1200; // 20 mins const DEFAULT_INDIVIDUAL_TEST_TIMEOUT = 10; // seconds const DEFAULT_COMMAND = "polkadot"; @@ -111,6 +118,10 @@ export { RPC_WS_PORT, RPC_HTTP_PORT, P2P_PORT, + JAEGER_AGENT_ZIPKIN_COMPACT_PORT, + JAEGER_AGENT_SERVE_CONFIGS_PORT, + JAEGER_AGENT_THRIFT_COMPACT_PORT, + JAEGER_AGENT_THRIFT_BINARY_PORT, DEFAULT_PORTS, DEFAULT_GLOBAL_TIMEOUT, DEFAULT_INDIVIDUAL_TEST_TIMEOUT, diff --git a/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts b/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts index 6a4f6af24..21734d13d 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts @@ -3,7 +3,8 @@ import { getUniqueName } from "../../configGenerator"; import { TMP_DONE, WAIT_UNTIL_SCRIPT_SUFIX } from "../../constants"; import { Network } from "../../network"; import { Node, ZombieRole } from "../../types"; -import { BootNodeResource, NodeResource } from "./resources"; +import { BootNodeResource, NodeResource, ServiceResource } from "./resources"; +import { PodSpec, ServiceSpec } from "./resources/types"; export async function genBootnodeDef( namespace: string, @@ -21,6 +22,11 @@ export async function genNodeDef( return nodeResource.generateSpec(); } +export function genServiceDef(podSpec: PodSpec): ServiceSpec { + const serviceResource = new ServiceResource(podSpec); + return serviceResource.generateSpec(); +} + export function replaceNetworkRef(podDef: any, network: Network) { // replace command if needed in containers for (const container of podDef.spec.containers) { diff --git a/javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts b/javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts index fb12987e2..4b28aba76 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts @@ -18,13 +18,14 @@ import { TRANSFER_CONTAINER_NAME, TRANSFER_CONTAINER_WAIT_LOG, } from "../../constants"; -import { fileMap } from "../../types"; +import { fileMap, ZombieRole } from "../../types"; import { Client, RunCommandOptions, RunCommandResponse, setClient, } from "../client"; +import { genServiceDef } from "./dynResourceDefinition"; const fs = require("fs").promises; const debug = require("debug")("zombie::kube::client"); @@ -135,6 +136,11 @@ export class KubeClient extends Client { logTable.print(); await this.createResource(podDef, true); + if (podDef.metadata.labels["zombie-role"] !== ZombieRole.Temp) { + const serviceDef = genServiceDef(podDef); + await this.createResource(serviceDef, true); + } + await this.waitTransferContainerReady(name); if (dbSnapshot) { diff --git a/javascript/packages/orchestrator/src/providers/k8s/resources/bootnodeResource.ts b/javascript/packages/orchestrator/src/providers/k8s/resources/bootNodeResource.ts similarity index 93% rename from javascript/packages/orchestrator/src/providers/k8s/resources/bootnodeResource.ts rename to javascript/packages/orchestrator/src/providers/k8s/resources/bootNodeResource.ts index 627ef8fd0..6b0bb9287 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/resources/bootnodeResource.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/resources/bootNodeResource.ts @@ -22,14 +22,13 @@ export class BootNodeResource extends NodeResource { "app.kubernetes.io/instance": "bootnode", "zombie-role": ZombieRole.BootNode, app: "zombienet", - "zombie-ns": this.namespace, }, }, spec: { hostname: "bootnode", containers, initContainers, - restartPolicy: "Never", + restartPolicy: "Always", volumes, securityContext: { fsGroup: 1000, diff --git a/javascript/packages/orchestrator/src/providers/k8s/resources/index.ts b/javascript/packages/orchestrator/src/providers/k8s/resources/index.ts index d65914a30..1a2cc8caf 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/resources/index.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/resources/index.ts @@ -1,2 +1,3 @@ -export { BootNodeResource } from "./bootnodeResource"; +export { BootNodeResource } from "./bootNodeResource"; export { NodeResource } from "./nodeResource"; +export { ServiceResource } from "./serviceResource"; diff --git a/javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts b/javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts index 0c5742178..c86eaf05d 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts @@ -8,7 +8,7 @@ import { TRANSFER_CONTAINER_NAME, TRANSFER_CONTAINER_WAIT_LOG, } from "../../../constants"; -import { Node, ZombieRole } from "../../../types"; +import { Node, ZombieRole, ZombieRoleLabel } from "../../../types"; import { Container, ContainerPort, @@ -160,12 +160,24 @@ export class NodeResource { return containers; } + private computeZombieRoleLabel(): ZombieRoleLabel { + const { validator, zombieRole } = this.nodeSetupConfig; + + if (zombieRole) { + return zombieRole; + } + + return validator ? "authority" : "full-node"; + } + protected generatePodSpec( initContainers: Container[], containers: Container[], volumes: Volume[], ): PodSpec { - const { name, validator } = this.nodeSetupConfig; + const { name, zombieRole } = this.nodeSetupConfig; + const zombieRoleLabel = this.computeZombieRoleLabel(); + const restartPolicy = zombieRole === ZombieRole.Temp ? "Never" : "Always"; return { apiVersion: "v1", @@ -173,11 +185,10 @@ export class NodeResource { metadata: { name, labels: { - "zombie-role": validator ? "authority" : "full-node", + "zombie-role": zombieRoleLabel, app: "zombienet", "app.kubernetes.io/name": this.namespace, "app.kubernetes.io/instance": name, - "zombie-ns": this.namespace, }, annotations: { "prometheus.io/scrape": "true", @@ -188,7 +199,7 @@ export class NodeResource { hostname: name, containers, initContainers, - restartPolicy: "Never", + restartPolicy, volumes, securityContext: { fsGroup: 1000, diff --git a/javascript/packages/orchestrator/src/providers/k8s/resources/serviceResource.ts b/javascript/packages/orchestrator/src/providers/k8s/resources/serviceResource.ts new file mode 100644 index 000000000..8b2ee4647 --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/k8s/resources/serviceResource.ts @@ -0,0 +1,103 @@ +import { + JAEGER_AGENT_SERVE_CONFIGS_PORT, + JAEGER_AGENT_THRIFT_BINARY_PORT, + JAEGER_AGENT_THRIFT_COMPACT_PORT, + JAEGER_AGENT_ZIPKIN_COMPACT_PORT, + P2P_PORT, + PROMETHEUS_PORT, + RPC_HTTP_PORT, + RPC_WS_PORT, +} from "../../../constants"; +import { PodSpec, ServiceSpec } from "./types"; + +export class ServiceResource { + constructor(private readonly podSpec: PodSpec) {} + + public generateSpec() { + const ports = this.generatePorts(); + const name = this.podSpec.metadata.name; + + return this.generateServiceSpec(name, ports); + } + + private shouldExposeJaegerPorts(): boolean { + return this.podSpec.spec.containers.some( + (container) => container.name === "jaeger-agent", + ); + } + + private generatePorts(): ServiceSpec["spec"]["ports"] { + let ports: ServiceSpec["spec"]["ports"] = [ + { + name: "prometheus", + protocol: "TCP", + port: PROMETHEUS_PORT, + targetPort: PROMETHEUS_PORT, + }, + { + name: "rpc-http", + protocol: "TCP", + port: RPC_HTTP_PORT, + targetPort: RPC_HTTP_PORT, + }, + { + name: "rpc-ws", + protocol: "TCP", + port: RPC_WS_PORT, + targetPort: RPC_WS_PORT, + }, + { + name: "p2p", + protocol: "TCP", + port: P2P_PORT, + targetPort: P2P_PORT, + }, + ]; + + if (this.shouldExposeJaegerPorts()) { + ports = ports.concat([ + { + name: "jaeger-agent-zipkin-compact", + protocol: "UDP", + port: JAEGER_AGENT_ZIPKIN_COMPACT_PORT, + targetPort: JAEGER_AGENT_ZIPKIN_COMPACT_PORT, + }, + { + name: "jaeger-agent-serve-configs", + protocol: "TCP", + port: JAEGER_AGENT_SERVE_CONFIGS_PORT, + targetPort: JAEGER_AGENT_SERVE_CONFIGS_PORT, + }, + { + name: "jaeger-agent-thrift-compact", + protocol: "UDP", + port: JAEGER_AGENT_THRIFT_COMPACT_PORT, + targetPort: JAEGER_AGENT_THRIFT_COMPACT_PORT, + }, + { + name: "jaeger-agent-thrift-binary", + protocol: "UDP", + port: JAEGER_AGENT_THRIFT_BINARY_PORT, + targetPort: JAEGER_AGENT_THRIFT_BINARY_PORT, + }, + ]); + } + + return ports; + } + + private generateServiceSpec( + name: string, + ports: ServiceSpec["spec"]["ports"], + ): ServiceSpec { + return { + apiVersion: "v1", + kind: "Service", + metadata: { name }, + spec: { + selector: { "app.kubernetes.io/instance": name }, + ports, + }, + }; + } +} diff --git a/javascript/packages/orchestrator/src/providers/k8s/resources/types.ts b/javascript/packages/orchestrator/src/providers/k8s/resources/types.ts index c663ef15a..e76c0b429 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/resources/types.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/resources/types.ts @@ -30,33 +30,55 @@ export interface Container { resources?: ContainerResource; } +export interface Labels { + "zombie-role": ZombieRoleLabel; + app: string; + "app.kubernetes.io/name": string; + "app.kubernetes.io/instance": string; +} + +export interface Annotations { + "prometheus.io/scrape": "true"; + "prometheus.io/port": string; +} + +export interface InnerPodSpec { + hostname: string; + restartPolicy: "Never" | "OnFailure" | "Always"; + containers: Container[]; + initContainers?: Container[]; + volumes?: Volume[]; + securityContext?: { + fsGroup: number; + runAsUser: number; + runAsGroup: number; + }; +} + export interface PodSpec { apiVersion: "v1"; kind: "Pod"; metadata: { name: string; - labels: { - "zombie-role": ZombieRoleLabel; - app: string; - "app.kubernetes.io/name": string; - "app.kubernetes.io/instance": string; - "zombie-ns": string; - }; - annotations?: { - "prometheus.io/scrape": "true"; - "prometheus.io/port": string; - }; + labels: Labels; + annotations?: Annotations; }; + spec: InnerPodSpec; +} + +export interface ServiceSpec { + apiVersion: "v1"; + kind: "Service"; + metadata: { name: string }; spec: { - hostname: string; - restartPolicy: "Never" | "OnFailure"; - containers: Container[]; - initContainers?: Container[]; - volumes?: Volume[]; - securityContext?: { - fsGroup: number; - runAsUser: number; - runAsGroup: number; + selector: { + "app.kubernetes.io/instance": string; }; + ports: { + name: string; + protocol: "TCP" | "UDP"; + port: number; + targetPort: number; + }[]; }; } diff --git a/javascript/packages/orchestrator/src/spawner.ts b/javascript/packages/orchestrator/src/spawner.ts index c89469cb7..6be19f948 100644 --- a/javascript/packages/orchestrator/src/spawner.ts +++ b/javascript/packages/orchestrator/src/spawner.ts @@ -121,14 +121,16 @@ export const spawnNode = async ( const endpointPort = RPC_WS_PORT; if (opts.inCI) { - const nodeIp = await client.getNodeIP(podDef.metadata.name); + // in CI we deploy a service (with the pod name) in front of each pod + // so here we can use the name (as short dns in the ns) to connect to pod. + const nodeDns = `${podDef.metadata.name}.${namespace}.svc.cluster.local`; networkNode = new NetworkNode( node.name, - WS_URI_PATTERN.replace("{{IP}}", nodeIp).replace( + WS_URI_PATTERN.replace("{{IP}}", nodeDns).replace( "{{PORT}}", endpointPort.toString(), ), - METRICS_URI_PATTERN.replace("{{IP}}", nodeIp).replace( + METRICS_URI_PATTERN.replace("{{IP}}", nodeDns).replace( "{{PORT}}", PROMETHEUS_PORT.toString(), ), @@ -136,7 +138,7 @@ export const spawnNode = async ( opts.userDefinedTypes, ); } else { - const nodeIdentifier = `${podDef.kind}/${podDef.metadata.name}`; + const nodeIdentifier = `service/${podDef.metadata.name}`; const fwdPort = await client.startPortForwarding( endpointPort, nodeIdentifier,