Feat/k8s service

javascript/packages/orchestrator/src/constants.ts

@@ -18,6 +18,13 @@ const DEFAULT_PORTS = {
   prometheusPort: PROMETHEUS_PORT,
 };
 
+// Jaeger agent container exposed port from:
+// https://www.jaegertracing.io/docs/1.6/getting-started/#all-in-one-docker-image
+const JAEGER_AGENT_ZIPKIN_COMPACT_PORT = 5775;
+const JAEGER_AGENT_SERVE_CONFIGS_PORT = 5778;
+const JAEGER_AGENT_THRIFT_COMPACT_PORT = 6831;
+const JAEGER_AGENT_THRIFT_BINARY_PORT = 6832;
+
 const DEFAULT_GLOBAL_TIMEOUT = 1200; // 20 mins
 const DEFAULT_INDIVIDUAL_TEST_TIMEOUT = 10; // seconds
 const DEFAULT_COMMAND = "polkadot";
@@ -111,6 +118,10 @@ export {
   RPC_WS_PORT,
   RPC_HTTP_PORT,
   P2P_PORT,
+  JAEGER_AGENT_ZIPKIN_COMPACT_PORT,
+  JAEGER_AGENT_SERVE_CONFIGS_PORT,
+  JAEGER_AGENT_THRIFT_COMPACT_PORT,
+  JAEGER_AGENT_THRIFT_BINARY_PORT,
   DEFAULT_PORTS,
   DEFAULT_GLOBAL_TIMEOUT,
   DEFAULT_INDIVIDUAL_TEST_TIMEOUT,

javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts

@@ -3,7 +3,8 @@ import { getUniqueName } from "../../configGenerator";
 import { TMP_DONE, WAIT_UNTIL_SCRIPT_SUFIX } from "../../constants";
 import { Network } from "../../network";
 import { Node, ZombieRole } from "../../types";
-import { BootNodeResource, NodeResource } from "./resources";
+import { BootNodeResource, NodeResource, ServiceResource } from "./resources";
+import { PodSpec, ServiceSpec } from "./resources/types";
 
 export async function genBootnodeDef(
   namespace: string,
@@ -21,6 +22,11 @@ export async function genNodeDef(
   return nodeResource.generateSpec();
 }
 
+export function genServiceDef(podSpec: PodSpec): ServiceSpec {
+  const serviceResource = new ServiceResource(podSpec);
+  return serviceResource.generateSpec();
+}
+
 export function replaceNetworkRef(podDef: any, network: Network) {
   // replace command if needed in containers
   for (const container of podDef.spec.containers) {

javascript/packages/orchestrator/src/providers/k8s/kubeClient.ts

@@ -18,13 +18,14 @@ import {
   TRANSFER_CONTAINER_NAME,
   TRANSFER_CONTAINER_WAIT_LOG,
 } from "../../constants";
-import { fileMap } from "../../types";
+import { fileMap, ZombieRole } from "../../types";
 import {
   Client,
   RunCommandOptions,
   RunCommandResponse,
   setClient,
 } from "../client";
+import { genServiceDef } from "./dynResourceDefinition";
 const fs = require("fs").promises;
 
 const debug = require("debug")("zombie::kube::client");
@@ -135,6 +136,11 @@ export class KubeClient extends Client {
     logTable.print();
 
     await this.createResource(podDef, true);
+    if (podDef.metadata.labels["zombie-role"] !== ZombieRole.Temp) {
+      const serviceDef = genServiceDef(podDef);
+      await this.createResource(serviceDef, true);
+    }
+
     await this.waitTransferContainerReady(name);
 
     if (dbSnapshot) {

javascript/packages/orchestrator/src/providers/k8s/resources/bootnodeResource.ts → javascript/packages/orchestrator/src/providers/k8s/resources/bootNodeResource.ts

@@ -22,14 +22,13 @@ export class BootNodeResource extends NodeResource {
           "app.kubernetes.io/instance": "bootnode",
           "zombie-role": ZombieRole.BootNode,
           app: "zombienet",
-          "zombie-ns": this.namespace,
         },
       },
       spec: {
         hostname: "bootnode",
         containers,
         initContainers,
-        restartPolicy: "Never",
+        restartPolicy: "Always",
         volumes,
         securityContext: {
           fsGroup: 1000,

javascript/packages/orchestrator/src/providers/k8s/resources/index.ts

@@ -1,2 +1,3 @@
-export { BootNodeResource } from "./bootnodeResource";
+export { BootNodeResource } from "./bootNodeResource";
 export { NodeResource } from "./nodeResource";
+export { ServiceResource } from "./serviceResource";

javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts

@@ -8,7 +8,7 @@ import {
   TRANSFER_CONTAINER_NAME,
   TRANSFER_CONTAINER_WAIT_LOG,
 } from "../../../constants";
-import { Node, ZombieRole } from "../../../types";
+import { Node, ZombieRole, ZombieRoleLabel } from "../../../types";
 import {
   Container,
   ContainerPort,
@@ -160,24 +160,35 @@ export class NodeResource {
     return containers;
   }
 
+  private computeZombieRoleLabel(): ZombieRoleLabel {
+    const { validator, zombieRole } = this.nodeSetupConfig;
+
+    if (zombieRole) {
+      return zombieRole;
+    }
+
+    return validator ? "authority" : "full-node";
+  }
+
   protected generatePodSpec(
     initContainers: Container[],
     containers: Container[],
     volumes: Volume[],
   ): PodSpec {
-    const { name, validator } = this.nodeSetupConfig;
+    const { name, zombieRole } = this.nodeSetupConfig;
+    const zombieRoleLabel = this.computeZombieRoleLabel();
+    const restartPolicy = zombieRole === ZombieRole.Temp ? "Never" : "Always";
 
     return {
       apiVersion: "v1",
       kind: "Pod",
       metadata: {
         name,
         labels: {
-          "zombie-role": validator ? "authority" : "full-node",
+          "zombie-role": zombieRoleLabel,
           app: "zombienet",
           "app.kubernetes.io/name": this.namespace,
           "app.kubernetes.io/instance": name,
-          "zombie-ns": this.namespace,
         },
         annotations: {
           "prometheus.io/scrape": "true",
@@ -188,7 +199,7 @@ export class NodeResource {
         hostname: name,
         containers,
         initContainers,
-        restartPolicy: "Never",
+        restartPolicy,
         volumes,
         securityContext: {
           fsGroup: 1000,

javascript/packages/orchestrator/src/providers/k8s/resources/serviceResource.ts

@@ -0,0 +1,103 @@
+import {
+  JAEGER_AGENT_SERVE_CONFIGS_PORT,
+  JAEGER_AGENT_THRIFT_BINARY_PORT,
+  JAEGER_AGENT_THRIFT_COMPACT_PORT,
+  JAEGER_AGENT_ZIPKIN_COMPACT_PORT,
+  P2P_PORT,
+  PROMETHEUS_PORT,
+  RPC_HTTP_PORT,
+  RPC_WS_PORT,
+} from "../../../constants";
+import { PodSpec, ServiceSpec } from "./types";
+
+export class ServiceResource {
+  constructor(private readonly podSpec: PodSpec) {}
+
+  public generateSpec() {
+    const ports = this.generatePorts();
+    const name = this.podSpec.metadata.name;
+
+    return this.generateServiceSpec(name, ports);
+  }
+
+  private shouldExposeJaegerPorts(): boolean {
+    return this.podSpec.spec.containers.some(
+      (container) => container.name === "jaeger-agent",
+    );
+  }
+
+  private generatePorts(): ServiceSpec["spec"]["ports"] {
+    let ports: ServiceSpec["spec"]["ports"] = [
+      {
+        name: "prometheus",
+        protocol: "TCP",
+        port: PROMETHEUS_PORT,
+        targetPort: PROMETHEUS_PORT,
+      },
+      {
+        name: "rpc-http",
+        protocol: "TCP",
+        port: RPC_HTTP_PORT,
+        targetPort: RPC_HTTP_PORT,
+      },
+      {
+        name: "rpc-ws",
+        protocol: "TCP",
+        port: RPC_WS_PORT,
+        targetPort: RPC_WS_PORT,
+      },
+      {
+        name: "p2p",
+        protocol: "TCP",
+        port: P2P_PORT,
+        targetPort: P2P_PORT,
+      },
+    ];
+
+    if (this.shouldExposeJaegerPorts()) {
+      ports = ports.concat([
+        {
+          name: "jaeger-agent-zipkin-compact",
+          protocol: "UDP",
+          port: JAEGER_AGENT_ZIPKIN_COMPACT_PORT,
+          targetPort: JAEGER_AGENT_ZIPKIN_COMPACT_PORT,
+        },
+        {
+          name: "jaeger-agent-serve-configs",
+          protocol: "TCP",
+          port: JAEGER_AGENT_SERVE_CONFIGS_PORT,
+          targetPort: JAEGER_AGENT_SERVE_CONFIGS_PORT,
+        },
+        {
+          name: "jaeger-agent-thrift-compact",
+          protocol: "UDP",
+          port: JAEGER_AGENT_THRIFT_COMPACT_PORT,
+          targetPort: JAEGER_AGENT_THRIFT_COMPACT_PORT,
+        },
+        {
+          name: "jaeger-agent-thrift-binary",
+          protocol: "UDP",
+          port: JAEGER_AGENT_THRIFT_BINARY_PORT,
+          targetPort: JAEGER_AGENT_THRIFT_BINARY_PORT,
+        },
+      ]);
+    }
+
+    return ports;
+  }
+
+  private generateServiceSpec(
+    name: string,
+    ports: ServiceSpec["spec"]["ports"],
+  ): ServiceSpec {
+    return {
+      apiVersion: "v1",
+      kind: "Service",
+      metadata: { name },
+      spec: {
+        selector: { "app.kubernetes.io/instance": name },
+        ports,
+      },
+    };
+  }
+}

javascript/packages/orchestrator/src/providers/k8s/resources/types.ts

@@ -30,33 +30,55 @@ export interface Container {
   resources?: ContainerResource;
 }
 
+export interface Labels {
+  "zombie-role": ZombieRoleLabel;
+  app: string;
+  "app.kubernetes.io/name": string;
+  "app.kubernetes.io/instance": string;
+}
+
+export interface Annotations {
+  "prometheus.io/scrape": "true";
+  "prometheus.io/port": string;
+}
+
+export interface InnerPodSpec {
+  hostname: string;
+  restartPolicy: "Never" | "OnFailure" | "Always";
+  containers: Container[];
+  initContainers?: Container[];
+  volumes?: Volume[];
+  securityContext?: {
+    fsGroup: number;
+    runAsUser: number;
+    runAsGroup: number;
+  };
+}
+
 export interface PodSpec {
   apiVersion: "v1";
   kind: "Pod";
   metadata: {
     name: string;
-    labels: {
-      "zombie-role": ZombieRoleLabel;
-      app: string;
-      "app.kubernetes.io/name": string;
-      "app.kubernetes.io/instance": string;
-      "zombie-ns": string;
-    };
-    annotations?: {
-      "prometheus.io/scrape": "true";
-      "prometheus.io/port": string;
-    };
+    labels: Labels;
+    annotations?: Annotations;
   };
+  spec: InnerPodSpec;
+}
+
+export interface ServiceSpec {
+  apiVersion: "v1";
+  kind: "Service";
+  metadata: { name: string };
   spec: {
-    hostname: string;
-    restartPolicy: "Never" | "OnFailure";
-    containers: Container[];
-    initContainers?: Container[];
-    volumes?: Volume[];
-    securityContext?: {
-      fsGroup: number;
-      runAsUser: number;
-      runAsGroup: number;
+    selector: {
+      "app.kubernetes.io/instance": string;
     };
+    ports: {
+      name: string;
+      protocol: "TCP" | "UDP";
+      port: number;
+      targetPort: number;
+    }[];
   };
 }

javascript/packages/orchestrator/src/spawner.ts

@@ -121,22 +121,24 @@ export const spawnNode = async (
 
   const endpointPort = RPC_WS_PORT;
   if (opts.inCI) {
-    const nodeIp = await client.getNodeIP(podDef.metadata.name);
+    // in CI we deploy a service (with the pod name) in front of each pod
+    // so here we can use the name (as short dns in the ns) to connect to pod.
+    const nodeDns = `${podDef.metadata.name}.${namespace}.svc.cluster.local`;
     networkNode = new NetworkNode(
       node.name,
-      WS_URI_PATTERN.replace("{{IP}}", nodeIp).replace(
+      WS_URI_PATTERN.replace("{{IP}}", nodeDns).replace(
         "{{PORT}}",
         endpointPort.toString(),
       ),
-      METRICS_URI_PATTERN.replace("{{IP}}", nodeIp).replace(
+      METRICS_URI_PATTERN.replace("{{IP}}", nodeDns).replace(
         "{{PORT}}",
         PROMETHEUS_PORT.toString(),
       ),
       nodeMultiAddress,
       opts.userDefinedTypes,
     );
   } else {
-    const nodeIdentifier = `${podDef.kind}/${podDef.metadata.name}`;
+    const nodeIdentifier = `service/${podDef.metadata.name}`;
     const fwdPort = await client.startPortForwarding(
       endpointPort,
       nodeIdentifier,