From d1c94c4204b5f265711714023a9bd7e52613397b Mon Sep 17 00:00:00 2001 From: l0r1s Date: Fri, 7 Apr 2023 16:10:19 +0300 Subject: [PATCH] feat: refactored into a NodeResource for kubernetes provider --- .../providers/k8s/dynResourceDefinition.ts | 87 +------- .../providers/k8s/resources/nodeResource.ts | 199 ++++++++++++++++++ .../src/providers/k8s/resources/types.ts | 62 ++++++ 3 files changed, 264 insertions(+), 84 deletions(-) create mode 100644 javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts create mode 100644 javascript/packages/orchestrator/src/providers/k8s/resources/types.ts diff --git a/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts b/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts index 6f5d9eeef..1e542a988 100644 --- a/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts +++ b/javascript/packages/orchestrator/src/providers/k8s/dynResourceDefinition.ts @@ -14,6 +14,7 @@ import { } from "../../constants"; import { Network } from "../../network"; import { Node } from "../../types"; +import { NodeResource } from "./resources/nodeResource"; export async function genBootnodeDef( namespace: string, @@ -53,50 +54,8 @@ export async function genNodeDef( namespace: string, nodeSetup: Node, ): Promise { - const [volume_mounts, devices] = make_volume_mounts(); - const container = await make_main_container(nodeSetup, volume_mounts); - const transferContainter = make_transfer_containter(); - - const containersToRun = [container]; - if ( - (nodeSetup.zombieRole === "node" || - nodeSetup.zombieRole === "cumulus-collator") && - nodeSetup.jaegerUrl && - nodeSetup.jaegerUrl === "localhost:6831" - ) { - // add sidecar - containersToRun.push(jaegerAgentDef()); - } - - return { - apiVersion: "v1", - kind: "Pod", - metadata: { - name: nodeSetup.name, - labels: { - "zombie-role": nodeSetup.validator ? "authority" : "full-node", - app: "zombienet", - "app.kubernetes.io/name": namespace, - "app.kubernetes.io/instance": nodeSetup.name, - }, - annotations: { - "prometheus.io/scrape": "true", - "prometheus.io/port": PROMETHEUS_PORT + "", //force string - }, - }, - spec: { - hostname: nodeSetup.name, - containers: containersToRun, - initContainers: [transferContainter], - restartPolicy: "Never", - volumes: devices, - securityContext: { - fsGroup: 1000, - runAsUser: 1000, - runAsGroup: 1000, - }, - }, - }; + const nodeResource = new NodeResource(namespace, nodeSetup); + return nodeResource.generateSpec(); } function make_transfer_containter(): any { @@ -179,46 +138,6 @@ async function make_main_container( return containerDef; } -function jaegerAgentDef() { - return { - name: "jaeger-agent", - image: "jaegertracing/jaeger-agent:1.28.0", - ports: [ - { - containerPort: 5775, - protocol: "UDP", - }, - { - containerPort: 5778, - protocol: "TCP", - }, - { - containerPort: 6831, - protocol: "UDP", - }, - { - containerPort: 6832, - protocol: "UDP", - }, - ], - command: [ - "/go/bin/agent-linux", - "--reporter.type=grpc", - "--reporter.grpc.host-port=tempo-tempo-distributed-distributor.tempo.svc.cluster.local:14250", - ], - resources: { - limits: { - memory: "50M", - cpu: "100m", - }, - requests: { - memory: "50M", - cpu: "100m", - }, - }, - }; -} - export function replaceNetworkRef(podDef: any, network: Network) { // replace command if needed in containers for (const container of podDef.spec.containers) { diff --git a/javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts b/javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts new file mode 100644 index 000000000..d6cf38642 --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/k8s/resources/nodeResource.ts @@ -0,0 +1,199 @@ +import { genCmd, genCumulusCollatorCmd } from "../../../cmdGenerator"; +import { + FINISH_MAGIC_FILE, + P2P_PORT, + PROMETHEUS_PORT, + RPC_HTTP_PORT, + RPC_WS_PORT, + TRANSFER_CONTAINER_NAME, + TRANSFER_CONTAINER_WAIT_LOG, +} from "../../../constants"; +import { Node } from "../../../types"; +import { + Container, + ContainerPort, + PodSpec, + Volume, + VolumeMount, +} from "./types"; + +export class NodeResource { + constructor( + protected readonly namespace: string, + protected readonly nodeSetupConfig: Node, + ) {} + + public async generateSpec() { + const volumes = await this.generateVolumes(); + const volumeMounts = this.generateVolumesMounts(); + const containersPorts = await this.generateContainersPorts(); + const initContainers = this.generateInitContainers(); + const containers = await this.generateContainers( + volumeMounts, + containersPorts, + ); + + return this.generatePodSpec(initContainers, containers, volumes); + } + + private async generateVolumes(): Promise { + return [ + { name: "tmp-cfg" }, + { name: "tmp-data" }, + { name: "tmp-relay-data" }, + ]; + } + + private generateVolumesMounts() { + return [ + { name: "tmp-cfg", mountPath: "/cfg", readOnly: false }, + { name: "tmp-data", mountPath: "/data", readOnly: false }, + { name: "tmp-relay-data", mountPath: "/relay-data", readOnly: false }, + ]; + } + + private async generateContainersPorts(): Promise { + return [ + { containerPort: PROMETHEUS_PORT, name: "prometheus" }, + { containerPort: RPC_HTTP_PORT, name: "rpc-http" }, + { containerPort: RPC_WS_PORT, name: "rpc-ws" }, + { containerPort: P2P_PORT, name: "p2p" }, + ]; + } + + private generateContainerCommand(): Promise { + if (this.nodeSetupConfig.zombieRole === "cumulus-collator") { + return genCumulusCollatorCmd(this.nodeSetupConfig); + } + + return genCmd(this.nodeSetupConfig); + } + + private generateInitContainers(): Container[] { + return [ + { + name: TRANSFER_CONTAINER_NAME, + image: "docker.io/alpine", + imagePullPolicy: "Always", + volumeMounts: [ + { name: "tmp-cfg", mountPath: "/cfg", readOnly: false }, + { name: "tmp-data", mountPath: "/data", readOnly: false }, + { name: "tmp-relay-data", mountPath: "/relay-data", readOnly: false }, + ], + command: [ + "ash", + "-c", + [ + "wget github.com/moparisthebest/static-curl/releases/download/v7.83.1/curl-amd64 -O /cfg/curl", + "echo downloaded", + "chmod +x /cfg/curl", + "echo chmoded", + "wget github.com/uutils/coreutils/releases/download/0.0.17/coreutils-0.0.17-x86_64-unknown-linux-musl.tar.gz -O /cfg/coreutils-0.0.17-x86_64-unknown-linux-musl.tar.gz", + "cd /cfg", + "tar -xvzf ./coreutils-0.0.17-x86_64-unknown-linux-musl.tar.gz", + "cp ./coreutils-0.0.17-x86_64-unknown-linux-musl/coreutils /cfg/coreutils", + "chmod +x /cfg/coreutils", + "rm -rf ./coreutils-0.0.17-x86_64-unknown-linux-musl", + "echo coreutils downloaded", + `until [ -f ${FINISH_MAGIC_FILE} ]; do echo ${TRANSFER_CONTAINER_WAIT_LOG}; sleep 1; done; echo copy files has finished`, + ].join(" && "), + ], + }, + ]; + } + + private shouldAddJaegerContainer() { + const { zombieRole, jaegerUrl } = this.nodeSetupConfig; + const isNodeOrCumulusCollator = + zombieRole === "node" || zombieRole === "cumulus-collator"; + const isJaegerUrlDefined = jaegerUrl && jaegerUrl === "localhost:6831"; + + return isNodeOrCumulusCollator && isJaegerUrlDefined; + } + + private generateJaegerContainer(): Container { + return { + name: "jaeger-agent", + image: "jaegertracing/jaeger-agent:1.28.0", + ports: [ + { containerPort: 5775, protocol: "UDP" }, + { containerPort: 5778, protocol: "TCP" }, + { containerPort: 6831, protocol: "UDP" }, + { containerPort: 6832, protocol: "UDP" }, + ], + command: [ + "/go/bin/agent-linux", + "--reporter.type=grpc", + "--reporter.grpc.host-port=tempo-tempo-distributed-distributor.tempo.svc.cluster.local:14250", + ], + resources: { + limits: { memory: "50M", cpu: "100m" }, + requests: { memory: "50M", cpu: "100m" }, + }, + }; + } + + private async generateContainers( + volumeMounts: VolumeMount[], + ports: ContainerPort[], + ): Promise { + const { image, name, env, resources } = this.nodeSetupConfig; + const containers: Container[] = [ + { + image, + name, + imagePullPolicy: "Always", + ports, + env, + volumeMounts, + command: await this.generateContainerCommand(), + resources: resources?.resources, + }, + ]; + + if (this.shouldAddJaegerContainer()) { + containers.push(this.generateJaegerContainer()); + } + + return containers; + } + + protected generatePodSpec( + initContainers: Container[], + containers: Container[], + volumes: Volume[], + ): PodSpec { + const { name, validator } = this.nodeSetupConfig; + + return { + apiVersion: "v1", + kind: "Pod", + metadata: { + name, + labels: { + "zombie-role": validator ? "authority" : "full-node", + app: "zombienet", + "app.kubernetes.io/name": this.namespace, + "app.kubernetes.io/instance": name, + "zombie-ns": this.namespace, + }, + annotations: { + "prometheus.io/scrape": "true", + "prometheus.io/port": `${PROMETHEUS_PORT}`, + }, + }, + spec: { + hostname: name, + containers, + initContainers, + restartPolicy: "Never", + volumes, + securityContext: { + fsGroup: 1000, + runAsUser: 1000, + runAsGroup: 1000, + }, + }, + }; + } +} diff --git a/javascript/packages/orchestrator/src/providers/k8s/resources/types.ts b/javascript/packages/orchestrator/src/providers/k8s/resources/types.ts new file mode 100644 index 000000000..fe5c59eae --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/k8s/resources/types.ts @@ -0,0 +1,62 @@ +import { envVars, Resources } from "../../../types"; + +type ContainerResource = Resources["resources"]; + +export interface VolumeMount { + name: string; + mountPath: string; + readOnly: boolean; +} + +export interface Volume { + name: string; +} + +export interface ContainerPort { + containerPort: number; + name?: "prometheus" | "rpc-http" | "rpc-ws" | "p2p"; + protocol?: "UDP" | "TCP"; +} + +export interface Container { + image: string; + name: string; + imagePullPolicy?: "Always"; + volumeMounts?: VolumeMount[]; + ports?: ContainerPort[]; + command: string[]; + args?: string[]; + env?: envVars[]; + resources?: ContainerResource; +} + +export interface PodSpec { + apiVersion: "v1"; + kind: "Pod"; + metadata: { + name: string; + labels: { + "zombie-role": string; + app: string; + "app.kubernetes.io/name": string; + "app.kubernetes.io/instance": string; + "zombie-ns": string; + }; + annotations?: { + "prometheus.io/scrape": "true"; + "prometheus.io/port": string; + }; + }; + spec: { + hostname: string; + restartPolicy: "Never" | "OnFailure"; + containers: Container[]; + initContainers?: Container[]; + volumes?: Volume[]; + securityContext?: { + fsGroup: number; + runAsUser: number; + runAsGroup: number; + }; + }; +}