From 3b134b4366dbacddfd859b9af00238a193cda64b Mon Sep 17 00:00:00 2001 From: Loris Moulin <45130584+l0r1s@users.noreply.github.com> Date: Thu, 6 Apr 2023 00:29:22 +0300 Subject: [PATCH] Refacto/podman resources (#898) --- .../providers/podman/dynResourceDefinition.ts | 434 ++---------------- .../podman/resources/bootnodeResource.ts | 36 ++ .../podman/resources/configs/grafana.yml | 17 + .../podman/resources/configs/prometheus.yml | 17 + .../podman/resources/configs/tempo.yaml | 45 ++ .../podman/resources/grafanaResource.ts | 132 ++++++ .../src/providers/podman/resources/index.ts | 6 + .../podman/resources/introspectorResource.ts | 61 +++ .../podman/resources/nodeResource.ts | 191 ++++++++ .../podman/resources/prometheusResource.ts | 133 ++++++ .../podman/resources/tempoResource.ts | 145 ++++++ .../src/providers/podman/resources/types.ts | 68 +++ 12 files changed, 885 insertions(+), 400 deletions(-) create mode 100644 javascript/packages/orchestrator/src/providers/podman/resources/bootnodeResource.ts create mode 100644 javascript/packages/orchestrator/src/providers/podman/resources/configs/grafana.yml create mode 100644 javascript/packages/orchestrator/src/providers/podman/resources/configs/prometheus.yml create mode 100644 javascript/packages/orchestrator/src/providers/podman/resources/configs/tempo.yaml create mode 100644 javascript/packages/orchestrator/src/providers/podman/resources/grafanaResource.ts create mode 100644 javascript/packages/orchestrator/src/providers/podman/resources/index.ts create mode 100644 javascript/packages/orchestrator/src/providers/podman/resources/introspectorResource.ts create mode 100644 javascript/packages/orchestrator/src/providers/podman/resources/nodeResource.ts create mode 100644 javascript/packages/orchestrator/src/providers/podman/resources/prometheusResource.ts create mode 100644 javascript/packages/orchestrator/src/providers/podman/resources/tempoResource.ts create mode 100644 javascript/packages/orchestrator/src/providers/podman/resources/types.ts diff --git a/javascript/packages/orchestrator/src/providers/podman/dynResourceDefinition.ts b/javascript/packages/orchestrator/src/providers/podman/dynResourceDefinition.ts index cedae3060..b6858af02 100644 --- a/javascript/packages/orchestrator/src/providers/podman/dynResourceDefinition.ts +++ b/javascript/packages/orchestrator/src/providers/podman/dynResourceDefinition.ts @@ -1,123 +1,34 @@ -import { getRandomPort, makeDir } from "@zombienet/utils"; -import { resolve } from "path"; -import { genCmd, genCumulusCollatorCmd } from "../../cmdGenerator"; +import { getRandomPort } from "@zombienet/utils"; import { getUniqueName } from "../../configGenerator"; -import { - INTROSPECTOR_POD_NAME, - P2P_PORT, - PROMETHEUS_PORT, - RPC_HTTP_PORT, - RPC_WS_PORT, -} from "../../constants"; import { Network } from "../../network"; import { Node } from "../../types"; import { getClient } from "../client"; - -const fs = require("fs").promises; +import { + BootNodeResource, + GrafanaResource, + IntrospectorResource, + NodeResource, + PrometheusResource, + TempoResource, +} from "./resources"; export async function genBootnodeDef( namespace: string, nodeSetup: Node, ): Promise { - const [volume_mounts, devices] = await make_volume_mounts(nodeSetup.name); - const container = await make_main_container(nodeSetup, volume_mounts); - return { - apiVersion: "v1", - kind: "Pod", - metadata: { - name: "bootnode", - namespace: namespace, - labels: { - "app.kubernetes.io/name": namespace, - "app.kubernetes.io/instance": "bootnode", - "zombie-role": "bootnode", - app: "zombienet", - "zombie-ns": namespace, - }, - }, - spec: { - hostname: "bootnode", - containers: [container], - initContainers: [], - restartPolicy: "OnFailure", - volumes: devices, - }, - }; + const client = getClient(); + const bootNodeResource = new BootNodeResource(client, namespace, nodeSetup); + const bootNodeResourceSpec = bootNodeResource.generateSpec(); + + return bootNodeResourceSpec; } export async function genPrometheusDef(namespace: string): Promise { const client = getClient(); - const volume_mounts = [ - { name: "prom-cfg", mountPath: "/etc/prometheus", readOnly: false }, - { name: "prom-data", mountPath: "/data", readOnly: false }, - ]; - const cfgPath = `${client.tmpDir}/prometheus/etc`; - const dataPath = `${client.tmpDir}/prometheus/data`; - await makeDir(cfgPath, true); - await makeDir(dataPath, true); - - const devices = [ - { name: "prom-cfg", hostPath: { type: "Directory", path: cfgPath } }, - { name: "prom-data", hostPath: { type: "Directory", path: dataPath } }, - ]; - - const config = `# config -global: - scrape_interval: 5s - external_labels: - monitor: 'zombienet-monitor' -# Scraping Prometheus itself -scrape_configs: -- job_name: 'prometheus' - static_configs: - - targets: ['localhost:9090'] -- job_name: 'dynamic' - file_sd_configs:\n\ - - files: - - /data/sd_config*.yaml - - /data/sd_config*.json - refresh_interval: 5s -`; - - await fs.writeFile(`${cfgPath}/prometheus.yml`, config); - - const ports = [ - { - containerPort: 9090, - name: "prometheus_endpoint", - hostPort: await getRandomPort(), - }, - ]; - - const containerDef = { - image: "docker.io/prom/prometheus", - name: "prometheus", - imagePullPolicy: "Always", - ports, - volumeMounts: volume_mounts, - }; + const prometheusResource = new PrometheusResource(client, namespace); + const prometheusResourceSpec = prometheusResource.generateSpec(); - return { - apiVersion: "v1", - kind: "Pod", - metadata: { - name: "prometheus", - namespace: namespace, - labels: { - "app.kubernetes.io/name": namespace, - "app.kubernetes.io/instance": "prometheus", - "zombie-role": "prometheus", - app: "zombienet", - "zombie-ns": namespace, - }, - }, - spec: { - hostname: "prometheus", - containers: [containerDef], - restartPolicy: "OnFailure", - volumes: devices, - }, - }; + return prometheusResourceSpec; } export async function genGrafanaDef( @@ -126,321 +37,44 @@ export async function genGrafanaDef( tempoIp: string, ): Promise { const client = getClient(); - const volume_mounts = [ - { - name: "datasources-cfg", - mountPath: "/etc/grafana/provisioning/datasources", - readOnly: false, - }, - ]; - const datasourcesPath = `${client.tmpDir}/grafana/datasources`; - await makeDir(datasourcesPath, true); - - const devices = [ - { - name: "datasources-cfg", - hostPath: { type: "Directory", path: datasourcesPath }, - }, - ]; - - const datasource = ` -# config file version -apiVersion: 1 -datasources: - - name: Prometheus - type: prometheus - access: proxy - orgId: 1 - url: http://${prometheusIp}:9090 - version: 1 - editable: true - - name: Tempo - type: tempo - access: proxy - orgId: 1 - url: http://${tempoIp}:3200 - version: 1 - editable: true -`; - - await fs.writeFile(`${datasourcesPath}/prometheus.yml`, datasource); - - const ports = [ - { - containerPort: 3000, - name: "grafana_web", - hostPort: await getRandomPort(), - }, - ]; - - const containerDef = { - image: "docker.io/grafana/grafana", - name: "grafana", - imagePullPolicy: "Always", - ports, - volumeMounts: volume_mounts, - }; + const grafanaResource = new GrafanaResource( + client, + namespace, + prometheusIp, + tempoIp, + ); + const grafanaResourceSpec = grafanaResource.generateSpec(); - return { - apiVersion: "v1", - kind: "Pod", - metadata: { - name: "grafana", - namespace: namespace, - labels: { - "app.kubernetes.io/name": namespace, - "app.kubernetes.io/instance": "grafana", - "zombie-role": "grafana", - app: "zombienet", - "zombie-ns": namespace, - }, - }, - spec: { - hostname: "grafana", - containers: [containerDef], - restartPolicy: "OnFailure", - volumes: devices, - }, - }; + return grafanaResourceSpec; } export async function getIntrospectorDef( namespace: string, wsUri: string, ): Promise { - const ports = [ - { - containerPort: 65432, - name: "prometheus", - hostPort: await getRandomPort(), - }, - ]; + const introspectorResource = new IntrospectorResource(namespace, wsUri); + const introspectorResourceSpec = introspectorResource.generateSpec(); - const containerDef = { - image: "docker.io/paritytech/polkadot-introspector:latest", - name: INTROSPECTOR_POD_NAME, - args: ["block-time-monitor", `--ws=${wsUri}`, "prometheus"], - imagePullPolicy: "Always", - ports, - volumeMounts: [], - }; - - return { - apiVersion: "v1", - kind: "Pod", - metadata: { - name: INTROSPECTOR_POD_NAME, - namespace: namespace, - labels: { - "app.kubernetes.io/name": namespace, - "app.kubernetes.io/instance": INTROSPECTOR_POD_NAME, - "zombie-role": INTROSPECTOR_POD_NAME, - app: "zombienet", - "zombie-ns": namespace, - }, - }, - spec: { - hostname: INTROSPECTOR_POD_NAME, - containers: [containerDef], - restartPolicy: "OnFailure", - }, - }; + return introspectorResourceSpec; } export async function genTempoDef(namespace: string): Promise { const client = getClient(); + const tempoResource = new TempoResource(client, namespace); + const tempoResourceSpec = tempoResource.generateSpec(); - const volume_mounts = [ - { name: "tempo-cfg", mountPath: "/etc/tempo", readOnly: false }, - { name: "tempo-data", mountPath: "/data", readOnly: false }, - ]; - const cfgPath = `${client.tmpDir}/tempo/etc`; - const dataPath = `${client.tmpDir}/tempo/data`; - await makeDir(cfgPath, true); - await makeDir(dataPath, true); - - const devices = [ - { name: "tempo-cfg", hostPath: { type: "Directory", path: cfgPath } }, - { name: "tempo-data", hostPath: { type: "Directory", path: dataPath } }, - ]; - - const tempoConfigPath = resolve( - __dirname, - `../../../static-configs/tempo.yaml`, - ); - await fs.copyFile(tempoConfigPath, `${cfgPath}/tempo.yaml`); - - const ports = [ - { - containerPort: 14268, - name: "jaeger_ingest", - hostPort: await getRandomPort(), - }, - { - containerPort: 3100, - name: "tempo", - hostPort: await getRandomPort(), - }, - { - containerPort: 4317, - name: "otlp_grpc", - hostPort: await getRandomPort(), - }, - { - containerPort: 4318, - name: "otlp_http", - hostPort: await getRandomPort(), - }, - { - containerPort: 9411, - name: "zipkin", - hostPort: await getRandomPort(), - }, - ]; - - const containerDef = { - image: "docker.io/grafana/tempo:latest", - name: "tempo", - args: ["-config.file=/etc/tempo/tempo.yaml"], - imagePullPolicy: "Always", - ports, - volumeMounts: volume_mounts, - }; - - return { - apiVersion: "v1", - kind: "Pod", - metadata: { - name: "tempo", - namespace: namespace, - labels: { - "app.kubernetes.io/name": namespace, - "app.kubernetes.io/instance": "tempo", - "zombie-role": "tempo", - app: "zombienet", - "zombie-ns": namespace, - }, - }, - spec: { - hostname: "tempo", - containers: [containerDef], - restartPolicy: "OnFailure", - volumes: devices, - }, - }; + return tempoResourceSpec; } export async function genNodeDef( namespace: string, nodeSetup: Node, ): Promise { - const [volume_mounts, devices] = await make_volume_mounts(nodeSetup.name); - const container = await make_main_container(nodeSetup, volume_mounts); - - return { - apiVersion: "v1", - kind: "Pod", - metadata: { - name: nodeSetup.name, - namespace: namespace, - labels: { - "zombie-role": nodeSetup.validator ? "authority" : "full-node", - app: "zombienet", - "zombie-ns": namespace, - "app.kubernetes.io/name": namespace, - "app.kubernetes.io/instance": nodeSetup.name, - }, - annotations: { - "prometheus.io/scrape": "true", - "prometheus.io/port": PROMETHEUS_PORT + "", //force string - }, - }, - spec: { - hostname: nodeSetup.name, - containers: [container], - initContainers: [], - restartPolicy: "OnFailure", - volumes: devices, - }, - }; -} - -async function make_volume_mounts(name: string): Promise<[any, any]> { - const volume_mounts = [ - { name: "tmp-cfg", mountPath: "/cfg:U", readOnly: false }, - { name: "tmp-data", mountPath: "/data:U", readOnly: false }, - { name: "tmp-relay-data", mountPath: "/relay-data:U", readOnly: false }, - ]; - const client = getClient(); - const cfgPath = `${client.tmpDir}/${name}/cfg`; - const dataPath = `${client.tmpDir}/${name}/data`; - const relayDataPath = `${client.tmpDir}/${name}/relay-data`; - await makeDir(cfgPath, true); - await makeDir(dataPath, true); - await makeDir(relayDataPath, true); - - const devices = [ - { name: "tmp-cfg", hostPath: { type: "Directory", path: cfgPath } }, - { name: "tmp-data", hostPath: { type: "Directory", path: dataPath } }, - { - name: "tmp-relay-data", - hostPath: { type: "Directory", path: relayDataPath }, - }, - ]; - - return [volume_mounts, devices]; -} - -async function make_main_container( - nodeSetup: Node, - volume_mounts: any[], -): Promise { - // @ts-ignore - const { rpcPort, wsPort, prometheusPort, p2pPort } = nodeSetup.externalPorts - ? nodeSetup.externalPorts - : {}; - const ports = [ - { - containerPort: PROMETHEUS_PORT, - name: "prometheus", - hostPort: prometheusPort || (await getRandomPort()), - }, - { - containerPort: RPC_HTTP_PORT, - name: "rpc", - hostPort: rpcPort || (await getRandomPort()), - }, - { - containerPort: RPC_WS_PORT, - name: "rpc-ws", - hostPort: wsPort || (await getRandomPort()), - }, - { - containerPort: P2P_PORT, - name: "p2p", - hostPort: p2pPort || (await getRandomPort()), - }, - ]; - - let computedCommand; - if (nodeSetup.zombieRole === "cumulus-collator") { - computedCommand = await genCumulusCollatorCmd(nodeSetup); - } else { - computedCommand = await genCmd(nodeSetup); - } - - let containerDef = { - image: nodeSetup.image, - name: nodeSetup.name, - imagePullPolicy: "Always", - ports, - env: nodeSetup.env, - volumeMounts: volume_mounts, - command: computedCommand, - }; + const nodeResource = new NodeResource(client, namespace, nodeSetup); + const nodeResourceSpec = nodeResource.generateSpec(); - return containerDef; + return nodeResourceSpec; } export function replaceNetworkRef(podDef: any, network: Network) { diff --git a/javascript/packages/orchestrator/src/providers/podman/resources/bootnodeResource.ts b/javascript/packages/orchestrator/src/providers/podman/resources/bootnodeResource.ts new file mode 100644 index 000000000..dd120a8a3 --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/podman/resources/bootnodeResource.ts @@ -0,0 +1,36 @@ +import { Node } from "../../../types"; +import { Client } from "../../client"; +import { NodeResource } from "./nodeResource"; +import { Container, PodSpec, Volume } from "./types"; + +export class BootNodeResource extends NodeResource { + constructor(client: Client, namespace: string, nodeSetupConfig: Node) { + super(client, namespace, nodeSetupConfig); + } + + protected generatePodSpec( + containers: Container[], + volumes: Volume[], + ): PodSpec { + return { + apiVersion: "v1", + kind: "Pod", + metadata: { + name: "bootnode", + namespace: this.namespace, + labels: { + "zombie-role": "bootnode", + app: "zombienet", + "zombie-ns": this.namespace, + }, + }, + spec: { + hostname: "bootnode", + initContainers: [], + restartPolicy: "OnFailure", + volumes, + containers, + }, + }; + } +} diff --git a/javascript/packages/orchestrator/src/providers/podman/resources/configs/grafana.yml b/javascript/packages/orchestrator/src/providers/podman/resources/configs/grafana.yml new file mode 100644 index 000000000..efbcc09d9 --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/podman/resources/configs/grafana.yml @@ -0,0 +1,17 @@ +# config file version +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://{{PROMETHEUS_IP}}:9090 + version: 1 + editable: true + - name: Tempo + type: tempo + access: proxy + orgId: 1 + url: http://{{TEMPO_IP}}:3200 + version: 1 + editable: true diff --git a/javascript/packages/orchestrator/src/providers/podman/resources/configs/prometheus.yml b/javascript/packages/orchestrator/src/providers/podman/resources/configs/prometheus.yml new file mode 100644 index 000000000..ee37054d0 --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/podman/resources/configs/prometheus.yml @@ -0,0 +1,17 @@ +# config +global: + scrape_interval: 5s + external_labels: + monitor: "zombienet-monitor" + +# Scraping Prometheus itself +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + - job_name: "dynamic" + file_sd_configs: + - files: + - /data/sd_config*.yaml + - /data/sd_config*.json + refresh_interval: 5s diff --git a/javascript/packages/orchestrator/src/providers/podman/resources/configs/tempo.yaml b/javascript/packages/orchestrator/src/providers/podman/resources/configs/tempo.yaml new file mode 100644 index 000000000..6a1a8808b --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/podman/resources/configs/tempo.yaml @@ -0,0 +1,45 @@ +server: + http_listen_port: 3100 + +distributor: + receivers: # this configuration will listen on all ports and protocols that tempo is capable of. + jaeger: # the receives all come from the OpenTelemetry collector. more configuration information can + protocols: # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver + thrift_http: # + grpc: # for a production deployment you should only enable the receivers you need! + thrift_binary: + thrift_compact: + zipkin: + otlp: + protocols: + http: + grpc: + opencensus: + +ingester: + trace_idle_period: 10s # the length of time after a trace has not received spans to consider it complete and flush it + max_block_bytes: 1_000_000 # cut the head block when it hits this size or ... + max_block_duration: 5m # this much time passes + +compactor: + compaction: + compaction_window: 1h # blocks in this time window will be compacted together + max_block_bytes: 100_000_000 # maximum size of compacted blocks + block_retention: 1h + compacted_block_retention: 10m + +storage: + trace: + backend: local # backend configuration to use + block: + bloom_filter_false_positive: .05 # bloom filter false positive rate. lower values create larger filters but fewer false positives + index_downsample_bytes: 1000 # number of bytes per index record + encoding: zstd # block encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 + wal: + path: /tmp/tempo/wal # where to store the the wal locally + encoding: snappy # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 + local: + path: /tmp/tempo/blocks + pool: + max_workers: 100 # worker pool determines the number of parallel requests to the object store backend + queue_depth: 10000 \ No newline at end of file diff --git a/javascript/packages/orchestrator/src/providers/podman/resources/grafanaResource.ts b/javascript/packages/orchestrator/src/providers/podman/resources/grafanaResource.ts new file mode 100644 index 000000000..072b25063 --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/podman/resources/grafanaResource.ts @@ -0,0 +1,132 @@ +import { getRandomPort, makeDir } from "@zombienet/utils"; +import fs from "fs/promises"; +import path from "path"; +import { Client } from "../../client"; +import { + Container, + ContainerPort, + PodSpec, + Volume, + VolumeMount, +} from "./types"; + +export class GrafanaResource { + private readonly dataSourcesPath: string; + + constructor( + client: Client, + private readonly namespace: string, + private readonly prometheusIp: string, + private readonly tempoIp: string, + ) { + this.dataSourcesPath = `${client.tmpDir}/grafana/datasources`; + } + + public async generateSpec() { + const volumes = await this.generateVolumes(); + const volumeMounts = this.generateVolumesMounts(); + const containersPorts = await this.generateContainersPorts(); + const containers = this.generateContainers(volumeMounts, containersPorts); + + return this.generatePodSpec(containers, volumes); + } + + private async createVolumeDirectories() { + try { + await makeDir(this.dataSourcesPath, true); + } catch { + throw new Error("Error creating directory for grafana resource"); + } + } + + private async generateGrafanaConfig() { + try { + const templateConfigPath = path.resolve( + __dirname, + "./configs/grafana.yml", + ); + const grafanaConfigBuffer = await fs.readFile(templateConfigPath); + + let grafanaConfig = grafanaConfigBuffer.toString("utf8"); + grafanaConfig = grafanaConfig + .replace("{{PROMETHEUS_IP}}", this.prometheusIp) + .replace("{{TEMPO_IP}}", this.tempoIp); + + await fs.writeFile( + `${this.dataSourcesPath}/prometheus.yml`, + grafanaConfig, + ); + } catch { + throw new Error("Error generating config for grafana resource"); + } + } + + private async generateVolumes(): Promise { + await this.createVolumeDirectories(); + await this.generateGrafanaConfig(); + + return [ + { + name: "datasources-cfg", + hostPath: { type: "Directory", path: this.dataSourcesPath }, + }, + ]; + } + + private generateVolumesMounts() { + return [ + { + name: "datasources-cfg", + mountPath: "/etc/grafana/provisioning/datasources", + readOnly: false, + }, + ]; + } + + private async generateContainersPorts(): Promise { + return [ + { + containerPort: 3000, + name: "grafana_web", + hostPort: await getRandomPort(), + }, + ]; + } + + private generateContainers( + volumeMounts: VolumeMount[], + ports: ContainerPort[], + ): Container[] { + return [ + { + image: "docker.io/grafana/grafana", + name: "grafana", + imagePullPolicy: "Always", + ports, + volumeMounts, + }, + ]; + } + + private generatePodSpec(containers: Container[], volumes: Volume[]): PodSpec { + return { + apiVersion: "v1", + kind: "Pod", + metadata: { + name: "grafana", + namespace: this.namespace, + labels: { + "zombie-role": "grafana", + app: "zombienet", + "zombie-ns": this.namespace, + }, + }, + spec: { + hostname: "grafana", + restartPolicy: "OnFailure", + volumes, + containers, + }, + }; + } +} diff --git a/javascript/packages/orchestrator/src/providers/podman/resources/index.ts b/javascript/packages/orchestrator/src/providers/podman/resources/index.ts new file mode 100644 index 000000000..5abc6a912 --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/podman/resources/index.ts @@ -0,0 +1,6 @@ +export { BootNodeResource } from "./bootnodeResource"; +export { GrafanaResource } from "./grafanaResource"; +export { IntrospectorResource } from "./introspectorResource"; +export { NodeResource } from "./nodeResource"; +export { PrometheusResource } from "./prometheusResource"; +export { TempoResource } from "./tempoResource"; diff --git a/javascript/packages/orchestrator/src/providers/podman/resources/introspectorResource.ts b/javascript/packages/orchestrator/src/providers/podman/resources/introspectorResource.ts new file mode 100644 index 000000000..f30e1bae7 --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/podman/resources/introspectorResource.ts @@ -0,0 +1,61 @@ +import { getRandomPort } from "@zombienet/utils"; +import { INTROSPECTOR_POD_NAME } from "../../../constants"; +import { Container, ContainerPort, PodSpec } from "./types"; + +export class IntrospectorResource { + constructor( + private readonly namespace: string, + private readonly wsUri: string, + ) {} + + public async generateSpec() { + const containerPorts = await this.generateContainersPorts(); + const containers = this.generateContainers(containerPorts); + + return this.generatePodSpec(containers); + } + + private async generateContainersPorts(): Promise { + return [ + { + containerPort: 65432, + name: "prometheus", + hostPort: await getRandomPort(), + }, + ]; + } + + private generateContainers(ports: ContainerPort[]): Container[] { + return [ + { + image: "docker.io/paritytech/polkadot-introspector:latest", + name: INTROSPECTOR_POD_NAME, + args: ["block-time-monitor", `--ws=${this.wsUri}`, "prometheus"], + imagePullPolicy: "Always", + ports, + volumeMounts: [], + }, + ]; + } + + private generatePodSpec(containers: Container[]): PodSpec { + return { + apiVersion: "v1", + kind: "Pod", + metadata: { + name: INTROSPECTOR_POD_NAME, + namespace: this.namespace, + labels: { + "zombie-role": INTROSPECTOR_POD_NAME, + app: "zombienet", + "zombie-ns": this.namespace, + }, + }, + spec: { + hostname: INTROSPECTOR_POD_NAME, + containers: containers, + restartPolicy: "OnFailure", + }, + }; + } +} diff --git a/javascript/packages/orchestrator/src/providers/podman/resources/nodeResource.ts b/javascript/packages/orchestrator/src/providers/podman/resources/nodeResource.ts new file mode 100644 index 000000000..903de5a4d --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/podman/resources/nodeResource.ts @@ -0,0 +1,191 @@ +import { getRandomPort, makeDir } from "@zombienet/utils"; +import { genCmd, genCumulusCollatorCmd } from "../../../cmdGenerator"; +import { + P2P_PORT, + PROMETHEUS_PORT, + RPC_HTTP_PORT, + RPC_WS_PORT, +} from "../../../constants"; +import { Node } from "../../../types"; +import { Client } from "../../client"; +import { + Container, + ContainerPort, + PodSpec, + Volume, + VolumeMount, +} from "./types"; + +export class NodeResource { + private readonly configPath: string; + private readonly dataPath: string; + private readonly relayDataPath: string; + + constructor( + client: Client, + protected readonly namespace: string, + protected readonly nodeSetupConfig: Node, + ) { + const nodeRootPath = `${client.tmpDir}/${nodeSetupConfig.name}`; + this.configPath = `${nodeRootPath}/cfg`; + this.dataPath = `${nodeRootPath}/data`; + this.relayDataPath = `${nodeRootPath}/relay-data`; + } + + public async generateSpec() { + const volumes = await this.generateVolumes(); + const volumeMounts = this.generateVolumesMounts(); + const containersPorts = await this.generateContainersPorts(); + const containers = await this.generateContainers( + volumeMounts, + containersPorts, + ); + + return this.generatePodSpec(containers, volumes); + } + + private async createVolumeDirectories() { + try { + await makeDir(this.configPath, true); + await makeDir(this.dataPath, true); + await makeDir(this.relayDataPath, true); + } catch { + throw new Error( + `Error generating directories for ${this.nodeSetupConfig.name} resource`, + ); + } + } + + private async generateVolumes(): Promise { + await this.createVolumeDirectories(); + + return [ + { + name: "tmp-cfg", + hostPath: { type: "Directory", path: this.configPath }, + }, + { + name: "tmp-data", + hostPath: { type: "Directory", path: this.dataPath }, + }, + { + name: "tmp-relay-data", + hostPath: { type: "Directory", path: this.relayDataPath }, + }, + ]; + } + + private generateVolumesMounts() { + return [ + { + name: "tmp-cfg", + mountPath: "/cfg:U", + readOnly: false, + }, + { + name: "tmp-data", + mountPath: "/data:U", + readOnly: false, + }, + { + name: "tmp-relay-data", + mountPath: "/relay-data:U", + readOnly: false, + }, + ]; + } + + private async portFromNodeSetupConfigOrDefault( + portProperty: keyof NonNullable, + ) { + const { externalPorts } = this.nodeSetupConfig; + + if (externalPorts && portProperty in externalPorts) { + return externalPorts[portProperty]; + } + + return getRandomPort(); + } + + private async generateContainersPorts(): Promise { + return [ + { + containerPort: PROMETHEUS_PORT, + name: "prometheus", + hostPort: await this.portFromNodeSetupConfigOrDefault("prometheusPort"), + }, + { + containerPort: RPC_HTTP_PORT, + name: "rpc", + hostPort: await this.portFromNodeSetupConfigOrDefault("rpcPort"), + }, + { + containerPort: RPC_WS_PORT, + name: "rpc-ws", + hostPort: await this.portFromNodeSetupConfigOrDefault("wsPort"), + }, + { + containerPort: P2P_PORT, + name: "p2p", + hostPort: await this.portFromNodeSetupConfigOrDefault("p2pPort"), + }, + ]; + } + + private generateContainerCommand(): Promise { + if (this.nodeSetupConfig.zombieRole === "cumulus-collator") { + return genCumulusCollatorCmd(this.nodeSetupConfig); + } + + return genCmd(this.nodeSetupConfig); + } + + private async generateContainers( + volumeMounts: VolumeMount[], + ports: ContainerPort[], + ): Promise { + return [ + { + image: this.nodeSetupConfig.image, + name: this.nodeSetupConfig.name, + imagePullPolicy: "Always", + env: this.nodeSetupConfig.env, + volumeMounts, + ports, + command: await this.generateContainerCommand(), + }, + ]; + } + + protected generatePodSpec( + containers: Container[], + volumes: Volume[], + ): PodSpec { + const { name, validator } = this.nodeSetupConfig; + + return { + apiVersion: "v1", + kind: "Pod", + metadata: { + name: name, + namespace: this.namespace, + labels: { + "zombie-role": validator ? "authority" : "full-node", + app: "zombienet", + "zombie-ns": this.namespace, + }, + annotations: { + "prometheus.io/scrape": "true", + "prometheus.io/port": `${PROMETHEUS_PORT}`, + }, + }, + spec: { + hostname: name, + initContainers: [], + restartPolicy: "OnFailure", + volumes, + containers, + }, + }; + } +} diff --git a/javascript/packages/orchestrator/src/providers/podman/resources/prometheusResource.ts b/javascript/packages/orchestrator/src/providers/podman/resources/prometheusResource.ts new file mode 100644 index 000000000..d46db7aed --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/podman/resources/prometheusResource.ts @@ -0,0 +1,133 @@ +import { getRandomPort, makeDir } from "@zombienet/utils"; +import fs from "fs/promises"; +import path from "path"; +import { Client } from "../../client"; +import { + Container, + ContainerPort, + PodSpec, + Volume, + VolumeMount, +} from "./types"; + +export class PrometheusResource { + private readonly configPath: string; + private readonly dataPath: string; + + constructor(client: Client, private readonly namespace: string) { + const nodeRootPath = `${client.tmpDir}/prometeus`; + this.configPath = `${nodeRootPath}/etc`; + this.dataPath = `${nodeRootPath}/data`; + } + + public async generateSpec() { + const volumes = await this.generateVolumes(); + const volumeMounts = this.generateVolumesMounts(); + const containersPorts = await this.generateContainersPorts(); + const containers = this.generateContainers(volumeMounts, containersPorts); + + return this.generatePodSpec(containers, volumes); + } + + private async createVolumeDirectories() { + try { + await makeDir(this.configPath, true); + await makeDir(this.dataPath, true); + } catch { + throw new Error("Error creating directories for prometheus resource"); + } + } + + private async generatePrometheusConfig() { + try { + const templateConfigPath = path.resolve( + __dirname, + "./configs/prometheus.yml", + ); + await fs.copyFile( + templateConfigPath, + `${this.configPath}/prometheus.yml`, + ); + } catch { + throw new Error("Error generating config for prometheus resource"); + } + } + + private async generateVolumes(): Promise { + await this.createVolumeDirectories(); + await this.generatePrometheusConfig(); + + return [ + { + name: "prom-cfg", + hostPath: { type: "Directory", path: this.configPath }, + }, + { + name: "prom-data", + hostPath: { type: "Directory", path: this.dataPath }, + }, + ]; + } + + private generateVolumesMounts() { + return [ + { + name: "prom-cfg", + mountPath: "/etc/prometheus", + readOnly: false, + }, + { + name: "prom-data", + mountPath: "/data", + readOnly: false, + }, + ]; + } + + private async generateContainersPorts(): Promise { + return [ + { + containerPort: 9090, + name: "prometheus_endpoint", + hostPort: await getRandomPort(), + }, + ]; + } + + private generateContainers( + volumeMounts: VolumeMount[], + ports: ContainerPort[], + ): Container[] { + return [ + { + image: "docker.io/prom/prometheus", + name: "prometheus", + imagePullPolicy: "Always", + ports, + volumeMounts, + }, + ]; + } + + private generatePodSpec(containers: Container[], volumes: Volume[]): PodSpec { + return { + apiVersion: "v1", + kind: "Pod", + metadata: { + name: "prometheus", + namespace: this.namespace, + labels: { + "zombie-role": "prometheus", + app: "zombienet", + "zombie-ns": this.namespace, + }, + }, + spec: { + hostname: "prometheus", + restartPolicy: "OnFailure", + volumes, + containers, + }, + }; + } +} diff --git a/javascript/packages/orchestrator/src/providers/podman/resources/tempoResource.ts b/javascript/packages/orchestrator/src/providers/podman/resources/tempoResource.ts new file mode 100644 index 000000000..aab14cc29 --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/podman/resources/tempoResource.ts @@ -0,0 +1,145 @@ +import { getRandomPort, makeDir } from "@zombienet/utils"; +import fs from "fs/promises"; +import path from "path"; +import { Client } from "../../client"; +import { Container, ContainerPort, Volume, VolumeMount } from "./types"; + +export class TempoResource { + private readonly configPath: string; + private readonly dataPath: string; + + constructor(client: Client, private readonly namespace: string) { + const nodeRootPath = `${client.tmpDir}/tempo`; + this.configPath = `${nodeRootPath}/etc`; + this.dataPath = `${nodeRootPath}/data`; + } + + public async generateSpec() { + const volumes = await this.generateVolumes(); + const volumeMounts = this.generateVolumesMounts(); + const containersPorts = await this.generateContainersPorts(); + const containers = this.generateContainers(volumeMounts, containersPorts); + + return this.generatePodSpec(containers, volumes); + } + + private async createVolumeDirectories() { + try { + await makeDir(this.configPath, true); + await makeDir(this.dataPath, true); + } catch { + throw new Error("Error creating directories for tempo resource"); + } + } + + private async generateTempoConfig() { + try { + const templateConfigPath = path.resolve( + __dirname, + `./configs/tempo.yaml`, + ); + await fs.copyFile(templateConfigPath, `${this.configPath}/tempo.yaml`); + } catch { + throw new Error("Error generating config for tempo resource"); + } + } + + private async generateVolumes(): Promise { + await this.createVolumeDirectories(); + await this.generateTempoConfig(); + + return [ + { + name: "tempo-cfg", + hostPath: { type: "Directory", path: this.configPath }, + }, + { + name: "tempo-data", + hostPath: { type: "Directory", path: this.dataPath }, + }, + ]; + } + + private generateVolumesMounts() { + return [ + { + name: "tempo-cfg", + mountPath: "/etc/tempo", + readOnly: false, + }, + { + name: "tempo-data", + mountPath: "/data", + readOnly: false, + }, + ]; + } + + private async generateContainersPorts(): Promise { + return [ + { + containerPort: 3100, + name: "tempo", + hostPort: await getRandomPort(), + }, + { + containerPort: 14268, + name: "jaeger_ingest", + hostPort: await getRandomPort(), + }, + { + containerPort: 4317, + name: "otlp_grpc", + hostPort: await getRandomPort(), + }, + { + containerPort: 4318, + name: "otlp_http", + hostPort: await getRandomPort(), + }, + { + containerPort: 9411, + name: "zipkin", + hostPort: await getRandomPort(), + }, + ]; + } + + private generateContainers( + volumeMounts: VolumeMount[], + ports: ContainerPort[], + ): Container[] { + return [ + { + image: "docker.io/grafana/tempo:latest", + name: "tempo", + args: ["-config.file=/etc/tempo/tempo.yaml"], + imagePullPolicy: "Always", + ports, + volumeMounts, + }, + ]; + } + + private generatePodSpec(containers: Container[], volumes: Volume[]) { + return { + apiVersion: "v1", + kind: "Pod", + metadata: { + name: "tempo", + namespace: this.namespace, + labels: { + "zombie-role": "tempo", + app: "zombienet", + "zombie-ns": this.namespace, + }, + }, + spec: { + hostname: "tempo", + restartPolicy: "OnFailure", + volumes, + containers, + }, + }; + } +} diff --git a/javascript/packages/orchestrator/src/providers/podman/resources/types.ts b/javascript/packages/orchestrator/src/providers/podman/resources/types.ts new file mode 100644 index 000000000..75797f9fd --- /dev/null +++ b/javascript/packages/orchestrator/src/providers/podman/resources/types.ts @@ -0,0 +1,68 @@ +import { envVars } from "../../../types"; + +export interface VolumeMount { + name: string; + mountPath: string; + readOnly: boolean; +} + +export interface Volume { + name: string; + hostPath: { + type: "Directory"; + path: string; + }; +} + +export interface ContainerPort { + containerPort: number; + name: + | "prometheus" + | "prometheus_endpoint" + | "grafana_web" + | "rpc" + | "rpc-ws" + | "p2p" + | "tempo" + | "jaeger_ingest" + | "otlp_grpc" + | "otlp_http" + | "zipkin"; + hostPort: number; +} + +export interface Container { + image: string; + name: string; + imagePullPolicy: "Always"; + volumeMounts: VolumeMount[]; + ports: ContainerPort[]; + command?: string[]; + args?: string[]; + env?: envVars[]; +} + +export interface PodSpec { + apiVersion: "v1"; + kind: "Pod"; + metadata: { + name: string; + namespace: string; + labels: { + "zombie-role": string; + app: string; + "zombie-ns": string; + }; + annotations?: { + "prometheus.io/scrape": "true"; + "prometheus.io/port": string; + }; + }; + spec: { + hostname: string; + restartPolicy: "OnFailure"; + containers: Container[]; + initContainers?: Container[]; + volumes?: Volume[]; + }; +}