added observability (#1026)

monitoring/Dockerfile

@@ -0,0 +1,10 @@
+FROM golang:alpine AS build
+WORKDIR /app
+COPY . .
+RUN go build -o main .
+
+# Stage 2: Run
+FROM alpine
+WORKDIR /app
+COPY --from=build /app/main /app/
+CMD ["/app/main"]

monitoring/app.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+var (
+	zombieNamespaceCount = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "zombie_namespace_count",
+		Help: "Number of namespaces with names starting with 'zombie-'",
+	})
+
+	zombiePodsCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "zombie_namespace_pods_count",
+		Help: "Number of pods in namespaces with names starting with 'zombie-'",
+	}, []string{"namespace"})
+
+	zombieNamespaceOld = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "zombie_namespace_old",
+		Help: "Indicates if a namespace with a name starting with 'zombie-' is older than 2 days",
+	}, []string{"namespace"})
+)
+
+func init() {
+	prometheus.MustRegister(zombieNamespaceCount)
+	prometheus.MustRegister(zombiePodsCount)
+	prometheus.MustRegister(zombieNamespaceOld)
+}
+
+func main() {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		panic(err)
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		panic(err)
+	}
+
+	go func() {
+		for {
+			updateMetrics(clientset)
+			time.Sleep(1 * time.Minute)
+		}
+	}()
+
+	http.Handle("/metrics", promhttp.Handler())
+	http.ListenAndServe(":9090", nil)
+}
+
+func updateMetrics(clientset *kubernetes.Clientset) {
+	namespaces, err := clientset.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{})
+	if err != nil {
+		fmt.Printf("Error listing namespaces: %v\n", err)
+		return
+	}
+
+	zombieNamespaceCounter := 0
+	twoDaysAgo := time.Now().Add(-48 * time.Hour)
+	for _, ns := range namespaces.Items {
+		if strings.HasPrefix(ns.Name, "zombie-") {
+			zombieNamespaceCounter++
+
+			pods, err := clientset.CoreV1().Pods(ns.Name).List(context.Background(), metav1.ListOptions{})
+			if err != nil {
+				fmt.Printf("Error listing pods in namespace %s: %v\n", ns.Name, err)
+				continue
+			}
+
+			zombiePodsCount.WithLabelValues(ns.Name).Set(float64(len(pods.Items)))
+
+			if ns.CreationTimestamp.Time.Before(twoDaysAgo) {
+				zombieNamespaceOld.WithLabelValues(ns.Name).Set(1)
+			} else {
+				zombieNamespaceOld.WithLabelValues(ns.Name).Set(0)
+			}
+		}
+	}
+
+	zombieNamespaceCount.Set(float64(zombieNamespaceCounter))
+}

monitoring/go.mod

@@ -0,0 +1,56 @@
+module zombienet_metrics
+
+go 1.19
+
+require (
+	github.com/prometheus/client_golang v1.15.1
+	k8s.io/client-go v0.27.1
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.1 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/oauth2 v0.5.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/term v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.27.1 // indirect
+	k8s.io/apimachinery v0.27.1 // indirect
+	k8s.io/klog/v2 v2.90.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect
+	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)