diff --git a/scripts/cluster_cleanup/Dockerfile b/scripts/cluster_cleanup/Dockerfile new file mode 100644 index 000000000..2e993ea29 --- /dev/null +++ b/scripts/cluster_cleanup/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.9-slim + +WORKDIR /app + +COPY requirements.txt /app + +RUN pip install --trusted-host pypi.python.org -r requirements.txt + +COPY cluster_cleanup.py /app + +CMD ["python", "cluster_cleanup.py"] diff --git a/scripts/cluster_cleanup/cluster_cleanup.py b/scripts/cluster_cleanup/cluster_cleanup.py new file mode 100644 index 000000000..c8fa9a65f --- /dev/null +++ b/scripts/cluster_cleanup/cluster_cleanup.py @@ -0,0 +1,63 @@ +import os +from datetime import datetime, timedelta +from kubernetes import client, config +import requests +import pytz + +def main(): + config.load_incluster_config() + + v1 = client.CoreV1Api() + + prefix = 'zombie-' + time_delta = timedelta(hours=12) + + now = datetime.utcnow().replace(tzinfo=pytz.UTC) + cutoff_time = now - time_delta + + namespace_list = v1.list_namespace().items + for ns in namespace_list: + if ns.metadata.name.startswith(prefix): + creation_time = ns.metadata.creation_timestamp.replace(tzinfo=pytz.UTC) + if creation_time < cutoff_time: + print(f"Found zombie namespace {ns.metadata.name} (created {now - creation_time} ago and matches the prefix).") + send_alert(f"namespace_cleanup_{ns.metadata.name}", "warning", f"Deleting zombie namespace {ns.metadata.name} (created {now - creation_time} ago") + v1.delete_namespace(ns.metadata.name) + + api_version = 'v1' + group = 'monitoring.coreos.com' + plural = 'podmonitors' + namespace = 'monitoring' + + custom_api = client.CustomObjectsApi() + pm_list = custom_api.list_namespaced_custom_object(group, api_version, namespace, plural)['items'] + + for pm in pm_list: + name = pm['metadata']['name'] + creation_time = datetime.strptime(pm['metadata']['creationTimestamp'], '%Y-%m-%dT%H:%M:%S%z').replace(tzinfo=None) + creation_time = creation_time.astimezone(pytz.UTC) + if creation_time < cutoff_time: + print(f"Found old PodMonitor {name} in namespace {namespace} (created {now - creation_time} ago).") + send_alert(f"podmonitor_cleanup_{name}", "warning", f"Deleting PodMonitor {name} in namespace {namespace} (created {now - creation_time} ago).") + custom_api.delete_namespaced_custom_object(group, api_version, namespace, plural, name, body={}, grace_period_seconds=0) + +def send_alert(alertname, severity, message): + url = 'https://alertmanager.parity-mgmt.parity.io/api/v1/alerts' + headers = {'Content-Type': 'application/json'} + payload = [ + { + "labels": { + "domain": "parity-zombienet", + "alertname": alertname, + "severity": severity + }, + "annotations": { + "message": f"{message}" + }, + "generatorURL": "https://grafana.parity-mgmt.parity.io/" + } + ] + response = requests.post(url, headers=headers, json=payload) + +if __name__ == "__main__": + main() diff --git a/scripts/cluster_cleanup/requirements.txt b/scripts/cluster_cleanup/requirements.txt new file mode 100644 index 000000000..30c74e3a7 --- /dev/null +++ b/scripts/cluster_cleanup/requirements.txt @@ -0,0 +1,3 @@ +kubernetes +pytz +requests \ No newline at end of file