forked from kubernetes/test-infra
-
Notifications
You must be signed in to change notification settings - Fork 0
/
prow_builds_enable_soft_eviction.py
executable file
·91 lines (74 loc) · 3.15 KB
/
prow_builds_enable_soft_eviction.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
#!/usr/bin/env python
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script hijacks the COS kubelet service definition to set soft eviction thresholds
# for nodefs on the prow builds cluster
# USE AT YOUR OWN RISK.
# TODO(bentheelder): delete this once dynamic kubelet config is available
# pylint: disable=line-too-long
from __future__ import print_function
import os
import sys
import subprocess
# xref prow/Makefile get-build-cluster-credentials
CLUSTER = 'prow'
ZONE = 'us-central1-f'
PROJECT = 'k8s-prow-builds'
AUTH_TO_CLUSTER_COMMAND = 'gcloud container clusters get-credentials %s --project=%s --zone=%s' % (CLUSTER, PROJECT, ZONE)
# this should be 20% more than the hard eviction threshold
# the grace period should be longer than the typical time for another pod to be cleaned up by sinker
KUBELET_ARGS_TO_ADD = '--eviction-soft=nodefs.available<30% --eviction-soft-grace-period=nodefs.available=2h'
# commands used *in order* to update the kubelet
KUBELET_UPDATE_COMMANDS = [
# this works because the ExecStart line normally ends with $KUBELET_OPTS
# so we replace `KUBELET_OPTS.*` (to the end of the line) with `KUBELET_OPTS --some --args ---we --want`
"sudo sed -i 's/KUBELET_OPTS.*/KUBELET_OPTS %s/' /etc/systemd/system/kubelet.service" % KUBELET_ARGS_TO_ADD,
"sudo systemctl daemon-reload",
"sudo systemctl restart kubelet"
]
def get_nodes():
command = ['kubectl', 'get', 'nodes']
res = subprocess.check_output(command)
nodes = []
for line in res.split('\n')[1:]:
node = line.split(' ')[0]
if node != '':
nodes.append(node)
return nodes
def run_on_node(node_name, command):
print("node: %s running: %s" % (node_name, command))
subprocess.call(['gcloud', 'compute', 'ssh', '--project='+PROJECT, '--zone='+ZONE, '--command='+command, node_name])
def main():
if sys.argv[-1] != "--yes-i-accept-that-this-is-very-risky":
print("This command is very risky and unsupported (!)")
print("Do not run this unless you know what you are doing and accept the consequences (!)")
sys.exit(-1)
# auth to the cluster
print('getting cluster auth...')
os.system(AUTH_TO_CLUSTER_COMMAND)
print('')
# get the list of nodes
print('getting nodes...')
nodes = get_nodes()
print("got %d nodes." % len(nodes))
print('')
# run our service patch command on the nodes
print('updating kubelet service on the nodes...')
for node in nodes:
print("\nupdating node: %s" % node)
for command in KUBELET_UPDATE_COMMANDS:
run_on_node(node, command)
print('\ndone')
if __name__ == '__main__':
main()