-
Notifications
You must be signed in to change notification settings - Fork 18
/
miniswarm
executable file
·484 lines (441 loc) · 13.1 KB
/
miniswarm
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
#!/bin/bash
set -e
###################
# Output formatting
###################
DEF_COLOR=""
GREEN=""
RED=""
if ! [[ ${MS_NO_COLOR:-""} ]]; then
DEF_COLOR="\x1b[0m"
GREEN="\x1b[32;01m"
RED="\x1b[31;01m"
fi
info() {
echo -e "${GREEN}INFO: ${*}${DEF_COLOR}"
}
err() {
echo -e "${RED}ERROR: ${*}${DEF_COLOR}"
}
line_prefix() {
sed -e "s/^/[$1] /"
}
##############
# Misc helpers
##############
# Keep running command until it succeeds or we timeout
wait_for_command() {
local command="$1"
local retry=${2:-60}
local sleep_time=${3:-1}
local tries=0
while (( tries < retry ));do
tries=$((tries+1))
if eval "$command";then
echo
return 0
fi
echo -n '.'
sleep "$sleep_time"
done
echo "TIMED OUT"
return 1
}
# Generates enumerated names
# generate_names "foo" 5
# output: foo0 foo1 foo2 foo3 foo4
generate_names() {
local name=$1
local count=$2
for ((i=0; i<count; i++)); do
echo "${name}${i}"
done
}
# Based on input, generate wanted and missing instance variables
set_wanted_instances() {
wanted_managers=( $(generate_names 'ms-manager' "$num_managers") )
wanted_workers=( $(generate_names 'ms-worker' "$num_workers") )
missing_managers=( $(missing_instances "ms-manager" "${wanted_managers[@]}") )
missing_workers=( $(missing_instances "ms-worker" "${wanted_workers[@]}") )
missing_instances=( "${missing_managers[@]}" "${missing_workers[@]}" )
extra_managers=( $(extra_instances "ms-manager" "${wanted_managers[@]}") )
extra_workers=( $(extra_instances "ms-worker" "${wanted_workers[@]}") )
extra_instances=( "${extra_managers[@]}" "${extra_workers[@]}" )
}
extra_instances() {
local filter=$1
local wanted_instances=( ${@:2} )
got_instances=$(docker-machine ls -q | grep "$filter" | sort)
comm -13 <(IFS=$'\n'; echo "${wanted_instances[*]}" | sort) <(echo -e "$got_instances" | sort)
}
missing_instances() {
local filter=$1
local wanted_instances=( ${@:2} )
got_instances=$(docker-machine ls -q | grep "$filter" | sort)
comm -23 <(IFS=$'\n'; echo "${wanted_instances[*]}" | sort) <(echo -e "$got_instances" | sort)
}
open_url() {
if [[ "$OSTYPE" == "darwin"* ]]; then
open "$1"
else
xdg-open "$1"
fi
}
################
# Docker helpers
################
create_instance() {
docker-machine create -d virtualbox "$1" 2>&1 | line_prefix "$1"
}
# Create missing instances
create_instances() {
for i in "${missing_instances[@]}"; do
# If boot2docker.iso doesn't exist, then run fist command serially otherwise we error
if ! [[ -e ~/.docker/machine/cache/boot2docker.iso ]]; then
create_instance "$i"
else
create_instance "$i" &
# Stagger to avoid VirtualBox errors when too many machines are launched in parallel
sleep 3
fi
done
wait
}
# Check if node in our cluster and in a ready state
node_in_cluster() {
docker-machine ssh ms-manager0 docker node ls 2>/dev/null | grep -q "$1 *Ready"
}
# Drain and leave the cluster
leave_cluster() {
if node_in_cluster "$1"; then
docker-machine ssh ms-manager0 docker node update --availability drain "$1" | line_prefix "$1"
# Give it a few sec to drain
# FIXME: There's probably a better way to do this
sleep 3
docker-machine ssh "$1" docker swarm leave --force 2>&1 | line_prefix "$1"
fi
}
# Cause instance to leave cluster, and remove node from swarm
delete_from_cluster() {
# First we leave
leave_cluster "$1"
# Use ids in-case there are multiple matches for a given name
ids=$(docker-machine ssh ms-manager0 docker node ls -qf name="$1")
for id in $ids; do
# Wait for node to be in down state so we can remove from swarm cluster
info "Waiting for node id=$id to be in a down state"
wait_for_command "docker-machine ssh ms-manager0 docker node ls -f id=$id |grep -q Down"
docker-machine ssh ms-manager0 docker node rm "$id" > /dev/null
done
}
# Delete extra instances
# call with --force to skip leaving the cluster gracefully
delete_instances() {
if [[ ${#extra_instances[@]} -eq 0 ]]; then
return 0
fi
# Gracefully leave cluster
if [[ ${1:-""} != "--force" ]]; then
for i in "${extra_instances[@]}"; do
delete_from_cluster "$i" &
done
wait
fi
docker-machine rm -y "${extra_instances[@]}"
}
# Stop instances in our miniswarm cluster
stop_instances() {
# Cluster gets out of wack when IP addresses change
# remove workers
for i in $(docker-machine ls -q | grep "ms-\(manager\|worker\)[0-9][0-9]*"); do
[[ $i == "ms-manager0" ]] && continue
{
# if we don't do this raft consensus breaks
if [[ $i == *manager* ]]; then
info "Demoting instance: $i"
docker-machine ssh ms-manager0 docker node demote "$i"
fi
info "Stopping instance: $i"
delete_from_cluster "$i" | line_prefix "$i";
docker-machine stop "$i" | line_prefix "$i";
} &
done
wait
# Kill the leader
leave_cluster "ms-manager0" | line_prefix "ms-manager0";
docker-machine stop "ms-manager0" | line_prefix "ms-manager0";
}
# Start the instance and regenerate its certs to avoid cert issues when IP changes
start_instance() {
info "Starting instance: $1"
docker-machine start "$1"
docker-machine regenerate-certs -f "$1"
info "Waiting for instance name=$1 to be in a running state"
wait_for_command "docker-machine ls --filter name=$1 --filter state=running -q | grep -q $1"
}
# Start all stopped instances that match our naming convention
start_instances() {
to_start=( $(docker-machine ls -q --filter state=Stopped | grep "ms-\(worker\|manager\)[0-9][0-9]*" || true) )
info "Starting: ${to_start[*]}"
for i in "${to_start[@]}"; do
start_instance "$i" | line_prefix "$i" &
done
wait
}
# Silly check to see if our cluster is initialized
cluster_is_initialized() {
docker-machine ssh ms-manager0 docker node ls > /dev/null 2>&1
}
# Initialize the cluster
init_cluster() {
if ! cluster_is_initialized; then
info "Initializing cluster"
docker-machine ssh ms-manager0 docker swarm init \
--advertise-addr "$manager_ip" --listen-addr "$manager_ip" > /dev/null
fi
}
# Join the swarm cluster
join_cluster() {
if ! node_in_cluster "$1"; then
node_ip=$(docker-machine ip "$1")
docker-machine ssh "$1" docker swarm leave --force 2> /dev/null | line_prefix "$1"
docker-machine ssh "$1" docker swarm join "$manager_ip" --listen-addr "$node_ip" --token "$2" 2>&1 | line_prefix "$1"
fi
}
# Get registration tokens
lookup_and_set_tokens() {
manager_token=$(docker-machine ssh ms-manager0 docker swarm join-token -q manager)
worker_token=$(docker-machine ssh ms-manager0 docker swarm join-token -q worker)
}
# Our main mainger ip
lookup_and_set_manager_ip() {
manager_ip=$(docker-machine ip ms-manager0)
}
# Initialize cluster and add managers/workers
create_cluster() {
lookup_and_set_manager_ip
init_cluster
lookup_and_set_tokens
for i in "${wanted_managers[@]}"; do
join_cluster "$i" "$manager_token" &
done
for i in "${wanted_workers[@]}"; do
join_cluster "$i" "$worker_token" &
done
wait
}
# Scale cluster to our desired size
scale_cluster() {
set_wanted_instances
[[ ${missing_instances[*]} ]] && info "Creating: ${missing_instances[*]}"
[[ ${extra_instances[*]} ]] && info "Deleting: ${extra_instances[*]}"
create_instances
delete_instances
start_instances
create_cluster
}
# Deploy swarm visualizer, because it's awesome
deploy_swarm_visualizer() {
lookup_and_set_manager_ip
(
eval "$(docker-machine env ms-manager0)"
if [[ $(docker ps -qa -f 'Status=running' -f Name='swarm_visualizer') ]]; then
return 0
fi
if [[ $(docker ps -qa -f 'Status=exited' -f Name='swarm_visualizer') ]]; then
docker rm -vf swarm_visualizer > /dev/null
fi
docker run -it -d --name swarm_visualizer -p 5000:5000 -e HOST="$manager_ip" -e PORT=5000 -v /var/run/docker.sock:/var/run/docker.sock manomarks/visualizer
)
# Wait for it to come up
wait_for_command "curl -s http://${manager_ip}:5000 -o /dev/null"
}
# Remove all services
remove_services() {
info 'Removing deployed services'
(
eval "$(docker-machine env ms-manager0)"
docker service rm $(docker service ls -q)
)
}
##############
# CLI commands
##############
# FIXME: implement
print_help() {
script_name=$(basename "$0")
echo -e "Usage: $script_name COMMAND"
echo -e "\nCluster management commands:"
echo -e " start - start miniswarm cluster"
echo -e " stop - stop miniswarm cluster"
echo -e " scale - scale miniswarm cluster"
echo -e " delete - delete miniswarm cluster"
echo -e "\nCluster helper commands:"
echo -e " vis - Start a cluster visualization GUI"
echo -e " service - Open service url"
echo -e " health - View the health of a service"
echo -e " logs - View the logs of a service"
echo -e "\nCommand Options/Examples:"
echo -e "\nstart/scale Options:"
echo -e "You can pass in either one or two arguments to configure the size of the cluster"
echo -e "If one argument (N) is passed in, we launch 1 manager and N-1 workers"
echo -e "\nExamples:"
echo -e "# Create 1 manager and 2 workers"
echo -e "$script_name start 3"
echo -e "\n# Create 2 manager and 3 workers"
echo -e "$script_name start 2 3"
echo -e "\n# Scale cluster down to 1 manager and 1 worker"
echo -e "$script_name scale 2"
echo -e "\n# Start previously stopped cluster, or if there isn't one start a 1 manager cluster"
echo -e "$script_name start"
echo -e "\nhealth options:"
echo -e "$script_name health SERVICE [-a]"
echo -e "-a - Print health of all tasks of a service, including exited ones"
echo -e "\nlogs options:"
echo -e "$script_name logs SERVICE [-f]"
echo -e "-f - Tail the logs of the service"
}
# Same as scale, but with connection instructions at the end
cmd_start() {
info "Starting miniswarm"
cmd_scale "$@"
echo -e "\n\n"
info "Stack starup complete. To connect to your stack, run the following command:"
info 'eval $(docker-machine env ms-manager0)'
}
# Run the visualizer
cmd_vis() {
deploy_swarm_visualizer
url="http://${manager_ip}:5000"
if [[ ${1:-""} == "--url" ]]; then
echo "$url"
else
info "Launching browser with URL: $url"
open_url "$url"
fi
}
# Scale our swarm
cmd_scale() {
info "Scaling miniswarm"
case "$#" in
0)
# lookup what's already there
num_managers=$(docker-machine ls -q | grep -c "ms-manager[0-9][0-9]*" || true)
num_workers=$(docker-machine ls -q | grep -c "ms-worker[0-9][0-9]*" || true)
if ((num_managers==0)) && ((num_workers==0)); then
num_managers=1
num_workers=0
fi
;;
1)
num_managers=1
num_workers=$((num_workers + $1 - num_managers))
;;
2)
num_managers=$1
num_workers=$2
;;
*)
print_help
exit 1
esac
scale_cluster
}
cmd_stop() {
info "Stopping miniswarm"
# FIXME: Stopping and starting a cluster causes services to hang
# for now removing all services until this is fixed
remove_services
stop_instances
}
cmd_delete() {
info "Deleting miniswarm"
num_managers=0
num_workers=0
set_wanted_instances
info "Deleting: ${extra_instances[*]}"
delete_instances --force
}
cmd_service() {
lookup_and_set_manager_ip
(
eval "$(docker-machine env ms-manager0)"
port=$(docker service inspect --format '{{(index .Endpoint.Ports 0).PublishedPort}}' "$1")
url="http://${manager_ip}:${port}"
if [[ ${2:-""} == "--url" ]]; then
echo "$url"
else
info "Launching browser with URL: $url"
open_url "$url"
fi
)
}
# View healthchecks of service
cmd_health() {
while read line; do
if [[ $2 != "-a" ]]; then
grep -q 'Running' <<<$line || continue
fi
node=$(grep -o 'ms-[^0-9][^0-9]*[0-9]*' <<<$line || true)
name=$(grep -o "$1\.[0-9][0-9]*" <<<$line || true)
id=$(cut -d ' ' -f 1 <<<$line || true)
(
# docker-machine ssh doesn't like the quotes in docker inspect -f
eval "$(docker-machine env "$node")"
docker inspect -f '{{ range .State.Health.Log }}{{ println "======\nStart:" .Start "\n======"}}{{ .Output }}{{end}}' "${name}.${id}" | line_prefix "${node} ${name}.${id}"
)
done < <(docker-machine ssh ms-manager0 docker service ps "$1" | tail -n +2)
}
# View logs of service
cmd_logs() {
while read line; do
node=$(grep -o 'ms-[^0-9][^0-9]*[0-9]*' <<<$line || true)
name=$(grep -o "$1\.[0-9][0-9]*" <<<$line || true)
id=$(cut -d ' ' -f 1 <<<$line || true)
[[ $2 == "-f" ]] && opts="-f"
(
# docker-machine ssh doesn't like the quotes in docker inspect -f
eval "$(docker-machine env "$node")"
docker logs $opts "${name}.${id}" 2>&1 | line_prefix "${node} ${name}.${id}"
) &
done < <(docker-machine ssh ms-manager0 docker service ps "$1" | tail -n +2)
wait
}
main() {
case "$1" in
start)
cmd_start "${@:2}"
;;
stop)
cmd_stop
;;
scale)
cmd_scale "${@:2}"
;;
delete)
cmd_delete
;;
vis|visualize|visualizer)
cmd_vis "${@:2}"
;;
service)
cmd_service "${@:2}"
;;
health)
cmd_health "${@:2}"
;;
logs)
cmd_logs "${@:2}"
;;
""|help)
print_help
;;
*)
print_help
echo -e "\n"
err "Unknown command: $1"
exit 1
;;
esac
}
main "$@"