forked from mozilla/addons-server
-
Notifications
You must be signed in to change notification settings - Fork 1
/
Makefile-os
179 lines (140 loc) · 6.63 KB
/
Makefile-os
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
####################################################################################################
# Our makefile makes use of docker compose commands. Our config files rely on environment variables
# both for passing configuration to the containers as well as configuring the compose file itself.
# Variables referenced in docker-compose*.yml should be read from .env, exported and saved in .env
####################################################################################################
DOCKER_BUILDER ?= container
DOCKER_PROGRESS ?= auto
DOCKER_PUSH ?=
BUILDX_BAKE_METADATA_FILE ?=
DOCKER_COMMIT ?= $(shell git rev-parse HEAD || echo "commit")
VERSION_BUILD_URL ?= build
BUILDX_BAKE_COMMAND := docker buildx bake web
override DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld
override BACKUPS_DIR = $(shell pwd)/backups
override EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S)
RESTORE_DIR ?= $(BACKUPS_DIR)/$(shell ls -1 backups | sort -r | head -n 1)
# Paths should be cleaned before mounting .:/data/olympia
# These are files which should be sourced from the container
# or should be fresh on every run of the project
CLEAN_PATHS := \
src/olympia.egg-info \
supervisord.pid \
version.json \
.PHONY: help_redirect
help_redirect:
@$(MAKE) help --no-print-directory
.PHONY: help_submake
help_submake:
@echo "Host only commands:"
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile-os | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@echo "\nAll other commands will be passed through to the docker 'web' container make:"
@make -f Makefile-docker help_submake
.PHONY: setup
setup: ## create configuration files version.json and .env required to run this project
for path in $(CLEAN_PATHS); do rm -rf "$(PWD)/$$path" && echo "$$path removed"; done
./scripts/setup.py
.PHONY: push_locales
push_locales: ## extracts and merges translation strings
bash ./scripts/push_l10n_extraction.sh $(ARGS)
.PHONY: update_docker
update_docker: data_export up data_restore ## update all the docker images
.PHONY: shell
shell: ## connect to a running addons-server docker shell
docker compose exec --user olympia web bash
.PHONY: rootshell
rootshell: ## connect to a running addons-server docker shell with root user
docker compose exec --user root web bash
.PHONY: data_export
data_export:
@ mkdir -p $(EXPORT_DIR)
# Extracting mysql database
docker compose exec mysqld /usr/bin/mysqldump olympia > $(EXPORT_DIR)/data_mysqld.sql
.PHONY: data_restore
data_restore:
@[ -d $(RESTORE_DIR) ] || (echo "Directory $(RESTORE_DIR) does not exist" && exit 1)
# Wait for MySQL server to be ready
docker compose exec mysqld bash \
-c 'while ! mysqladmin ping --silent; do echo "waiting"; sleep 1; done'
# Restoring mysql database
docker compose exec -T mysqld /usr/bin/mysql olympia < $(RESTORE_DIR)/data_mysqld.sql
$(MAKE) reindex_data
.PHONY: create_docker_builder
create_docker_builder: ## Create a custom builder for buildkit to efficiently build local images
docker buildx use $(DOCKER_BUILDER) 2>/dev/null || docker buildx create \
--name $(DOCKER_BUILDER) \
--driver=docker-container
BUILDX_BAKE_COMMAND += \
--progress=$(DOCKER_PROGRESS) \
--builder=$(DOCKER_BUILDER) \
ifneq ($(DOCKER_PUSH),)
BUILDX_BAKE_COMMAND += --push
else
BUILDX_BAKE_COMMAND += --load
endif
ifneq ($(BUILDX_BAKE_METADATA_FILE),)
BUILDX_BAKE_COMMAND += --metadata-file=$(BUILDX_BAKE_METADATA_FILE)
endif
.PHONY: docker_compose_config
docker_compose_config: ## Show the docker compose configuration
@docker compose config web --format json
.PHONY: docker_build_args
docker_build_args: ## Show the docker build configuration
@echo $(BUILDX_BAKE_COMMAND)
.PHONY: docker_build_config
docker_build_config:
@$(BUILDX_BAKE_COMMAND) --print
.PHONY: build_docker_image
build_docker_image: create_docker_builder ## Build the docker image
$(BUILDX_BAKE_COMMAND)
.PHONY: docker_mysqld_volume_create
docker_mysqld_volume_create: ## Create the mysqld volume
docker volume create $(DOCKER_MYSQLD_VOLUME)
.PHONY: docker_mysqld_volume_remove
docker_mysqld_volume_remove: ## Remove the mysqld volume
docker volume rm $(DOCKER_MYSQLD_VOLUME)
.PHONY: docker_compose_down
docker_compose_down: ## Stop the docker containers
docker compose down --rmi local --remove-orphans --volumes
.PHONY: docker_clean_volumes
docker_clean_volumes: ## Remove dangling volumes
docker volume prune --force
.PHONY: docker_clean_images
docker_clean_images: ## Remove dangling images
docker image prune --filter "dangling=true" --force
.PHONY: docker_clean_build_cache
docker_clean_build_cache: ## Remove buildx build cache
docker buildx prune -af
.PHONY: clean_deps
clean_deps: ## Remove all files in the ./deps directory
rm -rf ./deps/**
.PHONY: clean_docker
clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_images docker_clean_volumes docker_clean_build_cache clean_deps ## Remove all docker resources taking space on the host machine
.PHONY: docker_compose_up
docker_compose_up: docker_mysqld_volume_create ## Start the docker containers
docker compose up $(DOCKER_SERVICES) -d --wait --remove-orphans --force-recreate --quiet-pull $(ARGS)
.PHONY: docker_extract_deps
docker_extract_deps: ## Extract dependencies from the docker image to a local volume mount
# Run a fresh container from the base image to install deps. Since /deps is
# shared via a volume in docker-compose.yml, this installs deps for both web
# and worker containers, and does so without requiring the containers to be up.
# We just create dummy empty package.json and package-lock.json in deps/ so
# that docker compose doesn't create dummy ones itself, as they would be owned
# by root. They don't matter: the ones at the root directory are mounted
# instead.
touch deps/package.json
touch deps/package-lock.json
# mounting ./deps:/deps effectively removes dependencies from the /deps directory in the container
# running `update_deps` will install the dependencies in the /deps directory before running
docker compose run --rm --quiet-pull web make update_deps
.PHONY: up
up: setup docker_mysqld_volume_create docker_compose_up docker_clean_images docker_clean_volumes ## Create and start docker compose
.PHONY: down
down: docker_compose_down docker_clean_images docker_clean_volumes ## Stop the docker containers and clean up non-peristent dangling resources
.PHONY: initialize_docker
initialize_docker: up
docker compose exec --user olympia web make initialize
%: ## This directs any other recipe (command) to the web container's make.
docker compose exec --user olympia web make $(MAKECMDGOALS) ARGS=$(ARGS)
# You probably want to put new commands in Makefile-docker, unless they operate
# on multiple containers or are host-os specific.