-
Notifications
You must be signed in to change notification settings - Fork 34
/
Makefile.bootstrapit.make
611 lines (483 loc) · 15.8 KB
/
Makefile.bootstrapit.make
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
# Helpful Links
# http://clarkgrubb.com/makefile-style-guide
# https://explainshell.com
# https://stackoverflow.com/questions/448910
# https://shiroyasha.svbtle.com/escape-sequences-a-quick-guide-1
SHELL := /bin/bash
.SHELLFLAGS := -O extglob -eo pipefail -c
.DEFAULT_GOAL := help
.SUFFIXES:
PROJECT_DIR := $(notdir $(abspath .))
ifndef DEVELOPMENT_PYTHON_VERSION
DEVELOPMENT_PYTHON_VERSION := python=3.8
endif
ifndef SUPPORTED_PYTHON_VERSIONS
SUPPORTED_PYTHON_VERSIONS := $(DEVELOPMENT_PYTHON_VERSION)
endif
PKG_NAME := $(PACKAGE_NAME)
MODULE_NAME := $(shell echo $(subst -,_,$(PACKAGE_NAME)) | tr A-Z a-z)
# TODO (mb 2018-09-23): Support for bash on windows
# perhaps we need to install conda using this
# https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe
PLATFORM = $(shell uname -s)
# miniconda is shared between projects
CONDA_ROOT := $(shell if [[ -d /opt/conda/envs ]]; then echo "/opt/conda"; else echo "$$HOME/miniconda3"; fi;)
CONDA_BIN := $(CONDA_ROOT)/bin/conda
ENV_PREFIX := $(CONDA_ROOT)/envs
DEV_ENV_NAME := \
$(subst pypy,$(PKG_NAME)_pypy,$(subst python=,$(PKG_NAME)_py,$(subst .,,$(DEVELOPMENT_PYTHON_VERSION))))
CONDA_ENV_NAMES := \
$(subst pypy,$(PKG_NAME)_pypy,$(subst python=,$(PKG_NAME)_py,$(subst .,,$(SUPPORTED_PYTHON_VERSIONS))))
CONDA_ENV_PATHS := \
$(subst pypy,$(ENV_PREFIX)/$(PKG_NAME)_pypy,$(subst python=,$(ENV_PREFIX)/$(PKG_NAME)_py,$(subst .,,$(SUPPORTED_PYTHON_VERSIONS))))
# envname/bin/python is unfortunately not always the correct
# interpreter. In the case of pypy it is either envname/bin/pypy or
# envname/bin/pypy3
CONDA_ENV_BIN_PYTHON_PATHS := \
$(shell echo "$(CONDA_ENV_PATHS)" \
| sed 's!\(_py[[:digit:]]\{1,\}\)!\1/bin/python!g' \
| sed 's!\(_pypy2[[:digit:]]\)!\1/bin/pypy!g' \
| sed 's!\(_pypy3[[:digit:]]\)!\1/bin/pypy3!g' \
)
empty :=
literal_space := $(empty) $(empty)
BDIST_WHEEL_PYTHON_TAG := py2.py3
SDIST_FILE_CMD = ls -1t dist/*.tar.gz | head -n 1
BDIST_WHEEL_FILE_CMD = ls -1t dist/*.whl | head -n 1
# default version for development
DEV_ENV := $(ENV_PREFIX)/$(DEV_ENV_NAME)
DEV_ENV_PY := $(DEV_ENV)/bin/python
DOCKER := $(shell which docker)
DOCKER_BASE_IMAGE := registry.gitlab.com/mbarkhau/pycalver/base
GIT_HEAD_REV = $(shell git rev-parse --short HEAD)
DOCKER_IMAGE_VERSION = $(shell date -u +'%Y%m%dt%H%M%S')_$(GIT_HEAD_REV)
MAX_LINE_LEN = $(shell grep 'max-line-length' setup.cfg | sed 's![^0-9]\{1,\}!!')
build/envs.txt: requirements/conda.txt
@mkdir -p build/;
@if [[ ! -f $(CONDA_BIN) ]]; then \
echo "installing miniconda ..."; \
if [[ $(PLATFORM) == "Linux" ]]; then \
curl "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" --location \
> build/miniconda3.sh; \
elif [[ $(PLATFORM) == "MINGW64_NT-10.0" ]]; then \
curl "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" --location \
> build/miniconda3.sh; \
elif [[ $(PLATFORM) == "Darwin" ]]; then \
curl "https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh" --location \
> build/miniconda3.sh; \
fi; \
bash build/miniconda3.sh -b -p $(CONDA_ROOT); \
rm build/miniconda3.sh; \
fi
rm -f build/envs.txt.tmp;
@SUPPORTED_PYTHON_VERSIONS="$(SUPPORTED_PYTHON_VERSIONS)" \
CONDA_ENV_NAMES="$(CONDA_ENV_NAMES)" \
CONDA_ENV_PATHS="$(CONDA_ENV_PATHS)" \
CONDA_ENV_BIN_PYTHON_PATHS="$(CONDA_ENV_BIN_PYTHON_PATHS)" \
CONDA_BIN="$(CONDA_BIN)" \
bash scripts/setup_conda_envs.sh;
$(CONDA_BIN) env list \
| grep $(PKG_NAME) \
| rev | cut -d " " -f1 \
| rev | sort >> build/envs.txt.tmp;
mv build/envs.txt.tmp build/envs.txt;
build/deps.txt: build/envs.txt requirements/*.txt
@mkdir -p build/;
@SUPPORTED_PYTHON_VERSIONS="$(SUPPORTED_PYTHON_VERSIONS)" \
CONDA_ENV_NAMES="$(CONDA_ENV_NAMES)" \
CONDA_ENV_PATHS="$(CONDA_ENV_PATHS)" \
CONDA_ENV_BIN_PYTHON_PATHS="$(CONDA_ENV_BIN_PYTHON_PATHS)" \
CONDA_BIN="$(CONDA_BIN)" \
bash scripts/update_conda_env_deps.sh;
@echo "updating $(DEV_ENV_NAME) development deps ...";
@$(DEV_ENV_PY) -m pip install \
--disable-pip-version-check --upgrade \
--requirement=requirements/integration.txt;
@$(DEV_ENV_PY) -m pip install \
--disable-pip-version-check --upgrade \
--requirement=requirements/development.txt;
@echo "updating local vendor dep copies ...";
@$(DEV_ENV_PY) -m pip install \
--upgrade --disable-pip-version-check \
--no-deps --target=./vendor \
--requirement=requirements/vendor.txt;
@rm -f build/deps.txt.tmp;
@for env_py in $(CONDA_ENV_BIN_PYTHON_PATHS); do \
printf "\n# pip freeze for $${env_py}:\n" >> build/deps.txt.tmp; \
$${env_py} -m pip freeze >> build/deps.txt.tmp; \
printf "\n\n" >> build/deps.txt.tmp; \
done
@mv build/deps.txt.tmp build/deps.txt
## Short help message for each task.
.PHONY: help
help:
@awk '{ \
if ($$0 ~ /^.PHONY: [a-zA-Z\-\_0-9]+$$/) { \
helpCommand = substr($$0, index($$0, ":") + 2); \
if (helpMessage) { \
printf "\033[36m%-20s\033[0m %s\n", \
helpCommand, helpMessage; \
helpMessage = ""; \
} \
} else if ($$0 ~ /^[a-zA-Z\-\_0-9.\/]+:/) { \
helpCommand = substr($$0, 0, index($$0, ":")); \
if (helpMessage) { \
printf "\033[36m%-20s\033[0m %s\n", \
helpCommand, helpMessage; \
helpMessage = ""; \
} \
} else if ($$0 ~ /^##/) { \
if (! (helpMessage)) { \
helpMessage = substr($$0, 3); \
} \
} else { \
if (helpMessage) { \
print " "helpMessage \
} \
helpMessage = ""; \
} \
}' \
Makefile.bootstrapit.make Makefile
@if [[ ! -f $(DEV_ENV_PY) ]]; then \
echo "Missing python interpreter at $(DEV_ENV_PY) !"; \
echo "You problably want to first setup the virtual environments:"; \
echo ""; \
echo " make conda"; \
echo ""; \
exit 0; \
fi
@if [[ ! -f $(CONDA_BIN) ]]; then \
echo "No conda installation found!"; \
echo "You problably want to first setup the virtual environments:"; \
echo ""; \
echo " make conda"; \
echo ""; \
exit 0; \
fi
## Full help message for each task.
.PHONY: helpverbose
helpverbose:
@printf "Available make targets for \033[97m$(PKG_NAME)\033[0m:\n";
@awk '{ \
if ($$0 ~ /^.PHONY: [a-zA-Z\-\_0-9]+$$/) { \
helpCommand = substr($$0, index($$0, ":") + 2); \
if (helpMessage) { \
printf "\033[36m%-20s\033[0m %s\n", \
helpCommand, helpMessage; \
helpMessage = ""; \
} \
} else if ($$0 ~ /^[a-zA-Z\-\_0-9.\/]+:/) { \
helpCommand = substr($$0, 0, index($$0, ":")); \
if (helpMessage) { \
printf "\033[36m%-20s\033[0m %s\n", \
helpCommand, helpMessage; \
helpMessage = ""; \
} \
} else if ($$0 ~ /^##/) { \
if (helpMessage) { \
helpMessage = helpMessage"\n "substr($$0, 3); \
} else { \
helpMessage = substr($$0, 3); \
} \
} else { \
if (helpMessage) { \
print "\n "helpMessage"\n" \
} \
helpMessage = ""; \
} \
}' \
Makefile.bootstrapit.make Makefile
## -- Project Setup --
## Delete conda envs and cache 💩
.PHONY: clean
clean:
@for env_name in $(CONDA_ENV_NAMES); do \
env_py="$(ENV_PREFIX)/$${env_name}/bin/python"; \
if [[ -f $${env_py} ]]; then \
$(CONDA_BIN) env remove --name $${env_name} --yes; \
fi; \
done
rm -f build/envs.txt
rm -f build/deps.txt
rm -rf vendor/
rm -rf .mypy_cache/
rm -rf .pytest_cache/
rm -rf __pycache__/
rm -rf src/__pycache__/
rm -rf vendor/__pycache__/
@printf "\n setup/update completed ✨ 🍰 ✨ \n\n"
## Force update of dependencies by removing marker files
## Use this when you know an external dependency was
## updated, but that is not reflected in your
## requirements files.
##
## Usage: make force update
.PHONY: force
force:
rm -f build/envs.txt
rm -f build/deps.txt
rm -rf vendor/
rm -rf .mypy_cache/
rm -rf .pytest_cache/
rm -rf __pycache__/
rm -rf src/__pycache__/
rm -rf vendor/__pycache__/
## Create/Update python virtual environments
.PHONY: conda
conda: build/deps.txt
## Install git pre-push hooks
.PHONY: git_hooks
git_hooks:
@rm -f "$(PWD)/.git/hooks/pre-push"
ln -s "$(PWD)/scripts/pre-push-hook.sh" "$(PWD)/.git/hooks/pre-push"
## -- Integration --
## Run isort with --check-only
.PHONY: lint_isort
lint_isort:
@printf "isort ...\n"
@$(DEV_ENV)/bin/isort \
--check-only \
--line-width=$(MAX_LINE_LEN) \
--project $(MODULE_NAME) \
src/ test/
@printf "\e[1F\e[9C ok\n"
## Run sjfmt with --check
.PHONY: lint_fmt
lint_fmt:
@printf "sjfmt ...\n"
@$(DEV_ENV)/bin/sjfmt \
--target-version=py36 \
--skip-string-normalization \
--line-length=$(MAX_LINE_LEN) \
--check \
src/ test/ 2>&1 | sed "/All done/d" | sed "/left unchanged/d"
@printf "\e[1F\e[9C ok\n"
## Run flake8
.PHONY: lint_flake8
lint_flake8:
@rm -f reports/flake8*;
@mkdir -p "reports/";
@printf "flake8 ..\n"
@$(DEV_ENV)/bin/flake8 src/ --tee --output-file reports/flake8.txt || exit 0;
@$(DEV_ENV)/bin/flake8_junit reports/flake8.txt reports/flake8.xml >> /dev/null;
@$(DEV_ENV_PY) scripts/exit_0_if_empty.py reports/flake8.txt;
@printf "\e[1F\e[9C ok\n"
## Run pylint --errors-only.
.PHONY: lint_pylint_errors
lint_pylint_errors:
@printf "pylint ..\n";
@$(DEV_ENV)/bin/pylint --errors-only --jobs=4 --rcfile=setup.cfg \
src/ test/
@printf "\e[1F\e[9C ok\n"
## Run pylint.
.PHONY: lint_pylint
lint_pylint:
@mkdir -p "reports/";
@printf "pylint ..\n";
@$(DEV_ENV)/bin/pylint-ignore --rcfile=setup.cfg \
src/ test/
@printf "\e[1F\e[9C ok\n"
## Run pylint-ignore --update-ignorefile.
.PHONY: pylint_ignore
pylint_ignore:
$(DEV_ENV)/bin/pylint-ignore --rcfile=setup.cfg \
src/ test/ --update-ignorefile
## Run flake8 linter and check for fmt
.PHONY: lint
lint: lint_isort lint_fmt lint_flake8 lint_pylint
## Run mypy type checker
.PHONY: mypy
mypy:
@rm -rf ".mypy_cache";
@rm -rf "reports/mypycov";
@mkdir -p "reports/";
@printf "mypy ....\n"
@MYPYPATH=stubs/:vendor/ $(DEV_ENV_PY) -m mypy \
--html-report reports/mypycov \
--no-error-summary \
src/ | sed "/Generated HTML report/d"
@printf "\e[1F\e[9C ok\n"
## Run pytest unit and integration tests
.PHONY: test
test:
@rm -rf ".pytest_cache";
@rm -rf "src/__pycache__";
@rm -rf "test/__pycache__";
@rm -rf "reports/testcov/";
@rm -f "reports/pytest*";
@mkdir -p "reports/";
# First we test the local source tree using the dev environment
ENV=$${ENV-dev} \
PYTHONPATH=src/:vendor/:$$PYTHONPATH \
PATH=$(DEV_ENV)/bin:$$PATH \
$(DEV_ENV_PY) -m pytest -v \
--doctest-modules \
--verbose \
--cov-report "html:reports/testcov/" \
--cov-report term \
--html=reports/pytest/index.html \
--junitxml reports/pytest.xml \
-k "$${PYTEST_FILTER-$${FLTR}}" \
$(shell cd src/ && ls -1 */__init__.py | awk '{ sub(/\/__init__.py/, "", $$1); print "--cov "$$1 }') \
test/ src/;
# Next we install the package and run the test suite against it.
# IFS=' ' read -r -a env_py_paths <<< "$(CONDA_ENV_BIN_PYTHON_PATHS)"; \
# for i in $${!env_py_paths[@]}; do \
# env_py=$${env_py_paths[i]}; \
# $${env_py} -m pip install --upgrade .; \
# PYTHONPATH="" ENV=$${ENV-dev} $${env_py} -m pytest test/; \
# done;
@rm -rf ".pytest_cache";
@rm -rf "src/__pycache__";
@rm -rf "test/__pycache__";
## Run import sorting on src/ and test/
.PHONY: fmt_isort
fmt_isort:
@$(DEV_ENV)/bin/isort \
--line-width=$(MAX_LINE_LEN) \
--project $(MODULE_NAME) \
src/ test/;
## Run code formatter on src/ and test/
.PHONY: fmt_sjfmt
fmt_sjfmt:
@$(DEV_ENV)/bin/sjfmt \
--target-version=py36 \
--skip-string-normalization \
--line-length=$(MAX_LINE_LEN) \
src/ test/;
## Run code formatters
.PHONY: fmt
fmt: fmt_isort fmt_sjfmt
## -- Helpers --
## Shortcut for make fmt lint mypy devtest test
.PHONY: check
check: fmt lint mypy devtest test
## Start subshell with environ variables set.
.PHONY: env_subshell
env_subshell:
@bash --init-file <(echo '\
source $$HOME/.bashrc; \
source $(CONDA_ROOT)/etc/profile.d/conda.sh \
export ENV=$${ENV-dev}; \
export PYTHONPATH="src/:vendor/:$$PYTHONPATH"; \
conda activate $(DEV_ENV_NAME) \
')
## Usage: "source ./activate", to deactivate: "deactivate"
.PHONY: activate
activate:
@echo 'source $(CONDA_ROOT)/etc/profile.d/conda.sh;'
@echo 'if [[ -z $$ENV ]]; then'
@echo ' export _env_before_activate=$${ENV};'
@echo 'fi'
@echo 'if [[ -z $$PYTHONPATH ]]; then'
@echo ' export _pythonpath_before_activate=$${PYTHONPATH};'
@echo 'fi'
@echo 'export ENV=$${ENV-dev};'
@echo 'export PYTHONPATH="src/:vendor/:$$PYTHONPATH";'
@echo 'conda activate $(DEV_ENV_NAME);'
@echo 'function deactivate {'
@echo ' if [[ -z $${_env_before_activate} ]]; then'
@echo ' export ENV=$${_env_before_activate}; '
@echo ' else'
@echo ' unset ENV;'
@echo ' fi'
@echo ' if [[ -z $${_pythonpath_before_activate} ]]; then'
@echo ' export PYTHONPATH=$${_pythonpath_before_activate}; '
@echo ' else'
@echo ' unset PYTHONPATH;'
@echo ' fi'
@echo ' unset _env_before_activate;'
@echo ' unset _pythonpath_before_activate;'
@echo ' conda deactivate;'
@echo '};'
## Drop into an ipython shell with correct env variables set
.PHONY: ipy
ipy:
@ENV=$${ENV-dev} \
PYTHONPATH=src/:vendor/:$$PYTHONPATH \
PATH=$(DEV_ENV)/bin:$$PATH \
$(DEV_ENV)/bin/ipython
## Like `make test`, but with debug parameters
.PHONY: devtest
devtest:
@rm -rf "src/__pycache__";
@rm -rf "test/__pycache__";
ENV=$${ENV-dev} \
PYTHONPATH=src/:vendor/:$$PYTHONPATH \
PATH=$(DEV_ENV)/bin:$$PATH \
$(DEV_ENV_PY) -m pytest -v \
--doctest-modules \
--no-cov \
--durations 5 \
--verbose \
--capture=no \
--exitfirst \
--failed-first \
-k "$${PYTEST_FILTER-$${FLTR}}" \
test/ src/;
@rm -rf "src/__pycache__";
@rm -rf "test/__pycache__";
## Run `make lint mypy test` using docker
.PHONY: citest
citest:
$(DOCKER) build --file Dockerfile --tag tmp_citest_$(PKG_NAME) .
$(DOCKER) run --tty tmp_citest_$(PKG_NAME) make lint mypy test test_compat
## -- Build/Deploy --
# Generate Documentation
# .PHONY: doc
# doc:
# echo "Not Implemented"
## Freeze dependencies of the current development env.
## The requirements files this produces should be used
## in order to have reproducable builds, otherwise you
## should minimize the number of pinned versions in
## your requirements.
.PHONY: freeze
freeze:
$(DEV_ENV_PY) -m pip freeze \
> requirements/$(shell date -u +"%Y%m%dt%H%M%S")_freeze.txt
## Bump Version number in all files
.PHONY: bump_version
bump_version:
$(DEV_ENV)/bin/bumpver update;
## Create python sdist and bdist_wheel files
.PHONY: dist_build
dist_build:
@rm -rf build/lib3to6_out/
@rm -rf build/lib/
@rm -rf build/bdist*
$(DEV_ENV_PY) setup.py sdist;
$(DEV_ENV_PY) setup.py bdist_wheel --python-tag=$(BDIST_WHEEL_PYTHON_TAG);
@rm -rf src/*.egg-info
## Upload sdist and bdist files to pypi
.PHONY: dist_upload
dist_upload:
@if [[ "1" != "1" ]]; then \
echo "FAILSAFE! Not publishing a private package."; \
echo " To avoid this set IS_PUBLIC=1 in bootstrap.sh and run it."; \
exit 1; \
fi
$(DEV_ENV)/bin/twine check $$($(SDIST_FILE_CMD));
$(DEV_ENV)/bin/twine check $$($(BDIST_WHEEL_FILE_CMD));
$(DEV_ENV)/bin/twine upload --skip-existing \
--repository pypi-legacy \
$$($(SDIST_FILE_CMD)) $$($(BDIST_WHEEL_FILE_CMD));
## bump_version dist_build dist_upload
.PHONY: dist_publish
dist_publish: bump_version dist_build dist_upload
## Build docker images. Must be run when dependencies are added
## or updated. The main reasons this can fail are:
## 1. No ssh key at $(HOME)/.ssh/$(PKG_NAME)_gitlab_runner_id_rsa
## (which is needed to install packages from private repos
## and is copied into a temp container during the build).
## 2. Your docker daemon is not running
## 3. You're using WSL and docker is not exposed on tcp://localhost:2375
## 4. You're using WSL but didn't do export DOCKER_HOST="tcp://localhost:2375"
.PHONY: docker_build
docker_build:
$(DOCKER) build \
--file docker_base.Dockerfile \
--tag $(DOCKER_BASE_IMAGE):$(DOCKER_IMAGE_VERSION) \
--tag $(DOCKER_BASE_IMAGE) \
.;
$(DOCKER) push $(DOCKER_BASE_IMAGE)