-
Notifications
You must be signed in to change notification settings - Fork 38
/
config.yml
298 lines (256 loc) · 10.1 KB
/
config.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
version: 2
jobs:
build:
docker:
- image: buildpack-deps:bionic-scm
working_directory: ~/repo
steps:
- run:
name: Install base apt packages
command: |
apt-get update --yes -qq
apt-get install --yes -qq python3 python3-venv git-crypt jq docker.io
- checkout
- restore_cache:
keys:
- v1-dependencies-{{ checksum "requirements.txt" }}
# fallback to using the latest cache if no exact match is found
- v1-dependencies-
- run:
name: install dependencies
command: |
python3 -m venv venv
source venv/bin/activate
pip install --upgrade -r requirements.txt
# Install latest repo2docker to https://github.com/jupyter/repo2docker/pull/657
pip install --upgrade git+https://github.com/jupyter/repo2docker@e976627c1e238cf654ad178456b1bf81db7aac36
echo 'export PATH="${HOME}/repo/venv/bin:$PATH"' >> ${BASH_ENV}
- setup_remote_docker
- save_cache:
paths:
- ./venv
key: v1-dependencies-{{ checksum "requirements.txt" }}
- run:
name: Determine range of commits we are building
command: |
# CircleCI doesn't have equivalent to Travis' COMMIT_RANGE
COMMIT_RANGE=$(./.circleci/get-commit-range.py)
echo ${COMMIT_RANGE}
echo "export COMMIT_RANGE='${COMMIT_RANGE}'" >> ${BASH_ENV}
- run:
name: Test building support images we need
command: |
chartpress --commit-range ${COMMIT_RANGE}
- run:
name: Test building datahub image if needed
command: |
hubploy build datahub --commit-range ${COMMIT_RANGE}
- run:
name: Test building prob140 image if needed
command: |
hubploy build prob140 --commit-range ${COMMIT_RANGE}
- run:
name: Test building julia image if needed
command: |
hubploy build julia --commit-range ${COMMIT_RANGE}
- run:
name: Test building R image if needed
command: |
hubploy build r --commit-range ${COMMIT_RANGE}
- run:
name: Test building data8x image if needed
command: |
hubploy build data8x --commit-range ${COMMIT_RANGE}
- run:
name: Test building data100 image if needed
command: |
hubploy build data100 --commit-range ${COMMIT_RANGE}
- run:
name: Test building data102 image if needed
command: |
hubploy build data102 --commit-range ${COMMIT_RANGE}
- run:
name: Test building external image if needed
command: |
hubploy build external --commit-range ${COMMIT_RANGE}
deploy:
docker:
- image: buildpack-deps:bionic-scm
working_directory: ~/repo
steps:
- run:
name: Install base apt packages
command: |
apt-get update -qq --yes
apt-get install -qq --yes python3 python3-venv git-crypt lsb-release apt-transport-https docker.io
- checkout
# Download and cache dependencies
- restore_cache:
keys:
- v1-dependencies-gcloud-226-{{ checksum "requirements.txt" }}
# fallback to using the latest cache if no exact match is found
- v1-dependencies-gcloud-226-
- run:
name: install dependencies
command: |
python3 -m venv venv
source venv/bin/activate
pip install --upgrade -r requirements.txt
# Install latest repo2docker to https://github.com/jupyter/repo2docker/pull/657
pip install --upgrade git+https://github.com/jupyter/repo2docker@e976627c1e238cf654ad178456b1bf81db7aac36
# Can be removed once https://github.com/docker/docker-py/issues/2225 is merged and released
pip install --upgrade git+https://github.com/docker/docker-py.git@b6f6e7270ef1acfe7398b99b575d22d0d37ae8bf
curl -sSL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-226.0.0-linux-x86_64.tar.gz | tar -C venv/ -xzf -
# Be careful with quote ordering here. ${PATH} must not be expanded
# Don't use ~ here - bash can interpret PATHs containing ~, but most other things can't.
# Always use full PATHs in PATH!
echo 'export PATH="${HOME}/repo/venv/bin:${HOME}/repo/venv/google-cloud-sdk/bin:${PATH}"' >> ${BASH_ENV}
- run:
name: Set distribution codename
command: |
LSB_RELEASE_CODENAME=$(lsb_release -cs)
echo $LSB_RELEASE_CODENAME
echo "export LSB_RELEASE_CODENAME='${LSB_RELEASE_CODENAME}'" >> ${BASH_ENV}
# https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-apt?view=azure-cli-latest
- run:
name: Install azure client
command: |
echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $LSB_RELEASE_CODENAME main" | \
tee /etc/apt/sources.list.d/azure-cli.list
curl -sL https://packages.microsoft.com/keys/microsoft.asc | \
gpg --dearmor | \
tee /etc/apt/trusted.gpg.d/microsoft.asc.gpg > /dev/null
apt-get update -qq --yes
apt-get install -qq --yes azure-cli
- setup_remote_docker
- save_cache:
paths:
- ./venv
key: v1-dependencies-gcloud-226-{{ checksum "requirements.txt" }}
- run:
name: Unlock our secrets
command: |
echo "${GIT_CRYPT_KEY}" | base64 -d > ~/repo/key
git crypt unlock ~/repo/key
rm ~/repo/key
- run:
name: Build datahub image if needed
command: |
hubploy build datahub --check-registry --push
- run:
name: Build prob140 image if needed
command: |
hubploy build prob140 --check-registry --push
- run:
name: Build julia image if needed
command: |
hubploy build julia --check-registry --push
- run:
name: Build R image if needed
command: |
hubploy build r --check-registry --push
- run:
name: Build data8x image if needed
command: |
hubploy build data8x --check-registry --push
- run:
name: Build data100 image if needed
command: |
hubploy build data100 --check-registry --push
- run:
name: Build data102 image if needed
command: |
hubploy build data102 --check-registry --push
- run:
name: Build external image if needed
command: |
hubploy build external --check-registry --push
- run:
name: Install helm
command: |
curl https://storage.googleapis.com/kubernetes-helm/helm-v2.12.3-linux-amd64.tar.gz | \
tar -xzf -
mv linux-amd64/helm /usr/local/bin
helm init --client-only
helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/
helm repo update
- run:
name: Post annotation to Grafana
command: |
# We get GRAFANA_API_KEY from circle secret config. FIXME: put this inside git-crypt
export PULL_REQUEST_ID=$(git log -1 --pretty=%B | head -n1 | sed 's/^.*#\([0-9]*\).*/\1/')
export AUTHOR_NAME="$(git log -1 --pretty=%aN)"
export PULL_REQUEST_TITLE="$(git log --pretty=%B -1 | tail -n+3)"
python3 scripts/post-grafana-annotation.py \
--grafana-url http://grafana.datahub.berkeley.edu\
--tag deployment-start \
"$(echo -en ${PULL_REQUEST_TITLE}\\n\\n${AUTHOR_NAME}: https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pull/${PULL_REQUEST_ID})"
- run:
name: Build support images we need
command: |
# Authenticate to be able to push to our registry
# FIXME: Make a service account specifically for this?
gcloud auth activate-service-account --key-file deployments/datahub/secrets/gcr-key.json
# CircleCI doesn't have equivalent of Travis' COMMIT_RANGE, so we determine it ourself
COMMIT_RANGE=$(./.circleci/get-commit-range.py)
chartpress --push --commit-range $COMMIT_RANGE
- run:
name: Deploy datahub
command: |
hubploy deploy datahub hub ${CIRCLE_BRANCH}
- run:
name: Deploy prob140
command: |
hubploy deploy prob140 hub ${CIRCLE_BRANCH}
- run:
name: Deploy julia
command: |
hubploy deploy julia hub ${CIRCLE_BRANCH}
- run:
name: Deploy R
command: |
hubploy deploy r hub ${CIRCLE_BRANCH}
- run:
name: Deploy data8x
command: |
hubploy deploy data8x hub ${CIRCLE_BRANCH}
- run:
name: Deploy data100
command: |
hubploy deploy data100 hub ${CIRCLE_BRANCH}
- run:
name: Deploy data102
command: |
hubploy deploy data102 hub ${CIRCLE_BRANCH}
- run:
# FIXME: Only deploy on staging / prod?
name: Deploy support chart on datahub cluster (fall-2019)
command: |
gcloud auth \
activate-service-account \
--key-file deployments/datahub/secrets/gke-key.json
gcloud container clusters \
--region=us-central1 --project=ucb-datahub-2018 \
get-credentials fall-2019
helm dep up support
helm upgrade \
--install --wait \
--namespace=support \
support support/ \
-f support/secrets.yaml
workflows:
version: 2
build-and-deploy:
jobs:
- build:
filters:
branches:
ignore:
- staging
- prod
- deploy:
filters:
branches:
only:
- staging
- prod