Skip to content

Commit

Permalink
feat(IDX): enable bzlmod (#1959)
Browse files Browse the repository at this point in the history 
This turns on [Bazel modules](https://bazel.build/external/module) and
moves some of our (Bazel) dependency declarations to `MODULE.bazel`.
This means those dependencies are specified declaratively and not
imperatively as was the case in the `WORKSPACE`. This also allows us to
update some dependencies, like `rules_python`, which previously created
conflicts in transitive dependencies.

Some dependencies were also replaced (`rules_gazelle` -> `gazelle`).

This also adds a new workflow that ensures the Bazel lockfile,
`MODULE.bazel.lock`, is up to date on all the platforms we support
(Apple Intel, Apple Silicon, Linux x86). See workflow file for logic and
details.

---------

Co-authored-by: Marko Kosmerl <marko.kosmerl@dfinity.org>
  • Loading branch information
@nmattia @marko-k0
nmattia and marko-k0 authored Oct 10, 2024
1 parent dbb7b9d commit 292b462
Show file tree
Hide file tree
Showing 18 changed files with 4,807 additions and 729 deletions.
221 changes: 221 additions & 0 deletions .github/workflows/bazel-lockfile.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,221 @@
# Workflow to automatically update the bazel lockfile on pull requests and dev
# branches. In general, fails if lockfile is out of date.
# For each platform we support, we run a dry (--nobuild) Bazel build, and then
# upload the resulting lockfile as an artifact. The next platform downloads the
# lockfile artifact, and then also runs a dry build, rinse, repeat.
name: Update Bazel Lockfile
on:
# Merge groups should be checked for up-to-date lockfile
merge_group:
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
pull_request:
push:
branches:
# master is checked for up-to-date lockfile
- master
# dev branches get a new commit with updated lockfile (when necessary)
- 'dev-gh-*'

# runs for the same workflow are cancelled on PRs but not on master
# (logic copied from main workflow)
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true

jobs:
bazel-bzlmod-lockfile-apple-silicon:
name: Apple Silicon
runs-on: namespace-profile-darwin # profile created in namespace console
steps:
- uses: actions/checkout@v4
- name: Bazel dry run
run: |
bazel \
--noworkspace_rc \
--bazelrc=./bazel/conf/.bazelrc.build \
build \
--config=ci --config=macos_ci \
--test_tag_filters="test_macos,test_macos_slow,-upload" \
//rs/... //publish/binaries/... \
--lockfile_mode=update \
--nobuild
- uses: actions/upload-artifact@v4
with:
name: bazel-module-lock-apple-silicon
path: ./MODULE.bazel.lock

bazel-bzlmod-lockfile-linux:
name: Linux
container:
image: ghcr.io/dfinity/ic-build@sha256:115daa5ad5149182bb0416cbe5730f305be3bb2f48df576bc2c23067eefce84b
options: >-
-e NODE_NAME -v /cache:/cache
runs-on:
labels: dind-small
needs: bazel-bzlmod-lockfile-apple-silicon
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Before script
id: before-script
shell: bash
run: |
[ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME"
- name: Login to Dockerhub
shell: bash
run: ./ci/scripts/docker-login.sh
env:
DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }}
DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
- uses: actions/download-artifact@v4
with:
name: bazel-module-lock-apple-silicon

# Run a "build" with --nobuild so that Bazel updates the lockfile (if
# necessary). No targets will be built.
# The rest is copied from the main workflow.
- name: Bazel dry run
id: bazel-test-all
uses: ./.github/actions/bazel-test-all/
env:
CI_COMMIT_SHA: ${{ github.sha }}
CI_JOB_NAME: ${{ github.job }}
CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
CI_PIPELINE_SOURCE: ${{ github.event_name }}
CI_PROJECT_DIR: ${{ github.workspace }}
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
CI_RUN_ID: ${{ github.run_id }}
RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic"
BUILDEVENT_DATASET: "github-ci-dfinity"
with:
BAZEL_COMMAND: "build"
BAZEL_TARGETS: "//..."
BAZEL_EXTRA_ARGS: '--lockfile_mode=update --nobuild'
BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel"
BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}

- uses: actions/upload-artifact@v4
with:
name: bazel-module-lock-linux
path: ./MODULE.bazel.lock

# Same a Linux above but with our macOS self-hosted runners.
bazel-bzlmod-lockfile-macos-intel:
name: Apple Intel
needs: bazel-bzlmod-lockfile-linux
runs-on:
labels: macOS
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set PATH
run: |
echo "/usr/local/bin" >> $GITHUB_PATH
echo "$HOME/.cargo/bin:" >> $GITHUB_PATH
- name: Login to Dockerhub
shell: bash
run: ./ci/scripts/docker-login.sh
env:
DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }}
DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }}
- uses: actions/download-artifact@v4
with:
name: bazel-module-lock-linux
- name: Run Bazel Test Darwin x86-64
id: bazel-test-darwin-x86-64
uses: ./.github/actions/bazel-test-all/
env:
CI_COMMIT_SHA: ${{ github.sha }}
CI_JOB_NAME: ${{ github.job }}
CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
CI_PIPELINE_SOURCE: ${{ github.event_name }}
CI_PROJECT_DIR: ${{ github.workspace }}
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
CI_RUN_ID: ${{ github.run_id }}
RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic"
BUILDEVENT_DATASET: "github-ci-dfinity"
with:
BAZEL_CI_CONFIG: "--config=ci --config macos_ci"
BAZEL_COMMAND: build
BAZEL_EXTRA_ARGS: '--test_tag_filters=test_macos --lockfile_mode=update --nobuild'
BAZEL_STARTUP_ARGS: "--output_base /var/tmp/bazel-output/${CI_RUN_ID}"
BAZEL_TARGETS: "//rs/... //publish/binaries/..."
BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }}
- uses: actions/upload-artifact@v4
with:
name: bazel-module-lock-apple-intel
path: ./MODULE.bazel.lock

# Finally download the lockfile that went through every platform and check it
# for changes. See steps for actual logic depending on GHA event.
bazel-bzlmod-lockfile-update:
name: Check lockfile changes
needs: bazel-bzlmod-lockfile-macos-intel # the last platform
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref || github.ref }}
# set the token in case we need to push the updated lockfile
token: ${{ secrets.IDX_PUSH_TO_PR }}
- uses: actions/download-artifact@v4
with:
name: bazel-module-lock-apple-intel
- name: check and push
run: |
set -e
echo git status
git status
echo checking state
if git diff --quiet; then
# if diff is clean, then the lockfile did not need updating
echo "MODULE.bazel.lock is up to date"
exit 0
fi
# on merge groups & on master, fail the job
if [ "${{ github.event_name }}" == "merge_group" ]; then
echo "denying merge because of out-of-date MODULE.bazel.lock"
exit 1
fi
if [ "${{ github.event_name }}" == "push" ] && \
[ "${{ github.ref_name }}" == "master" ]; then
echo "MODULE.bazel.lock is out of date on master branch"
exit 1
fi
# on dev branches and PRs, update the lockfile
if [ "${{ github.event_name }}" == "push" ] && \
[[ "${{ github.ref_name }}" =~ ^dev-gh-* ]]; then
echo "updating MODULE.bazel.lock"
git config user.email "infra+github-automation@dfinity.org"
git config user.name "IDX GitHub Automation"
git add MODULE.bazel.lock
git commit -m "Update MODULE.bazel.lock"
git push
exit 1
fi
if [ "${{ github.event_name }}" == "pull_request" ]; then
echo "updating MODULE.bazel.lock"
git config user.email "infra+github-automation@dfinity.org"
git config user.name "IDX GitHub Automation"
git add MODULE.bazel.lock
git commit -m "Update MODULE.bazel.lock"
git push
exit 1
fi
# Unknown case; just fail
echo "MODULE.bazel.lock is out of date"
exit 1
16 changes: 8 additions & 8 deletions BUILD.bazel
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
load("@bazel_gazelle//:def.bzl", "gazelle")
load("@bazel_skylib//rules:common_settings.bzl", "string_setting")
load("@gazelle//:def.bzl", "gazelle")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")

package(default_visibility = ["//visibility:public"])
Expand Down Expand Up @@ -79,19 +79,19 @@ gazelle(

alias(
name = "gobin",
actual = "@go_sdk//:bin/go",
actual = "@rules_go//go",
visibility = ["//visibility:public"],
)

# Builds python dependencies
# Builds python dependencies. To update the lockfile:
# $ bazel run //:python-requirements.update
compile_pip_requirements(
name = "python-requirements",
timeout = "moderate",
requirements_in = "requirements.in",
src = "requirements.in",
# NOTE: the requirements.txt reports the system python's version, not
# the one specified in MODULE.bazel.
# https://rules-python.readthedocs.io/en/latest/pypi-dependencies.html#pip-rules
requirements_txt = "requirements.txt",
tags = [
"requires-network",
],
)

test_suite(
Expand Down
Loading

0 comments on commit 292b462

Please sign in to comment.