From 2769d6b9cc08038c2fe4ac657a7991a720f6f3c0 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sat, 30 Jul 2022 16:53:18 -0400 Subject: [PATCH 1/4] Minor refactoring --- src/bootstrap/config.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 203db2d3876ec..dc0fd2bdc6511 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -993,7 +993,6 @@ impl Config { // This is currently all tier 1 targets and tier 2 targets with host tools // (since others may not have CI artifacts) // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 - // FIXME: this is duplicated in bootstrap.py let supported_platforms = [ // tier 1 "aarch64-unknown-linux-gnu", @@ -1176,6 +1175,7 @@ impl Config { if config.llvm_from_ci { let triple = &config.build.triple; + let ci_llvm_bin = config.ci_llvm_root().join("bin"); let mut build_target = config .target_config .entry(config.build) @@ -1183,7 +1183,6 @@ impl Config { check_ci_llvm!(build_target.llvm_config); check_ci_llvm!(build_target.llvm_filecheck); - let ci_llvm_bin = config.out.join(&*config.build.triple).join("ci-llvm/bin"); build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); } From 8ca14aa7ff9d0c4adde9c9058acbb2edf59bb05f Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sat, 30 Jul 2022 17:11:50 -0400 Subject: [PATCH 2/4] Refactor if-available setting to work in CI This verifies if the HEAD sha matches with the detected LLVM SHA, and if not, permits usage of the detected LLVM. Otherwise, we fallback on regular non-downloaded LLVM (currently still cached with sccache, though that's still 10+ minutes on macOS). --- src/bootstrap/config.rs | 36 +----------------- src/bootstrap/native.rs | 83 ++++++++++++++++++++++++++++++++++++----- src/bootstrap/util.rs | 4 ++ 3 files changed, 79 insertions(+), 44 deletions(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index dc0fd2bdc6511..56a0c1b2d9158 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -990,41 +990,7 @@ impl Config { config.llvm_from_ci = match llvm.download_ci_llvm { Some(StringOrBool::String(s)) => { assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s); - // This is currently all tier 1 targets and tier 2 targets with host tools - // (since others may not have CI artifacts) - // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 - let supported_platforms = [ - // tier 1 - "aarch64-unknown-linux-gnu", - "i686-pc-windows-gnu", - "i686-pc-windows-msvc", - "i686-unknown-linux-gnu", - "x86_64-unknown-linux-gnu", - "x86_64-apple-darwin", - "x86_64-pc-windows-gnu", - "x86_64-pc-windows-msvc", - // tier 2 with host tools - "aarch64-apple-darwin", - "aarch64-pc-windows-msvc", - "aarch64-unknown-linux-musl", - "arm-unknown-linux-gnueabi", - "arm-unknown-linux-gnueabihf", - "armv7-unknown-linux-gnueabihf", - "mips-unknown-linux-gnu", - "mips64-unknown-linux-gnuabi64", - "mips64el-unknown-linux-gnuabi64", - "mipsel-unknown-linux-gnu", - "powerpc-unknown-linux-gnu", - "powerpc64-unknown-linux-gnu", - "powerpc64le-unknown-linux-gnu", - "riscv64gc-unknown-linux-gnu", - "s390x-unknown-linux-gnu", - "x86_64-unknown-freebsd", - "x86_64-unknown-illumos", - "x86_64-unknown-linux-musl", - "x86_64-unknown-netbsd", - ]; - supported_platforms.contains(&&*config.build.triple) + crate::native::is_ci_llvm_available(&config) } Some(StringOrBool::Bool(b)) => b, None => false, diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 1bba5e68be446..6a33b264015f3 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -114,23 +114,20 @@ pub fn prebuilt_llvm_config( Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() }) } -pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { - let config = &builder.config; - if !config.llvm_from_ci { - return; - } +/// This retrieves the LLVM sha we *want* to use, according to git history. +pub(crate) fn detect_llvm_sha(config: &crate::config::Config) -> String { let mut rev_list = config.git(); rev_list.args(&[ PathBuf::from("rev-list"), - format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(), + format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(), "-n1".into(), "--first-parent".into(), "HEAD".into(), "--".into(), - builder.src.join("src/llvm-project"), - builder.src.join("src/bootstrap/download-ci-llvm-stamp"), + config.src.join("src/llvm-project"), + config.src.join("src/bootstrap/download-ci-llvm-stamp"), // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` - builder.src.join("src/version"), + config.src.join("src/version"), ]); let llvm_sha = output(&mut rev_list); let llvm_sha = llvm_sha.trim(); @@ -143,8 +140,76 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { panic!(); } + llvm_sha.to_owned() +} + +/// Returns whether the CI-found LLVM is currently usable. +/// +/// This checks both the build triple platform to confirm we're usable at all, +/// and then verifies if the current HEAD matches the detected LLVM SHA head, +/// in which case LLVM is indicated as not available. +pub(crate) fn is_ci_llvm_available(config: &crate::config::Config) -> bool { + // This is currently all tier 1 targets and tier 2 targets with host tools + // (since others may not have CI artifacts) + // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 + let supported_platforms = [ + // tier 1 + "aarch64-unknown-linux-gnu", + "i686-pc-windows-gnu", + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", + // tier 2 with host tools + "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-musl", + "arm-unknown-linux-gnueabi", + "arm-unknown-linux-gnueabihf", + "armv7-unknown-linux-gnueabihf", + "mips-unknown-linux-gnu", + "mips64-unknown-linux-gnuabi64", + "mips64el-unknown-linux-gnuabi64", + "mipsel-unknown-linux-gnu", + "powerpc-unknown-linux-gnu", + "powerpc64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "riscv64gc-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-unknown-freebsd", + "x86_64-unknown-illumos", + "x86_64-unknown-linux-musl", + "x86_64-unknown-netbsd", + ]; + if !supported_platforms.contains(&&*config.build.triple) { + return false; + } + + if crate::util::CiEnv::is_ci() { + let llvm_sha = detect_llvm_sha(config); + let head_sha = output(config.git().arg("rev-parse").arg("HEAD")); + let head_sha = head_sha.trim(); + if llvm_sha == head_sha { + eprintln!( + "Detected LLVM as non-available: running in CI and modified LLVM in this change" + ); + return false; + } + } + + true +} + +pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { + let config = &builder.config; + if !config.llvm_from_ci { + return; + } let llvm_root = config.ci_llvm_root(); let llvm_stamp = llvm_root.join(".llvm-stamp"); + let llvm_sha = detect_llvm_sha(&config); let key = format!("{}{}", llvm_sha, config.llvm_assertions); if program_out_of_date(&llvm_stamp, &key) && !config.dry_run { download_ci_llvm(builder, &llvm_sha); diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index 1895e2901489e..3a00e258e00e5 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -258,6 +258,10 @@ impl CiEnv { } } + pub fn is_ci() -> bool { + Self::current() != CiEnv::None + } + /// If in a CI environment, forces the command to run with colors. pub fn force_coloring_in_ci(self, cmd: &mut Command) { if self != CiEnv::None { From 6c7f30114515111c53224e2de1434a324f73c0e8 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 31 Jul 2022 17:45:15 -0400 Subject: [PATCH 3/4] Enable downloading prebuilt LLVM in test builders See comment added for details on the test builder restriction. This is primarily intended for macOS CI, but is likely to be a slight win on other builders too. --- .github/workflows/ci.yml | 6 ++++++ src/bootstrap/config.rs | 2 +- src/bootstrap/native.rs | 8 +++++++- .../docker/host-x86_64/arm-android/Dockerfile | 4 ++++ .../host-x86_64/i686-gnu-nopt/Dockerfile | 4 ++++ src/ci/docker/host-x86_64/i686-gnu/Dockerfile | 3 +++ .../docker/host-x86_64/test-various/Dockerfile | 4 ++++ .../host-x86_64/x86_64-gnu-aux/Dockerfile | 4 ++++ .../host-x86_64/x86_64-gnu-debug/Dockerfile | 3 +++ .../x86_64-gnu-distcheck/Dockerfile | 4 ++++ .../x86_64-gnu-llvm-13-stage1/Dockerfile | 6 +++++- .../host-x86_64/x86_64-gnu-llvm-13/Dockerfile | 6 +++++- .../host-x86_64/x86_64-gnu-tools/Dockerfile | 4 ++++ src/ci/github-actions/ci.yml | 18 ++++++++++++++++++ src/ci/run.sh | 12 ++++++++++++ src/ci/scripts/install-clang.sh | 4 ++++ 16 files changed, 88 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7eb41fec36aba..11b115cb2ff24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -391,24 +391,28 @@ jobs: env: RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --set llvm.allow-old-toolchain" SCRIPT: make ci-mingw-subset-1 + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 os: windows-latest-xl - name: i686-mingw-2 env: RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --set llvm.allow-old-toolchain" SCRIPT: make ci-mingw-subset-2 + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 os: windows-latest-xl - name: x86_64-mingw-1 env: SCRIPT: make ci-mingw-subset-1 RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain" + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 os: windows-latest-xl - name: x86_64-mingw-2 env: SCRIPT: make ci-mingw-subset-2 RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain" + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 os: windows-latest-xl - name: dist-x86_64-msvc @@ -433,6 +437,7 @@ jobs: - name: dist-i686-mingw env: RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain" + NO_DOWNLOAD_CI_LLVM: 1 SCRIPT: python x.py dist CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 @@ -441,6 +446,7 @@ jobs: env: SCRIPT: python x.py dist RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain" + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 os: windows-latest-xl diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 56a0c1b2d9158..13e3049f2c81f 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -990,7 +990,7 @@ impl Config { config.llvm_from_ci = match llvm.download_ci_llvm { Some(StringOrBool::String(s)) => { assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s); - crate::native::is_ci_llvm_available(&config) + crate::native::is_ci_llvm_available(&config, llvm_assertions.unwrap_or(false)) } Some(StringOrBool::Bool(b)) => b, None => false, diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 6a33b264015f3..7ecf74d3068d0 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -148,7 +148,7 @@ pub(crate) fn detect_llvm_sha(config: &crate::config::Config) -> String { /// This checks both the build triple platform to confirm we're usable at all, /// and then verifies if the current HEAD matches the detected LLVM SHA head, /// in which case LLVM is indicated as not available. -pub(crate) fn is_ci_llvm_available(config: &crate::config::Config) -> bool { +pub(crate) fn is_ci_llvm_available(config: &crate::config::Config, asserts: bool) -> bool { // This is currently all tier 1 targets and tier 2 targets with host tools // (since others may not have CI artifacts) // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 @@ -187,6 +187,12 @@ pub(crate) fn is_ci_llvm_available(config: &crate::config::Config) -> bool { return false; } + let triple = &*config.build.triple; + if (triple == "aarch64-unknown-linux-gnu" || triple.contains("i686")) && asserts { + // No alt builder for aarch64-unknown-linux-gnu today. + return false; + } + if crate::util::CiEnv::is_ci() { let llvm_sha = detect_llvm_sha(config); let head_sha = output(config.git().arg("rev-parse").arg("HEAD")); diff --git a/src/ci/docker/host-x86_64/arm-android/Dockerfile b/src/ci/docker/host-x86_64/arm-android/Dockerfile index a2335687ab335..d0d367b39b493 100644 --- a/src/ci/docker/host-x86_64/arm-android/Dockerfile +++ b/src/ci/docker/host-x86_64/arm-android/Dockerfile @@ -29,6 +29,10 @@ ENV PATH=$PATH:/android/sdk/platform-tools ENV TARGETS=arm-linux-androideabi +# We are intentionally allowing an old toolchain on this builder (and that's +# incompatible with LLVM downloads today). +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS --arm-linux-androideabi-ndk=/android/ndk/arm-14 \ --set llvm.allow-old-toolchain diff --git a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile index 1f8d0a64ea48e..71ea13071bd1f 100644 --- a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile +++ b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile @@ -27,6 +27,10 @@ RUN mkdir -p /config RUN echo "[rust]" > /config/nopt-std-config.toml RUN echo "optimize = false" >> /config/nopt-std-config.toml +# We are intentionally allowing an old toolchain on this builder (and that's +# incompatible with LLVM downloads today). +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu --disable-optimize-tests \ --set llvm.allow-old-toolchain ENV SCRIPT python3 ../x.py test --stage 0 --config /config/nopt-std-config.toml library/std \ diff --git a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile index 7bca0398dea7c..ebeab3dbdb799 100644 --- a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile @@ -23,6 +23,9 @@ RUN sh /scripts/sccache.sh COPY scripts/cmake.sh /scripts/ RUN /scripts/cmake.sh +# We are intentionally allowing an old toolchain on this builder (and that's +# incompatible with LLVM downloads today). +ENV NO_DOWNLOAD_CI_LLVM 1 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu \ --set llvm.allow-old-toolchain # Exclude some tests that are unlikely to be platform specific, to speed up diff --git a/src/ci/docker/host-x86_64/test-various/Dockerfile b/src/ci/docker/host-x86_64/test-various/Dockerfile index 4d554a2852a09..b75e2f085cd3b 100644 --- a/src/ci/docker/host-x86_64/test-various/Dockerfile +++ b/src/ci/docker/host-x86_64/test-various/Dockerfile @@ -30,6 +30,10 @@ WORKDIR / COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +# We are disabling CI LLVM since this builder needs to build LLD, which is +# currently unsupported when downloading pre-built LLVM. +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS \ --musl-root-x86_64=/usr/local/x86_64-linux-musl \ --set build.nodejs=/node-v15.14.0-linux-x64/bin/node \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile index bfc6975c19dc9..321b3d6ace486 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile @@ -26,6 +26,10 @@ RUN sh /scripts/sccache.sh COPY scripts/cmake.sh /scripts/ RUN /scripts/cmake.sh +# We are intentionally allowing an old toolchain on this builder (and that's +# incompatible with LLVM downloads today). +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu \ --set llvm.allow-old-toolchain ENV RUST_CHECK_TARGET check-aux diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile index 13d440423b20b..739045248fea2 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile @@ -31,6 +31,9 @@ RUN sh /scripts/sccache.sh ENV RUSTBUILD_FORCE_CLANG_BASED_TESTS 1 ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1 +# llvm.use-linker conflicts with downloading CI LLVM +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --enable-debug \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile index 4bb4042cd7e2f..be4def85f14c4 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile @@ -22,6 +22,10 @@ RUN sh /scripts/sccache.sh COPY scripts/cmake.sh /scripts/ RUN /scripts/cmake.sh +# We are intentionally allowing an old toolchain on this builder (and that's +# incompatible with LLVM downloads today). +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --set rust.ignore-git=false \ --set llvm.allow-old-toolchain ENV SCRIPT python3 ../x.py --stage 2 test distcheck diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile index 21891c1244d7c..23f2215c2d93c 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile @@ -26,7 +26,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh -# using llvm-link-shared due to libffi issues -- see #34486 +# We are disabling CI LLVM since this builder is intentionally using a host +# LLVM, rather than the typical src/llvm-project LLVM. +ENV NO_DOWNLOAD_CI_LLVM 1 + +# Using llvm-link-shared due to libffi issues -- see #34486 ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --llvm-root=/usr/lib/llvm-13 \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile index 5f3af2f47caec..8f6831bc54e63 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile @@ -37,7 +37,11 @@ RUN apt-get update && \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh -# using llvm-link-shared due to libffi issues -- see #34486 +# We are disabling CI LLVM since this builder is intentionally using a host +# LLVM, rather than the typical src/llvm-project LLVM. +ENV NO_DOWNLOAD_CI_LLVM 1 + +# Using llvm-link-shared due to libffi issues -- see #34486 ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --llvm-root=/usr/lib/llvm-13 \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index f442a477a7b1e..e2ec4f275001b 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -80,6 +80,10 @@ COPY host-x86_64/x86_64-gnu-tools/browser-ui-test.version /tmp/ # the local version of the package is different than the one used by the CI. RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --unsafe-perm=true +# We are intentionally allowing an old toolchain on this builder (and that's +# incompatible with LLVM downloads today). +ENV NO_DOWNLOAD_CI_LLVM 1 + ENV RUST_CONFIGURE_ARGS \ --set llvm.allow-old-toolchain \ --build=x86_64-unknown-linux-gnu \ diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 91204eb0b0572..a6bdd9f3186b1 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -600,6 +600,9 @@ jobs: --build=i686-pc-windows-gnu --set llvm.allow-old-toolchain SCRIPT: make ci-mingw-subset-1 + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 <<: *job-windows-xl @@ -609,6 +612,9 @@ jobs: --build=i686-pc-windows-gnu --set llvm.allow-old-toolchain SCRIPT: make ci-mingw-subset-2 + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 <<: *job-windows-xl @@ -619,6 +625,9 @@ jobs: --build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 <<: *job-windows-xl @@ -629,6 +638,9 @@ jobs: --build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 <<: *job-windows-xl @@ -678,6 +690,9 @@ jobs: --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 SCRIPT: python x.py dist CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 @@ -691,6 +706,9 @@ jobs: --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain + # We are intentionally allowing an old toolchain on this builder (and that's + # incompatible with LLVM downloads today). + NO_DOWNLOAD_CI_LLVM: 1 CUSTOM_MINGW: 1 DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows-xl diff --git a/src/ci/run.sh b/src/ci/run.sh index 6545475d99641..091ccf58e2b81 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -102,6 +102,18 @@ else fi RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" + + # We enable this for non-dist builders, since those aren't trying to produce + # fresh binaries. We currently don't entirely support distributing a fresh + # copy of the compiler (including llvm tools, etc.) if we haven't actually + # built LLVM, since not everything necessary is copied into the + # local-usage-only LLVM artifacts. If that changes, this could maybe be made + # true for all builds. In practice it's probably a good idea to keep building + # LLVM continuously on at least some builders to ensure it works, though. + # (And PGO is its own can of worms). + if [ "$NO_DOWNLOAD_CI_LLVM" = "" ]; then + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.download-ci-llvm=if-available" + fi fi if [ "$RUST_RELEASE_CHANNEL" = "nightly" ] || [ "$DIST_REQUIRE_ALL_TOOLS" = "" ]; then diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh index 0bc8a0389a8d4..02b72625d6eb5 100755 --- a/src/ci/scripts/install-clang.sh +++ b/src/ci/scripts/install-clang.sh @@ -61,6 +61,10 @@ elif isWindows && [[ ${CUSTOM_MINGW-0} -ne 1 ]]; then 7z x -oclang-rust/ "LLVM-${LLVM_VERSION}-win64.exe" ciCommandSetEnv RUST_CONFIGURE_ARGS \ "${RUST_CONFIGURE_ARGS} --set llvm.clang-cl=$(pwd)/clang-rust/bin/clang-cl.exe" + + # Disable downloading CI LLVM on this builder; + # setting up clang-cl just above conflicts with the default if-available option. + ciCommandSetEnv NO_DOWNLOAD_CI_LLVM 1 fi if isWindows; then From d7b91c31207a1440732b01d357181c68e7e72a97 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Wed, 10 Aug 2022 07:16:25 -0400 Subject: [PATCH 4/4] Add global safe.directory for CI --- src/ci/docker/host-x86_64/armhf-gnu/Dockerfile | 2 ++ src/ci/docker/host-x86_64/wasm32/Dockerfile | 2 ++ src/ci/docker/static/gitconfig | 2 ++ src/ci/run.sh | 10 ++++++++++ 4 files changed, 16 insertions(+) create mode 100644 src/ci/docker/static/gitconfig diff --git a/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile b/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile index e2dbc7cfd7c64..69f88e4952019 100644 --- a/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/armhf-gnu/Dockerfile @@ -78,6 +78,8 @@ RUN curl -O https://ci-mirrors.rust-lang.org/rustc/vexpress-v2p-ca15-tc1.dtb COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +COPY static/gitconfig /etc/gitconfig + ENV RUST_CONFIGURE_ARGS --qemu-armhf-rootfs=/tmp/rootfs ENV SCRIPT python3 ../x.py --stage 2 test --host='' --target arm-unknown-linux-gnueabihf diff --git a/src/ci/docker/host-x86_64/wasm32/Dockerfile b/src/ci/docker/host-x86_64/wasm32/Dockerfile index 878c4e341589d..9e37c2822e8bd 100644 --- a/src/ci/docker/host-x86_64/wasm32/Dockerfile +++ b/src/ci/docker/host-x86_64/wasm32/Dockerfile @@ -48,6 +48,8 @@ ENV TARGETS=wasm32-unknown-emscripten # Use -O1 optimizations in the link step to reduce time spent optimizing. ENV EMCC_CFLAGS=-O1 +COPY static/gitconfig /etc/gitconfig + # Emscripten installation is user-specific ENV NO_CHANGE_USER=1 diff --git a/src/ci/docker/static/gitconfig b/src/ci/docker/static/gitconfig new file mode 100644 index 0000000000000..6bad35f01ce71 --- /dev/null +++ b/src/ci/docker/static/gitconfig @@ -0,0 +1,2 @@ +[safe] +directory = * diff --git a/src/ci/run.sh b/src/ci/run.sh index 091ccf58e2b81..9a247fb60a8ee 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -11,6 +11,16 @@ if [ "$NO_CHANGE_USER" = "" ]; then useradd --shell /bin/bash -u $LOCAL_USER_ID -o -c "" -m user export HOME=/home/user unset LOCAL_USER_ID + + # Ensure that runners are able to execute git commands in the worktree, + # overriding the typical git protections. In our docker container we're running + # as root, while the user owning the checkout is not root. + # This is only necessary when we change the user, otherwise we should + # already be running with the right user. + # + # For NO_CHANGE_USER done in the small number of Dockerfiles affected. + echo -e '[safe]\n\tdirectory = *' > /home/user/gitconfig + exec su --preserve-environment -c "env PATH=$PATH \"$0\"" user fi fi