treewide: Try to put together darwin cross bootsttrapping

NixOS · May 6, 2021 · 08d1131 · 08d1131
1 parent f529df0
commit 08d1131
Show file tree

Hide file tree

Showing 11 changed files with 209 additions and 20 deletions.
diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix
@@ -64,6 +64,7 @@ let
 
   useGccForLibs = isClang
     && libcxx == null
+    && !stdenv.targetPlatform.isDarwin
     && !(stdenv.targetPlatform.useLLVM or false)
     && !(stdenv.targetPlatform.useAndroidPrebuilt or false)
     && !(stdenv.targetPlatform.isiOS or false)

diff --git a/pkgs/development/compilers/llvm/10/default.nix b/pkgs/development/compilers/llvm/10/default.nix
@@ -1,5 +1,5 @@
 { lowPrio, newScope, pkgs, lib, stdenv, cmake, gccForLibs
-, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapBintoolsWith
+, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapCCWithoutLibc, wrapBintoolsWith
 , buildLlvmTools # tools, but from the previous stage, for cross
 , targetLlvmLibraries # libraries, but from the next stage, for cross
 }:
@@ -80,6 +80,14 @@ let
       extraBuildCommands = mkExtraBuildCommands cc;
     };
 
+    clangNoLibc = wrapCCWithoutLibc rec {
+      cc = tools.clang-unwrapped;
+      extraPackages = [
+        targetLlvmLibraries.compiler-rt
+      ];
+      extraBuildCommands = mkExtraBuildCommands cc;
+    };
+
     lld = callPackage ./lld {};
 
     lldb = callPackage ./lldb {};

diff --git a/pkgs/development/compilers/llvm/11/default.nix b/pkgs/development/compilers/llvm/11/default.nix
@@ -1,5 +1,5 @@
 { lowPrio, newScope, pkgs, lib, stdenv, cmake, gccForLibs
-, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapBintoolsWith
+, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapCCWithoutLibc, wrapBintoolsWith
 , buildLlvmTools # tools, but from the previous stage, for cross
 , targetLlvmLibraries # libraries, but from the next stage, for cross
 }:
@@ -82,6 +82,14 @@ let
       extraBuildCommands = mkExtraBuildCommands cc;
     };
 
+    clangNoLibc = wrapCCWithoutLibc rec {
+      cc = tools.clang-unwrapped;
+      extraPackages = [
+        targetLlvmLibraries.compiler-rt
+      ];
+      extraBuildCommands = mkExtraBuildCommands cc;
+    };
+
     lld = callPackage ./lld {};
 
     lldb = callPackage ./lldb {};

diff --git a/pkgs/development/compilers/llvm/5/default.nix b/pkgs/development/compilers/llvm/5/default.nix
@@ -1,5 +1,5 @@
 { lowPrio, newScope, pkgs, lib, stdenv, cmake, gccForLibs
-, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith
+, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapCCWithoutLibc
 , buildLlvmTools # tools, but from the previous stage, for cross
 , targetLlvmLibraries # libraries, but from the next stage, for cross
 }:
@@ -76,6 +76,14 @@ let
       extraBuildCommands = mkExtraBuildCommands cc;
     };
 
+    clangNoLibc = wrapCCWithoutLibc rec {
+      cc = tools.clang-unwrapped;
+      extraPackages = [
+        targetLlvmLibraries.compiler-rt
+      ];
+      extraBuildCommands = mkExtraBuildCommands cc;
+    };
+
     lld = callPackage ./lld {};
 
     lldb = callPackage ./lldb {};

diff --git a/pkgs/development/compilers/llvm/6/default.nix b/pkgs/development/compilers/llvm/6/default.nix
@@ -1,5 +1,5 @@
 { lowPrio, newScope, pkgs, lib, stdenv, cmake, gccForLibs
-, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith
+, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapCCWithoutLibc
 , buildLlvmTools # tools, but from the previous stage, for cross
 , targetLlvmLibraries # libraries, but from the next stage, for cross
 }:
@@ -77,6 +77,14 @@ let
       extraBuildCommands = mkExtraBuildCommands cc;
     };
 
+    clangNoLibc = wrapCCWithoutLibc rec {
+      cc = tools.clang-unwrapped;
+      extraPackages = [
+        targetLlvmLibraries.compiler-rt
+      ];
+      extraBuildCommands = mkExtraBuildCommands cc;
+    };
+
     lld = callPackage ./lld {};
 
     lldb = callPackage ./lldb {};

diff --git a/pkgs/development/compilers/llvm/7/default.nix b/pkgs/development/compilers/llvm/7/default.nix
@@ -1,5 +1,5 @@
 { lowPrio, newScope, pkgs, lib, stdenv, cmake, gccForLibs
-, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapBintoolsWith
+, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapCCWithoutLibc, wrapBintoolsWith
 , buildLlvmTools # tools, but from the previous stage, for cross
 , targetLlvmLibraries # libraries, but from the next stage, for cross
 }:
@@ -82,6 +82,52 @@ let
       extraBuildCommands = mkExtraBuildCommands cc;
     };
 
+    clangNoLibcxx = wrapCCWith rec {
+      cc = tools.clang-unwrapped;
+      libcxx = null;
+      bintools = wrapBintoolsWith {
+        inherit (tools) bintools;
+      };
+      extraPackages = [
+        targetLlvmLibraries.compiler-rt
+      ];
+      extraBuildCommands = ''
+        echo "-rtlib=compiler-rt" >> $out/nix-support/cc-cflags
+        echo "-B${targetLlvmLibraries.compiler-rt}/lib" >> $out/nix-support/cc-cflags
+        echo "-nostdlib++" >> $out/nix-support/cc-cflags
+      '' + mkExtraBuildCommands cc;
+    };
+
+    clangNoLibcYesCompilerRt = wrapCCWithoutLibc rec {
+      cc = tools.clang-unwrapped;
+      extraPackages = [
+        targetLlvmLibraries.compiler-rt
+      ];
+      extraBuildCommands = ''
+        echo "-rtlib=compiler-rt" >> $out/nix-support/cc-cflags
+        echo "-B${targetLlvmLibraries.compiler-rt}/lib" >> $out/nix-support/cc-cflags
+      '' + mkExtraBuildCommands cc;
+    };
+
+    # Darwin "builds" Libsystem then compiler-rt
+    clangYesLibcNoCompilerRt = wrapCCWith {
+      cc = tools.clang-unwrapped;
+      libcxx = null;
+      extraPackages = [ ];
+      extraBuildCommands = ''
+        echo "-nostartfiles" >> $out/nix-support/cc-cflags
+      '';
+    };
+
+    clangNoLibcNoCompilerRt = wrapCCWithoutLibc {
+      cc = tools.clang-unwrapped;
+      libcxx = null;
+      extraPackages = [ ];
+      extraBuildCommands = ''
+        echo "-nostartfiles" >> $out/nix-support/cc-cflags
+      '';
+    };
+
     lld = callPackage ./lld {};
 
     lldb = callPackage ./lldb {};
@@ -167,21 +213,30 @@ let
   in {
 
     compiler-rt = callPackage ./compiler-rt {
-      stdenv = if stdenv.hostPlatform.useLLVM or false
-               then overrideCC stdenv buildLlvmTools.lldClangNoCompilerRt
-               else stdenv;
+      stdenv =
+        /**/ if stdenv.hostPlatform.useLLVM or false
+          then overrideCC stdenv buildLlvmTools.lldClangNoCompilerRt
+        else if stdenv.hostPlatform != stdenv.buildPlatform && stdenv.hostPlatform.isDarwin
+          then overrideCC stdenv buildLlvmTools.clangYesLibcNoCompilerRt
+        else stdenv;
     };
 
     stdenv = overrideCC stdenv buildLlvmTools.clang;
 
     libcxxStdenv = overrideCC stdenv buildLlvmTools.libcxxClang;
 
     libcxx = callPackage ./libc++ ({} //
+      (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && stdenv.hostPlatform.isDarwin) {
+        stdenv = overrideCC stdenv buildLlvmTools.clangNoLibcxx;
+      }) //
       (lib.optionalAttrs (stdenv.hostPlatform.useLLVM or false) {
         stdenv = overrideCC stdenv buildLlvmTools.lldClangNoLibcxx;
       }));
 
     libcxxabi = callPackage ./libc++abi ({} //
+      (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && stdenv.hostPlatform.isDarwin) {
+        stdenv = overrideCC stdenv buildLlvmTools.clangYesLibcNoCompilerRt;
+      }) //
       (lib.optionalAttrs (stdenv.hostPlatform.useLLVM or false) {
         stdenv = overrideCC stdenv buildLlvmTools.lldClangNoLibcxx;
         libunwind = libraries.libunwind;

diff --git a/pkgs/development/compilers/llvm/8/default.nix b/pkgs/development/compilers/llvm/8/default.nix
@@ -1,5 +1,5 @@
 { lowPrio, newScope, pkgs, lib, stdenv, cmake, gccForLibs
-, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapBintoolsWith
+, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapCCWithoutLibc, wrapBintoolsWith
 , buildLlvmTools # tools, but from the previous stage, for cross
 , targetLlvmLibraries # libraries, but from the next stage, for cross
 }:
@@ -83,6 +83,14 @@ let
       extraBuildCommands = mkExtraBuildCommands cc;
     };
 
+    clangNoLibc = wrapCCWithoutLibc rec {
+      cc = tools.clang-unwrapped;
+      extraPackages = [
+        targetLlvmLibraries.compiler-rt
+      ];
+      extraBuildCommands = mkExtraBuildCommands cc;
+    };
+
     lld = callPackage ./lld {};
 
     lldb = callPackage ./lldb {};

diff --git a/pkgs/development/compilers/llvm/9/default.nix b/pkgs/development/compilers/llvm/9/default.nix
@@ -1,5 +1,5 @@
 { lowPrio, newScope, pkgs, lib, stdenv, cmake, gccForLibs
-, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapBintoolsWith
+, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapCCWithoutLibc, wrapBintoolsWith
 , buildLlvmTools # tools, but from the previous stage, for cross
 , targetLlvmLibraries # libraries, but from the next stage, for cross
 }:
@@ -83,6 +83,14 @@ let
       extraBuildCommands = mkExtraBuildCommands cc;
     };
 
+    clangNoLibc = wrapCCWithoutLibc rec {
+      cc = tools.clang-unwrapped;
+      extraPackages = [
+        targetLlvmLibraries.compiler-rt
+      ];
+      extraBuildCommands = mkExtraBuildCommands cc;
+    };
+
     lld = callPackage ./lld {};
 
     lldb = callPackage ./lldb {};

diff --git a/pkgs/os-specific/darwin/apple-source-releases/default.nix b/pkgs/os-specific/darwin/apple-source-releases/default.nix
@@ -240,6 +240,27 @@ let
   # Only used for bootstrapping. It’s convenient because it was the last version to come with a real makefile.
   adv_cmds-boot = applePackage "adv_cmds/boot.nix" "osx-10.5.8" "102ssayxbg9wb35mdmhswbnw0bg7js3pfd8fcbic83c5q3bqa6c6" {};
 
+  # Contains a headers-only libsystem
+  stdenvHeadersLibsystem = pkgs.crossLibcStdenv.override (old: {
+     cc = old.cc.override rec {
+       bintools = old.cc.bintools.override {
+         libc = pkgs.darwin.LibsystemCrossHeaders;
+         noLibc = false;
+       };
+       libc = bintools.libc;
+       noLibc = false;
+     };
+  });
+
+  # Contains a libsystem withot libresolv
+  stdenvBasicLibsystem = stdenvHeadersLibsystem.override (old: {
+     cc = old.cc.override rec {
+       bintools = old.cc.bintools.override {
+         libc = pkgs.darwin.LibsystemCrossBasic;
+       };
+       libc = bintools.libc;
+     };
+  });
 in
 
 developerToolsPackages_11_3_1 // macosPackages_11_0_1 // {
@@ -307,5 +328,44 @@ developerToolsPackages_11_3_1 // macosPackages_11_0_1 // {
 
     # TODO(matthewbauer):
     # To be removed, once I figure out how to build a newer Security version.
-    Security        = applePackage "Security/boot.nix" "osx-10.9.5"      "1nv0dczf67dhk17hscx52izgdcyacgyy12ag0jh6nl5hmfzsn8yy" {};
+    Security      = applePackage "Security/boot.nix" "osx-10.9.5"      "1nv0dczf67dhk17hscx52izgdcyacgyy12ag0jh6nl5hmfzsn8yy" {};
+
+    # TODO(@Ericson2314): Dedup after fewer overrides are needed, as described below
+    LibsystemCrossHeaders = pkgs.darwin.Libsystem.override {
+      # TODO most of these could unconditionally use stdenvNoCC; do that next mass rebuild.
+      stdenv = pkgs.crossLibcStdenv;
+      xnu = pkgs.darwin.xnu.override { stdenv = pkgs.crossLibcStdenv; };
+      ncurses = pkgs.buildPackages.ncurses; # TODO we are taking headers from a non-darwin package? Find a better way to get headers.
+      libpthread = pkgs.darwin.libpthread.override {
+        xnu = pkgs.darwin.xnu.override { stdenv = pkgs.crossLibcStdenv; };
+      };
+
+      headersOnly = true;
+      withLibresolv = false;
+    };
+    LibsystemCrossBasic = pkgs.darwin.LibsystemCrossHeaders.override {
+      headersOnly = false;
+      Csu = pkgs.darwin.Csu.override {
+        stdenv = stdenvHeadersLibsystem;
+      };
+    };
+    LibsystemCross = pkgs.darwin.LibsystemCrossBasic.override {
+      withLibresolv = true;
+      libresolv = pkgs.darwin.libresolv.override {
+        stdenv = stdenvBasicLibsystem;
+        configd = pkgs.darwin.configd.override {
+          stdenv = stdenvBasicLibsystem;
+          ppp = pkgs.darwin.ppp.override { stdenv = stdenvBasicLibsystem; };
+          IOKit = pkgs.darwin.IOKit.override {
+            stdenv = stdenvBasicLibsystem;
+            xnu = pkgs.darwin.xnu.override { stdenv = pkgs.crossLibcStdenv; };
+          };
+          eap8021x = pkgs.darwin.eap8021x.override { stdenv = stdenvBasicLibsystem; };
+          Security = pkgs.darwin.Security.override {
+            stdenv = stdenvBasicLibsystem;
+          };
+          xnu = pkgs.darwin.xnu.override { stdenv = pkgs.crossLibcStdenv; };
+        };
+      };
+    };
 }
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
@@ -44,11 +44,25 @@ in
       extraPackages = [];
       inherit bintools;
     };
-    allowedRequisites =
-      lib.mapNullable (rs: rs ++ [ bintools ]) (stdenv.allowedRequisites or null);
+    allowedRequisites = null;
   };
 
-  stdenvNoLibs = mkStdenvNoLibs stdenv;
+  stdenvNoLibs =
+    if stdenv.hostPlatform != stdenv.buildPlatform && (stdenv.hostPlatform.isDarwin || stdenv.hostPlatform.isDarwin.useLLVM or false)
+    then
+      # We cannot touch binutils or cc themselves, because that will cause
+      # infinite recursion. So instead, we just choose a libc based on the
+      # current platform. That meanse we won't respect whatever compiler was
+      # passed in with the stdenv stage argument.
+      #
+      # TODO It would be much better to pass the `stdenvNoCC` and *unwrapped*
+      # cc, bintools, compiler-rt equivalent, etc. and create all final stdenvs
+      # as part of the stage. Then we would never be tempted to override a
+      # later thing to to create an earlier thing (leading to infinite
+      # recursion) and we also would still respect the stage arguments choices
+      # for these things.
+      overrideCC stdenv buildPackages.llvmPackages.clangNoCompilerRt
+    else mkStdenvNoLibs stdenv;
 
   gccStdenvNoLibs = mkStdenvNoLibs gccStdenv;
   clangStdenvNoLibs = mkStdenvNoLibs clangStdenv;
@@ -10392,8 +10406,10 @@ in
   gccCrossLibcStdenv = overrideCC stdenv buildPackages.gccCrossStageStatic;
 
   crossLibcStdenv =
-    if stdenv.hostPlatform.useLLVM or false
-    then overrideCC stdenv buildPackages.llvmPackages_8.lldClangNoLibc
+    /**/ if stdenv.hostPlatform.useLLVM or false
+      then overrideCC stdenv buildPackages.llvmPackages.lldClangNoLibc
+    else if stdenv.hostPlatform.isDarwin
+      then overrideCC stdenv buildPackages.llvmPackages.clangNoLibcNoCompilerRt
     else gccCrossLibcStdenv;
 
   # The GCC used to build libc for the target platform. Normal gccs will be
@@ -11710,6 +11726,10 @@ in
     inherit cc bintools libc libcxx extraPackages zlib;
   } // extraArgs; in self);
 
+  wrapCCWithoutLibc = args: wrapCCWith ({
+    bintools = if stdenv.targetPlatform.isDarwin then darwin.binutilsNoLibc else binutilsNoLibc;
+  } // args);
+
   wrapCC = cc: wrapCCWith {
     inherit cc;
   };
@@ -12537,7 +12557,10 @@ in
     bintools = binutils-unwrapped;
     libc =
       /**/ if stdenv.targetPlatform.libc == "msvcrt" then targetPackages.windows.mingw_w64_headers
-      else if stdenv.targetPlatform.libc == "libSystem" then darwin.xcode
+      else if stdenv.targetPlatform.libc == "libSystem" then
+        if stdenv.targetPlatform.useiOSPrebuilt or false
+        then targetPackages.darwin.xcode
+        else null
       else if stdenv.targetPlatform.libc == "nblibc" then targetPackages.netbsdCross.headers
       else null;
   };
@@ -14685,7 +14708,10 @@ in
     else if name == "musl" then targetPackages.muslCross or muslCross
     else if name == "msvcrt" then targetPackages.windows.mingw_w64 or windows.mingw_w64
     else if stdenv.targetPlatform.useiOSPrebuilt then targetPackages.darwin.iosSdkPkgs.libraries or darwin.iosSdkPkgs.libraries
-    else if name == "libSystem" then targetPackages.darwin.xcode
+    else if name == "libSystem" then
+      if stdenv.targetPlatform.useiOSPrebuilt or false
+      then targetPackages.darwin.xcode
+      else targetPackages.darwin.LibsystemCross or darwin.LibsystemCross
     else if name == "nblibc" then targetPackages.netbsdCross.libc or netbsdCross.libc
     else if name == "wasilibc" then targetPackages.wasilibc or wasilibc
     else if name == "relibc" then targetPackages.relibc or relibc

diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix
@@ -216,10 +216,9 @@ let
         pkgsStatic = super';
       })] ++ overlays;
       crossOverlays = [ (import ./static.nix) ];
-    } // lib.optionalAttrs stdenv.hostPlatform.isLinux {
       crossSystem = {
         isStatic = true;
-        parsed = stdenv.hostPlatform.parsed // {
+        parsed = stdenv.hostPlatform.parsed // lib.optionalAttrs stdenv.hostPlatform.isLinux {
           abi = {
             gnu = lib.systems.parse.abis.musl;
             gnueabi = lib.systems.parse.abis.musleabi;