From ef06fe8e50ccc3b3d0077d1066623dfefc0d0d78 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Feb 2023 23:16:22 +0100 Subject: [PATCH 01/31] make-derivation.nix: Inline the call to derivation --- pkgs/stdenv/generic/make-derivation.nix | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 6bf319d07308a..9f0fe1bb45318 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -584,7 +584,30 @@ lib.extendDerivation # should be made available to Nix expressions using the # derivation (e.g., in assertions). passthru) - (derivation (derivationArg // lib.optionalAttrs envIsExportable checkedEnv)); + ( + let + drvAttrs = derivationArg // lib.optionalAttrs envIsExportable checkedEnv; + strict = builtins.derivationStrict drvAttrs; + + commonAttrs = drvAttrs // (builtins.listToAttrs outputsList) // + { all = map (x: x.value) outputsList; + inherit drvAttrs; + }; + + outputToAttrListElement = outputName: + { name = outputName; + value = commonAttrs // { + outPath = builtins.getAttr outputName strict; + drvPath = strict.drvPath; + type = "derivation"; + inherit outputName; + }; + }; + + outputsList = map outputToAttrListElement outputs; + + in (builtins.head outputsList).value + ); in fnOrAttrs: From db2571bbc9da8634d6dd141820d1e40c9e8e106c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Feb 2023 23:33:18 +0100 Subject: [PATCH 02/31] lib.extendDerivation: Allow derivationStrict-style simple outputs --- lib/customisation.nix | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/lib/customisation.nix b/lib/customisation.nix index cb3a4b561151f..20fea784fe32e 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix @@ -1,5 +1,10 @@ { lib }: - +let + inherit (lib) + isAttrs + isString + ; +in rec { @@ -209,10 +214,23 @@ rec { outputToAttrListElement = outputName: { name = outputName; value = commonAttrs // { - inherit (drv.${outputName}) type outputName; + type = drv.${outputName}.type or "derivation"; + inherit outputName; outputSpecified = true; - drvPath = assert condition; drv.${outputName}.drvPath; - outPath = assert condition; drv.${outputName}.outPath; + drvPath = assert condition; + if isAttrs drv.${outputName} + then + drv.${outputName}.drvPath + else + assert isString drv.${outputName}; + drv.${outputName}; + outPath = assert condition; + if isAttrs drv.${outputName} + then + drv.${outputName}.outPath + else + assert isString drv.${outputName}; + drv.${outputName}; } // # TODO: give the derivation control over the outputs. # `overrideAttrs` may not be the only attribute that needs From 3cbfa159bdc4d4739daad616f8b023245dfbf654 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Feb 2023 23:33:58 +0100 Subject: [PATCH 03/31] make-derivation.nix: Remove intermediate derivation output wrappers --- pkgs/stdenv/generic/make-derivation.nix | 30 +++++++++---------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 9f0fe1bb45318..81803f2052e6b 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -588,25 +588,17 @@ lib.extendDerivation let drvAttrs = derivationArg // lib.optionalAttrs envIsExportable checkedEnv; strict = builtins.derivationStrict drvAttrs; - - commonAttrs = drvAttrs // (builtins.listToAttrs outputsList) // - { all = map (x: x.value) outputsList; - inherit drvAttrs; - }; - - outputToAttrListElement = outputName: - { name = outputName; - value = commonAttrs // { - outPath = builtins.getAttr outputName strict; - drvPath = strict.drvPath; - type = "derivation"; - inherit outputName; - }; - }; - - outputsList = map outputToAttrListElement outputs; - - in (builtins.head outputsList).value + outputs = drvAttrs.outputs or ["out"]; + outputName = lib.head outputs; + in + drvAttrs + // lib.genAttrs outputs (o: strict.${o}) + // { + type = "derivation"; + inherit drvAttrs outputName; + inherit (strict) drvPath; + outPath = strict.${outputName}; + } ); in From 59db5a55570a16ab678a0b81766470043c5f5789 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Feb 2023 23:47:25 +0100 Subject: [PATCH 04/31] mkDerivation: Add __cleanAttrs ? false This removes unnecessary implementation details from the package attrset. --- pkgs/applications/misc/hello/default.nix | 2 ++ pkgs/stdenv/generic/make-derivation.nix | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkgs/applications/misc/hello/default.nix b/pkgs/applications/misc/hello/default.nix index e9b9e4f4b96e7..37f718d71cae6 100644 --- a/pkgs/applications/misc/hello/default.nix +++ b/pkgs/applications/misc/hello/default.nix @@ -18,6 +18,8 @@ stdenv.mkDerivation (finalAttrs: { doCheck = true; + __cleanAttrs = true; + passthru.tests = { version = testers.testVersion { package = hello; }; diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 81803f2052e6b..6bdff2e8e6bcc 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -288,6 +288,7 @@ else let "nativeCheckInputs" "nativeInstallCheckInputs" "__darwinAllowLocalNetworking" "__impureHostDeps" "__propagatedImpureHostDeps" + "__cleanAttrs" "sandboxProfile" "propagatedSandboxProfile"] ++ lib.optional (__structuredAttrs || envIsExportable) "env")) // (lib.optionalAttrs (attrs ? name || (attrs ? pname && attrs ? version)) { @@ -591,12 +592,13 @@ lib.extendDerivation outputs = drvAttrs.outputs or ["out"]; outputName = lib.head outputs; in - drvAttrs + lib.optionalAttrs (! attrs.__cleanAttrs or false) drvAttrs // lib.genAttrs outputs (o: strict.${o}) // { type = "derivation"; inherit drvAttrs outputName; inherit (strict) drvPath; + inherit (drvAttrs) name; outPath = strict.${outputName}; } ); From 89ff7055be569c62fbac08bcd585d244bd22245d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Feb 2023 23:54:11 +0100 Subject: [PATCH 05/31] mkDerivation: Do not expose passthru input attribute when __cleanAttrs --- pkgs/stdenv/generic/make-derivation.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 6bdff2e8e6bcc..eb69a01301468 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -579,12 +579,13 @@ lib.extendDerivation disallowedRequisites = [ ]; }); - inherit meta passthru overrideAttrs; - } // + inherit meta overrideAttrs; + } + // lib.optionalAttrs (! attrs.__cleanAttrs or false) { inherit passthru; } # Pass through extra attributes that are not inputs, but # should be made available to Nix expressions using the # derivation (e.g., in assertions). - passthru) + // passthru) ( let drvAttrs = derivationArg // lib.optionalAttrs envIsExportable checkedEnv; From 9b45ecd2a6ef5c65a4d1892d1302b307e53b1d78 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 00:07:31 +0100 Subject: [PATCH 06/31] mkDerivation: Remove inputDerivation from __cleanAttrs packages --- pkgs/stdenv/generic/make-derivation.nix | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index eb69a01301468..2c1550b62e075 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -547,7 +547,8 @@ in lib.extendDerivation validity.handled - ({ + ( + lib.optionalAttrs (! attrs.__cleanAttrs or false) { # A derivation that always builds successfully and whose runtime # dependencies are the original derivations build time dependencies # This allows easy building and distributing of all derivations @@ -578,10 +579,11 @@ lib.extendDerivation disallowedReferences = [ ]; disallowedRequisites = [ ]; }); - + inherit passthru; + } + // { inherit meta overrideAttrs; } - // lib.optionalAttrs (! attrs.__cleanAttrs or false) { inherit passthru; } # Pass through extra attributes that are not inputs, but # should be made available to Nix expressions using the # derivation (e.g., in assertions). @@ -593,11 +595,11 @@ lib.extendDerivation outputs = drvAttrs.outputs or ["out"]; outputName = lib.head outputs; in - lib.optionalAttrs (! attrs.__cleanAttrs or false) drvAttrs + lib.optionalAttrs (! attrs.__cleanAttrs or false) (drvAttrs // { inherit drvAttrs; }) // lib.genAttrs outputs (o: strict.${o}) // { type = "derivation"; - inherit drvAttrs outputName; + inherit outputName; inherit (strict) drvPath; inherit (drvAttrs) name; outPath = strict.${outputName}; From b9e72585a7b11cf78fa06afc3b9b3706edcd320c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 00:09:22 +0100 Subject: [PATCH 07/31] mkDerivation: Add name and version to __cleanAttrs packages --- pkgs/stdenv/generic/make-derivation.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 2c1550b62e075..e68810dbb38af 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -597,11 +597,12 @@ lib.extendDerivation in lib.optionalAttrs (! attrs.__cleanAttrs or false) (drvAttrs // { inherit drvAttrs; }) // lib.genAttrs outputs (o: strict.${o}) + // builtins.intersectAttrs { version = null; } drvAttrs // { type = "derivation"; inherit outputName; inherit (strict) drvPath; - inherit (drvAttrs) name; + name = drvAttrs.pname or drvAttrs.name; outPath = strict.${outputName}; } ); From 7a0bba883e4e923fce72a4b8b512cd3da091dcbb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 00:15:26 +0100 Subject: [PATCH 08/31] lib.inspectMkDerivationArgs: init, alternative to .drvAttrs for __cleanArgs packages --- lib/default.nix | 2 +- lib/derivations.nix | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/default.nix b/lib/default.nix index 0424db36b2e99..9230c3ddb2ccc 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -113,7 +113,7 @@ let inherit (self.customisation) overrideDerivation makeOverridable callPackageWith callPackagesWith extendDerivation hydraJob makeScope makeScopeWithSplicing; - inherit (self.derivations) lazyDerivation; + inherit (self.derivations) inspectMkDerivationArgs lazyDerivation; inherit (self.meta) addMetaAttrs dontDistribute setName updateName appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio hiPrioSet getLicenseFromSpdxId getExe; diff --git a/lib/derivations.nix b/lib/derivations.nix index dce98b46ddb98..ef8122f440a3a 100644 --- a/lib/derivations.nix +++ b/lib/derivations.nix @@ -98,4 +98,7 @@ in # `lazyDerivation` caller knew a shortcut, be taken from there. meta = args.meta or checked.meta; } // passthru; + + inspectMkDerivationArgs = drv: + (drv.overrideAttrs (o: { passthru.__inspectArgs = o; })).__inspectArgs; } From 3abc47ff878460e83f0bd90e6061b9826d06bead Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 00:27:22 +0100 Subject: [PATCH 09/31] pkgs.inputDerivation: Add pkg.inputDerivation alternative for __cleanAttrs packages --- pkgs/build-support/trivial-builders.nix | 33 +++++++++++++++++++++++++ pkgs/stdenv/generic/make-derivation.nix | 3 +++ 2 files changed, 36 insertions(+) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index d8b4ae5dd97f8..fb39cad750d0e 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -886,4 +886,37 @@ rec { outputHash = "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5"; preferLocalBuild = true; } "mkdir $out"; + + + # A derivation that always builds successfully and whose runtime + # dependencies are the original derivations build time dependencies + # This allows easy building and distributing of all derivations + # needed to enter a nix-shell with + # nix-build shell.nix -A inputDerivation + inputDerivation = pkg: pkg.overrideAttrs (derivationArg: { + # Add a name in case the original drv didn't have one + name = derivationArg.name or (if derivationArg?pname then derivationArg.pname + (if derivationArg?version then "-${derivationArg.version}" else "") else "inputDerivation"); + # This always only has one output + outputs = [ "out" ]; + + # Propagate the original builder and arguments, since we override + # them and they might contain references to build inputs + # NOTE: keep in sync with mkDerivation + _derivation_original_builder = derivationArg.realBuilder or stdenv.shell; + _derivation_original_args = derivationArg.args or ["-e" (derivationArg.builder or ../stdenv/generic/default-builder.sh)]; + + builder = stdenv.shell; + # The bash builtin `export` dumps all current environment variables, + # which is where all build input references end up (e.g. $PATH for + # binaries). By writing this to $out, Nix can find and register + # them as runtime dependencies (since Nix greps for store paths + # through $out to find them) + args = [ "-c" "export > $out" ]; + + # inputDerivation produces the inputs; not the outputs, so any + # restrictions on what used to be the outputs don't serve a purpose + # anymore. + disallowedReferences = [ ]; + disallowedRequisites = [ ]; + }); } diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index e68810dbb38af..1a9dc749c3147 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -315,7 +315,9 @@ else let else "${attrs.pname}${staticMarker}${hostSuffix}-${attrs.version}" ); }) // lib.optionalAttrs __structuredAttrs { env = checkedEnv; } // { + # NOTE: keep in sync with trivial-builders.nix#inputDerivation builder = attrs.realBuilder or stdenv.shell; + # NOTE: keep in sync with trivial-builders.nix#inputDerivation args = attrs.args or ["-e" (attrs.builder or ./default-builder.sh)]; inherit stdenv; @@ -549,6 +551,7 @@ lib.extendDerivation validity.handled ( lib.optionalAttrs (! attrs.__cleanAttrs or false) { + # TODO: reuse pkgs.inputDerivation instead # A derivation that always builds successfully and whose runtime # dependencies are the original derivations build time dependencies # This allows easy building and distributing of all derivations From 416741d3753f1fb19dd0db852dfa538eb0f05c1b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 11:45:35 +0100 Subject: [PATCH 10/31] mkDerivation: Add outputs to __cleanAttrs packages --- pkgs/stdenv/generic/make-derivation.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 1a9dc749c3147..50f6ab621fcb3 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -603,7 +603,7 @@ lib.extendDerivation // builtins.intersectAttrs { version = null; } drvAttrs // { type = "derivation"; - inherit outputName; + inherit outputName outputs; inherit (strict) drvPath; name = drvAttrs.pname or drvAttrs.name; outPath = strict.${outputName}; From dc32ea5daf9005bf5b9a6208793a4c36270a28fa Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 16:14:25 +0100 Subject: [PATCH 11/31] lib.hydraJob: Accept __cleanAttrs packages --- lib/customisation.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/customisation.nix b/lib/customisation.nix index 20fea784fe32e..d820dc140c4fb 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix @@ -256,7 +256,8 @@ rec { outputs = drv.outputs or ["out"]; commonAttrs = - { inherit (drv) name system meta; inherit outputs; } + { inherit (drv) name meta; inherit outputs; } + // lib.optionalAttrs (drv?system) { inherit (drv) system; } // lib.optionalAttrs (drv._hydraAggregate or false) { _hydraAggregate = true; constituents = map hydraJob (lib.flatten drv.constituents); From cd0de0e00e4ff04ee6070709d3ac5d72063cc9d2 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 17:38:48 +0100 Subject: [PATCH 12/31] hello: Expose src --- pkgs/applications/misc/hello/default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/applications/misc/hello/default.nix b/pkgs/applications/misc/hello/default.nix index 37f718d71cae6..08759f156abf5 100644 --- a/pkgs/applications/misc/hello/default.nix +++ b/pkgs/applications/misc/hello/default.nix @@ -32,6 +32,9 @@ stdenv.mkDerivation (finalAttrs: { passthru.tests.run = callPackage ./test.nix { hello = finalAttrs.finalPackage; }; + # We allow `hello.src` to be used in tests and examples. + passthru.src = finalAttrs.src; + meta = with lib; { description = "A program that produces a familiar, friendly greeting"; longDescription = '' From d02174bce3eb7ed1b562df06e51b2b9ba9d3bc55 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 19:52:56 +0100 Subject: [PATCH 13/31] mkDerivation: Keep legacy name semantics This avoids a mass-rebuild as many expressions currently use the name+version pair instead of just a name. The old meaning of name is based on the abused "name part" of store paths, but there is no real reason for the attrset to use this odd definition of name, except perhaps strong backward compatibility. --- pkgs/stdenv/generic/make-derivation.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 50f6ab621fcb3..f96f68e0910c4 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -605,7 +605,7 @@ lib.extendDerivation type = "derivation"; inherit outputName outputs; inherit (strict) drvPath; - name = drvAttrs.pname or drvAttrs.name; + inherit (drvAttrs) name; outPath = strict.${outputName}; } ); From 2377feb7cbe01dd412959381b654f27d7a581388 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 20:09:42 +0100 Subject: [PATCH 14/31] make-derivation.nix: Remove a `//` --- pkgs/stdenv/generic/make-derivation.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index f96f68e0910c4..93a5ca798e1d3 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -598,9 +598,12 @@ lib.extendDerivation outputs = drvAttrs.outputs or ["out"]; outputName = lib.head outputs; in - lib.optionalAttrs (! attrs.__cleanAttrs or false) (drvAttrs // { inherit drvAttrs; }) + ( + if attrs.__cleanAttrs or false + then builtins.intersectAttrs { version = null; } drvAttrs + else drvAttrs // { inherit drvAttrs; } + ) // lib.genAttrs outputs (o: strict.${o}) - // builtins.intersectAttrs { version = null; } drvAttrs // { type = "derivation"; inherit outputName outputs; From fac41dda05a7c63ec043d1d41e1042a701145fdd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 20:13:42 +0100 Subject: [PATCH 15/31] make-derivation.nix: Remove a `let` --- pkgs/stdenv/generic/make-derivation.nix | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 93a5ca798e1d3..80b47d0d631fa 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -545,6 +545,10 @@ else let "The β€˜env’ attribute set can only contain derivation, string, boolean or integer attributes. The β€˜${n}’ attribute is of type ${builtins.typeOf v}."; v) env; + drvAttrs = derivationArg // lib.optionalAttrs envIsExportable checkedEnv; + strict = builtins.derivationStrict drvAttrs; + outputName = lib.head outputs; + in lib.extendDerivation @@ -592,12 +596,6 @@ lib.extendDerivation # derivation (e.g., in assertions). // passthru) ( - let - drvAttrs = derivationArg // lib.optionalAttrs envIsExportable checkedEnv; - strict = builtins.derivationStrict drvAttrs; - outputs = drvAttrs.outputs or ["out"]; - outputName = lib.head outputs; - in ( if attrs.__cleanAttrs or false then builtins.intersectAttrs { version = null; } drvAttrs From b7477b9332556d90f834806fb839e3071ad35c34 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 20:16:38 +0100 Subject: [PATCH 16/31] cutomisation.nix: Remove a `//` --- lib/customisation.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/customisation.nix b/lib/customisation.nix index d820dc140c4fb..ea7e234e79a21 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix @@ -256,8 +256,9 @@ rec { outputs = drv.outputs or ["out"]; commonAttrs = - { inherit (drv) name meta; inherit outputs; } - // lib.optionalAttrs (drv?system) { inherit (drv) system; } + { inherit (drv) name meta; inherit outputs; + ${if drv?system then "system" else null} = drv.system; + } // lib.optionalAttrs (drv._hydraAggregate or false) { _hydraAggregate = true; constituents = map hydraJob (lib.flatten drv.constituents); From bc10d5956aeedc8c6352eda9e07db1137df42444 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 20:56:49 +0100 Subject: [PATCH 17/31] lib/customisation.nix: Refactor: convert if to or Should be faster. --- lib/customisation.nix | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/lib/customisation.nix b/lib/customisation.nix index ea7e234e79a21..9c00d2e047664 100644 --- a/lib/customisation.nix +++ b/lib/customisation.nix @@ -218,19 +218,15 @@ rec { inherit outputName; outputSpecified = true; drvPath = assert condition; - if isAttrs drv.${outputName} - then - drv.${outputName}.drvPath - else - assert isString drv.${outputName}; - drv.${outputName}; + drv.${outputName}.drvPath or ( + assert isString drv.drvPath; + drv.drvPath + ); outPath = assert condition; - if isAttrs drv.${outputName} - then - drv.${outputName}.outPath - else + drv.${outputName}.outPath or ( assert isString drv.${outputName}; - drv.${outputName}; + drv.${outputName} + ); } // # TODO: give the derivation control over the outputs. # `overrideAttrs` may not be the only attribute that needs From 2db600e96b636773d003202a73ca9b72372a0160 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 22:20:14 +0100 Subject: [PATCH 18/31] make-derivation.nix: Inline genAttrs Hoping to squeeze some performance out of this. --- pkgs/stdenv/generic/make-derivation.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 80b47d0d631fa..562b9aebcc5a2 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -601,7 +601,10 @@ lib.extendDerivation then builtins.intersectAttrs { version = null; } drvAttrs else drvAttrs // { inherit drvAttrs; } ) - // lib.genAttrs outputs (o: strict.${o}) + // builtins.listToAttrs (map (outputName: { + name = outputName; + value = strict.${outputName}; + }) outputs) // { type = "derivation"; inherit outputName outputs; From 7458ee07ca738c298519c9597e808007fa508734 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Feb 2023 23:07:27 +0100 Subject: [PATCH 19/31] make-derivation.nix: Save another `//` Hoping to squeeze some performance out of this. --- pkgs/stdenv/generic/make-derivation.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 562b9aebcc5a2..8a8b4d88cd01c 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -599,7 +599,7 @@ lib.extendDerivation ( if attrs.__cleanAttrs or false then builtins.intersectAttrs { version = null; } drvAttrs - else drvAttrs // { inherit drvAttrs; } + else drvAttrs ) // builtins.listToAttrs (map (outputName: { name = outputName; @@ -610,6 +610,7 @@ lib.extendDerivation inherit outputName outputs; inherit (strict) drvPath; inherit (drvAttrs) name; + ${if attrs.__cleanAttrs or false then null else "drvAttrs"} = drvAttrs; outPath = strict.${outputName}; } ); From b2c0a997ebe294e84c5334675cc915570fcc9bef Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Feb 2023 15:26:16 +0100 Subject: [PATCH 20/31] hello: Simplify --- pkgs/applications/misc/hello/default.nix | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/pkgs/applications/misc/hello/default.nix b/pkgs/applications/misc/hello/default.nix index 08759f156abf5..5585468c9a5c2 100644 --- a/pkgs/applications/misc/hello/default.nix +++ b/pkgs/applications/misc/hello/default.nix @@ -2,7 +2,6 @@ , lib , stdenv , fetchurl -, nixos , testers , hello }: @@ -18,23 +17,11 @@ stdenv.mkDerivation (finalAttrs: { doCheck = true; - __cleanAttrs = true; - passthru.tests = { version = testers.testVersion { package = hello; }; - - invariant-under-noXlibs = - testers.testEqualDerivation - "hello must not be rebuilt when environment.noXlibs is set." - hello - (nixos { environment.noXlibs = true; }).pkgs.hello; + run = callPackage ./test.nix { hello = finalAttrs.finalPackage; }; }; - passthru.tests.run = callPackage ./test.nix { hello = finalAttrs.finalPackage; }; - - # We allow `hello.src` to be used in tests and examples. - passthru.src = finalAttrs.src; - meta = with lib; { description = "A program that produces a familiar, friendly greeting"; longDescription = '' From 36582b07ad0dc038edf325cc60357245ca5e5100 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Feb 2023 16:22:24 +0100 Subject: [PATCH 21/31] salve-mundi: init A very fancy wrapped GNU hello, showcasing various Nixpkgs features. --- pkgs/applications/misc/hello/salve-mundi.nix | 93 ++++++++++++++++++++ pkgs/top-level/all-packages.nix | 2 + 2 files changed, 95 insertions(+) create mode 100644 pkgs/applications/misc/hello/salve-mundi.nix diff --git a/pkgs/applications/misc/hello/salve-mundi.nix b/pkgs/applications/misc/hello/salve-mundi.nix new file mode 100644 index 0000000000000..d88994cde3452 --- /dev/null +++ b/pkgs/applications/misc/hello/salve-mundi.nix @@ -0,0 +1,93 @@ +{ callPackage +, lib +, stdenv +, fetchurl +, nixos +, testers +, salve-mundi +, makeWrapper +, runCommand +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "salve-mundi"; + version = "2.12.1"; + + src = fetchurl { + url = "mirror://gnu/hello/hello-${finalAttrs.version}.tar.gz"; + sha256 = "sha256-jZkUKv2SV28wsM18tCqNxoCZmLxdYH2Idh9RLibH2yA="; + }; + + doCheck = true; + + + /* Wrap the program */ + + postInstall = lib.optionalString (finalAttrs.passthru.greeting != null) '' + wrapProgram $out/bin/hello --append-flags --greeting=${lib.escapeShellArg (lib.escapeShellArg finalAttrs.passthru.greeting)} + ''; + nativeBuildInputs = [ makeWrapper ]; + passthru.greeting = "SALVE MUNDI"; + + + /* Tidy up the package attributes and make them useful */ + + __cleanAttrs = true; + passthru.exe = lib.getExe finalAttrs.finalPackage; + # We allow `hello.src` to be used in tests and examples, despite __cleanAttrs + passthru.src = finalAttrs.src; + + + /* Strict deps enforce a separation of concerns that also benefits cross compilation + and, for shells, shell completions support via nativeBuildInputs */ + + strictDeps = true; + + + /* A fairly extensive suite of extra tests that we like to hold either for + the package in the Nixpkgs package set, or even for all possible overrides + of the package. */ + + passthru.tests = { + version = testers.testVersion { package = salve-mundi; }; + + # We use Nixpkgs attributes instead of `finalAttrs.finalPackage` here + # because overriding is not supported. Running the test on an overridden + # finalPackage wouldn't work, and is a bit unnecessary anyway. + invariant-under-noXlibs = + testers.testEqualDerivation + "hello must not be rebuilt when environment.noXlibs is set." + salve-mundi + (nixos { environment.noXlibs = true; }).pkgs.salve-mundi; + + run = runCommand "salve-mundi-test-run" { + nativeBuildInputs = [ finalAttrs.finalPackage ]; + } '' + diff -U3 --color=auto <(hello) <(echo 'SALVE MUNDI') + touch $out + ''; + + restore-default-greeting = callPackage ./test.nix { + hello = finalAttrs.finalPackage.overrideAttrs (o: { + passthru = o.passthru // { + greeting = null; + }; + }); + }; + }; + + meta = with lib; { + description = "A showcase of Nixpkgs features that produces an unfamiliar, archaic greeting"; + longDescription = '' + GNU Hello is a program that prints "Hello, world!" when you run it. + It is fully customizable. This package proves it, and showcases some + of the fancier things you can do with Nixpkgs. + ''; + mainProgram = "hello"; + homepage = "https://www.gnu.org/software/hello/manual/"; + changelog = "https://git.savannah.gnu.org/cgit/hello.git/plain/NEWS?h=v${finalAttrs.version}"; + license = licenses.gpl3Plus; + maintainers = [ maintainers.roberth ]; + platforms = platforms.all; + }; +}) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index fbcbb60bfcb32..5d2304dae4af9 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -30431,6 +30431,8 @@ with pkgs; hello = callPackage ../applications/misc/hello { }; + salve-mundi = callPackage ../applications/misc/hello/salve-mundi.nix { }; + hello-wayland = callPackage ../applications/graphics/hello-wayland { }; hello-unfree = callPackage ../applications/misc/hello-unfree { }; From 2295009ca39d35552807e6a2c1c7d2bbdf29b7ea Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Mar 2023 22:41:23 +0100 Subject: [PATCH 22/31] Revert "pkgs.inputDerivation: Add pkg.inputDerivation alternative for __cleanAttrs packages" Do not add this until we've decided what inputDerivation is and how it fits in with cleanAttrs and the possible devShell attribute - https://github.com/NixOS/nixpkgs/pull/206728 This reverts commit 99e1be309a70bfb8476b1892b8d102065860b2be. --- pkgs/build-support/trivial-builders.nix | 33 ------------------------- pkgs/stdenv/generic/make-derivation.nix | 3 --- 2 files changed, 36 deletions(-) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index fb39cad750d0e..d8b4ae5dd97f8 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -886,37 +886,4 @@ rec { outputHash = "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5"; preferLocalBuild = true; } "mkdir $out"; - - - # A derivation that always builds successfully and whose runtime - # dependencies are the original derivations build time dependencies - # This allows easy building and distributing of all derivations - # needed to enter a nix-shell with - # nix-build shell.nix -A inputDerivation - inputDerivation = pkg: pkg.overrideAttrs (derivationArg: { - # Add a name in case the original drv didn't have one - name = derivationArg.name or (if derivationArg?pname then derivationArg.pname + (if derivationArg?version then "-${derivationArg.version}" else "") else "inputDerivation"); - # This always only has one output - outputs = [ "out" ]; - - # Propagate the original builder and arguments, since we override - # them and they might contain references to build inputs - # NOTE: keep in sync with mkDerivation - _derivation_original_builder = derivationArg.realBuilder or stdenv.shell; - _derivation_original_args = derivationArg.args or ["-e" (derivationArg.builder or ../stdenv/generic/default-builder.sh)]; - - builder = stdenv.shell; - # The bash builtin `export` dumps all current environment variables, - # which is where all build input references end up (e.g. $PATH for - # binaries). By writing this to $out, Nix can find and register - # them as runtime dependencies (since Nix greps for store paths - # through $out to find them) - args = [ "-c" "export > $out" ]; - - # inputDerivation produces the inputs; not the outputs, so any - # restrictions on what used to be the outputs don't serve a purpose - # anymore. - disallowedReferences = [ ]; - disallowedRequisites = [ ]; - }); } diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 8a8b4d88cd01c..a75c0c45ddd23 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -315,9 +315,7 @@ else let else "${attrs.pname}${staticMarker}${hostSuffix}-${attrs.version}" ); }) // lib.optionalAttrs __structuredAttrs { env = checkedEnv; } // { - # NOTE: keep in sync with trivial-builders.nix#inputDerivation builder = attrs.realBuilder or stdenv.shell; - # NOTE: keep in sync with trivial-builders.nix#inputDerivation args = attrs.args or ["-e" (attrs.builder or ./default-builder.sh)]; inherit stdenv; @@ -555,7 +553,6 @@ lib.extendDerivation validity.handled ( lib.optionalAttrs (! attrs.__cleanAttrs or false) { - # TODO: reuse pkgs.inputDerivation instead # A derivation that always builds successfully and whose runtime # dependencies are the original derivations build time dependencies # This allows easy building and distributing of all derivations From 165a2de2476cc5160e48ba76bcab2752f7510cf0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Mar 2023 22:47:36 +0100 Subject: [PATCH 23/31] make-derivation.nix: Use formal for __cleanAttrs --- pkgs/stdenv/generic/make-derivation.nix | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index a75c0c45ddd23..92daad00f6fce 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -161,6 +161,11 @@ let # but for anything complex, be prepared to debug if enabling. , __structuredAttrs ? config.structuredAttrsByDefault or false +# Opt-in for those who want a clear separation between attributes that should +# be used and those that are internal. Also expected to perform slightly better +# when applied widely. +, __cleanAttrs ? false + , env ? { } , ... } @ attrs: @@ -552,7 +557,7 @@ in lib.extendDerivation validity.handled ( - lib.optionalAttrs (! attrs.__cleanAttrs or false) { + lib.optionalAttrs (! __cleanAttrs) { # A derivation that always builds successfully and whose runtime # dependencies are the original derivations build time dependencies # This allows easy building and distributing of all derivations @@ -594,7 +599,7 @@ lib.extendDerivation // passthru) ( ( - if attrs.__cleanAttrs or false + if __cleanAttrs then builtins.intersectAttrs { version = null; } drvAttrs else drvAttrs ) @@ -607,7 +612,7 @@ lib.extendDerivation inherit outputName outputs; inherit (strict) drvPath; inherit (drvAttrs) name; - ${if attrs.__cleanAttrs or false then null else "drvAttrs"} = drvAttrs; + ${if __cleanAttrs then null else "drvAttrs"} = drvAttrs; outPath = strict.${outputName}; } ); From ce3908327cfb40918d419476724429ae66b6fd96 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Mar 2023 22:49:14 +0100 Subject: [PATCH 24/31] make-derivation.nix: Do expose `internals` when `__cleanAttrs` Realistically, we'll want easier access than having to call a function to extract these bits. Having to go through a function means that the CLI becomes almost useless for troubleshooting, and the same for tab completion in the repl. --- pkgs/stdenv/generic/make-derivation.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 92daad00f6fce..9a9b1150565d1 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -612,7 +612,7 @@ lib.extendDerivation inherit outputName outputs; inherit (strict) drvPath; inherit (drvAttrs) name; - ${if __cleanAttrs then null else "drvAttrs"} = drvAttrs; + ${if __cleanAttrs then "internals" else "drvAttrs"} = drvAttrs; outPath = strict.${outputName}; } ); From c24b772883a6c9d0ecbb494850353d25d7b4196d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Mar 2023 22:51:33 +0100 Subject: [PATCH 25/31] Revert "lib.inspectMkDerivationArgs: init, alternative to .drvAttrs for __cleanArgs packages" This was too unergonomic to be the only method for retrieving these. By introducing the `internals` attribute for this purpose instead, we restore easy access without setting any wrong expectations regarding the stability of the contents. This reverts commit 01eea717c474d25b1636f9ee93c9f7bf95907fc6. --- lib/default.nix | 2 +- lib/derivations.nix | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/default.nix b/lib/default.nix index 9230c3ddb2ccc..0424db36b2e99 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -113,7 +113,7 @@ let inherit (self.customisation) overrideDerivation makeOverridable callPackageWith callPackagesWith extendDerivation hydraJob makeScope makeScopeWithSplicing; - inherit (self.derivations) inspectMkDerivationArgs lazyDerivation; + inherit (self.derivations) lazyDerivation; inherit (self.meta) addMetaAttrs dontDistribute setName updateName appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio hiPrioSet getLicenseFromSpdxId getExe; diff --git a/lib/derivations.nix b/lib/derivations.nix index ef8122f440a3a..dce98b46ddb98 100644 --- a/lib/derivations.nix +++ b/lib/derivations.nix @@ -98,7 +98,4 @@ in # `lazyDerivation` caller knew a shortcut, be taken from there. meta = args.meta or checked.meta; } // passthru; - - inspectMkDerivationArgs = drv: - (drv.overrideAttrs (o: { passthru.__inspectArgs = o; })).__inspectArgs; } From c09aa935a1232b82c8eeee83eaba3a991e7f4ebf Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Mar 2023 16:49:38 +0100 Subject: [PATCH 26/31] salve-mundi.nix: Explain what the file is for --- pkgs/applications/misc/hello/salve-mundi.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/applications/misc/hello/salve-mundi.nix b/pkgs/applications/misc/hello/salve-mundi.nix index d88994cde3452..b2335d46f28f0 100644 --- a/pkgs/applications/misc/hello/salve-mundi.nix +++ b/pkgs/applications/misc/hello/salve-mundi.nix @@ -1,3 +1,8 @@ +/* + This file showcases various techniques and features of Nixpkgs, whereas the + regular hello serves as more of an introductory package definition. + Salve mundi is latin for hello world. +*/ { callPackage , lib , stdenv From dbb87455bc0676dc285394de5ca06a42d7d6dd8b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Mar 2023 17:17:25 +0100 Subject: [PATCH 27/31] doc/stdenv: Document __cleanAttrs --- doc/stdenv/stdenv.chapter.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/stdenv/stdenv.chapter.md b/doc/stdenv/stdenv.chapter.md index b3f9f681da4c6..38d7b424e4ea1 100644 --- a/doc/stdenv/stdenv.chapter.md +++ b/doc/stdenv/stdenv.chapter.md @@ -4,7 +4,9 @@ The standard build environment in the Nix Packages collection provides an enviro ## Using `stdenv` {#sec-using-stdenv} -To build a package with the standard environment, you use the function `stdenv.mkDerivation`, instead of the primitive built-in function `derivation`, e.g. +To build a package with the standard environment, you use the function `stdenv.mkDerivation`. It calls the built-in `derivation` function for you, and turns its result into a package attribute set. + +A minimal invocation looks as follows. ```nix stdenv.mkDerivation { @@ -362,6 +364,19 @@ Unless set to `false`, some build systems with good support for parallel buildin ### Special variables {#special-variables} +#### `__cleanAttrs` {#var-__cleanAttrs} + +By default, `mkDerivation` will expose its arguments in the returned package attribute set. This is unnecessary and leads to some confusion and doubt. + +When `__cleanAttrs = true;` is passed to `mkDerivation`, it will return a minimal set of package attributes, which package authors can extend via [`passthru`](#var-stdenv-passthru). + +Benefits of `__cleanAttrs`: + + - Users don't have to sift through unnecessary attributes when exploring a package in the `nix repl`. + - Users can confidently use non-standard attributes that a package provides. + - Package authors know which variables are intended for use by the builder. They can change the builder environment with confidence that they don't break consumers of their package. + - It is a little bit more efficient in terms of CPU cycles and memory use. + #### `passthru` {#var-stdenv-passthru} This is an attribute set which can be filled with arbitrary values. For example: From b38a3649fc6311cf7d193424fd9a8aad9654f01a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Mar 2023 17:26:18 +0100 Subject: [PATCH 28/31] doc/stdenv: Document __structuredAttrs --- doc/stdenv/stdenv.chapter.md | 11 +++++++++++ pkgs/top-level/config.nix | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/stdenv/stdenv.chapter.md b/doc/stdenv/stdenv.chapter.md index 38d7b424e4ea1..b126529370898 100644 --- a/doc/stdenv/stdenv.chapter.md +++ b/doc/stdenv/stdenv.chapter.md @@ -377,6 +377,17 @@ Benefits of `__cleanAttrs`: - Package authors know which variables are intended for use by the builder. They can change the builder environment with confidence that they don't break consumers of their package. - It is a little bit more efficient in terms of CPU cycles and memory use. +#### `__structuredAttrs` {#var-__structuredAttrs} + +When `__structuredAttrs = true` is passed to `mkDerivation`, derivation attributes are serialized in JSON format and made available to the builder through a `.attrs.json` file. This will + + - allow large values to be passed, + - eliminate the need for `passAsFile`, + - allow for finer grained settings such as [`outputChecks`](https://nixos.org/manual/nix/unstable/language/advanced-attributes.html?highlight=__struct#adv-attr-outputChecks), + - turn non-nested arrays into bash arrays rather than a concatenated string. + +See [`__structuredAttrs` in the Nix manual](https://nixos.org/manual/nix/unstable/language/advanced-attributes.html#adv-attr-structuredAttrs). + #### `passthru` {#var-stdenv-passthru} This is an attribute set which can be filled with arbitrary values. For example: diff --git a/pkgs/top-level/config.nix b/pkgs/top-level/config.nix index 1de93a9f3fdea..f75a4167bd846 100644 --- a/pkgs/top-level/config.nix +++ b/pkgs/top-level/config.nix @@ -48,7 +48,7 @@ let }; structuredAttrsByDefault = mkMassRebuild { - feature = "set `__structuredAttrs` to true by default"; + feature = "set [`__structuredAttrs`](#var-__structuredAttrs) to true by default"; }; enableParallelBuildingByDefault = mkMassRebuild { From 854693c49316988e720cd2e72dedfafb46f45109 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Mar 2023 17:58:56 +0100 Subject: [PATCH 29/31] pkgs.tests.stdenv: Test __cleanAttrs --- pkgs/test/stdenv/default.nix | 55 ++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/pkgs/test/stdenv/default.nix b/pkgs/test/stdenv/default.nix index c7bb07f625ed6..00df0e156500b 100644 --- a/pkgs/test/stdenv/default.nix +++ b/pkgs/test/stdenv/default.nix @@ -6,6 +6,7 @@ , lib , runCommand , testers +, emptyFile }: let @@ -273,4 +274,58 @@ in }; }; + + cleanAttrs_version_is_optional = + assert (stdenv.mkDerivation { + __cleanAttrs = true; + name = "hi"; + })?version == false; + + # test ok, now make nix-build happy. + emptyFile; + + /* + Changes to mkDerivation and friends must not cause attributes to be added + unless absolutely necessary. + */ + cleanAttrs_does_not_leak_impl_details_and_is_lazy = + assert builtins.attrNames (stdenv.mkDerivation { + name = "hello"; + version = "1.0"; + __cleanAttrs = true; + outputs = ["out" "dev"]; + passthru.my-foo = throw "no need to eval this passthru.foo value"; + someNonStandardAttr = throw "no need to eval most derivation attributes"; + meta.maintainers = with lib.maintainers; [ roberth ]; + }) == [ + "all" + "dev" + + # drvPath is an implementation detail, but still required by Nix as of 2023 + # https://github.com/NixOS/nix/issues/6507 + "drvPath" + + "internals" + "meta" + + # a public attribute from passthru + "my-foo" + + "name" + "out" "outPath" "outputName" "outputs" + + # overriding is always messing with internals, but not + # covered by cleanAttrs. It would be disproportionately + # disruptive. + "overrideAttrs" + + # type = "derivation"; + "type" + + # optional, but specified in our case + "version" + ]; + + # test ok, now make nix-build happy. + emptyFile; } From a4655ce3247d7f69d75d76d9abc66fb980b19558 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 12 Mar 2023 18:09:44 +0100 Subject: [PATCH 30/31] doc/rl-2305: Add __cleanAttrs --- nixos/doc/manual/release-notes/rl-2305.section.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2305.section.md b/nixos/doc/manual/release-notes/rl-2305.section.md index b5e157cdb76ea..24a5ac94942fe 100644 --- a/nixos/doc/manual/release-notes/rl-2305.section.md +++ b/nixos/doc/manual/release-notes/rl-2305.section.md @@ -165,6 +165,10 @@ In addition to numerous new and upgraded packages, this release has the followin - Pantheon now defaults to Mutter 42 and GNOME settings daemon 42, all Pantheon packages are now tracking elementary OS 7 updates. +- `mkDerivation` does not have to leak implementation details anymore. + + Passing [`__cleanAttrs = true`](https://nixos.org/manual/nixpkgs/unstable/#var-__cleanAttrs) to `mkDerivation` returns a minimal set of package attributes that can be extended via [`passthru`](https://nixos.org/manual/nixpkgs/unstable/#var-stdenv-passthru). This reduces confusion for package users and allows for confident use of non-standard attributes provided by a package. It also gives package authors more confidence in modifying the builder environment without breaking consumers of their package, and it is slightly more efficient. + - The module for the application firewall `opensnitch` got the ability to configure rules. Available as [services.opensnitch.rules](#opt-services.opensnitch.rules) - The module `usbmuxd` now has the ability to change the package used by the daemon. In case you're experiencing issues with `usbmuxd` you can try an alternative program like `usbmuxd2`. Available as [services.usbmuxd.package](#opt-services.usbmuxd.package) From 56a414180fdd4c13462d9d5ff447b49d0fea9017 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 14 Mar 2023 17:23:42 +0100 Subject: [PATCH 31/31] mkDerivation: Add `__cleanAttrs = "warn";` The implementation with dynamic attributes is a little weird, but it performs better by avoiding some attrset copying and sorting, `//`. --- doc/stdenv/stdenv.chapter.md | 14 ++++++++++- pkgs/stdenv/generic/make-derivation.nix | 32 +++++++++++++++++++++---- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/doc/stdenv/stdenv.chapter.md b/doc/stdenv/stdenv.chapter.md index b126529370898..279e19a7d6134 100644 --- a/doc/stdenv/stdenv.chapter.md +++ b/doc/stdenv/stdenv.chapter.md @@ -368,7 +368,9 @@ Unless set to `false`, some build systems with good support for parallel buildin By default, `mkDerivation` will expose its arguments in the returned package attribute set. This is unnecessary and leads to some confusion and doubt. -When `__cleanAttrs = true;` is passed to `mkDerivation`, it will return a minimal set of package attributes, which package authors can extend via [`passthru`](#var-stdenv-passthru). +New packages may pass `__cleanAttrs = true;` to `mkDerivation`, so that it will return a minimal set of package attributes, which package authors can extend via [`passthru`](#var-stdenv-passthru). + +Existing packages may be modified to pass `__cleanAttrs = "warn";`, so that the legacy attributes remain available, but they will print a helpful warning when they are accessed. Doing this is best avoided until the packaging function used supports recursively defined arguments like [`mkDerivation` does](#mkderivation-recursive-attributes). Benefits of `__cleanAttrs`: @@ -377,6 +379,16 @@ Benefits of `__cleanAttrs`: - Package authors know which variables are intended for use by the builder. They can change the builder environment with confidence that they don't break consumers of their package. - It is a little bit more efficient in terms of CPU cycles and memory use. +##### What to do when warned {#warning-package-attr-impl-detail} + +If you encounter the warning `The attribute ... of package ... is an implementation detail`, you are invited to help us explicitly support ways in which a package attribute set may used, so that Nixpkgs contributors and users will be aware of your use case. + +Ideally the package can add support for whatever is the high level goal you are trying to achieve. This usually involves adding an attribute explicitly to the package attribute set using [`passthru`](#var-stdenv-passthru). Perhaps some commonly applied logic can be added to its value. The public attributes defined in `passthru` can make use of a [recursive package definition](#mkderivation-recursive-attributes) in order to access other parts of the `mkDerivation`-based package in a way that works with `overrideAttrs`. + +If you do not know why you got the warning, you may set environment variable `NIX_ABORT_ON_WARN=true` and pass `--show-trace` to Nix. This will cause an evaluation trace to be printed for the location of the warning. From this trace, you can usually derive which expression is responsible for getting the attribute. If you use the `nix-command` experimental feature, you also need to pass `--impure`. + +If the attribute can not reasonably be supported by the community, you may use the `internals` attribute to find the same value without the warning. We hope that this will be rare, and if you need `internals` to make some temporary hack bearable, it's perfectly reasonable to use it. + #### `__structuredAttrs` {#var-__structuredAttrs} When `__structuredAttrs = true` is passed to `mkDerivation`, derivation attributes are serialized in JSON format and made available to the builder through a `.attrs.json` file. This will diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 9a9b1150565d1..41528bf9311be 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -552,12 +552,33 @@ else let strict = builtins.derivationStrict drvAttrs; outputName = lib.head outputs; + name = attrs.name or attrs.pname; + + # begin __cleanAttrs helpers + + whatNow = + ( + if meta.maintainers != [] + then "You may work with the maintainer(s) of ${name}: ${lib.concatMapStringsSep ", " (m: m.github or m.name or m.email) meta.maintainers} (and/or the community), to add explicit support for your use case." + else "You may work with the community to add explicit support for your use case to ${name}." + ) + + " See https://nixos.org/manual/nixpkgs/unstable/#warning-package-attr-impl-detail" + ; + + warnCleanAttrs = lib.mapAttrs (k: lib.warn "The attribute ${lib.strings.escapeNixIdentifier k} of package ${name} ${if k == "passthru" then "is" else "seems to be"} an implementation detail and may be removed from the package attribute set in the future. ${whatNow}"); + + warnCleanDrvAttrs = lib.warn "The attribute drvAttrs of package ${name} is an implementation detail and may be removed from the package attribute set in the future. ${whatNow}"; + + maybeWarnCleanAttrs = if __cleanAttrs == "warn" then warnCleanAttrs else x: x; + + # end __cleanAttrs helpers + in lib.extendDerivation validity.handled ( - lib.optionalAttrs (! __cleanAttrs) { + lib.optionalAttrs (__cleanAttrs != true) (maybeWarnCleanAttrs { # A derivation that always builds successfully and whose runtime # dependencies are the original derivations build time dependencies # This allows easy building and distributing of all derivations @@ -589,7 +610,7 @@ lib.extendDerivation disallowedRequisites = [ ]; }); inherit passthru; - } + }) // { inherit meta overrideAttrs; } @@ -599,8 +620,10 @@ lib.extendDerivation // passthru) ( ( - if __cleanAttrs + if __cleanAttrs == true then builtins.intersectAttrs { version = null; } drvAttrs + else if __cleanAttrs == "warn" + then warnCleanAttrs drvAttrs else drvAttrs ) // builtins.listToAttrs (map (outputName: { @@ -612,7 +635,8 @@ lib.extendDerivation inherit outputName outputs; inherit (strict) drvPath; inherit (drvAttrs) name; - ${if __cleanAttrs then "internals" else "drvAttrs"} = drvAttrs; + ${if __cleanAttrs != false then "internals" else "drvAttrs"} = if __cleanAttrs == "warn" then warnCleanDrvAttrs drvAttrs else drvAttrs; + ${if __cleanAttrs == "warn" then "drvAttrs" else null} = warnCleanDrvAttrs drvAttrs; outPath = strict.${outputName}; } );