Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

rustPlatform.fetchCargoTarball: support pname+version #332975

Merged
merged 5 commits into from
Sep 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 4 additions & 8 deletions doc/languages-frameworks/rust.section.md
Original file line number Diff line number Diff line change
Expand Up @@ -567,8 +567,7 @@ buildPythonPackage rec {
};

cargoDeps = rustPlatform.fetchCargoTarball {
inherit src sourceRoot;
name = "${pname}-${version}";
inherit pname version src sourceRoot;
hash = "sha256-miW//pnOmww2i6SOGbkrAIdc/JMDT4FJLqdMFojZeoY=";
};

Expand Down Expand Up @@ -611,9 +610,8 @@ buildPythonPackage rec {
};

cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
inherit pname version src;
sourceRoot = "${pname}-${version}/${cargoRoot}";
name = "${pname}-${version}";
hash = "sha256-PS562W4L1NimqDV2H0jl5vYhL08H9est/pbIxSdYVfo=";
};

Expand Down Expand Up @@ -652,8 +650,7 @@ buildPythonPackage rec {
};

cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
name = "${pname}-${version}";
inherit pname version src;
hash = "sha256-heOBK8qi2nuc/Ib+I/vLzZ1fUUD/G/KTw9d7M4Hz5O0=";
};

Expand Down Expand Up @@ -697,8 +694,7 @@ stdenv.mkDerivation rec {
};

cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
name = "${pname}-${version}";
inherit pname version src;
hash = "sha256-8fa3fa+sFi5H+49B5sr2vYPkp9C9s6CcE0zv4xB8gww=";
};

Expand Down
2 changes: 1 addition & 1 deletion pkgs/applications/audio/gnome-podcasts/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ stdenv.mkDerivation rec {
};

cargoDeps = rustPlatform.fetchCargoTarball {
inherit pname version src;
inherit src;
hash = "sha256-XTfKqKs7874ak7Lzscxw8E2qcnJOWMZaaol8TpIB6Vw=";
};

Expand Down
270 changes: 155 additions & 115 deletions pkgs/build-support/rust/fetch-cargo-tarball/default.nix
Original file line number Diff line number Diff line change
@@ -1,116 +1,156 @@
{ lib, stdenv, cacert, git, cargo, python3 }:
let cargo-vendor-normalise = stdenv.mkDerivation {
name = "cargo-vendor-normalise";
src = ./cargo-vendor-normalise.py;
nativeBuildInputs = [ python3.pkgs.wrapPython ];
dontUnpack = true;
installPhase = "install -D $src $out/bin/cargo-vendor-normalise";
pythonPath = [ python3.pkgs.toml ];
postFixup = "wrapPythonPrograms";
doInstallCheck = true;
installCheckPhase = ''
# check that ../fetchcargo-default-config.toml is a fix point
reference=${../fetchcargo-default-config.toml}
< $reference $out/bin/cargo-vendor-normalise > test;
cmp test $reference
'';
preferLocalBuild = true;
};
{
lib,
stdenv,
cacert,
git,
cargo,
python3,
}:
let
cargo-vendor-normalise = stdenv.mkDerivation {
name = "cargo-vendor-normalise";
src = ./cargo-vendor-normalise.py;
nativeBuildInputs = [ python3.pkgs.wrapPython ];
dontUnpack = true;
installPhase = "install -D $src $out/bin/cargo-vendor-normalise";
pythonPath = [ python3.pkgs.toml ];
postFixup = "wrapPythonPrograms";
doInstallCheck = true;
installCheckPhase = ''
# check that ../fetchcargo-default-config.toml is a fix point
reference=${../fetchcargo-default-config.toml}
< $reference $out/bin/cargo-vendor-normalise > test;
cmp test $reference
'';
preferLocalBuild = true;
};
in
{ name ? "cargo-deps"
, src ? null
, srcs ? []
, patches ? []
, sourceRoot ? ""
, cargoUpdateHook ? ""
, nativeBuildInputs ? []
, ...
} @ args:

let hash_ =
if args ? hash then
{
outputHashAlgo = if args.hash == "" then "sha256" else null;
outputHash = args.hash;
}
else if args ? sha256 then { outputHashAlgo = "sha256"; outputHash = args.sha256; }
else throw "fetchCargoTarball requires a hash for ${name}";
in stdenv.mkDerivation ({
name = "${name}-vendor.tar.gz";
nativeBuildInputs = [ cacert git cargo-vendor-normalise cargo ] ++ nativeBuildInputs;

buildPhase = ''
runHook preBuild

# Ensure deterministic Cargo vendor builds
export SOURCE_DATE_EPOCH=1

if [[ ! -f Cargo.lock ]]; then
echo
echo "ERROR: The Cargo.lock file doesn't exist"
echo
echo "Cargo.lock is needed to make sure that cargoHash/cargoSha256 doesn't change"
echo "when the registry is updated."
echo

exit 1
fi

# Keep the original around for copyLockfile
cp Cargo.lock Cargo.lock.orig

export CARGO_HOME=$(mktemp -d cargo-home.XXX)
CARGO_CONFIG=$(mktemp cargo-config.XXXX)

if [[ -n "$NIX_CRATES_INDEX" ]]; then
cat >$CARGO_HOME/config.toml <<EOF
[source.crates-io]
replace-with = 'mirror'
[source.mirror]
registry = "$NIX_CRATES_INDEX"
EOF
fi

${cargoUpdateHook}

# Override the `http.cainfo` option usually specified in `.cargo/config`.
export CARGO_HTTP_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt

if grep '^source = "git' Cargo.lock; then
echo
echo "ERROR: The Cargo.lock contains git dependencies"
echo
echo "This is currently not supported in the fixed-output derivation fetcher."
echo "Use cargoLock.lockFile / importCargoLock instead."
echo

exit 1
fi

cargo vendor $name --respect-source-config | cargo-vendor-normalise > $CARGO_CONFIG

# Create an empty vendor directory when there is no dependency to vendor
mkdir -p $name
# Add the Cargo.lock to allow hash invalidation
cp Cargo.lock.orig $name/Cargo.lock

# Packages with git dependencies generate non-default cargo configs, so
# always install it rather than trying to write a standard default template.
install -D $CARGO_CONFIG $name/.cargo/config;

runHook postBuild
'';

# Build a reproducible tar, per instructions at https://reproducible-builds.org/docs/archives/
installPhase = ''
tar --owner=0 --group=0 --numeric-owner --format=gnu \
--sort=name --mtime="@$SOURCE_DATE_EPOCH" \
-czf $out $name
'';

inherit (hash_) outputHashAlgo outputHash;

impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ "NIX_CRATES_INDEX" ];
} // (builtins.removeAttrs args [
"name" "sha256" "cargoUpdateHook" "nativeBuildInputs"
]))
{
pname ? null,
version ? null,
name ? if args ? pname && args ? version then "${pname}-${version}" else "cargo-deps",
emilazy marked this conversation as resolved.
Show resolved Hide resolved
src ? null,
srcs ? [ ],
patches ? [ ],
sourceRoot ? "",
cargoUpdateHook ? "",
nativeBuildInputs ? [ ],
...
}@args:

assert lib.assertMsg (
(args ? pname || args ? version) -> !(args ? name)
) "Either specify `pname` with `version`, or specify `name` only, not a mix of both.";
assert lib.assertMsg (
args ? pname == args ? version
) "If `pname` is specified, `version` must be also, and vice versa.";
let
# args to remove from the final call to stdenv.mkDerivation, as we've already handled them
removedArgs = [
"name"
"pname"
"version"
"sha256"
"cargoUpdateHook"
"nativeBuildInputs"
];

hash_ =
if args ? hash then
{
outputHashAlgo = if args.hash == "" then "sha256" else null;
outputHash = args.hash;
}
else if args ? sha256 then
{
outputHashAlgo = "sha256";
outputHash = args.sha256;
}
else
throw "fetchCargoTarball requires a hash for ${name}";
in
stdenv.mkDerivation (
{
name = "${name}-vendor.tar.gz";
nativeBuildInputs = [
cacert
git
cargo-vendor-normalise
cargo
] ++ nativeBuildInputs;

buildPhase = ''
runHook preBuild

# Ensure deterministic Cargo vendor builds
export SOURCE_DATE_EPOCH=1

if [[ ! -f Cargo.lock ]]; then
echo
echo "ERROR: The Cargo.lock file doesn't exist"
echo
echo "Cargo.lock is needed to make sure that cargoHash/cargoSha256 doesn't change"
echo "when the registry is updated."
echo

exit 1
Comment on lines +89 to +96
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In Nix files, I do like my shell to have two-space indent. .editorconfig says four, but @emilazy and I might connive to effect a two-space-coup on shell.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you hide whitespace (https://github.com/NixOS/nixpkgs/pull/332975/files?diff=unified&w=1) you'll see I didn't substantively touch these lines at all. These lines are indented in the diff, but only by the same amount that the surroundings are indented (and was done by nixfmt, not manually).

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh yes, this is just me noticing formatting due to, well, the formatting being impacted with nixfmt.

fi

# Keep the original around for copyLockfile
cp Cargo.lock Cargo.lock.orig

export CARGO_HOME=$(mktemp -d cargo-home.XXX)
CARGO_CONFIG=$(mktemp cargo-config.XXXX)

if [[ -n "$NIX_CRATES_INDEX" ]]; then
cat >$CARGO_HOME/config.toml <<EOF
[source.crates-io]
replace-with = 'mirror'
[source.mirror]
registry = "$NIX_CRATES_INDEX"
EOF
Comment on lines +106 to +111
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Needs one more level of indent.

Copy link
Contributor Author

@eclairevoyant eclairevoyant Sep 5, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The indentation error is present in the base branch. , but sure, I can fix it

Actually if I indent this then the contents of the toml file will be indented.
The only line that's indentable is L93 but that might look even more confusing 🫤

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wow, load-bearing odd indentation. Thanks Bash.

fi

${cargoUpdateHook}

# Override the `http.cainfo` option usually specified in `.cargo/config`.
export CARGO_HTTP_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt

if grep '^source = "git' Cargo.lock; then
echo
echo "ERROR: The Cargo.lock contains git dependencies"
echo
echo "This is currently not supported in the fixed-output derivation fetcher."
echo "Use cargoLock.lockFile / importCargoLock instead."
echo

exit 1
fi

cargo vendor $name --respect-source-config | cargo-vendor-normalise > $CARGO_CONFIG

# Create an empty vendor directory when there is no dependency to vendor
mkdir -p $name
# Add the Cargo.lock to allow hash invalidation
cp Cargo.lock.orig $name/Cargo.lock

# Packages with git dependencies generate non-default cargo configs, so
# always install it rather than trying to write a standard default template.
install -D $CARGO_CONFIG $name/.cargo/config;

runHook postBuild
'';

# Build a reproducible tar, per instructions at https://reproducible-builds.org/docs/archives/
installPhase = ''
tar --owner=0 --group=0 --numeric-owner --format=gnu \
--sort=name --mtime="@$SOURCE_DATE_EPOCH" \
-czf $out $name
'';

inherit (hash_) outputHashAlgo outputHash;

impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ "NIX_CRATES_INDEX" ];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Future: document this environment variable somewhere. Maybe in the Rust section of the nixpkgs-manual?

}
// (removeAttrs args removedArgs)
)
2 changes: 1 addition & 1 deletion pkgs/by-name/ch/chance/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ stdenv.mkDerivation (finalAttrs: {
};

cargoDeps = rustPlatform.fetchCargoTarball {
inherit (finalAttrs) pname version src;
inherit (finalAttrs) src;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to the PR headmatter, this is removed in order to not cause hash mismatches.

hash = "sha256-Q4CfDQxlhspjg7Et+0zHwZ/iSnp0CnwwpW/gT7htlL8=";
};

Expand Down
2 changes: 1 addition & 1 deletion pkgs/by-name/ge/geopard/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ stdenv.mkDerivation (finalAttrs: {
};

cargoDeps = rustPlatform.fetchCargoTarball {
inherit (finalAttrs) pname version src;
inherit (finalAttrs) src;
hash = "sha256-YVbaXGGwQaqjUkA47ryW1VgJpZTx5ApRGdCcB5aA71M=";
};

Expand Down
2 changes: 1 addition & 1 deletion pkgs/by-name/ke/key-rack/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ stdenv.mkDerivation (finalAttrs: {
'';

cargoDeps = rustPlatform.fetchCargoTarball {
inherit (finalAttrs) pname version src;
inherit (finalAttrs) src;
hash = "sha256-wCJTm0W+g3+O1t1fR4maqJoxpPM0NeJG7d54MMAH33c=";
};

Expand Down
3 changes: 1 addition & 2 deletions pkgs/by-name/mo/mousai/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@ stdenv.mkDerivation rec {
};

cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
name = "${pname}-${version}";
inherit pname version src;
hash = "sha256-FjnRI1vHA9YF/Uw2+hDtMJmeJVa5RcxaYoG4XgXa9Ds=";
};

Expand Down
2 changes: 1 addition & 1 deletion pkgs/by-name/su/surrealist/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ in stdenv.mkDerivation (finalAttrs: {
};

cargoDeps = rustPlatform.fetchCargoTarball {
inherit (finalAttrs) patches src sourceRoot version;
inherit (finalAttrs) patches src sourceRoot;
name = "${finalAttrs.pname}-${finalAttrs.version}";
hash = "sha256-LtQS0kH+2P4odV7BJYiH6T51+iZHAM9W9mV96rNfNWs=";
};
Expand Down
2 changes: 1 addition & 1 deletion pkgs/development/libraries/libkrun/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ stdenv.mkDerivation (finalAttrs: {
outputs = [ "out" "dev" ];

cargoDeps = rustPlatform.fetchCargoTarball {
inherit (finalAttrs) pname version src;
inherit (finalAttrs) src;
hash = "sha256-33s62iOWYh1a8ETY/fbPRxvnj8dR4/UfG8mjFyWwz5k=";
};

Expand Down
2 changes: 1 addition & 1 deletion pkgs/development/python-modules/datafusion/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ buildPythonPackage rec {

cargoDeps = rustPlatform.fetchCargoTarball {
name = "datafusion-cargo-deps";
inherit src pname version;
inherit src;
hash = "sha256-M2ZNAFWdsnN9C4+YbqFxZVH9fHR10Bimf1Xzrd9oy9E=";
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ buildPythonPackage rec {
};

cargoDeps = rustPlatform.fetchCargoTarball {
inherit pname version src;
inherit src;
hash = "sha256-KrEBr998AV/bKcIoq0tX72/QwPD9bQplrS0Zw+JiSMQ=";
};

Expand Down
Loading