Skip to content

Commit

Permalink
Merge pull request #4056 from tweag/non-ca-depending-on-ca
Browse files Browse the repository at this point in the history 
Allow non-CA derivations to depend on CA ones
  • Loading branch information
@edolstra
edolstra authored Oct 27, 2020
2 parents 731edf0 + ab21ab6 commit 02a1fac
Show file tree
Hide file tree
Showing 8 changed files with 162 additions and 38 deletions.
39 changes: 28 additions & 11 deletions src/libexpr/primops.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1089,18 +1089,35 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *

// Regular, non-CA derivation should always return a single hash and not
// hash per output.
Hash h = std::get<0>(hashDerivationModulo(*state.store, Derivation(drv), true));
auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
std::visit(overloaded {
[&](Hash h) {
for (auto & i : outputs) {
auto outPath = state.store->makeOutputPath(i, h, drvName);
drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(i,
DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(outPath),
},
});
}
},
[&](CaOutputHashes) {
// Shouldn't happen as the toplevel derivation is not CA.
assert(false);
},
[&](UnknownHashes) {
for (auto & i : outputs) {
drv.outputs.insert_or_assign(i,
DerivationOutput {
.output = DerivationOutputDeferred{},
});
}
},
},
hashModulo);

for (auto & i : outputs) {
auto outPath = state.store->makeOutputPath(i, h, drvName);
drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(i,
DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(outPath),
},
});
}
}

/* Write the resulting term into the Nix store directory. */
Expand Down
12 changes: 11 additions & 1 deletion src/libstore/build/derivation-goal.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -493,7 +493,8 @@ void DerivationGoal::inputsRealised()
if (useDerivation) {
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());

if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
if ((!fullDrv.inputDrvs.empty() &&
fullDrv.type() == DerivationType::CAFloating) || fullDrv.type() == DerivationType::DeferredInputAddressed) {
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a stub goal
aliasing that resolved derivation goal */
Expand Down Expand Up @@ -3166,6 +3167,15 @@ void DerivationGoal::registerOutputs()
[&](DerivationOutputCAFloating dof) {
return newInfoFromCA(dof);
},
[&](DerivationOutputDeferred) {
// No derivation should reach that point without having been
// rewritten first
assert(false);
// Ugly, but the compiler insists on having this return a value
// of type `ValidPathInfo` despite the `assert(false)`, so
// let's provide it
return *(ValidPathInfo*)0;
},
}, output.output);

/* Calculate where we'll move the output files. In the checking case we
Expand Down
63 changes: 56 additions & 7 deletions src/libstore/derivations.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
return std::nullopt;
},
[](DerivationOutputDeferred) -> std::optional<StorePath> {
return std::nullopt;
},
}, output);
}

Expand All @@ -37,6 +40,7 @@ bool derivationIsCA(DerivationType dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return true;
case DerivationType::DeferredInputAddressed: return false;
};
// Since enums can have non-variant values, but making a `default:` would
// disable exhaustiveness warnings.
Expand All @@ -48,6 +52,7 @@ bool derivationIsFixed(DerivationType dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
case DerivationType::DeferredInputAddressed: return false;
};
assert(false);
}
Expand All @@ -57,6 +62,7 @@ bool derivationIsImpure(DerivationType dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
case DerivationType::DeferredInputAddressed: return false;
};
assert(false);
}
Expand Down Expand Up @@ -180,6 +186,11 @@ static DerivationOutput parseDerivationOutput(const Store & store,
};
}
} else {
if (pathS == "") {
return DerivationOutput {
.output = DerivationOutputDeferred { }
};
}
validatePath(pathS);
return DerivationOutput {
.output = DerivationOutputInputAddressed {
Expand Down Expand Up @@ -325,6 +336,11 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, "");
},
[&](DerivationOutputDeferred) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
}
}, i.second.output);
s += ')';
}
Expand Down Expand Up @@ -389,7 +405,7 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName

DerivationType BasicDerivation::type() const
{
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
std::optional<HashType> floatingHashType;
for (auto & i : outputs) {
std::visit(overloaded {
Expand All @@ -408,22 +424,27 @@ DerivationType BasicDerivation::type() const
throw Error("All floating outputs must use the same hash type");
}
},
[&](DerivationOutputDeferred _) {
deferredIAOutputs.insert(i.first);
},
}, i.second.output);
}

if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
throw Error("Must have at least one output");
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
return DerivationType::InputAddressed;
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("Only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out")
throw Error("Single fixed output must be named \"out\"");
return DerivationType::CAFixed;
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) {
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
return DerivationType::CAFloating;
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
return DerivationType::DeferredInputAddressed;
} else {
throw Error("Can't mix derivation output types");
}
Expand Down Expand Up @@ -476,7 +497,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
/* Return a fixed hash for fixed-output derivations. */
switch (drv.type()) {
case DerivationType::CAFloating:
throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
return UnknownHashes {};
case DerivationType::CAFixed: {
std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputs) {
Expand All @@ -491,12 +512,15 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
}
case DerivationType::InputAddressed:
break;
case DerivationType::DeferredInputAddressed:
break;
}

/* For other derivations, replace the inputs paths with recursive
calls to this function. */
std::map<std::string, StringSet> inputs2;
for (auto & i : drv.inputDrvs) {
bool hasUnknownHash = false;
const auto & res = pathDerivationModulo(store, i.first);
std::visit(overloaded {
// Regular non-CA derivation, replace derivation
Expand All @@ -514,7 +538,13 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
justOut);
}
},
[&](UnknownHashes) {
hasUnknownHash = true;
},
}, res);
if (hasUnknownHash) {
return UnknownHashes {};
}
}

return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
Expand Down Expand Up @@ -620,6 +650,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
<< "";
},
[&](DerivationOutputDeferred) {
out << ""
<< ""
<< "";
},
}, i.second.output);
}
worker_proto::write(store, out, drv.inputSrcs);
Expand All @@ -645,7 +680,6 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
}


// N.B. Outputs are left unchanged
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {

debug("Rewriting the derivation");
Expand All @@ -666,6 +700,21 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
newEnv.emplace(envName, envValue);
}
drv.env = newEnv;

auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
for (auto & [outputName, output] : drv.outputs) {
if (std::holds_alternative<DerivationOutputDeferred>(output.output)) {
Hash h = std::get<Hash>(hashModulo);
auto outPath = store.makeOutputPath(outputName, h, drv.name);
drv.env[outputName] = store.printStorePath(outPath);
output = DerivationOutput {
.output = DerivationOutputInputAddressed {
.path = std::move(outPath),
},
};
}
}

}


Expand Down
14 changes: 12 additions & 2 deletions src/libstore/derivations.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,18 @@ struct DerivationOutputCAFloating
HashType hashType;
};

/* Input-addressed output which depends on a (CA) derivation whose hash isn't
* known atm
*/
struct DerivationOutputDeferred {};

struct DerivationOutput
{
std::variant<
DerivationOutputInputAddressed,
DerivationOutputCAFixed,
DerivationOutputCAFloating
DerivationOutputCAFloating,
DerivationOutputDeferred
> output;
std::optional<HashType> hashAlgoOpt(const Store & store) const;
/* Note, when you use this function you should make sure that you're passing
Expand All @@ -72,6 +78,7 @@ typedef std::map<string, string> StringPairs;

enum struct DerivationType : uint8_t {
InputAddressed,
DeferredInputAddressed,
CAFixed,
CAFloating,
};
Expand Down Expand Up @@ -167,9 +174,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;

struct UnknownHashes {};

typedef std::variant<
Hash, // regular DRV normalized hash
CaOutputHashes
CaOutputHashes, // Fixed-output derivation hashes
UnknownHashes // Deferred hashes for floating outputs drvs and their dependencies
> DrvHashModulo;

/* Returns hashes with the details of fixed-output subderivations
Expand Down
4 changes: 3 additions & 1 deletion src/libstore/local-store.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
[&](DerivationOutputCAFloating _) {
/* Nothing to check */
},
[&](DerivationOutputDeferred) {
},
}, i.second.output);
}
}
Expand Down Expand Up @@ -817,7 +819,7 @@ std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivati
}
/* can't just use else-if instead of `!haveCached` because we need to unlock
`drvPathResolutions` before it is locked in `Derivation::resolve`. */
if (!haveCached && drv.type() == DerivationType::CAFloating) {
if (!haveCached && (drv.type() == DerivationType::CAFloating || drv.type() == DerivationType::DeferredInputAddressed)) {
/* Try resolve drv and use that path instead. */
auto attempt = drv.tryResolve(*this);
if (!attempt)
Expand Down
1 change: 1 addition & 0 deletions src/nix/show-derivation.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ struct CmdShowDerivation : InstallablesCommand
[&](DerivationOutputCAFloating dof) {
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
},
[&](DerivationOutputDeferred) {},
}, output.output);
}
}
Expand Down
11 changes: 10 additions & 1 deletion tests/content-addressed.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ rec {
'';
};
rootCA = mkDerivation {
name = "dependent";
name = "rootCA";
outputs = [ "out" "dev" ];
buildCommand = ''
echo "building a CA derivation"
Expand Down Expand Up @@ -51,4 +51,13 @@ rec {
outputHashMode = "recursive";
outputHashAlgo = "sha256";
};
dependentNonCA = mkDerivation {
name = "dependent-non-ca";
buildCommand = ''
echo "Didn't cut-off"
echo "building dependent-non-ca"
mkdir -p $out
echo ${rootCA}/non-ca-hello > $out/dep
'';
};
}
Loading

0 comments on commit 02a1fac

Please sign in to comment.