From 26d0c609cc3e4b5b8e72af719c76123b1503b2a6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 25 Mar 2020 00:37:52 +0000 Subject: [PATCH] Continue generalizing, and fix some probable bugs from the last version --- src/libexpr/primops.cc | 52 ++------ src/libstore/derivations.cc | 245 +++++++++++++++++++++++++++++++----- src/libstore/derivations.hh | 57 ++++++++- src/libstore/local-store.cc | 15 +-- 4 files changed, 284 insertions(+), 85 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2215da68c64..5197be360c0 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -718,45 +718,15 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Set "out" output with hash algo and hash for fixed-output derivations. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName); - - HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); - Hash h(*outputHash, ht); - drv.outputs.insert_or_assign("out", DerivationOutputT( - NoPath(), - (outputHashRecursive ? "r:" : "") + printHashType(ht), - h.to_string(Base16, false))); } /* Compute the final derivation, which additionally contains the outputs paths created from the hash of the initial one. */ - Derivation drvFinal; - //drvFinal.inputSrcs = drv.inputSrcs; - for (auto & i : drv.inputSrcs) - drvFinal.inputSrcs.insert(i.clone()); - drvFinal.platform = drv.platform; - drvFinal.builder = drv.builder; - drvFinal.args = drv.args; - drvFinal.env = drv.env; - { - const auto drvOrPseudo = derivationModulo(*state.store, drv); - if (const DerivationT *pval = std::get_if<0>(&drvOrPseudo)) { - Hash hash = hashDerivation(*state.store, *pval); - for (auto & i : outputs) { - auto outPath = state.store->makeOutputPath(i, hash, drvName); - if (!jsonObject) drvFinal.env[i] = state.store->printStorePath(outPath); - drvFinal.outputs.insert_or_assign(i, - DerivationOutput(std::move(outPath), "", "")); - } - } else if (std::get_if<1>(&drvOrPseudo)) { - HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); - Hash h(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); - auto & output = drv.outputs.find("out")->second; - drvFinal.outputs.insert_or_assign("out", DerivationOutput( - std::move(outPath), - std::string(output.hashAlgo), - std::string(output.hash))); - if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); + Derivation drvFinal = bakeDerivationPaths(*state.store, drv, drvName); + + if (!jsonObject) { + for (const auto & i : drvFinal.outputs) { + drvFinal.env[i.first] = state.store->printStorePath(i.second.path); } } @@ -770,17 +740,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * case we don't actually write store derivations, so we can't read them later. */ { - const auto drvOrPseudo = derivationModulo(*state.store, drvFinal); - Hash hash; - if (const DerivationT *pval = std::get_if<0>(&drvOrPseudo)) { - hash = hashDerivation(*state.store, *pval); - } else if (const std::string *pval = std::get_if<1>(&drvOrPseudo)) { - hash = hashString(htSHA256, *pval); - } - // Cache it + const std::variant, std::string> drvOrPseudo = + derivationModuloOrOutput(*state.store, drvFinal); + auto hash = hashDerivationOrPseudo(*state.store, std::move(drvOrPseudo)); drvHashes.insert_or_assign(drvPath.clone(), std::move(hash)); } - state.mkAttrs(v, 1 + drvFinal.outputs.size()); mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); for (auto & i : drvFinal.outputs) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 2f58cecb1e5..feae25dd26b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -358,10 +358,44 @@ bool BasicDerivationT::isFixedOutput() const outputs.begin()->second.hash != ""; } +template +static std::string printLogicalPrefix(const DerivationOutputT & output) { + return "fixed:out:" + + output.hashAlgo + ":" + + output.hash + ":"; +} + +std::optional printLogicalOutput( + Store & store, + const DerivationOutputT & output) +{ + return printLogicalPrefix(output) + store.printStorePath(output.path); +} + +static StorePath printPhysicalOutput( + Store & store, + const DerivationOutputT & output, + const std::string & drvName) +{ + bool recursive; + Hash h; + output.parseHashInfo(recursive, h); + return store.makeStorePath("output:out", h, std::string_view(printLogicalPrefix(output) + drvName)); +} + +std::optional printLogicalOutput( + Store & store, + const DerivationOutputT & output, + const std::string & drvName) +{ + return printLogicalPrefix(output) + + store.printStorePath(printPhysicalOutput(store, output, drvName)); +} + DrvHashes drvHashes; -/* pathDerivationModulo and hashDerivationModulo are mutually recursive +/* pathDerivationModulo and derivationModulo are mutually recursive */ /* Look up the derivation by value and memoize the @@ -371,24 +405,55 @@ static const Hash & pathDerivationModulo(Store & store, const StorePath & drvPat { auto h = drvHashes.find(drvPath); if (h == drvHashes.end()) { - assert(store.isValidPath(drvPath)); - const std::variant, std::string> drvOrPseudo = derivationModulo( - store, - readDerivation( + const std::variant, std::string> drvOrPseudo = + derivationModuloOrOutput( store, - store.toRealPath(store.printStorePath(drvPath)))); - Hash hash; - if (const DerivationT *pval = std::get_if<0>(&drvOrPseudo)) { - hash = hashDerivation(store, *pval); - } else if (const std::string *pval = std::get_if<1>(&drvOrPseudo)) { - hash = hashString(htSHA256, *pval); - } - // Cache it - h = drvHashes.insert_or_assign(drvPath.clone(), hash).first; + readDerivation( + store, + store.toRealPath(store.printStorePath(drvPath)))); + const auto hash = hashDerivationOrPseudo(store, std::move(drvOrPseudo)); + h = drvHashes.insert_or_assign(drvPath.clone(), std::move(hash)).first; } + // Cache it return h->second; } +template +Hash hashDerivationOrPseudo( + Store & store, + typename std::variant, std::string> drvOrPseudo) +{ + Hash hash; + if (const DerivationT *pval = std::get_if<0>(&drvOrPseudo)) { + hash = hashDerivation(store, *pval); + } else if (const std::string *pval = std::get_if<1>(&drvOrPseudo)) { + hash = hashString(htSHA256, *pval); + } + return hash; +} + +template +DerivationT derivationModulo( + Store & store, + const DerivationT & drv) +{ + DerivationT drvNorm((BasicDerivationT)drv); + for (auto & i : drv.inputDrvs) { + const auto h = pathDerivationModulo(store, i.first); + drvNorm.inputDrvs.insert_or_assign(h, i.second); + } + + return drvNorm; +} + +template +DerivationT derivationModulo( + Store & store, + const DerivationT & drv) +{ + return drv; +} + /* Returns the hash of a derivation modulo fixed-output subderivations. A fixed-output derivation is a derivation with one output (`out') for which an expected hash and hash algorithm are @@ -409,31 +474,97 @@ static const Hash & pathDerivationModulo(Store & store, const StorePath & drvPat paths have been replaced by the result of a recursive call to this function, and that for fixed-output derivations we return a hash of its output path. */ -template -std::variant, string> derivationModulo( + +template +std::variant, string> derivationModuloOrOutput( Store & store, - const DerivationT & drv) + const DerivationT & drv) { /* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) { - typename DerivationOutputsT::const_iterator i = drv.outputs.begin(); - return "fixed:out:" - + i->second.hashAlgo + ":" - + i->second.hash + ":" - + printStorePath(store, i->second.path); + DerivationOutputsT::const_iterator i = drv.outputs.begin(); + auto res = printLogicalOutput(store, i->second); + assert(res); + return *res; } /* For other derivations, replace the inputs paths with recursive calls to this function. */ - DerivationT drvNorm((BasicDerivationT)drv); - for (auto & i : drv.inputDrvs) { - const auto h = pathDerivationModulo(store, i.first); - drvNorm.inputDrvs.insert_or_assign(h, i.second); + return derivationModulo(store, drv); +} + +template +std::variant, string> derivationModuloOrOutput( + Store & store, + const DerivationT & drv, + const std::string & drvName) +{ + if (drv.isFixedOutput()) { + DerivationOutputsT::const_iterator i = drv.outputs.begin(); + auto res = printLogicalOutput(store, i->second, drvName); + assert(res); + return *res; } + return derivationModulo(store, drv); +} - return drvNorm; +template +DerivationT bakeDerivationPaths( + Store & store, + const DerivationT & drv, + const std::string & drvName) +{ + DerivationT drvFinal; + //drvFinal.inputSrcs = drv.inputSrcs; + for (auto & i : drv.inputSrcs) + drvFinal.inputSrcs.insert(i.clone()); + drvFinal.platform = drv.platform; + drvFinal.builder = drv.builder; + drvFinal.args = drv.args; + drvFinal.env = drv.env; + const auto drvOrPseudo = derivationModuloOrOutput(store, drv, drvName); + if (const DerivationT *pval = std::get_if<0>(&drvOrPseudo)) { + Hash hash = hashDerivation(store, *pval); + for (const auto i : drv.outputs) { + auto outPath = store.makeOutputPath(i.first, hash, drvName); + drvFinal.outputs.insert_or_assign(i.first, + DerivationOutput(std::move(outPath), "", "")); + } + } else if (std::get_if<1>(&drvOrPseudo)) { + DerivationOutputsT::const_iterator out = drv.outputs.find("out"); + if (out == drv.outputs.end()) + throw Error("derivation does not have an output named 'out'"); + auto & output = out->second; + auto outPath = printPhysicalOutput(store, output, drvName); + drvFinal.outputs.insert_or_assign("out", DerivationOutput( + std::move(outPath), + std::string(output.hashAlgo), + std::string(output.hash))); + } + return drvFinal; } +template +DerivationT stripDerivationPaths( + Store & store, + const DerivationT & drv) +{ + DerivationT drvInitial; + //drvInitial.inputSrcs = drv.inputSrcs; + for (auto & i : drv.inputSrcs) + drvInitial.inputSrcs.insert(i.clone()); + drvInitial.platform = drv.platform; + drvInitial.builder = drv.builder; + drvInitial.args = drv.args; + drvInitial.env = drv.env; + for (const auto & i : drv.outputs) { + drvInitial.outputs.insert_or_assign(i.first, DerivationOutputT( + NoPath(), + std::string(i.second.hashAlgo), + std::string(i.second.hash))); + } + return drvInitial; +} std::string StorePathWithOutputs::to_string(const Store & store) const { @@ -515,15 +646,71 @@ template struct DerivationT; template struct DerivationT; template -std::variant, std::string> derivationModulo( +DerivationT derivationModulo( Store & store, const DerivationT & drv); template -std::variant, std::string> derivationModulo( +DerivationT derivationModulo( Store & store, const DerivationT & drv); +template +DerivationT derivationModulo( + Store & store, + const DerivationT & drv); +template +DerivationT derivationModulo( + Store & store, + const DerivationT & drv); + +template +std::variant, std::string> derivationModuloOrOutput( + Store & store, + const DerivationT & drv); +template +std::variant, std::string> derivationModuloOrOutput( + Store & store, + const DerivationT & drv); +template +std::variant, std::string> derivationModuloOrOutput( + Store & store, + const DerivationT & drv, + const std::string & drvName); +template +std::variant, std::string> derivationModuloOrOutput( + Store & store, + const DerivationT & drv, + const std::string & drvName); + +template +Hash hashDerivationOrPseudo( + Store & store, + typename std::variant, std::string> drvOrPseudo); +template +Hash hashDerivationOrPseudo( + Store & store, + typename std::variant, std::string> drvOrPseudo); template Hash hashDerivation(Store & store, const DerivationT & drv); template Hash hashDerivation(Store & store, const DerivationT & drv); +template +DerivationT bakeDerivationPaths( + Store & store, + const DerivationT & drv, + const std::string & drvName); +template +DerivationT bakeDerivationPaths( + Store & store, + const DerivationT & drv, + const std::string & drvName); + +template +DerivationT stripDerivationPaths( + Store & store, + const DerivationT & drv); +template +DerivationT stripDerivationPaths( + Store & store, + const DerivationT & drv); + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index e6bef0f712c..5aea3d417df 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -19,11 +19,13 @@ struct DerivationOutputT Path path; std::string hashAlgo; /* hash used for expected hash computation */ std::string hash; /* expected hash, may be null */ + DerivationOutputT(Path && path, std::string && hashAlgo, std::string && hash) : path(std::move(path)) , hashAlgo(std::move(hashAlgo)) , hash(std::move(hash)) { } + void parseHashInfo(bool & recursive, Hash & hash) const; }; @@ -111,16 +113,69 @@ Derivation readDerivation(const Store & store, const Path & drvPath); // FIXME: remove bool isDerivation(const string & fileName); + +// TODO dedup some with `Store::makeFixedOutputPath` after DerivationOutput +// doesn't just contain strings but actual `Hash` and recursive vs flat enum. + +/// Print the symbolic output path if it is fixed output +std::optional printLogicalOutput( + Store & store, + const DerivationOutputT & output); +std::optional printLogicalOutput( + Store & store, + const DerivationOutputT & output, + const std::string & drvName); + +/// Reduce a derivation down to a resolved normal form template -std::variant, string> derivationModulo( +DerivationT derivationModulo( Store & store, const DerivationT & drv); +/// Identity function, but useful to be called from polymorphic code. +template +DerivationT derivationModulo( + Store & store, + const DerivationT & drv); + +/* Reduce a derivation down to a resolved normal form if it is regular, or + symbolic output path if it is fixed output. */ +template +std::variant, string> derivationModuloOrOutput( + Store & store, + const DerivationT & drv); +template +std::variant, string> derivationModuloOrOutput( + Store & store, + const DerivationT & drv, + const std::string & drvName); + +/// Turn the output of derivationModuloOrOutput into a plain hash. +template +Hash hashDerivationOrPseudo( + Store & store, + typename std::variant, std::string> drvOrPseudo); + +/* Hash a resolved normal form derivation. */ template Hash hashDerivation( Store & store, const DerivationT & drv); +/* Compute a "baked" derivation, made from the which additionally contains the + outputs paths created from the hash of the initial one. */ +template +DerivationT bakeDerivationPaths( + Store & store, + const DerivationT & drv, + const std::string & drvName); + +/* Opposite of bakeDerivationPaths */ +template +DerivationT stripDerivationPaths( + Store & store, + const DerivationT & drv); + /* Memoisation of hashDerivationModulo(). */ typedef std::map DrvHashes; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 408e0b27c4a..6cc9f523abd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -556,19 +556,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - - bool recursive; Hash h; - out->second.parseHashInfo(recursive, h); - - check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out"); } - else { - const Hash h = hashDerivation(*this, - std::get<0>(derivationModulo(*this, drv))); - for (auto & i : drv.outputs) - check(makeOutputPath(i.first, h, drvName), i.second.path, i.first); - } + auto drvOracle = bakeDerivationPaths(*this, stripDerivationPaths(*this, drv), drvName); + + for (auto & i : drv.outputs) + check(drvOracle.outputs.find(i.first)->second.path, i.second.path, i.first); }