Skip to content

Commit

Permalink
Continue generalizing, and fix some probable bugs from the last version
Browse files Browse the repository at this point in the history
  • Loading branch information
@Ericson2314
Ericson2314 committed Mar 25, 2020
1 parent 28d27c2 commit 26d0c60
Show file tree
Hide file tree
Showing 4 changed files with 284 additions and 85 deletions.
52 changes: 8 additions & 44 deletions src/libexpr/primops.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -718,45 +718,15 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Set "out" output with hash algo and hash for fixed-output derivations. */
if (outputs.size() != 1 || *(outputs.begin()) != "out")
throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName);

HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
Hash h(*outputHash, ht);
drv.outputs.insert_or_assign("out", DerivationOutputT(
NoPath(),
(outputHashRecursive ? "r:" : "") + printHashType(ht),
h.to_string(Base16, false)));
}

/* Compute the final derivation, which additionally contains the outputs
paths created from the hash of the initial one. */
Derivation drvFinal;
//drvFinal.inputSrcs = drv.inputSrcs;
for (auto & i : drv.inputSrcs)
drvFinal.inputSrcs.insert(i.clone());
drvFinal.platform = drv.platform;
drvFinal.builder = drv.builder;
drvFinal.args = drv.args;
drvFinal.env = drv.env;
{
const auto drvOrPseudo = derivationModulo(*state.store, drv);
if (const DerivationT<Hash, NoPath> *pval = std::get_if<0>(&drvOrPseudo)) {
Hash hash = hashDerivation(*state.store, *pval);
for (auto & i : outputs) {
auto outPath = state.store->makeOutputPath(i, hash, drvName);
if (!jsonObject) drvFinal.env[i] = state.store->printStorePath(outPath);
drvFinal.outputs.insert_or_assign(i,
DerivationOutput(std::move(outPath), "", ""));
}
} else if (std::get_if<1>(&drvOrPseudo)) {
HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
Hash h(*outputHash, ht);
auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
auto & output = drv.outputs.find("out")->second;
drvFinal.outputs.insert_or_assign("out", DerivationOutput(
std::move(outPath),
std::string(output.hashAlgo),
std::string(output.hash)));
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
Derivation drvFinal = bakeDerivationPaths(*state.store, drv, drvName);

if (!jsonObject) {
for (const auto & i : drvFinal.outputs) {
drvFinal.env[i.first] = state.store->printStorePath(i.second.path);
}
}

Expand All @@ -770,17 +740,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
case we don't actually write store derivations, so we can't
read them later. */
{
const auto drvOrPseudo = derivationModulo<StorePath>(*state.store, drvFinal);
Hash hash;
if (const DerivationT<Hash, StorePath> *pval = std::get_if<0>(&drvOrPseudo)) {
hash = hashDerivation(*state.store, *pval);
} else if (const std::string *pval = std::get_if<1>(&drvOrPseudo)) {
hash = hashString(htSHA256, *pval);
}
// Cache it
const std::variant<DerivationT<Hash, StorePath>, std::string> drvOrPseudo =
derivationModuloOrOutput(*state.store, drvFinal);
auto hash = hashDerivationOrPseudo(*state.store, std::move(drvOrPseudo));
drvHashes.insert_or_assign(drvPath.clone(), std::move(hash));
}

state.mkAttrs(v, 1 + drvFinal.outputs.size());
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
for (auto & i : drvFinal.outputs) {
Expand Down
245 changes: 216 additions & 29 deletions src/libstore/derivations.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,10 +358,44 @@ bool BasicDerivationT<Path>::isFixedOutput() const
outputs.begin()->second.hash != "";
}

template<typename Path>
static std::string printLogicalPrefix(const DerivationOutputT<Path> & output) {
return "fixed:out:"
+ output.hashAlgo + ":"
+ output.hash + ":";
}

std::optional<std::string> printLogicalOutput(
Store & store,
const DerivationOutputT<StorePath> & output)
{
return printLogicalPrefix(output) + store.printStorePath(output.path);
}

static StorePath printPhysicalOutput(
Store & store,
const DerivationOutputT<NoPath> & output,
const std::string & drvName)
{
bool recursive;
Hash h;
output.parseHashInfo(recursive, h);
return store.makeStorePath("output:out", h, std::string_view(printLogicalPrefix(output) + drvName));
}

std::optional<std::string> printLogicalOutput(
Store & store,
const DerivationOutputT<NoPath> & output,
const std::string & drvName)
{
return printLogicalPrefix(output)
+ store.printStorePath(printPhysicalOutput(store, output, drvName));
}


DrvHashes drvHashes;

/* pathDerivationModulo and hashDerivationModulo are mutually recursive
/* pathDerivationModulo and derivationModulo are mutually recursive
*/

/* Look up the derivation by value and memoize the
Expand All @@ -371,24 +405,55 @@ static const Hash & pathDerivationModulo(Store & store, const StorePath & drvPat
{
auto h = drvHashes.find(drvPath);
if (h == drvHashes.end()) {
assert(store.isValidPath(drvPath));
const std::variant<DerivationT<Hash, StorePath>, std::string> drvOrPseudo = derivationModulo(
store,
readDerivation(
const std::variant<DerivationT<Hash, StorePath>, std::string> drvOrPseudo =
derivationModuloOrOutput(
store,
store.toRealPath(store.printStorePath(drvPath))));
Hash hash;
if (const DerivationT<Hash, StorePath> *pval = std::get_if<0>(&drvOrPseudo)) {
hash = hashDerivation(store, *pval);
} else if (const std::string *pval = std::get_if<1>(&drvOrPseudo)) {
hash = hashString(htSHA256, *pval);
}
// Cache it
h = drvHashes.insert_or_assign(drvPath.clone(), hash).first;
readDerivation(
store,
store.toRealPath(store.printStorePath(drvPath))));
const auto hash = hashDerivationOrPseudo(store, std::move(drvOrPseudo));
h = drvHashes.insert_or_assign(drvPath.clone(), std::move(hash)).first;
}
// Cache it
return h->second;
}

template<typename OutPath>
Hash hashDerivationOrPseudo(
Store & store,
typename std::variant<DerivationT<Hash, OutPath>, std::string> drvOrPseudo)
{
Hash hash;
if (const DerivationT<Hash, OutPath> *pval = std::get_if<0>(&drvOrPseudo)) {
hash = hashDerivation(store, *pval);
} else if (const std::string *pval = std::get_if<1>(&drvOrPseudo)) {
hash = hashString(htSHA256, *pval);
}
return hash;
}

template<typename OutPath>
DerivationT<Hash, OutPath> derivationModulo(
Store & store,
const DerivationT<StorePath, OutPath> & drv)
{
DerivationT<Hash, OutPath> drvNorm((BasicDerivationT<OutPath>)drv);
for (auto & i : drv.inputDrvs) {
const auto h = pathDerivationModulo(store, i.first);
drvNorm.inputDrvs.insert_or_assign(h, i.second);
}

return drvNorm;
}

template<typename OutPath>
DerivationT<Hash, OutPath> derivationModulo(
Store & store,
const DerivationT<Hash, OutPath> & drv)
{
return drv;
}

/* Returns the hash of a derivation modulo fixed-output
subderivations. A fixed-output derivation is a derivation with one
output (`out') for which an expected hash and hash algorithm are
Expand All @@ -409,31 +474,97 @@ static const Hash & pathDerivationModulo(Store & store, const StorePath & drvPat
paths have been replaced by the result of a recursive call to this
function, and that for fixed-output derivations we return a hash of
its output path. */
template<typename OutPath>
std::variant<DerivationT<Hash, OutPath>, string> derivationModulo(

template<typename InputDrvPath>
std::variant<DerivationT<Hash, StorePath>, string> derivationModuloOrOutput(
Store & store,
const DerivationT<StorePath, OutPath> & drv)
const DerivationT<InputDrvPath, StorePath> & drv)
{
/* Return a fixed hash for fixed-output derivations. */
if (drv.isFixedOutput()) {
typename DerivationOutputsT<OutPath>::const_iterator i = drv.outputs.begin();
return "fixed:out:"
+ i->second.hashAlgo + ":"
+ i->second.hash + ":"
+ printStorePath(store, i->second.path);
DerivationOutputsT<StorePath>::const_iterator i = drv.outputs.begin();
auto res = printLogicalOutput(store, i->second);
assert(res);
return *res;
}

/* For other derivations, replace the inputs paths with recursive
calls to this function. */
DerivationT<Hash, OutPath> drvNorm((BasicDerivationT<OutPath>)drv);
for (auto & i : drv.inputDrvs) {
const auto h = pathDerivationModulo(store, i.first);
drvNorm.inputDrvs.insert_or_assign(h, i.second);
return derivationModulo(store, drv);
}

template<typename InputDrvPath>
std::variant<DerivationT<Hash, NoPath>, string> derivationModuloOrOutput(
Store & store,
const DerivationT<InputDrvPath, NoPath> & drv,
const std::string & drvName)
{
if (drv.isFixedOutput()) {
DerivationOutputsT<NoPath>::const_iterator i = drv.outputs.begin();
auto res = printLogicalOutput(store, i->second, drvName);
assert(res);
return *res;
}
return derivationModulo(store, drv);
}

return drvNorm;
template<typename InputDrvPath>
DerivationT<InputDrvPath, StorePath> bakeDerivationPaths(
Store & store,
const DerivationT<InputDrvPath, NoPath> & drv,
const std::string & drvName)
{
DerivationT<InputDrvPath, StorePath> drvFinal;
//drvFinal.inputSrcs = drv.inputSrcs;
for (auto & i : drv.inputSrcs)
drvFinal.inputSrcs.insert(i.clone());
drvFinal.platform = drv.platform;
drvFinal.builder = drv.builder;
drvFinal.args = drv.args;
drvFinal.env = drv.env;
const auto drvOrPseudo = derivationModuloOrOutput(store, drv, drvName);
if (const DerivationT<Hash, NoPath> *pval = std::get_if<0>(&drvOrPseudo)) {
Hash hash = hashDerivation(store, *pval);
for (const auto i : drv.outputs) {
auto outPath = store.makeOutputPath(i.first, hash, drvName);
drvFinal.outputs.insert_or_assign(i.first,
DerivationOutput(std::move(outPath), "", ""));
}
} else if (std::get_if<1>(&drvOrPseudo)) {
DerivationOutputsT<NoPath>::const_iterator out = drv.outputs.find("out");
if (out == drv.outputs.end())
throw Error("derivation does not have an output named 'out'");
auto & output = out->second;
auto outPath = printPhysicalOutput(store, output, drvName);
drvFinal.outputs.insert_or_assign("out", DerivationOutput(
std::move(outPath),
std::string(output.hashAlgo),
std::string(output.hash)));
}
return drvFinal;
}

template<typename InputDrvPath>
DerivationT<InputDrvPath, NoPath> stripDerivationPaths(
Store & store,
const DerivationT<InputDrvPath, StorePath> & drv)
{
DerivationT<InputDrvPath, NoPath> drvInitial;
//drvInitial.inputSrcs = drv.inputSrcs;
for (auto & i : drv.inputSrcs)
drvInitial.inputSrcs.insert(i.clone());
drvInitial.platform = drv.platform;
drvInitial.builder = drv.builder;
drvInitial.args = drv.args;
drvInitial.env = drv.env;
for (const auto & i : drv.outputs) {
drvInitial.outputs.insert_or_assign(i.first, DerivationOutputT(
NoPath(),
std::string(i.second.hashAlgo),
std::string(i.second.hash)));
}
return drvInitial;
}

std::string StorePathWithOutputs::to_string(const Store & store) const
{
Expand Down Expand Up @@ -515,15 +646,71 @@ template struct DerivationT<Hash, StorePath>;
template struct DerivationT<Hash, NoPath>;

template
std::variant<DerivationT<Hash, StorePath>, std::string> derivationModulo(
DerivationT<Hash, StorePath> derivationModulo(
Store & store,
const DerivationT<StorePath, StorePath> & drv);
template
std::variant<DerivationT<Hash, NoPath>, std::string> derivationModulo(
DerivationT<Hash, NoPath> derivationModulo(
Store & store,
const DerivationT<StorePath, NoPath> & drv);
template
DerivationT<Hash, StorePath> derivationModulo(
Store & store,
const DerivationT<Hash, StorePath> & drv);
template
DerivationT<Hash, NoPath> derivationModulo(
Store & store,
const DerivationT<Hash, NoPath> & drv);

template
std::variant<DerivationT<Hash, StorePath>, std::string> derivationModuloOrOutput(
Store & store,
const DerivationT<StorePath, StorePath> & drv);
template
std::variant<DerivationT<Hash, StorePath>, std::string> derivationModuloOrOutput(
Store & store,
const DerivationT<Hash, StorePath> & drv);
template
std::variant<DerivationT<Hash, NoPath>, std::string> derivationModuloOrOutput(
Store & store,
const DerivationT<StorePath, NoPath> & drv,
const std::string & drvName);
template
std::variant<DerivationT<Hash, NoPath>, std::string> derivationModuloOrOutput(
Store & store,
const DerivationT<Hash, NoPath> & drv,
const std::string & drvName);

template
Hash hashDerivationOrPseudo(
Store & store,
typename std::variant<DerivationT<Hash, StorePath>, std::string> drvOrPseudo);
template
Hash hashDerivationOrPseudo(
Store & store,
typename std::variant<DerivationT<Hash, NoPath>, std::string> drvOrPseudo);

template Hash hashDerivation(Store & store, const DerivationT<Hash, StorePath> & drv);
template Hash hashDerivation(Store & store, const DerivationT<Hash, NoPath> & drv);

template
DerivationT<StorePath, StorePath> bakeDerivationPaths(
Store & store,
const DerivationT<StorePath, NoPath> & drv,
const std::string & drvName);
template
DerivationT<Hash, StorePath> bakeDerivationPaths(
Store & store,
const DerivationT<Hash, NoPath> & drv,
const std::string & drvName);

template
DerivationT<StorePath, NoPath> stripDerivationPaths(
Store & store,
const DerivationT<StorePath, StorePath> & drv);
template
DerivationT<Hash, NoPath> stripDerivationPaths(
Store & store,
const DerivationT<Hash, StorePath> & drv);

}
Loading

0 comments on commit 26d0c60

Please sign in to comment.