Skip to content

Commit

Permalink
Separate derivation resolving from derivation hashing
Browse files Browse the repository at this point in the history 
Use the newly parameterized derivation types to enforce the invariants
needed to split these.
  • Loading branch information
@Ericson2314
Ericson2314 committed Mar 25, 2020
1 parent e00a077 commit 28d27c2
Show file tree
Hide file tree
Showing 4 changed files with 118 additions and 70 deletions.
81 changes: 49 additions & 32 deletions src/libexpr/primops.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -557,7 +557,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
ignoreNulls = state.forceBool(*attr->value, pos);

/* Build the derivation expression by processing the attributes. */
Derivation drv;
DerivationT<StorePath, NoPath> drv;

PathSet context;

Expand Down Expand Up @@ -715,58 +715,75 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
throw EvalError("derivation names are not allowed to end in '%s', at %s", drvExtension, posDrvName);

if (outputHash) {
/* Handle fixed-output derivations. */
/* Set "out" output with hash algo and hash for fixed-output derivations. */
if (outputs.size() != 1 || *(outputs.begin()) != "out")
throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName);

HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
Hash h(*outputHash, ht);

auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath),
(outputHashRecursive ? "r:" : "") + printHashType(h.type),
drv.outputs.insert_or_assign("out", DerivationOutputT(
NoPath(),
(outputHashRecursive ? "r:" : "") + printHashType(ht),
h.to_string(Base16, false)));
}

else {
/* Compute a hash over the "masked" store derivation, which is
the final one except that in the list of outputs, the
output paths are empty strings, and the corresponding
environment variables have an empty value. This ensures
that changes in the set of output names do get reflected in
the hash. */
for (auto & i : outputs) {
if (!jsonObject) drv.env[i] = "";
drv.outputs.insert_or_assign(i,
DerivationOutput(StorePath::dummy.clone(), "", ""));
}

Hash h = hashDerivationModulo(*state.store, Derivation(drv), true);

for (auto & i : outputs) {
auto outPath = state.store->makeOutputPath(i, h, drvName);
if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(i,
DerivationOutput(std::move(outPath), "", ""));
/* Compute the final derivation, which additionally contains the outputs
paths created from the hash of the initial one. */
Derivation drvFinal;
//drvFinal.inputSrcs = drv.inputSrcs;
for (auto & i : drv.inputSrcs)
drvFinal.inputSrcs.insert(i.clone());
drvFinal.platform = drv.platform;
drvFinal.builder = drv.builder;
drvFinal.args = drv.args;
drvFinal.env = drv.env;
{
const auto drvOrPseudo = derivationModulo(*state.store, drv);
if (const DerivationT<Hash, NoPath> *pval = std::get_if<0>(&drvOrPseudo)) {
Hash hash = hashDerivation(*state.store, *pval);
for (auto & i : outputs) {
auto outPath = state.store->makeOutputPath(i, hash, drvName);
if (!jsonObject) drvFinal.env[i] = state.store->printStorePath(outPath);
drvFinal.outputs.insert_or_assign(i,
DerivationOutput(std::move(outPath), "", ""));
}
} else if (std::get_if<1>(&drvOrPseudo)) {
HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo);
Hash h(*outputHash, ht);
auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName);
auto & output = drv.outputs.find("out")->second;
drvFinal.outputs.insert_or_assign("out", DerivationOutput(
std::move(outPath),
std::string(output.hashAlgo),
std::string(output.hash)));
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
}
}

/* Write the resulting term into the Nix store directory. */
auto drvPath = writeDerivation(state.store, drv, drvName, state.repair);
auto drvPath = writeDerivation(state.store, drvFinal, drvName, state.repair);
auto drvPathS = state.store->printStorePath(drvPath);

printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS);

/* Optimisation, but required in read-only mode! because in that
case we don't actually write store derivations, so we can't
read them later. */
drvHashes.insert_or_assign(drvPath.clone(),
hashDerivationModulo(*state.store, Derivation(drv), false));
{
const auto drvOrPseudo = derivationModulo<StorePath>(*state.store, drvFinal);
Hash hash;
if (const DerivationT<Hash, StorePath> *pval = std::get_if<0>(&drvOrPseudo)) {
hash = hashDerivation(*state.store, *pval);
} else if (const std::string *pval = std::get_if<1>(&drvOrPseudo)) {
hash = hashString(htSHA256, *pval);
}
// Cache it
drvHashes.insert_or_assign(drvPath.clone(), std::move(hash));
}

state.mkAttrs(v, 1 + drv.outputs.size());
state.mkAttrs(v, 1 + drvFinal.outputs.size());
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
for (auto & i : drv.outputs) {
for (auto & i : drvFinal.outputs) {
mkString(*state.allocAttr(v, state.symbols.create(i.first)),
state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS});
}
Expand Down
90 changes: 56 additions & 34 deletions src/libstore/derivations.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ BasicDerivationT<Path>::BasicDerivationT(const BasicDerivationT<Path> & other)
}


template<typename InputDrvPath, typename OutputPath>
DerivationT<InputDrvPath, OutputPath>::DerivationT(const BasicDerivationT<OutputPath> & other)
: BasicDerivationT<OutputPath>(other)
{
}

template<typename InputDrvPath, typename OutputPath>
DerivationT<InputDrvPath, OutputPath>::DerivationT(const DerivationT<InputDrvPath, OutputPath> & other)
: BasicDerivationT<OutputPath>(other)
Expand Down Expand Up @@ -78,7 +84,7 @@ StorePath writeDerivation(ref<Store> store,
(that can be missing (of course) and should not necessarily be
held during a garbage collection). */
auto suffix = std::string(name) + drvExtension;
auto contents = drv.unparse(*store, false);
auto contents = drv.unparse(*store);
return settings.readOnlyMode
? store->computeStorePathForText(suffix, contents, references)
: store->addTextToStore(suffix, contents, references, repair);
Expand Down Expand Up @@ -283,8 +289,7 @@ static string printStoreDrvPath(const Store & store, const Hash & hash) {
}

template<typename InputDrvPath, typename OutputPath>
string DerivationT<InputDrvPath, OutputPath>::unparse(const Store & store, bool maskOutputs,
std::map<std::string, StringSet> * actualInputs) const
string DerivationT<InputDrvPath, OutputPath>::unparse(const Store & store) const
{
string s;
s.reserve(65536);
Expand All @@ -294,28 +299,19 @@ string DerivationT<InputDrvPath, OutputPath>::unparse(const Store & store, bool
for (auto & i : this->outputs) {
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedString(s, maskOutputs ? "" : printStorePath(store, i.second.path));
s += ','; printUnquotedString(s, printStorePath(store, i.second.path));
s += ','; printUnquotedString(s, i.second.hashAlgo);
s += ','; printUnquotedString(s, i.second.hash);
s += ')';
}

s += "],[";
first = true;
if (actualInputs) {
for (auto & i : *actualInputs) {
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
s += ')';
}
} else {
for (auto & i : inputDrvs) {
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, printStoreDrvPath(store, i.first));
s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
s += ')';
}
for (auto & i : inputDrvs) {
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, printStoreDrvPath(store, i.first));
s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end());
s += ')';
}

s += "],";
Expand All @@ -330,8 +326,10 @@ string DerivationT<InputDrvPath, OutputPath>::unparse(const Store & store, bool
first = true;
for (auto & i : this->env) {
if (first) first = false; else s += ',';
auto o = this->outputs.find(i.first);
s += '('; printString(s, i.first);
s += ','; printString(s, maskOutputs && this->outputs.count(i.first) ? "" : i.second);
// TODO use proper placeholders
s += ','; printString(s, o != this->outputs.end() ? "" : i.second);
s += ')';
}

Expand All @@ -340,6 +338,10 @@ string DerivationT<InputDrvPath, OutputPath>::unparse(const Store & store, bool
return s;
}

template<typename OutPath>
Hash hashDerivation(Store & store, const DerivationT<Hash, OutPath> & drv) {
return hashString(htSHA256, drv.unparse(store));
}

// FIXME: remove
bool isDerivation(const string & fileName)
Expand Down Expand Up @@ -370,15 +372,19 @@ static const Hash & pathDerivationModulo(Store & store, const StorePath & drvPat
auto h = drvHashes.find(drvPath);
if (h == drvHashes.end()) {
assert(store.isValidPath(drvPath));
// Cache it
h = drvHashes.insert_or_assign(
drvPath.clone(),
hashDerivationModulo(
const std::variant<DerivationT<Hash, StorePath>, std::string> drvOrPseudo = derivationModulo(
store,
readDerivation(
store,
readDerivation(
store,
store.toRealPath(store.printStorePath(drvPath))),
false)).first;
store.toRealPath(store.printStorePath(drvPath))));
Hash hash;
if (const DerivationT<Hash, StorePath> *pval = std::get_if<0>(&drvOrPseudo)) {
hash = hashDerivation(store, *pval);
} else if (const std::string *pval = std::get_if<1>(&drvOrPseudo)) {
hash = hashString(htSHA256, *pval);
}
// Cache it
h = drvHashes.insert_or_assign(drvPath.clone(), hash).first;
}
return h->second;
}
Expand All @@ -403,26 +409,29 @@ static const Hash & pathDerivationModulo(Store & store, const StorePath & drvPat
paths have been replaced by the result of a recursive call to this
function, and that for fixed-output derivations we return a hash of
its output path. */
Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
template<typename OutPath>
std::variant<DerivationT<Hash, OutPath>, string> derivationModulo(
Store & store,
const DerivationT<StorePath, OutPath> & drv)
{
/* Return a fixed hash for fixed-output derivations. */
if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator i = drv.outputs.begin();
return hashString(htSHA256, "fixed:out:"
typename DerivationOutputsT<OutPath>::const_iterator i = drv.outputs.begin();
return "fixed:out:"
+ i->second.hashAlgo + ":"
+ i->second.hash + ":"
+ store.printStorePath(i->second.path));
+ printStorePath(store, i->second.path);
}

/* For other derivations, replace the inputs paths with recursive
calls to this function. */
std::map<std::string, StringSet> inputs2;
DerivationT<Hash, OutPath> drvNorm((BasicDerivationT<OutPath>)drv);
for (auto & i : drv.inputDrvs) {
const auto h = pathDerivationModulo(store, i.first);
inputs2.insert_or_assign(h.to_string(Base16, false), i.second);
drvNorm.inputDrvs.insert_or_assign(h, i.second);
}

return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
return drvNorm;
}


Expand Down Expand Up @@ -502,6 +511,19 @@ template struct BasicDerivationT<StorePath>;
template struct BasicDerivationT<NoPath>;

template struct DerivationT<StorePath, StorePath>;
template struct DerivationT<Hash, StorePath>;
template struct DerivationT<Hash, NoPath>;

template
std::variant<DerivationT<Hash, StorePath>, std::string> derivationModulo(
Store & store,
const DerivationT<StorePath, StorePath> & drv);
template
std::variant<DerivationT<Hash, NoPath>, std::string> derivationModulo(
Store & store,
const DerivationT<StorePath, NoPath> & drv);

template Hash hashDerivation(Store & store, const DerivationT<Hash, StorePath> & drv);
template Hash hashDerivation(Store & store, const DerivationT<Hash, NoPath> & drv);

}
14 changes: 11 additions & 3 deletions src/libstore/derivations.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,11 @@ struct DerivationT : BasicDerivationT<OutputPath>
DerivationInputsT<InputDrvPath> inputDrvs; /* inputs that are sub-derivations */

/* Print a derivation. */
std::string unparse(const Store & store, bool maskOutputs,
std::map<std::string, StringSet> * actualInputs = nullptr) const;
std::string unparse(const Store & store) const;

DerivationT() { }
DerivationT(DerivationT<InputDrvPath, OutputPath> && other) = default;
DerivationT(const BasicDerivationT<OutputPath> & other);
explicit DerivationT(const DerivationT<InputDrvPath, OutputPath> & other);
};

Expand All @@ -111,7 +111,15 @@ Derivation readDerivation(const Store & store, const Path & drvPath);
// FIXME: remove
bool isDerivation(const string & fileName);

Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
template<typename OutPath>
std::variant<DerivationT<Hash, OutPath>, string> derivationModulo(
Store & store,
const DerivationT<StorePath, OutPath> & drv);

template<typename OutPath>
Hash hashDerivation(
Store & store,
const DerivationT<Hash, OutPath> & drv);

/* Memoisation of hashDerivationModulo(). */
typedef std::map<StorePath, Hash> DrvHashes;
Expand Down
3 changes: 2 additions & 1 deletion src/libstore/local-store.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
}

else {
Hash h = hashDerivationModulo(*this, drv, true);
const Hash h = hashDerivation(*this,
std::get<0>(derivationModulo<StorePath>(*this, drv)));
for (auto & i : drv.outputs)
check(makeOutputPath(i.first, h, drvName), i.second.path, i.first);
}
Expand Down

0 comments on commit 28d27c2

Please sign in to comment.