diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 357962007a4..08f96285050 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -263,7 +263,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource stats.narInfoWrite++; } -bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) +bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath, const std::string ca) { // FIXME: this only checks whether a .narinfo with a matching hash // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even @@ -271,7 +271,7 @@ bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) return fileExists(narInfoFileFor(storePath)); } -void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) +void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink, const std::string ca) { auto info = queryPathInfo(storePath).cast(); @@ -298,7 +298,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) } void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, - Callback> callback) noexcept + Callback> callback, const std::string ca) noexcept { auto uri = getUri(); auto storePathS = printStorePath(storePath); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 52ef8aa7ac2..b153d904ef1 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -66,10 +66,11 @@ private: public: - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(const StorePath & path, const std::string ca) override; void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override; + Callback> callback, + const std::string ca) noexcept override; std::optional queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } @@ -85,7 +86,7 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink, const std::string ca) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f5c132a8397..56caa45c642 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2703,7 +2703,7 @@ struct RestrictedStore : public LocalFSStore } void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override + Callback> callback, const std::string ca) noexcept override { if (goal.isAllowed(path)) { try { @@ -2762,11 +2762,11 @@ struct RestrictedStore : public LocalFSStore return path; } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(const StorePath & path, Sink & sink, std::string ca) override { if (!goal.isAllowed(path)) throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); - LocalFSStore::narFromPath(path, sink); + LocalFSStore::narFromPath(path, sink, ca); } void ensurePath(const StorePath & path) override diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 486babf1427..254cb4fce2c 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -58,20 +58,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) } }; - /* Try the hashed mirrors first. */ - if (getAttr("outputHashMode") == "flat") - for (auto hashedMirror : settings.hashedMirrors.get()) - try { - if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - auto ht = parseHashType(getAttr("outputHashAlgo")); - auto h = Hash(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); - return; - } catch (Error & e) { - debug(e.what()); - } - - /* Otherwise try the specified URL. */ fetch(mainUrl); } diff --git a/src/libstore/hashed-mirror-store.cc b/src/libstore/hashed-mirror-store.cc new file mode 100644 index 00000000000..3d5cdfa79fe --- /dev/null +++ b/src/libstore/hashed-mirror-store.cc @@ -0,0 +1,202 @@ +#include "binary-cache-store.hh" +#include "filetransfer.hh" +#include "globals.hh" +#include "archive.hh" + +namespace nix { + +class HashedMirrorStore : public Store +{ +private: + + Path cacheUri; + Path cacheDir; + +public: + + HashedMirrorStore( + const Params & params, const Path & _cacheUri) + : Store(params) + , cacheUri(_cacheUri) + { + if (hasPrefix(cacheUri, "hashed-mirror+")) + cacheUri = cacheUri.substr(14); + + if (cacheUri.back() == '/') + cacheUri.pop_back(); + + if (!hasPrefix(cacheUri, "file://")) + throw Error("only file:// cache is currently supported in hashed mirror store"); + + cacheDir = cacheUri.substr(7); + } + + std::string getUri() override + { + return cacheUri; + } + + void init() + { + } + + void narFromPath(const StorePath & storePath, Sink & sink, const std::string ca) override + { + dumpPath(cacheDir + getPath(ca), sink); + } + + static Hash getHash(std::string ca) + { + if (ca == "") + throw Error("ca cannot be empty in hashed mirror store"); + + if (!hasPrefix(ca, "fixed:")) + throw Error("hashed mirror must be fixed-output"); + + ca = ca.substr(6); + + if (hasPrefix(ca, "r:")) + throw Error("hashed mirror cannot be recursive"); + + return Hash(ca); + } + + static std::string getPath(std::string ca) + { + Hash h = getHash(ca); + + return "/" + printHashType(h.type) + "/" + h.to_string(Base16, false); + } + + bool isValidPathUncached(const StorePath & storePath, std::string ca) override + { + return fileExists(getPath(ca)); + } + + void queryPathInfoUncached(const StorePath & path, + Callback> callback, const std::string ca) noexcept override + { + auto info = std::make_shared(path.clone()); + + // not efficient! + StringSink sink; + dumpPath(cacheDir + getPath(ca), sink); + info->narHash = hashString(htSHA256, *sink.s); + info->narSize = sink.s->size(); + + info->ca = ca; + callback(std::move(info)); + } + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { + unsupported("queryPathFromHashPart"); + } + + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) override + { + if (info.references.size() > 0) + throw Error("references are not supported in a hashed mirror store"); + + auto dirname = std::string(dirOf(cacheDir + getPath(info.ca))); + if (!pathExists(dirname)) + if (mkdir(dirname.c_str(), 0777) == -1) + throw SysError(format("creating directory '%1%'") % dirname); + + auto path = cacheDir + getPath(info.ca); + restorePath(path, source); + + Hash h(info.ca.substr(6)); + + HashSink hashSink(h.type); + readFile(path, hashSink); + + Hash gotHash = hashSink.finish().first; + if (gotHash != h) + throw Error("path '%s' does not have correct hash: expected %s, got %s", path, h.to_string(), gotHash.to_string()); + } + + StorePath addToStore(const string & name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + PathFilter & filter, RepairFlag repair) override + { + unsupported("addToStore"); + } + + void ensurePath(const StorePath & path) override + { + unsupported("ensurePath"); + } + + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair = NoRepair) override + { unsupported("addTextToStore"); } + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) override + { unsupported("buildDerivation"); } + + StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute, std::map pathsInfo) override + { + StorePathSet res; + for (auto & i : paths) { + auto ca = pathsInfo.find(printStorePath(i)); + if (isValidPath(i, ca != pathsInfo.end() ? ca->second : "")) + res.insert(i.clone()); + } + return res; + } + + // Taken from local-binary-cache.cc, should make this also support http + + static void atomicWrite(const Path & path, const std::string & s) + { + Path tmp = path + ".tmp." + std::to_string(getpid()); + AutoDelete del(tmp, false); + writeFile(tmp, s); + if (rename(tmp.c_str(), path.c_str())) + throw SysError(format("renaming '%1%' to '%2%'") % tmp % path); + del.cancel(); + } + + bool fileExists(const std::string & path) + { + return pathExists(cacheDir + "/" + path); + } + + void upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) + { + atomicWrite(cacheDir + "/" + path, data); + } + + void getFile(const std::string & path, Sink & sink) + { + try { + readFile(cacheDir + "/" + path, sink); + } catch (SysError & e) { + if (e.errNo == ENOENT) + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); + } + } + + bool supportsOtherStoreDir() { + return true; + } + +}; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + if (std::string(uri, 0, 14) != "hashed-mirror+") + return 0; + auto store = std::make_shared(params, uri); + store->init(); + return store; +}); + +} diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index af20d389b06..135e6a13b4b 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -88,7 +88,8 @@ struct LegacySSHStore : public Store } void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override + Callback> callback, + const std::string ca) noexcept override { try { auto conn(connections->get()); @@ -182,7 +183,7 @@ struct LegacySSHStore : public Store throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); } - void narFromPath(const StorePath & path, Sink & sink) override + void narFromPath(const StorePath & path, Sink & sink, const std::string ca) override { auto conn(connections->get()); @@ -260,7 +261,8 @@ struct LegacySSHStore : public Store } StorePathSet queryValidPaths(const StorePathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override + SubstituteFlag maybeSubstitute = NoSubstitute, + std::map pathsInfo = {}) override { auto conn(connections->get()); diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index aa5abd835d7..42ae3d45baa 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -77,7 +77,7 @@ ref LocalFSStore::getFSAccessor() std::dynamic_pointer_cast(shared_from_this()))); } -void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) +void LocalFSStore::narFromPath(const StorePath & path, Sink & sink, const std::string) { if (!isValidPath(path)) throw Error("path '%s' is not valid", printStorePath(path)); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80851b59106..efab321b15b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -624,7 +624,7 @@ uint64_t LocalStore::addValidPath(State & state, void LocalStore::queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept + Callback> callback, const std::string ca) noexcept { try { auto info = std::make_shared(path.clone()); @@ -704,7 +704,7 @@ bool LocalStore::isValidPath_(State & state, const StorePath & path) } -bool LocalStore::isValidPathUncached(const StorePath & path) +bool LocalStore::isValidPathUncached(const StorePath & path, const std::string ca) { return retrySQLite([&]() { auto state(_state.lock()); @@ -713,7 +713,7 @@ bool LocalStore::isValidPathUncached(const StorePath & path) } -StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute, std::map pathsInfo) { StorePathSet res; for (auto & i : paths) @@ -835,7 +835,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) for (auto & sub : getDefaultSubstituters()) { if (remaining.empty()) break; - if (sub->storeDir != storeDir) continue; + if (sub->storeDir != storeDir && !sub->supportsOtherStoreDir()) continue; if (!sub->wantMassQuery) continue; auto valid = sub->queryValidPaths(remaining); @@ -855,16 +855,38 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) + SubstitutablePathInfos & infos, std::map drvs) { if (!settings.useSubstitutes) return; for (auto & sub : getDefaultSubstituters()) { - if (sub->storeDir != storeDir) continue; + if (sub->storeDir != storeDir && sub->supportsOtherStoreDir()) continue; for (auto & path : paths) { if (infos.count(path)) continue; debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); try { - auto info = sub->queryPathInfo(path); + std::string ca = ""; + + auto drv = drvs.find(printStorePath(path)); + if (drv != drvs.end()) { + auto env = drv->second.env; + auto outputHashMode = env.find("outputHashMode"); + auto outputHashAlgo = env.find("outputHashAlgo"); + auto outputHash = env.find("outputHash"); + if (outputHashMode != env.end() && outputHashAlgo != env.end() && outputHash != env.end()) { + auto ht = parseHashType(outputHashAlgo->second); + auto h = Hash(outputHash->second, ht); + FileIngestionMethod ingestionMethod; + if (outputHashMode->second == "recursive") + ingestionMethod = FileIngestionMethod::Recursive; + else if (outputHashMode->second == "flat") + ingestionMethod = FileIngestionMethod::Flat; + else + throw Error("unknown ingestion method: '%s'", outputHashMode->second); + ca = makeFixedOutputCA(ingestionMethod, h); + } + } + + auto info = sub->queryPathInfo(path, ca); auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); infos.insert_or_assign(path.clone(), SubstitutablePathInfo{ diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c1e75390c5d..9fd39fa7d0b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -25,6 +25,7 @@ const int nixSchemaVersion = 10; struct Derivation; +class HashedMirrorStore; struct OptimiseStats { @@ -119,15 +120,17 @@ public: std::string getUri() override; - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(const StorePath & path, const std::string ca) override; StorePathSet queryValidPaths(const StorePathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override; + SubstituteFlag maybeSubstitute = NoSubstitute, + std::map pathsInfo = {}) override; StorePathSet queryAllValidPaths() override; void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override; + Callback> callback, + const std::string ca) noexcept override; void queryReferrers(const StorePath & path, StorePathSet & referrers) override; @@ -142,7 +145,8 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) override; + SubstitutablePathInfos & infos, + std::map drvs = {}) override; void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 9c47fe52454..7d3b5d9d9b1 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -159,7 +159,8 @@ void Store::queryMissing(const std::vector & targets, SubstitutablePathInfos infos; StorePathSet paths; // FIXME paths.insert(outPath.clone()); - querySubstitutablePathInfos(paths, infos); + + querySubstitutablePathInfos(paths, infos, { {outPathS, Derivation(*drv) } }); if (infos.empty()) { drvState_->lock()->done = true; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5c36693e699..d483b342d30 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -253,7 +253,7 @@ ConnectionHandle RemoteStore::getConnection() } -bool RemoteStore::isValidPathUncached(const StorePath & path) +bool RemoteStore::isValidPathUncached(const StorePath & path, const std::string ca) { auto conn(getConnection()); conn->to << wopIsValidPath << printStorePath(path); @@ -262,7 +262,7 @@ bool RemoteStore::isValidPathUncached(const StorePath & path) } -StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute, std::map pathsInfo) { auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { @@ -309,7 +309,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) + SubstitutablePathInfos & infos, std::map drvs) { if (paths.empty()) return; @@ -353,7 +353,8 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, void RemoteStore::queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept + Callback> callback, + const std::string ca) noexcept { try { std::shared_ptr info; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3c86b4524eb..71b186426fe 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -35,15 +35,17 @@ public: /* Implementations of abstract store API methods. */ - bool isValidPathUncached(const StorePath & path) override; + bool isValidPathUncached(const StorePath & path, const std::string ca) override; StorePathSet queryValidPaths(const StorePathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override; + SubstituteFlag maybeSubstitute = NoSubstitute, + std::map pathsInfo = {}) override; StorePathSet queryAllValidPaths() override; void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override; + Callback> callback, + const std::string ca) noexcept override; void queryReferrers(const StorePath & path, StorePathSet & referrers) override; @@ -58,7 +60,7 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) override; + SubstitutablePathInfos & infos, std::map drvs = {}) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs, diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index b24e7b7d621..8e2ddad8f24 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -223,7 +223,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore fetches the .narinfo file, rather than first checking for its existence via a HEAD request. Since .narinfos are small, doing a GET is unlikely to be slower than HEAD. */ - bool isValidPathUncached(const StorePath & storePath) override + bool isValidPathUncached(const StorePath & storePath, const std::string ca) override { try { queryPathInfo(storePath); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index caae6b596c7..55ea9c71670 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -40,7 +40,7 @@ class SSHStore : public RemoteStore bool sameMachine() override { return false; } - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink, const std::string ca) override; ref getFSAccessor() override; @@ -68,7 +68,7 @@ class SSHStore : public RemoteStore }; }; -void SSHStore::narFromPath(const StorePath & path, Sink & sink) +void SSHStore::narFromPath(const StorePath & path, Sink & sink, const std::string ca) { auto conn(connections->get()); conn->to << wopNarFromPath << printStorePath(path); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 095363d0c9b..79d798f48b6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -241,7 +241,7 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } -bool Store::isValidPath(const StorePath & storePath) +bool Store::isValidPath(const StorePath & storePath, const std::string ca) { auto hashPart = storePathToHash(printStorePath(storePath)); @@ -265,7 +265,7 @@ bool Store::isValidPath(const StorePath & storePath) } } - bool valid = isValidPathUncached(storePath); + bool valid = isValidPathUncached(storePath, ca); if (diskCache && !valid) // FIXME: handle valid = true case. @@ -277,7 +277,7 @@ bool Store::isValidPath(const StorePath & storePath) /* Default implementation for stores that only implement queryPathInfoUncached(). */ -bool Store::isValidPathUncached(const StorePath & path) +bool Store::isValidPathUncached(const StorePath & path, const std::string ca) { try { queryPathInfo(path); @@ -288,7 +288,7 @@ bool Store::isValidPathUncached(const StorePath & path) } -ref Store::queryPathInfo(const StorePath & storePath) +ref Store::queryPathInfo(const StorePath & storePath, const std::string ca) { std::promise> promise; @@ -299,14 +299,14 @@ ref Store::queryPathInfo(const StorePath & storePath) } catch (...) { promise.set_exception(std::current_exception()); } - }}); + }}, ca); return promise.get_future().get(); } void Store::queryPathInfo(const StorePath & storePath, - Callback> callback) noexcept + Callback> callback, const std::string ca) noexcept { std::string hashPart; @@ -364,11 +364,11 @@ void Store::queryPathInfo(const StorePath & storePath, (*callbackPtr)(ref(info)); } catch (...) { callbackPtr->rethrow(); } - }}); + }}, ca); } -StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) +StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute, std::map pathsInfo) { struct State { @@ -582,7 +582,7 @@ void copyStorePath(ref srcStore, ref dstStore, if (!info->narHash) { StringSink sink; - srcStore->narFromPath({storePath}, sink); + srcStore->narFromPath({storePath}, sink, info->ca); auto info2 = make_ref(*info); info2->narHash = hashString(htSHA256, *sink.s); if (!info->narSize) info2->narSize = sink.s->size(); @@ -606,7 +606,7 @@ void copyStorePath(ref srcStore, ref dstStore, total += len; act.progress(total, info->narSize); }); - srcStore->narFromPath(storePath, wrapperSink); + srcStore->narFromPath(storePath, wrapperSink, info->ca); }, [&]() { throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); }); @@ -618,7 +618,10 @@ void copyStorePath(ref srcStore, ref dstStore, void copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { - auto valid = dstStore->queryValidPaths(storePaths, substitute); + std::map pathsInfo; + for (auto & path : storePaths) + pathsInfo.emplace(dstStore->printStorePath(path), std::string(srcStore->queryPathInfo(path)->ca)); + auto valid = dstStore->queryValidPaths(storePaths, substitute, pathsInfo); PathSet missing; for (auto & path : storePaths) @@ -642,15 +645,15 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st processGraph(pool, PathSet(missing.begin(), missing.end()), - [&](const Path & storePath) { - if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) { + [&](const Path & storePathS) { + auto storePath = dstStore->parseStorePath(storePathS); + auto info = srcStore->queryPathInfo(storePath); + if (dstStore->isValidPath(storePath, info->ca)) { nrDone++; showProgress(); return PathSet(); } - auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); - bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); @@ -662,7 +665,8 @@ void copyPaths(ref srcStore, ref dstStore, const StorePathSet & st auto storePath = dstStore->parseStorePath(storePathS); - if (!dstStore->isValidPath(storePath)) { + auto info = srcStore->queryPathInfo(storePath); + if (!dstStore->isValidPath(storePath, info->ca)) { MaintainCount mc(nrRunning); showProgress(); try { @@ -944,6 +948,10 @@ std::list> getDefaultSubstituters() for (auto uri : settings.extraSubstituters.get()) addStore(uri); + for (auto uri : settings.hashedMirrors.get()) { + addStore("hashed-mirror+" + uri); + } + stores.sort([](ref & a, ref & b) { return a->priority < b->priority; }); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b1e25fc7d66..7d66256907c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -298,6 +298,8 @@ public: virtual std::string getUri() = 0; + virtual bool supportsOtherStoreDir() { return false; }; + StorePath parseStorePath(std::string_view path) const; std::optional maybeParseStorePath(std::string_view path) const; @@ -379,18 +381,19 @@ public: const StorePathSet & references) const; /* Check whether a path is valid. */ - bool isValidPath(const StorePath & path); + bool isValidPath(const StorePath & path, const std::string ca = ""); protected: - virtual bool isValidPathUncached(const StorePath & path); + virtual bool isValidPathUncached(const StorePath & path, const std::string ca = ""); public: /* Query which of the given paths is valid. Optionally, try to substitute missing paths. */ virtual StorePathSet queryValidPaths(const StorePathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute); + SubstituteFlag maybeSubstitute = NoSubstitute, + std::map pathsInfo = {}); /* Query the set of all valid paths. Note that for some store backends, the name part of store paths may be omitted @@ -402,16 +405,16 @@ public: /* Query information about a valid path. It is permitted to omit the name part of the store path. */ - ref queryPathInfo(const StorePath & path); + ref queryPathInfo(const StorePath & path, const std::string ca = ""); /* Asynchronous version of queryPathInfo(). */ void queryPathInfo(const StorePath & path, - Callback> callback) noexcept; + Callback> callback, const std::string ca = "") noexcept; protected: virtual void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept = 0; + Callback> callback, const std::string ca = "") noexcept = 0; public: @@ -445,7 +448,7 @@ public: sizes) of a set of paths. If a path does not have substitute info, it's omitted from the resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) { return; }; + SubstitutablePathInfos & infos, std::map drvs = {}) { return; }; /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, @@ -473,7 +476,7 @@ public: const StorePathSet & references, RepairFlag repair = NoRepair) = 0; /* Write a NAR dump of a store path. */ - virtual void narFromPath(const StorePath & path, Sink & sink) = 0; + virtual void narFromPath(const StorePath & path, Sink & sink, std::string ca = "") = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If @@ -713,7 +716,7 @@ public: LocalFSStore(const Params & params); - void narFromPath(const StorePath & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink, std::string ca) override; ref getFSAccessor() override; /* Register a permanent GC root. */