Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

More templated STL support for the daemon protocol #3895

Merged
merged 18 commits into from
Oct 5, 2020
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
2f2ae99
WIP systematize more of the worker protocol
Ericson2314 Aug 4, 2020
1bab8a3
Remove unneeded definition
Ericson2314 Aug 4, 2020
46cafb1
Merge branch 'drv-outputs-map-allow-missing' into templated-daemon-pr…
Ericson2314 Aug 4, 2020
89bda7f
Merge branch 'drv-outputs-map-allow-missing' into templated-daemon-pr…
Ericson2314 Aug 4, 2020
6d003d8
Merge branch 'drv-outputs-map-allow-missing' into templated-daemon-pr…
Ericson2314 Aug 4, 2020
7302761
Merge remote-tracking branch 'obsidian/drv-outputs-map-allow-missing'…
Ericson2314 Aug 5, 2020
f795f0f
Merge branch 'drv-outputs-map-allow-missing-namespace' of github.com:…
meditans Aug 6, 2020
3d8240c
Remove leftover commented code
meditans Aug 6, 2020
9ab07e9
Use template structs instead of phantoms
meditans Aug 6, 2020
46f9dd5
Fix bug due to non-deterministic arg eval order
meditans Aug 6, 2020
8f92bb5
Merge branch 'drv-outputs-map-allow-missing' of github.com:obsidiansy…
Ericson2314 Aug 7, 2020
be0d429
Merge branch 'master' of github.com:NixOS/nix into templated-daemon-p…
Ericson2314 Aug 19, 2020
c265e0e
Merge remote-tracking branch 'upstream/master' into templated-daemon-…
Ericson2314 Aug 20, 2020
b92d3b2
Merge remote-tracking branch 'upstream/master' into templated-daemon-…
Ericson2314 Sep 22, 2020
e9fc203
Merge remote-tracking branch 'upstream/master' into templated-daemon-…
Ericson2314 Sep 22, 2020
45a0ed8
Revert "Use template structs instead of phantoms"
Ericson2314 Sep 30, 2020
b759701
nix::worker_proto -> worker_proto
Ericson2314 Sep 30, 2020
69afaea
Merge remote-tracking branch 'upstream/master' into templated-daemon-…
Ericson2314 Sep 30, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/libstore/build.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1864,11 +1864,11 @@ HookReply DerivationGoal::tryBuildHook()

/* Tell the hook all the inputs that have to be copied to the
remote system. */
writeStorePaths(worker.store, hook->sink, inputPaths);
WorkerProto<StorePathSet>::write(worker.store, hook->sink, inputPaths);

/* Tell the hooks the missing outputs that have to be copied back
from the remote system. */
writeStorePaths(worker.store, hook->sink, missingPaths);
WorkerProto<StorePathSet>::write(worker.store, hook->sink, missingPaths);

hook->sink = FdSink();
hook->toHook.writeSide = -1;
Expand Down
36 changes: 18 additions & 18 deletions src/libstore/daemon.cc
Original file line number Diff line number Diff line change
Expand Up @@ -256,11 +256,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}

case wopQueryValidPaths: {
auto paths = readStorePaths<StorePathSet>(*store, from);
auto paths = WorkerProto<StorePathSet>::read(*store, from);
logger->startWork();
auto res = store->queryValidPaths(paths);
logger->stopWork();
writeStorePaths(*store, to, res);
WorkerProto<StorePathSet>::write(*store, to, res);
break;
}

Expand All @@ -276,11 +276,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}

case wopQuerySubstitutablePaths: {
auto paths = readStorePaths<StorePathSet>(*store, from);
auto paths = WorkerProto<StorePathSet>::read(*store, from);
logger->startWork();
auto res = store->querySubstitutablePaths(paths);
logger->stopWork();
writeStorePaths(*store, to, res);
WorkerProto<StorePathSet>::write(*store, to, res);
break;
}

Expand Down Expand Up @@ -309,7 +309,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
paths = store->queryValidDerivers(path);
else paths = store->queryDerivationOutputs(path);
logger->stopWork();
writeStorePaths(*store, to, paths);
WorkerProto<StorePathSet>::write(*store, to, paths);
break;
}

Expand All @@ -327,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto outputs = store->queryDerivationOutputMap(path);
logger->stopWork();
worker_proto::write(*store, to, outputs);
WorkerProto<std::map<std::string, std::optional<StorePath>>>::write(*store, to, outputs);
break;
}

Expand Down Expand Up @@ -397,7 +397,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopAddTextToStore: {
string suffix = readString(from);
string s = readString(from);
auto refs = readStorePaths<StorePathSet>(*store, from);
auto refs = WorkerProto<StorePathSet>::read(*store, from);
logger->startWork();
auto path = store->addTextToStore(suffix, s, refs, NoRepair);
logger->stopWork();
Expand Down Expand Up @@ -518,7 +518,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopCollectGarbage: {
GCOptions options;
options.action = (GCOptions::GCAction) readInt(from);
options.pathsToDelete = readStorePaths<StorePathSet>(*store, from);
options.pathsToDelete = WorkerProto<StorePathSet>::read(*store, from);
from >> options.ignoreLiveness >> options.maxFreed;
// obsolete fields
readInt(from);
Expand Down Expand Up @@ -587,7 +587,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
else {
to << 1
<< (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
writeStorePaths(*store, to, i->second.references);
WorkerProto<StorePathSet>::write(*store, to, i->second.references);
to << i->second.downloadSize
<< i->second.narSize;
}
Expand All @@ -598,19 +598,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
SubstitutablePathInfos infos;
StorePathCAMap pathsMap = {};
if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
auto paths = readStorePaths<StorePathSet>(*store, from);
auto paths = WorkerProto<StorePathSet>::read(*store, from);
for (auto & path : paths)
pathsMap.emplace(path, std::nullopt);
} else
pathsMap = readStorePathCAMap(*store, from);
pathsMap = WorkerProto<StorePathCAMap>::read(*store, from);
logger->startWork();
store->querySubstitutablePathInfos(pathsMap, infos);
logger->stopWork();
to << infos.size();
for (auto & i : infos) {
to << store->printStorePath(i.first)
<< (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
writeStorePaths(*store, to, i.second.references);
WorkerProto<StorePathSet>::write(*store, to, i.second.references);
to << i.second.downloadSize << i.second.narSize;
}
break;
Expand All @@ -620,7 +620,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto paths = store->queryAllValidPaths();
logger->stopWork();
writeStorePaths(*store, to, paths);
WorkerProto<StorePathSet>::write(*store, to, paths);
break;
}

Expand All @@ -639,7 +639,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
to << 1;
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
<< info->narHash->to_string(Base16, false);
writeStorePaths(*store, to, info->references);
WorkerProto<StorePathSet>::write(*store, to, info->references);
to << info->registrationTime << info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
to << info->ultimate
Expand Down Expand Up @@ -699,7 +699,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (deriver != "")
info.deriver = store->parseStorePath(deriver);
info.narHash = Hash::parseAny(readString(from), htSHA256);
info.references = readStorePaths<StorePathSet>(*store, from);
info.references = WorkerProto<StorePathSet>::read(*store, from);
from >> info.registrationTime >> info.narSize >> info.ultimate;
info.sigs = readStrings<StringSet>(from);
info.ca = parseContentAddressOpt(readString(from));
Expand Down Expand Up @@ -799,9 +799,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
uint64_t downloadSize, narSize;
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
logger->stopWork();
writeStorePaths(*store, to, willBuild);
writeStorePaths(*store, to, willSubstitute);
writeStorePaths(*store, to, unknown);
WorkerProto<StorePathSet>::write(*store, to, willBuild);
WorkerProto<StorePathSet>::write(*store, to, willSubstitute);
WorkerProto<StorePathSet>::write(*store, to, unknown);
to << downloadSize << narSize;
break;
}
Expand Down
4 changes: 2 additions & 2 deletions src/libstore/derivations.cc
Original file line number Diff line number Diff line change
Expand Up @@ -605,7 +605,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
drv.outputs.emplace(std::move(name), std::move(output));
}

drv.inputSrcs = readStorePaths<StorePathSet>(store, in);
drv.inputSrcs = WorkerProto<StorePathSet>::read(store, in);
in >> drv.platform >> drv.builder;
drv.args = readStrings<Strings>(in);

Expand Down Expand Up @@ -640,7 +640,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
},
}, i.second.output);
}
writeStorePaths(store, out, drv.inputSrcs);
WorkerProto<StorePathSet>::write(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
out << drv.env.size();
for (auto & i : drv.env)
Expand Down
4 changes: 2 additions & 2 deletions src/libstore/export-import.cc
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
teeSink
<< exportMagic
<< printStorePath(path);
writeStorePaths(*this, teeSink, info->references);
WorkerProto<StorePathSet>::write(*this, teeSink, info->references);
teeSink
<< (info->deriver ? printStorePath(*info->deriver) : "")
<< 0;
Expand Down Expand Up @@ -73,7 +73,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)

//Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path);

info.references = readStorePaths<StorePathSet>(*this, source);
info.references = WorkerProto<StorePathSet>::read(*this, source);

auto deriver = readString(source);
if (deriver != "")
Expand Down
14 changes: 7 additions & 7 deletions src/libstore/legacy-ssh-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ struct LegacySSHStore : public Store
auto deriver = readString(conn->from);
if (deriver != "")
info->deriver = parseStorePath(deriver);
info->references = readStorePaths<StorePathSet>(*this, conn->from);
info->references = WorkerProto<StorePathSet>::read(*this, conn->from);
readLongLong(conn->from); // download size
info->narSize = readLongLong(conn->from);

Expand Down Expand Up @@ -139,7 +139,7 @@ struct LegacySSHStore : public Store
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash->to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
WorkerProto<StorePathSet>::write(*this, conn->to, info.references);
conn->to
<< info.registrationTime
<< info.narSize
Expand Down Expand Up @@ -168,7 +168,7 @@ struct LegacySSHStore : public Store
conn->to
<< exportMagic
<< printStorePath(info.path);
writeStorePaths(*this, conn->to, info.references);
WorkerProto<StorePathSet>::write(*this, conn->to, info.references);
conn->to
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< 0
Expand Down Expand Up @@ -251,10 +251,10 @@ struct LegacySSHStore : public Store
conn->to
<< cmdQueryClosure
<< includeOutputs;
writeStorePaths(*this, conn->to, paths);
WorkerProto<StorePathSet>::write(*this, conn->to, paths);
conn->to.flush();

for (auto & i : readStorePaths<StorePathSet>(*this, conn->from))
for (auto & i : WorkerProto<StorePathSet>::read(*this, conn->from))
out.insert(i);
}

Expand All @@ -267,10 +267,10 @@ struct LegacySSHStore : public Store
<< cmdQueryValidPaths
<< false // lock
<< maybeSubstitute;
writeStorePaths(*this, conn->to, paths);
WorkerProto<StorePathSet>::write(*this, conn->to, paths);
conn->to.flush();

return readStorePaths<StorePathSet>(*this, conn->from);
return WorkerProto<StorePathSet>::read(*this, conn->from);
}

void connect() override
Expand Down
Loading