Skip to content

Commit

Permalink
Build against nix-master
Browse files Browse the repository at this point in the history
  • Loading branch information
@edolstra
edolstra committed Dec 30, 2019
1 parent 5cac40a commit e7f2139
Show file tree
Hide file tree
Showing 14 changed files with 172 additions and 145 deletions.
12 changes: 6 additions & 6 deletions flake.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion src/hydra-eval-jobs/Makefile.am
View file
@@ -1,5 +1,5 @@
bin_PROGRAMS = hydra-eval-jobs

hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc
hydra_eval_jobs_LDADD = $(NIX_LIBS)
hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixrust
hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra
5 changes: 3 additions & 2 deletions src/hydra-eval-jobs/hydra-eval-jobs.cc
View file
Expand Up @@ -115,8 +115,9 @@ static void findJobsWrapped(EvalState & state, JSONObject & top,
done. */
auto localStore = state.store.dynamic_pointer_cast<LocalFSStore>();
if (gcRootsDir != "" && localStore) {
Path root = gcRootsDir + "/" + baseNameOf(drvPath);
if (!pathExists(root)) localStore->addPermRoot(drvPath, root, false);
Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath));
if (!pathExists(root))
localStore->addPermRoot(localStore->parseStorePath(drvPath), root, false);
}

auto res2 = res.object("outputs");
Expand Down
2 changes: 1 addition & 1 deletion src/hydra-queue-runner/Makefile.am
View file
Expand Up @@ -3,5 +3,5 @@ bin_PROGRAMS = hydra-queue-runner
hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \
builder.cc build-result.cc build-remote.cc \
build-result.hh counter.hh token-server.hh state.hh db.hh
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx
hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -lnixrust
hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations
76 changes: 42 additions & 34 deletions src/hydra-queue-runner/build-remote.cc
View file
Expand Up @@ -82,10 +82,10 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil


static void copyClosureTo(std::timed_mutex & sendMutex, ref<Store> destStore,
FdSource & from, FdSink & to, const PathSet & paths,
FdSource & from, FdSink & to, const StorePathSet & paths,
bool useSubstitutes = false)
{
PathSet closure;
StorePathSet closure;
for (auto & path : paths)
destStore->computeFSClosure(path, closure);

Expand All @@ -94,20 +94,21 @@ static void copyClosureTo(std::timed_mutex & sendMutex, ref<Store> destStore,
garbage-collect paths that are already there. Optionally, ask
the remote host to substitute missing paths. */
// FIXME: substitute output pollutes our build log
to << cmdQueryValidPaths << 1 << useSubstitutes << closure;
to << cmdQueryValidPaths << 1 << useSubstitutes;
writeStorePaths(*destStore, to, closure);
to.flush();

/* Get back the set of paths that are already valid on the remote
host. */
auto present = readStorePaths<PathSet>(*destStore, from);
auto present = readStorePaths<StorePathSet>(*destStore, from);

if (present.size() == closure.size()) return;

Paths sorted = destStore->topoSortPaths(closure);
auto sorted = destStore->topoSortPaths(closure);

Paths missing;
StorePathSet missing;
for (auto i = sorted.rbegin(); i != sorted.rend(); ++i)
if (present.find(*i) == present.end()) missing.push_back(*i);
if (!present.count(*i)) missing.insert(i->clone());

printMsg(lvlDebug, format("sending %1% missing paths") % missing.size());

Expand All @@ -131,7 +132,7 @@ void State::buildRemote(ref<Store> destStore,
{
assert(BuildResult::TimedOut == 8);

string base = baseNameOf(step->drvPath);
string base(step->drvPath.to_string());
result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2);
AutoDelete autoDelete(result.logFile, false);

Expand Down Expand Up @@ -217,22 +218,22 @@ void State::buildRemote(ref<Store> destStore,
outputs of the input derivations. */
updateStep(ssSendingInputs);

PathSet inputs;
BasicDerivation basicDrv(step->drv);
StorePathSet inputs;
BasicDerivation basicDrv(*step->drv);

if (sendDerivation)
inputs.insert(step->drvPath);
inputs.insert(step->drvPath.clone());
else
for (auto & p : step->drv.inputSrcs)
inputs.insert(p);
for (auto & p : step->drv->inputSrcs)
inputs.insert(p.clone());

for (auto & input : step->drv.inputDrvs) {
Derivation drv2 = readDerivation(input.first);
for (auto & input : step->drv->inputDrvs) {
Derivation drv2 = readDerivation(*localStore, localStore->printStorePath(input.first));
for (auto & name : input.second) {
auto i = drv2.outputs.find(name);
if (i == drv2.outputs.end()) continue;
inputs.insert(i->second.path);
basicDrv.inputSrcs.insert(i->second.path);
inputs.insert(i->second.path.clone());
basicDrv.inputSrcs.insert(i->second.path.clone());
}
}

Expand All @@ -241,14 +242,15 @@ void State::buildRemote(ref<Store> destStore,
this will copy the inputs to the binary cache from the local
store. */
if (localStore != std::shared_ptr<Store>(destStore))
copyClosure(ref<Store>(localStore), destStore, step->drv.inputSrcs, NoRepair, NoCheckSigs);
copyClosure(ref<Store>(localStore), destStore, step->drv->inputSrcs, NoRepair, NoCheckSigs);

/* Copy the input closure. */
if (!machine->isLocalhost()) {
auto mc1 = std::make_shared<MaintainCount<counter>>(nrStepsWaiting);
mc1.reset();
MaintainCount<counter> mc2(nrStepsCopyingTo);
printMsg(lvlDebug, format("sending closure of ‘%1%’ to ‘%2%’") % step->drvPath % machine->sshName);
printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’",
localStore->printStorePath(step->drvPath), machine->sshName);

auto now1 = std::chrono::steady_clock::now();

Expand All @@ -272,14 +274,19 @@ void State::buildRemote(ref<Store> destStore,
logFD = -1;

/* Do the build. */
printMsg(lvlDebug, format("building ‘%1%’ on ‘%2%’") % step->drvPath % machine->sshName);
printMsg(lvlDebug, "building ‘%s’ on ‘%s’",
localStore->printStorePath(step->drvPath),
machine->sshName);

updateStep(ssBuilding);

if (sendDerivation)
to << cmdBuildPaths << PathSet({step->drvPath});
else
to << cmdBuildDerivation << step->drvPath << basicDrv;
if (sendDerivation) {
to << cmdBuildPaths;
writeStorePaths(*localStore, to, singleton(step->drvPath));
} else {
to << cmdBuildDerivation << localStore->printStorePath(step->drvPath);
writeDerivation(to, *localStore, basicDrv);
}
to << maxSilentTime << buildTimeout;
if (GET_PROTOCOL_MINOR(remoteVersion) >= 2)
to << maxLogSize;
Expand Down Expand Up @@ -380,7 +387,8 @@ void State::buildRemote(ref<Store> destStore,
/* If the path was substituted or already valid, then we didn't
get a build log. */
if (result.isCached) {
printMsg(lvlInfo, format("outputs of ‘%1%’ substituted or already valid on ‘%2%’") % step->drvPath % machine->sshName);
printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’",
localStore->printStorePath(step->drvPath), machine->sshName);
unlink(result.logFile.c_str());
result.logFile = "";
}
Expand All @@ -395,13 +403,12 @@ void State::buildRemote(ref<Store> destStore,

auto now1 = std::chrono::steady_clock::now();

PathSet outputs;
for (auto & output : step->drv.outputs)
outputs.insert(output.second.path);
auto outputs = step->drv->outputPaths();

/* Query the size of the output paths. */
size_t totalNarSize = 0;
to << cmdQueryPathInfos << outputs;
to << cmdQueryPathInfos;
writeStorePaths(*localStore, to, outputs);
to.flush();
while (true) {
if (readString(from) == "") break;
Expand All @@ -416,8 +423,8 @@ void State::buildRemote(ref<Store> destStore,
return;
}

printMsg(lvlDebug, format("copying outputs of ‘%s’ from ‘%s’ (%d bytes)")
% step->drvPath % machine->sshName % totalNarSize);
printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)",
localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize);

/* Block until we have the required amount of memory
available, which is twice the NAR size (namely the
Expand All @@ -431,10 +438,11 @@ void State::buildRemote(ref<Store> destStore,

auto resMs = std::chrono::duration_cast<std::chrono::milliseconds>(resStop - resStart).count();
if (resMs >= 1000)
printMsg(lvlError, format("warning: had to wait %d ms for %d memory tokens for %s")
% resMs % totalNarSize % step->drvPath);
printMsg(lvlError, "warning: had to wait %d ms for %d memory tokens for %s",
resMs, totalNarSize, localStore->printStorePath(step->drvPath));

to << cmdExportPaths << 0 << outputs;
to << cmdExportPaths << 0;
writeStorePaths(*localStore, to, outputs);
to.flush();
destStore->importPaths(from, result.accessor, NoCheckSigs);

Expand Down
28 changes: 14 additions & 14 deletions src/hydra-queue-runner/build-result.cc
View file
Expand Up @@ -14,16 +14,14 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
BuildOutput res;

/* Compute the closure size. */
PathSet outputs;
for (auto & output : drv.outputs)
outputs.insert(output.second.path);
PathSet closure;
auto outputs = drv.outputPaths();
StorePathSet closure;
for (auto & output : outputs)
store->computeFSClosure(output, closure);
store->computeFSClosure(singleton(output), closure);
for (auto & path : closure) {
auto info = store->queryPathInfo(path);
res.closureSize += info->narSize;
if (outputs.find(path) != outputs.end()) res.size += info->narSize;
if (outputs.count(path)) res.size += info->narSize;
}

/* Get build products. */
Expand All @@ -39,11 +37,13 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
, std::regex::extended);

for (auto & output : outputs) {
Path failedFile = output + "/nix-support/failed";
auto outputS = store->printStorePath(output);

Path failedFile = outputS + "/nix-support/failed";
if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular)
res.failed = true;

Path productsFile = output + "/nix-support/hydra-build-products";
Path productsFile = outputS + "/nix-support/hydra-build-products";
if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular)
continue;

Expand Down Expand Up @@ -72,7 +72,7 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
auto st = accessor->stat(product.path);
if (st.type == FSAccessor::Type::tMissing) continue;

product.name = product.path == output ? "" : baseNameOf(product.path);
product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path);

if (st.type == FSAccessor::Type::tRegular) {
product.isRegular = true;
Expand All @@ -91,22 +91,22 @@ BuildOutput getBuildOutput(nix::ref<Store> store,
if (!explicitProducts) {
for (auto & output : drv.outputs) {
BuildProduct product;
product.path = output.second.path;
product.path = store->printStorePath(output.second.path);
product.type = "nix-build";
product.subtype = output.first == "out" ? "" : output.first;
product.name = storePathToName(product.path);
product.name = output.second.path.name();

auto st = accessor->stat(product.path);
if (st.type == FSAccessor::Type::tMissing)
throw Error(format("getting status of ‘%1%’") % product.path);
throw Error("getting status of ‘%s’", product.path);
if (st.type == FSAccessor::Type::tDirectory)
res.products.push_back(product);
}
}

/* Get the release name from $output/nix-support/hydra-release-name. */
for (auto & output : outputs) {
Path p = output + "/nix-support/hydra-release-name";
auto p = store->printStorePath(output) + "/nix-support/hydra-release-name";
if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue;
try {
res.releaseName = trim(accessor->readFile(p));
Expand All @@ -116,7 +116,7 @@ BuildOutput getBuildOutput(nix::ref<Store> store,

/* Get metrics. */
for (auto & output : outputs) {
Path metricsFile = output + "/nix-support/hydra-metrics";
auto metricsFile = store->printStorePath(output) + "/nix-support/hydra-metrics";
if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue;
for (auto & line : tokenizeString<Strings>(accessor->readFile(metricsFile), "\n")) {
auto fields = tokenizeString<std::vector<std::string>>(line);
Expand Down

0 comments on commit e7f2139

Please sign in to comment.