Permalink
Browse files

exportReferencesGraph: Export more complete info in JSON format

This writes info about every path in the closure in the same format as
‘nix path-info --json’. Thus it also includes NAR hashes and sizes.

Example:

  [
    {
      "path": "/nix/store/10h6li26i7g6z3mdpvra09yyf10mmzdr-hello-2.10",
      "narHash": "sha256:0ckdc4z20kkmpqdilx0wl6cricxv90lh85xpv2qljppcmz6vzcxl",
      "narSize": 197648,
      "references": [
        "/nix/store/10h6li26i7g6z3mdpvra09yyf10mmzdr-hello-2.10",
        "/nix/store/27binbdy296qvjycdgr1535v8872vz3z-glibc-2.24"
      ],
      "closureSize": 20939776
    },
    {
      "path": "/nix/store/27binbdy296qvjycdgr1535v8872vz3z-glibc-2.24",
      "narHash": "sha256:1nfn3m3p98y1c0kd0brp80dn9n5mycwgrk183j17rajya0h7gax3",
      "narSize": 20742128,
      "references": [
        "/nix/store/27binbdy296qvjycdgr1535v8872vz3z-glibc-2.24"
      ],
      "closureSize": 20742128
    }
  ]

Fixes #1134.
  • Loading branch information...
edolstra committed Jan 26, 2017
1 parent 6de33a9 commit c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a
Showing with 90 additions and 50 deletions.
  1. +11 −1 src/libstore/build.cc
  2. +59 −0 src/libstore/store-api.cc
  3. +14 −0 src/libstore/store-api.hh
  4. +6 −49 src/nix/path-info.cc
@@ -10,6 +10,7 @@
#include "builtins.hh"
#include "finally.hh"
#include "compression.hh"
#include "json.hh"
#include <algorithm>
#include <iostream>
@@ -2273,9 +2274,18 @@ void DerivationGoal::doExportReferencesGraph()
}
}
/* Write closure info to `fileName'. */
/* Write closure info to <fileName>. */
writeFile(tmpDir + "/" + fileName,
worker.store.makeValidityRegistration(paths, false, false));
/* Write a more comprehensive JSON serialisation to
<fileName>.json. */
std::ostringstream str;
{
JSONPlaceholder jsonRoot(str, true);
worker.store.pathInfoToJSON(jsonRoot, paths, false, true);
}
writeFile(tmpDir + "/" + fileName + ".json", str.str());
}
}
@@ -4,6 +4,7 @@
#include "util.hh"
#include "nar-info-disk-cache.hh"
#include "thread-pool.hh"
#include "json.hh"
#include <future>
@@ -439,6 +440,64 @@ string Store::makeValidityRegistration(const PathSet & paths,
}
void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
bool includeImpureInfo, bool showClosureSize)
{
auto jsonList = jsonOut.list();
for (auto storePath : storePaths) {
auto info = queryPathInfo(storePath);
storePath = info->path;
auto jsonPath = jsonList.object();
jsonPath
.attr("path", storePath)
.attr("narHash", info->narHash.to_string())
.attr("narSize", info->narSize);
{
auto jsonRefs = jsonPath.list("references");
for (auto & ref : info->references)
jsonRefs.elem(ref);
}
if (info->ca != "")
jsonPath.attr("ca", info->ca);
if (showClosureSize)
jsonPath.attr("closureSize", getClosureSize(storePath));
if (!includeImpureInfo) continue;
if (info->deriver != "")
jsonPath.attr("deriver", info->deriver);
if (info->registrationTime)
jsonPath.attr("registrationTime", info->registrationTime);
if (info->ultimate)
jsonPath.attr("ultimate", info->ultimate);
if (!info->sigs.empty()) {
auto jsonSigs = jsonPath.list("signatures");
for (auto & sig : info->sigs)
jsonSigs.elem(sig);
}
}
}
unsigned long long Store::getClosureSize(const Path & storePath)
{
unsigned long long totalSize = 0;
PathSet closure;
computeFSClosure(storePath, closure, false, false);
for (auto & p : closure)
totalSize += queryPathInfo(p)->narSize;
return totalSize;
}
const Store::Stats & Store::getStats()
{
{
@@ -22,6 +22,7 @@ struct Derivation;
class FSAccessor;
class NarInfoDiskCache;
class Store;
class JSONPlaceholder;
/* Size of the hash part of store paths, in base-32 characters. */
@@ -469,6 +470,19 @@ public:
string makeValidityRegistration(const PathSet & paths,
bool showDerivers, bool showHash);
/* Write a JSON representation of store path metadata, such as the
hash and the references. If ‘includeImpureInfo’ is true,
variable elements such as the registration time are
included. If ‘showClosureSize’ is true, the closure size of
each path is included. */
void pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths,
bool includeImpureInfo, bool showClosureSize);
/* Return the size of the closure of the specified path, that is,
the sum of the size of the NAR serialisation of each path in
the closure. */
unsigned long long getClosureSize(const Path & storePath);
/* Optimise the disk space usage of the Nix store by hard-linking files
with the same contents. */
virtual void optimiseStore() = 0;
@@ -65,55 +65,12 @@ struct CmdPathInfo : StorePathsCommand
for (auto & storePath : storePaths)
pathLen = std::max(pathLen, storePath.size());
auto getClosureSize = [&](const Path & storePath) -> unsigned long long {
unsigned long long totalSize = 0;
PathSet closure;
store->computeFSClosure(storePath, closure, false, false);
for (auto & p : closure)
totalSize += store->queryPathInfo(p)->narSize;
return totalSize;
};
if (json) {
JSONList jsonRoot(std::cout, true);
for (auto storePath : storePaths) {
auto info = store->queryPathInfo(storePath);
storePath = info->path;
auto jsonPath = jsonRoot.object();
jsonPath
.attr("path", storePath)
.attr("narHash", info->narHash.to_string())
.attr("narSize", info->narSize);
if (showClosureSize)
jsonPath.attr("closureSize", getClosureSize(storePath));
if (info->deriver != "")
jsonPath.attr("deriver", info->deriver);
{
auto jsonRefs = jsonPath.list("references");
for (auto & ref : info->references)
jsonRefs.elem(ref);
}
if (info->registrationTime)
jsonPath.attr("registrationTime", info->registrationTime);
if (info->ultimate)
jsonPath.attr("ultimate", info->ultimate);
if (info->ca != "")
jsonPath.attr("ca", info->ca);
if (!info->sigs.empty()) {
auto jsonSigs = jsonPath.list("signatures");
for (auto & sig : info->sigs)
jsonSigs.elem(sig);
}
}
JSONPlaceholder jsonRoot(std::cout, true);
store->pathInfoToJSON(jsonRoot,
// FIXME: preserve order?
PathSet(storePaths.begin(), storePaths.end()),
true, showClosureSize);
}
else {
@@ -128,7 +85,7 @@ struct CmdPathInfo : StorePathsCommand
std::cout << '\t' << std::setw(11) << info->narSize;
if (showClosureSize)
std::cout << '\t' << std::setw(11) << getClosureSize(storePath);
std::cout << '\t' << std::setw(11) << store->getClosureSize(storePath);
if (showSigs) {
std::cout << '\t';

5 comments on commit c2b0d87

@dezgeg

This comment has been minimized.

Contributor

dezgeg replied Jan 31, 2017

Doesn't this break existing nixpkgs code?

@edolstra

This comment has been minimized.

Member

edolstra replied Jan 31, 2017

No, because it also writes the data in the old format.

@volth

This comment has been minimized.

Contributor

volth replied Feb 22, 2018

Is this feature broken as by now?

runCommand "closure" {
  exportReferencesGraph = ["refs.json" toplevel];
  __json = true;
} "cp refs.json $out";

fails with

error: cannot process __json attribute of '/nix/store/35brslvh1j0wbqr17d08839c1lfm2b5i-closure.drv': [json.exception.type_error.305] cannot use operator[] with number

runCommand "closure" {
  exportReferencesGraph = ["refs" toplevel];
} "cp refs.json $out";

fails with
cannot stat 'refs.json': No such file or directory

@dtzWill

This comment has been minimized.

Contributor

dtzWill replied Feb 23, 2018

Looked into this because I wanted to check if the nlohmann update I proposed (and was merged) was to blame.

Here's what I found:

  1. Nope, still happens after reverting that. Good :).
  2. 1351b0d makes it so you want __structuredAttrs = true
  3. Look at tests/config.nix and tests/structured-attrs/*. I've never used exportReferencesGraph before but from these and some poking around I came up with this... thing:
with import <nixpkgs> {};
let
  toplevel = bash;
in
derivation {
  name = "refs";
  builder = stdenv.shell;
  inherit system;
  exportReferencesGraph.refs = [toplevel];
  __structuredAttrs = true;
  args = ["-e" (builtins.toFile "builder.sh" ''source .attrs.sh; cp .attrs.json ''${outputs[out]}'')];
  buildCommand = ''cp .attrs.json ''${outputs[out]}'';
  PATH = "${coreutils}/bin";
}

Which I can point nix-build at and get something like:

{
  "PATH": "/nix/store/6m7ldqalxz3818rzc557zh17szy6bqf6-coreutils-8.29/bin",
  "buildCommand": "cp .attrs.json ${outputs[out]}",
  "builder": "/nix/store/yq03c2ny43mc24j7dq5riznzb09ddhpq-bash-4.4-p12/bin/bash",
  "exportReferencesGraph": {
    "refs": [
      "/nix/store/yq03c2ny43mc24j7dq5riznzb09ddhpq-bash-4.4-p12"
    ]
  },
  "name": "refs",
  "outputs": {
    "out": "/nix/store/dfig8jibs5xjcs0aymf0yh3jiwxzh1jm-refs"
  },
  "refs": [
    {
      "closureSize": 21221664,
      "narHash": "sha256:010qr8kbfdgfm2nsr9l7rb3hxmkkpvf5pvd00vvqdyl0l2k31jwx",
      "narSize": 21221664,
      "path": "/nix/store/4h0allsz28x9mdirzb4wv51algacxzr7-glibc-2.26-131",
      "references": [
        "/nix/store/4h0allsz28x9mdirzb4wv51algacxzr7-glibc-2.26-131"
      ]
    },
    {
      "closureSize": 22381552,
      "narHash": "sha256:07dkqwmsk4ij85kcbk8d78f9a0q89zb5s3nv1xsv90hnarq1dmxv",
      "narSize": 1159888,
      "path": "/nix/store/yq03c2ny43mc24j7dq5riznzb09ddhpq-bash-4.4-p12",
      "references": [
        "/nix/store/4h0allsz28x9mdirzb4wv51algacxzr7-glibc-2.26-131",
        "/nix/store/yq03c2ny43mc24j7dq5riznzb09ddhpq-bash-4.4-p12"
      ]
    }
  ],
  "system": "x86_64-linux"
}

(my attempts to use runCommand as you do were unsuccessful because it tried to source $stdenv/setup and stdenv wasn't set, or something?)

Anyway hope this helps until a better answer is available! :)

@volth

This comment has been minimized.

Contributor

volth replied Feb 23, 2018

Please sign in to comment.