Skip to content

Commit

Permalink
Switch to new Nix bindings
Browse files Browse the repository at this point in the history 
Implements support for Nix's new Perl bindings[1]. The current state
basically does `openStore()`, but always uses `auto` and doesn't support
stores at other URIs.

Even though the stores are cached inside the Perl implementation, I
decided to instantiate those once in the Nix helper module. That way
store openings aren't cluttered across the entire codebase. Also, there
are two stores used later on - MACHINE_LOCAL_STORE for `auto`,
BINARY_CACHE_STORE for the one from `store_uri` in `hydra.conf` - and
using consistent names should make the intent clearer then.

This doesn't contain any behavioral changes, i.e. the build product
availability issue from #1352 isn't fixed. This patch only contains the
migration to the new API.

[1] NixOS/nix#9863
  • Loading branch information
@Ma27
Ma27 committed Feb 12, 2024
1 parent 878c0f2 commit d3a9595
Show file tree
Hide file tree
Showing 16 changed files with 49 additions and 56 deletions.
11 changes: 6 additions & 5 deletions flake.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion flake.nix
View file
Expand Up @@ -2,7 +2,7 @@
description = "A Nix-based continuous build system";

inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small";
inputs.nix.url = "github:NixOS/nix";
inputs.nix.url = "github:obsidiansystems/nix/perl-open-other-store";
inputs.nix.inputs.nixpkgs.follows = "nixpkgs";

# TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is
Expand Down
3 changes: 1 addition & 2 deletions src/lib/Hydra/Base/Controller/NixChannel.pm
View file
Expand Up @@ -4,7 +4,6 @@ use strict;
use warnings;
use base 'Hydra::Base::Controller::REST';
use List::SomeUtils qw(any);
use Nix::Store;
use Hydra::Helper::Nix;
use Hydra::Helper::CatalystUtils;

Expand All @@ -30,7 +29,7 @@ sub getChannelData {
my $outputs = {};
foreach my $output (@outputs) {
my $outPath = $output->get_column("outpath");
next if $checkValidity && !isValidPath($outPath);
next if $checkValidity && !$MACHINE_LOCAL_STORE->isValidPath($outPath);
$outputs->{$output->get_column("outname")} = $outPath;
push @storePaths, $outPath;
# Put the system type in the manifest (for top-level
Expand Down
18 changes: 8 additions & 10 deletions src/lib/Hydra/Controller/Build.pm
View file
Expand Up @@ -10,8 +10,6 @@ use File::Basename;
use File::LibMagic;
use File::stat;
use Data::Dump qw(dump);
use Nix::Store;
use Nix::Config;
use List::SomeUtils qw(all);
use Encode;
use JSON::PP;
Expand Down Expand Up @@ -82,9 +80,9 @@ sub build_GET {
# false because `$_->path` will be empty
$c->stash->{available} =
$c->stash->{isLocalStore}
? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all
? all { $_->path && $MACHINE_LOCAL_STORE->isValidPath($_->path) } $build->buildoutputs->all
: 1;
$c->stash->{drvAvailable} = isValidPath $build->drvpath;
$c->stash->{drvAvailable} = $MACHINE_LOCAL_STORE->isValidPath($build->drvpath);

if ($build->finished && $build->iscachedbuild) {
my $path = ($build->buildoutputs)[0]->path or undef;
Expand Down Expand Up @@ -308,7 +306,7 @@ sub output : Chained('buildChain') PathPart Args(1) {
error($c, "This build is not finished yet.") unless $build->finished;
my $output = $build->buildoutputs->find({name => $outputName});
notFound($c, "This build has no output named ‘$outputName’") unless defined $output;
gone($c, "Output is no longer available.") unless isValidPath $output->path;
gone($c, "Output is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($output->path);

$c->response->header('Content-Disposition', "attachment; filename=\"build-${\$build->id}-${\$outputName}.nar.bz2\"");
$c->stash->{current_view} = 'NixNAR';
Expand Down Expand Up @@ -425,15 +423,15 @@ sub getDependencyGraph {
};
$$done{$path} = $node;
my @refs;
foreach my $ref (queryReferences($path)) {
foreach my $ref ($MACHINE_LOCAL_STORE->queryReferences($path)) {
next if $ref eq $path;
next unless $runtime || $ref =~ /\.drv$/;
getDependencyGraph($self, $c, $runtime, $done, $ref);
push @refs, $ref;
}
# Show in reverse topological order to flatten the graph.
# Should probably do a proper BFS.
my @sorted = reverse topoSortPaths(@refs);
my @sorted = reverse $MACHINE_LOCAL_STORE->topoSortPaths(@refs);
$node->{refs} = [map { $$done{$_} } @sorted];
}

Expand All @@ -446,7 +444,7 @@ sub build_deps : Chained('buildChain') PathPart('build-deps') {
my $build = $c->stash->{build};
my $drvPath = $build->drvpath;

error($c, "Derivation no longer available.") unless isValidPath $drvPath;
error($c, "Derivation no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($drvPath);

$c->stash->{buildTimeGraph} = getDependencyGraph($self, $c, 0, {}, $drvPath);

Expand All @@ -461,7 +459,7 @@ sub runtime_deps : Chained('buildChain') PathPart('runtime-deps') {

requireLocalStore($c);

error($c, "Build outputs no longer available.") unless all { isValidPath($_) } @outPaths;
error($c, "Build outputs no longer available.") unless all { $MACHINE_LOCAL_STORE->isValidPath($_) } @outPaths;

my $done = {};
$c->stash->{runtimeGraph} = [ map { getDependencyGraph($self, $c, 1, $done, $_) } @outPaths ];
Expand All @@ -481,7 +479,7 @@ sub nix : Chained('buildChain') PathPart('nix') CaptureArgs(0) {
if (isLocalStore) {
foreach my $out ($build->buildoutputs) {
notFound($c, "Path " . $out->path . " is no longer available.")
unless isValidPath($out->path);
unless $MACHINE_LOCAL_STORE->isValidPath($out->path);
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/lib/Hydra/Controller/Root.pm
View file
Expand Up @@ -395,7 +395,7 @@ sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) {
my ($hash) = $narinfo =~ NARINFO_REGEX;

die("Hash length was not 32") if length($hash) != 32;
my $path = queryPathFromHashPart($hash);
my $path = $MACHINE_LOCAL_STORE->queryPathFromHashPart($hash);

if (!$path) {
$c->response->status(404);
Expand Down
5 changes: 4 additions & 1 deletion src/lib/Hydra/Helper/Nix.pm
View file
Expand Up @@ -40,8 +40,11 @@ our @EXPORT = qw(
registerRoot
restartBuilds
run
$MACHINE_LOCAL_STORE
);

our $MACHINE_LOCAL_STORE = Nix::Store->new();


sub getHydraHome {
my $dir = $ENV{"HYDRA_HOME"} or die "The HYDRA_HOME directory does not exist!\n";
Expand Down Expand Up @@ -494,7 +497,7 @@ sub restartBuilds {
$builds = $builds->search({ finished => 1 });

foreach my $build ($builds->search({}, { columns => ["drvpath"] })) {
next if !isValidPath($build->drvpath);
next if !$MACHINE_LOCAL_STORE->isValidPath($build->drvpath);
registerRoot $build->drvpath;
}

Expand Down
7 changes: 3 additions & 4 deletions src/lib/Hydra/Plugin/BazaarInput.pm
View file
Expand Up @@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
use File::Path;
use Hydra::Helper::Exec;
use Hydra::Helper::Nix;
use Nix::Store;

sub supportedInputTypes {
my ($self, $inputTypes) = @_;
Expand Down Expand Up @@ -38,9 +37,9 @@ sub fetchInput {
(my $cachedInput) = $self->{db}->resultset('CachedBazaarInputs')->search(
{uri => $uri, revision => $revision});

addTempRoot($cachedInput->storepath) if defined $cachedInput;
$MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;

if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
} else {
Expand All @@ -58,7 +57,7 @@ sub fetchInput {
($sha256, $storePath) = split ' ', $stdout;

# FIXME: time window between nix-prefetch-bzr and addTempRoot.
addTempRoot($storePath);
$MACHINE_LOCAL_STORE->addTempRoot($storePath);

$self->{db}->txn_do(sub {
$self->{db}->resultset('CachedBazaarInputs')->create(
Expand Down
7 changes: 3 additions & 4 deletions src/lib/Hydra/Plugin/DarcsInput.pm
View file
Expand Up @@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
use File::Path;
use Hydra::Helper::Exec;
use Hydra::Helper::Nix;
use Nix::Store;

sub supportedInputTypes {
my ($self, $inputTypes) = @_;
Expand Down Expand Up @@ -58,7 +57,7 @@ sub fetchInput {
{uri => $uri, revision => $revision},
{rows => 1});

if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$revision = $cachedInput->revision;
Expand All @@ -75,8 +74,8 @@ sub fetchInput {
die "darcs changes --count failed" if $? != 0;

system "rm", "-rf", "$tmpDir/export/_darcs";
$storePath = addToStore("$tmpDir/export", 1, "sha256");
$sha256 = queryPathHash($storePath);
$storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/export", 1, "sha256");
$sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath);
$sha256 =~ s/sha256://;

$self->{db}->txn_do(sub {
Expand Down
6 changes: 3 additions & 3 deletions src/lib/Hydra/Plugin/GitInput.pm
View file
Expand Up @@ -186,9 +186,9 @@ sub fetchInput {
{uri => $uri, branch => $branch, revision => $revision, isdeepclone => defined($deepClone) ? 1 : 0},
{rows => 1});

addTempRoot($cachedInput->storepath) if defined $cachedInput;
$MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;

if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$revision = $cachedInput->revision;
Expand Down Expand Up @@ -217,7 +217,7 @@ sub fetchInput {
($sha256, $storePath) = split ' ', grab(cmd => ["nix-prefetch-git", $clonePath, $revision], chomp => 1);

# FIXME: time window between nix-prefetch-git and addTempRoot.
addTempRoot($storePath);
$MACHINE_LOCAL_STORE->addTempRoot($storePath);

$self->{db}->txn_do(sub {
$self->{db}->resultset('CachedGitInputs')->update_or_create(
Expand Down
7 changes: 3 additions & 4 deletions src/lib/Hydra/Plugin/MercurialInput.pm
View file
Expand Up @@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
use File::Path;
use Hydra::Helper::Nix;
use Hydra::Helper::Exec;
use Nix::Store;
use Fcntl qw(:flock);

sub supportedInputTypes {
Expand Down Expand Up @@ -68,9 +67,9 @@ sub fetchInput {
(my $cachedInput) = $self->{db}->resultset('CachedHgInputs')->search(
{uri => $uri, branch => $branch, revision => $revision});

addTempRoot($cachedInput->storepath) if defined $cachedInput;
$MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;

if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
} else {
Expand All @@ -85,7 +84,7 @@ sub fetchInput {
($sha256, $storePath) = split ' ', $stdout;

# FIXME: time window between nix-prefetch-hg and addTempRoot.
addTempRoot($storePath);
$MACHINE_LOCAL_STORE->addTempRoot($storePath);

$self->{db}->txn_do(sub {
$self->{db}->resultset('CachedHgInputs')->update_or_create(
Expand Down
5 changes: 2 additions & 3 deletions src/lib/Hydra/Plugin/PathInput.pm
View file
Expand Up @@ -5,7 +5,6 @@ use warnings;
use parent 'Hydra::Plugin';
use POSIX qw(strftime);
use Hydra::Helper::Nix;
use Nix::Store;

sub supportedInputTypes {
my ($self, $inputTypes) = @_;
Expand All @@ -30,7 +29,7 @@ sub fetchInput {
{srcpath => $uri, lastseen => {">", $timestamp - $timeout}},
{rows => 1, order_by => "lastseen DESC"});

if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
$sha256 = $cachedInput->sha256hash;
$timestamp = $cachedInput->timestamp;
Expand All @@ -46,7 +45,7 @@ sub fetchInput {
}
chomp $storePath;

$sha256 = (queryPathInfo($storePath, 0))[1] or die;
$sha256 = ($MACHINE_LOCAL_STORE->queryPathInfo($storePath, 0))[1] or die;

($cachedInput) = $self->{db}->resultset('CachedPathInputs')->search(
{srcpath => $uri, sha256hash => $sha256});
Expand Down
9 changes: 4 additions & 5 deletions src/lib/Hydra/Plugin/SubversionInput.pm
View file
Expand Up @@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex);
use Hydra::Helper::Exec;
use Hydra::Helper::Nix;
use IPC::Run;
use Nix::Store;

sub supportedInputTypes {
my ($self, $inputTypes) = @_;
Expand Down Expand Up @@ -45,7 +44,7 @@ sub fetchInput {
(my $cachedInput) = $self->{db}->resultset('CachedSubversionInputs')->search(
{uri => $uri, revision => $revision});

addTempRoot($cachedInput->storepath) if defined $cachedInput;
$MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput;

if (defined $cachedInput && isValidPath($cachedInput->storepath)) {
$storePath = $cachedInput->storepath;
Expand All @@ -62,16 +61,16 @@ sub fetchInput {
die "error checking out Subversion repo at `$uri':\n$stderr" if $res;

if ($type eq "svn-checkout") {
$storePath = addToStore($wcPath, 1, "sha256");
$storePath = $MACHINE_LOCAL_STORE->addToStore($wcPath, 1, "sha256");
} else {
# Hm, if the Nix Perl bindings supported filters in
# addToStore(), then we wouldn't need to make a copy here.
my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die;
(system "svn", "export", $wcPath, "$tmpDir/source", "--quiet") == 0 or die "svn export failed";
$storePath = addToStore("$tmpDir/source", 1, "sha256");
$storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/source", 1, "sha256");
}

$sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://;
$sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath); $sha256 =~ s/sha256://;

$self->{db}->txn_do(sub {
$self->{db}->resultset('CachedSubversionInputs')->update_or_create(
Expand Down
9 changes: 4 additions & 5 deletions src/lib/Hydra/View/NARInfo.pm
View file
Expand Up @@ -6,8 +6,7 @@ use File::Basename;
use Hydra::Helper::CatalystUtils;
use MIME::Base64;
use Nix::Manifest;
use Nix::Store;
use Nix::Utils;
use Hydra::Helper::Nix;
use base qw/Catalyst::View/;

sub process {
Expand All @@ -17,7 +16,7 @@ sub process {

$c->response->content_type('text/x-nix-narinfo'); # !!! check MIME type

my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
my ($deriver, $narHash, $time, $narSize, $refs) = $MACHINE_LOCAL_STORE->queryPathInfo($storePath, 1);

my $info;
$info .= "StorePath: $storePath\n";
Expand All @@ -28,8 +27,8 @@ sub process {
$info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
if (defined $deriver) {
$info .= "Deriver: " . basename $deriver . "\n";
if (isValidPath($deriver)) {
my $drv = derivationFromPath($deriver);
if ($MACHINE_LOCAL_STORE->isValidPath($deriver)) {
my $drv = $MACHINE_LOCAL_STORE->derivationFromPath($deriver);
$info .= "System: $drv->{platform}\n";
}
}
Expand Down
8 changes: 4 additions & 4 deletions src/script/hydra-eval-jobset
View file
Expand Up @@ -85,14 +85,14 @@ sub attrsToSQL {
# Fetch a store path from 'eval_substituter' if not already present.
sub getPath {
my ($path) = @_;
return 1 if isValidPath($path);
return 1 if $MACHINE_LOCAL_STORE->isValidPath($path);

my $substituter = $config->{eval_substituter};

system("nix", "--experimental-features", "nix-command", "copy", "--from", $substituter, "--", $path)
if defined $substituter;

return isValidPath($path);
return $MACHINE_LOCAL_STORE->isValidPath($path);
}


Expand Down Expand Up @@ -143,7 +143,7 @@ sub fetchInputBuild {
, version => $version
, outputName => $mainOutput->name
};
if (isValidPath($prevBuild->drvpath)) {
if ($MACHINE_LOCAL_STORE->isValidPath($prevBuild->drvpath)) {
$result->{drvPath} = $prevBuild->drvpath;
}

Expand Down Expand Up @@ -233,7 +233,7 @@ sub fetchInputEval {
my $out = $build->buildoutputs->find({ name => "out" });
next unless defined $out;
# FIXME: Should we fail if the path is not valid?
next unless isValidPath($out->path);
next unless $MACHINE_LOCAL_STORE->isValidPath($out->path);
$jobs->{$build->get_column('job')} = $out->path;
}

Expand Down

0 comments on commit d3a9595

Please sign in to comment.