Skip to content

Commit

Permalink
Merge 535b2b1 into d510103
Browse files Browse the repository at this point in the history
  • Loading branch information
exarkun committed May 31, 2023
2 parents d510103 + 535b2b1 commit 1b12368
Show file tree
Hide file tree
Showing 10 changed files with 313 additions and 23 deletions.
45 changes: 31 additions & 14 deletions .circleci/config.yml
Expand Up @@ -89,7 +89,7 @@ workflows:

- "nixos":
name: "<<matrix.pythonVersion>>"
nixpkgs: "22.11"
nixpkgs: "22_11"
matrix:
parameters:
pythonVersion:
Expand Down Expand Up @@ -536,13 +536,18 @@ executors:
docker:
# Run in a highly Nix-capable environment.
- <<: *DOCKERHUB_AUTH
image: "nixos/nix:2.10.3"
image: "nixos/nix:2.15.0"
environment:
# CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us
# to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push
# to.
CACHIX_NAME: "tahoe-lafs-opensource"

# Let us use features marked "experimental". For example, most/all of
# the `nix <subcommand>` forms.
NIX_CONFIG: "experimental-features = nix-command flakes"


commands:
nix-build:
parameters:
Expand All @@ -565,15 +570,20 @@ commands:
# Get cachix for Nix-friendly caching.
name: "Install Basic Dependencies"
command: |
NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz"
# It shouldn't much matter which nixpkgs we use here. We're just
# installing a few small CI infrastructure tools. We *don't* take
# build or runtime dependencies for the Tahoe-LAFS packages or
# tests from this package set.
NIXPKGS=https://github.com/nixos/nixpkgs/archive/nixos-22.11.tar.gz
nix-env \
--file $NIXPKGS \
--install \
-A cachix bash
# Activate it for "binary substitution". This sets up
# configuration tht lets Nix download something from the cache
# instead of building it locally, if possible.
cachix use "${CACHIX_NAME}"
-A cachix bash jp
# Activate it for "binary substitution". This sets up
# configuration tht lets Nix download something from the cache
# instead of building it locally, if possible.
cachix use "${CACHIX_NAME}"
- "checkout"

Expand All @@ -588,17 +598,20 @@ commands:
- "run":
name: "Build Dependencies"
command: |
source .circleci/lib.sh
# CircleCI build environment looks like it has a zillion and a
# half cores. Don't let Nix autodetect this high core count
# because it blows up memory usage and fails the test run. Pick a
# number of cores that suits the build environment we're paying
# for (the free one!).
source .circleci/lib.sh
CORES=3
# nix-shell will build all of the dependencies of the target but
# not the target itself.
cache_if_able nix-shell \
--run "" \
--cores 3 \
--cores ${CORES} \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
--argstr pythonVersion "<<parameters.pythonVersion>>" \
./default.nix
Expand All @@ -607,10 +620,14 @@ commands:
name: "Build Package"
command: |
source .circleci/lib.sh
cache_if_able nix-build \
# Translate the nixpkgs selection into a flake reference we can
# use to override the default nixpkgs input.
NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<<parameters.nixpkgs>>)
cache_if_able nix build -v \
--cores 4 \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
--argstr pythonVersion "<<parameters.pythonVersion>>" \
./default.nix
--override-input nixpkgs "${NIXPKGS}" \
.#<<parameters.pythonVersion>>-tahoe-lafs
- steps: "<<parameters.buildSteps>>"
19 changes: 19 additions & 0 deletions .circleci/lib.sh
Expand Up @@ -117,3 +117,22 @@ function describe_build() {
echo "Cache not writeable."
fi
}

# Inspect the flake input metadata for an input of a given name and return the
# revision at which that input is pinned. If the input does not exist then
# return garbage (probably "null").
read_input_revision() {
input_name=$1
shift

nix flake metadata --json | jp --unquoted 'locks.nodes."'"$input_name"'".locked.rev'
}

# Return a flake reference that refers to a certain revision of nixpkgs. The
# certain revision is the revision to which the specified input is pinned.
nixpkgs_flake_reference() {
input_name=$1
shift

echo "github:NixOS/nixpkgs?rev=$(read_input_revision $input_name)"
}
2 changes: 1 addition & 1 deletion default.nix
Expand Up @@ -15,7 +15,7 @@ let
sources = import nix/sources.nix;
in
{
pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the
pkgsVersion ? "nixpkgs-22_11" # a string which chooses a nixpkgs from the
# niv-managed sources data

, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself
Expand Down
80 changes: 80 additions & 0 deletions flake.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

96 changes: 96 additions & 0 deletions flake.nix
@@ -0,0 +1,96 @@
{
description = "Tahoe-LAFS, free and open decentralized data store";

inputs = {
# Two alternate nixpkgs pins. Ideally these could be selected easily from
# the command line but there seems to be no syntax/support for that.
# However, these at least cause certain revisions to be pinned in our lock
# file where you *can* dig them out - and the CI configuration does.
"nixpkgs-22_11" = {
url = github:NixOS/nixpkgs?ref=nixos-22.11;
};
"nixpkgs-unstable" = {
url = github:NixOS/nixpkgs;
};

# Point the default nixpkgs at one of those. This avoids having getting a
# _third_ package set involved and gives a way to provide what should be a
# working experience by default (that is, if nixpkgs doesn't get
# overridden).
nixpkgs.follows = "nixpkgs-22_11";

flake-utils.url = "github:numtide/flake-utils";
};

# Provide a Python package override that adds a working Tahoe-LAFS to the
# package set.
#
# "Python package override" is not a formally recognized kind of flake
# output but it provides the best results here:
#
# 1. It composes. It is possible to apply N package overrides to a Python
# derivation to produce a new one that integrates them all.
#
# 1b. It avoids creating a combinatorial explosion of outputs. We don't
# need to provide a Tahoe-LAFS package for every version of Python.
# Downstream consumers can apply the override to whichever Python
# derivation they like.
#
# 1c. It avoids locking downstream consumers into some *specific* Python
# derivations (the ones we pick). They can bring their own.
#
# 2. It avoids unnecessary rebuilds. It is an override but by itself it
# does not override anything in nixpkgs. This means it does not impact
# any derivations that depend on Python itself. Since nixpkgs includes
# many such derivations and many of them are expensive to build, this is
# a significant advantage.
#
# 2b. Maybe it is actually possible to avoid these rebuilds if we
# override python in nixpkgs. Study
# https://github.com/SomeoneSerge/pkgs/blob/0b9e433013b37b71f358dd5dfec00a96ca0dab7e/overlay.nix#L21-L27
# and see if you can make any sense of what's going on there.
#
outputs = { self, nixpkgs, flake-utils, ... }:
{
# An overlay to add Tahoe-LAFS to the Python package sets.
overlays.default = import ./nix/overlay.nix { inherit (nixpkgs) lib; };
} // flake-utils.lib.eachSystem [ "x86_64-linux" ] (system:
{
packages =
let
pkgs = import nixpkgs {
inherit system;
overlays = [ self.overlays.default ];
};
# Create a Python runtime with the Tahoe-LAFS package installed.
makeTahoe = py: py.withPackages (ps: [ps.tahoe-lafs]);
in
with pkgs; {
# Okay, above I said we would avoid a combinatorial explosion
# but here we have a Tahoe-LAFS package for every Python
# derivation.
#
# Both things are true. These packages let us test that our
# package actually works against these different versions of
# Python but downstream consumers can largely ignore them and
# pick their own Python derivation (of course, we will not have
# tested that such an integration works but at least this gives
# us a good idea about which major/minor versions of which
# Python interpreters Tahoe-LAFS works with).
#
# Also note that these are not *Python packages*. They are
# whole Python runtimes which happen to have Tahoe-LAFS
# available to them.
python3-tahoe-lafs = makeTahoe python3;

python38-tahoe-lafs = makeTahoe python38;
python39-tahoe-lafs = makeTahoe python39;
python310-tahoe-lafs = makeTahoe python310;
python311-tahoe-lafs = makeTahoe python311;

pypy38-tahoe-lafs = makeTahoe pypy38;
pypy39-tahoe-lafs = makeTahoe pypy39;
};
}
);
}
1 change: 1 addition & 0 deletions newsfragments/3903.installation
@@ -0,0 +1 @@
There is now a `Nix flake <https://nixos.wiki/wiki/Flakes>`_ which provides a Python package override defining a Tahoe-LAFS package (as well as some pre-composed Python runtime packages including Tahoe-LAFS).
32 changes: 32 additions & 0 deletions nix/overlay.nix
@@ -0,0 +1,32 @@
{ lib }:
let
makePython = final: prev: py:
let
self = prev.${py}.override {
inherit self;
packageOverrides = lib.composeManyExtensions final.pythonPackageOverlays;
};
in {
${py} = self;
"${py}Packages" = final.${py}.pkgs;
};
in
# This overlay adds Tahoe-LAFS and all of its properly-configured Python
# package dependencies to a Python package set. Downstream consumers
# can apply it to their own nixpkgs derivation to produce a Tahoe-LAFS
# package.
final: prev: ({
pythonPackageOverlays = (prev.pythonPackageOverlays or []) ++ [
(import ./python-overrides.nix)
];
}
// (makePython final prev "python3")

// (makePython final prev "python38")
// (makePython final prev "python39")
// (makePython final prev "python310")
// (makePython final prev "python311")

// (makePython final prev "pypy38")
// (makePython final prev "pypy39")
)

0 comments on commit 1b12368

Please sign in to comment.