Skip to content

Commit

Permalink
Merge 15efb3f into e9f6882
Browse files Browse the repository at this point in the history
  • Loading branch information
exarkun committed Jul 26, 2023
2 parents e9f6882 + 15efb3f commit 97840c2
Show file tree
Hide file tree
Showing 13 changed files with 780 additions and 417 deletions.
176 changes: 114 additions & 62 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,23 +87,23 @@ workflows:
- "oraclelinux-8":
{}

- "nixos":
name: "<<matrix.pythonVersion>>"
nixpkgs: "22.11"
- "unittest":
name: "<<matrix.pythonVersion>>-unittest"
nixpkgs: "nixpkgs-unstable"
matrix:
parameters:
pythonVersion:
- "python38"
- "python39"
- "python310"
- "python311"

- "nixos":
name: "<<matrix.pythonVersion>>"
nixpkgs: "unstable"
- "integrationtest":
name: "<<matrix.pythonVersion>>-integrationtest"
nixpkgs: "nixpkgs-unstable"
matrix:
parameters:
pythonVersion:
- "python311"
- "python310"

# Eventually, test against PyPy 3.8
#- "pypy27-buster":
Expand All @@ -120,15 +120,6 @@ workflows:
- "another-locale":
{}

- "integration":
# Run even the slow integration tests here. We need the `--` to
# sneak past tox and get to pytest.
tox-args: "-- --runslow integration"
requires:
# If the unit test suite doesn't pass, don't bother running the
# integration tests.
- "debian-11"

- "typechecks":
{}
- "docs":
Expand Down Expand Up @@ -381,12 +372,75 @@ jobs:
image: "tahoelafsci/fedora:35-py3"
user: "nobody"

nixos:
# A parameterized job which runs the integration test suite in the
# Nix-defined environment.
integrationtest:
parameters:
nixpkgs:
description: >-
Reference the name of a niv-managed nixpkgs source (see `niv show`
and nix/sources.json)
Reference the name of a flake-managed nixpkgs input (see `nix flake
metadata` and flake.nix)
type: "string"
pythonVersion:
description: >-
Reference the name of a Python package in nixpkgs to use.
type: "string"

executor: "nix"

parallelism: 4

steps:
- "nix-build":
nixpkgs: "<<parameters.nixpkgs>>"
pythonVersion: "<<parameters.pythonVersion>>"
buildSteps:
- "run":
name: "Integration Test"
command: |
source .circleci/lib.sh
# Translate the nixpkgs selection into a flake reference we
# can use to override the default nixpkgs input.
NIXPKGS=$(nixpkgs_flake_reference <<parameters.nixpkgs>>)
# circleci "installs" circleci-agent directly into /bin.
# That's not typically on our path but we can execute it
# directly. Docs suggest running `circleci`, from
# `circleci-cli`, to invoke this behavior but then we'd have
# to fiddle with PATH to make it work. Let's avoid that and
# call the agent executable directly.
AGENT=/bin/circleci-agent
TESTS=$(
$AGENT tests glob "integration/test_*.py" |
$AGENT tests split --split-by=timings
)
echo $TESTS
# Run the integration tests, even the slow ones. Generate
# junit xml results for CircleCI to ingest.
cache_if_able nix run \
--verbose \
--print-build-logs \
--override-input nixpkgs "$NIXPKGS" \
.#<<parameters.pythonVersion>>-integration \
-- \
--verbose \
--runslow \
--capture=no \
--junitxml=integration-results.xml \
$TESTS
- "store_test_results":
path: "integration-results.xml"

# A parameterized job which runs the unit test suite in the Nix-defined
# environment.
unittest:
parameters:
nixpkgs:
description: >-
Reference the name of a flake-managed nixpkgs input (see `nix flake
metadata` and flake.nix)
type: "string"
pythonVersion:
description: >-
Expand All @@ -403,14 +457,19 @@ jobs:
- "run":
name: "Unit Test"
command: |
# The dependencies are all built so we can allow more
# parallelism here.
source .circleci/lib.sh
cache_if_able nix-build \
--cores 8 \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
--argstr pythonVersion "<<parameters.pythonVersion>>" \
nix/tests.nix
# Translate the nixpkgs selection into a flake reference we
# can use to override the default nixpkgs input.
NIXPKGS=$(nixpkgs_flake_reference <<parameters.nixpkgs>>)
cache_if_able nix run \
--verbose \
--print-build-logs \
--override-input nixpkgs "$NIXPKGS" \
.#<<parameters.pythonVersion>>-unittest -- \
--jobs $UNITTEST_CORES \
allmydata
typechecks:
docker:
Expand Down Expand Up @@ -536,20 +595,23 @@ executors:
docker:
# Run in a highly Nix-capable environment.
- <<: *DOCKERHUB_AUTH
image: "nixos/nix:2.10.3"
image: "nixos/nix:2.16.1"
environment:
# CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us
# to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push
# to.
CACHIX_NAME: "tahoe-lafs-opensource"
# Let us use features marked "experimental". For example, most/all of
# the `nix <subcommand>` forms.
NIX_CONFIG: "experimental-features = nix-command flakes"

commands:
nix-build:
parameters:
nixpkgs:
description: >-
Reference the name of a niv-managed nixpkgs source (see `niv show`
and nix/sources.json)
Reference the name of a flake-managed nixpkgs input (see `nix flake
metadata` and flake.nix)
type: "string"
pythonVersion:
description: >-
Expand All @@ -565,15 +627,21 @@ commands:
# Get cachix for Nix-friendly caching.
name: "Install Basic Dependencies"
command: |
NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz"
nix-env \
--file $NIXPKGS \
--install \
-A cachix bash
# Activate it for "binary substitution". This sets up
# configuration tht lets Nix download something from the cache
# instead of building it locally, if possible.
cachix use "${CACHIX_NAME}"
# Get some build environment dependencies and let them float on a
# certain release branch. These aren't involved in the actual
# package build (only in CI environment setup) so the fact that
# they float shouldn't hurt reproducibility.
NIXPKGS="nixpkgs/nixos-23.05"
nix profile install \
$NIXPKGS#cachix \
$NIXPKGS#bash \
$NIXPKGS#jp \
$NIXPKGS#circleci-cli
# Activate our cachix cache for "binary substitution". This sets
# up configuration tht lets Nix download something from the cache
# instead of building it locally, if possible.
cachix use "${CACHIX_NAME}"
- "checkout"

Expand All @@ -585,32 +653,16 @@ commands:
-p 'python3.withPackages (ps: [ ps.setuptools ])' \
--run 'python setup.py update_version'
- "run":
name: "Build Dependencies"
command: |
# CircleCI build environment looks like it has a zillion and a
# half cores. Don't let Nix autodetect this high core count
# because it blows up memory usage and fails the test run. Pick a
# number of cores that suits the build environment we're paying
# for (the free one!).
source .circleci/lib.sh
# nix-shell will build all of the dependencies of the target but
# not the target itself.
cache_if_able nix-shell \
--run "" \
--cores 3 \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
--argstr pythonVersion "<<parameters.pythonVersion>>" \
./default.nix
- "run":
name: "Build Package"
command: |
source .circleci/lib.sh
cache_if_able nix-build \
--cores 4 \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
--argstr pythonVersion "<<parameters.pythonVersion>>" \
./default.nix
NIXPKGS=$(nixpkgs_flake_reference <<parameters.nixpkgs>>)
cache_if_able nix build \
--verbose \
--print-build-logs \
--cores "$DEPENDENCY_CORES" \
--override-input nixpkgs "$NIXPKGS" \
.#<<parameters.pythonVersion>>-tahoe-lafs
- steps: "<<parameters.buildSteps>>"
29 changes: 29 additions & 0 deletions .circleci/lib.sh
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
# CircleCI build environment looks like it has a zillion and a half cores.
# Don't let Nix autodetect this high core count because it blows up memory
# usage and fails the test run. Pick a number of cores that suits the build
# environment we're paying for (the free one!).
DEPENDENCY_CORES=3

# Once dependencies are built, we can allow some more concurrency for our own
# test suite.
UNITTEST_CORES=8

# Run a command, enabling cache writes to cachix if possible. The command is
# accepted as a variable number of positional arguments (like argv).
function cache_if_able() {
Expand Down Expand Up @@ -117,3 +127,22 @@ function describe_build() {
echo "Cache not writeable."
fi
}

# Inspect the flake input metadata for an input of a given name and return the
# revision at which that input is pinned. If the input does not exist then
# return garbage (probably "null").
read_input_revision() {
input_name=$1
shift

nix flake metadata --json | jp --unquoted 'locks.nodes."'"$input_name"'".locked.rev'
}

# Return a flake reference that refers to a certain revision of nixpkgs. The
# certain revision is the revision to which the specified input is pinned.
nixpkgs_flake_reference() {
input_name=$1
shift

echo "github:NixOS/nixpkgs?rev=$(read_input_revision $input_name)"
}
62 changes: 13 additions & 49 deletions default.nix
Original file line number Diff line number Diff line change
@@ -1,49 +1,13 @@
let
# sources.nix contains information about which versions of some of our
# dependencies we should use. since we use it to pin nixpkgs, all the rest
# of our dependencies are *also* pinned - indirectly.
#
# sources.nix is managed using a tool called `niv`. as an example, to
# update to the most recent version of nixpkgs from the 21.11 maintenance
# release, in the top-level tahoe-lafs checkout directory you run:
#
# niv update nixpkgs-21.11
#
# niv also supports chosing a specific revision, following a different
# branch, etc. find complete documentation for the tool at
# https://github.com/nmattia/niv
sources = import nix/sources.nix;
in
{
pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the
# niv-managed sources data

, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself

, pythonVersion ? "python310" # a string choosing the python derivation from
# nixpkgs to target

, extrasNames ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
# the dependencies of which the resulting
# package will also depend on. Include all of the
# runtime extras by default because the incremental
# cost of including them is a lot smaller than the
# cost of re-building the whole thing to add them.

}:
with (pkgs.${pythonVersion}.override {
packageOverrides = import ./nix/python-overrides.nix;
}).pkgs;
callPackage ./nix/tahoe-lafs.nix {
# Select whichever package extras were requested.
inherit extrasNames;

# Define the location of the Tahoe-LAFS source to be packaged (the same
# directory as contains this file). Clean up as many of the non-source
# files (eg the `.git` directory, `~` backup files, nix's own `result`
# symlink, etc) as possible to avoid needing to re-build when files that
# make no difference to the package have changed.
tahoe-lafs-src = pkgs.lib.cleanSource ./.;

doCheck = false;
}
# This is the flake-compat glue code. It loads the flake and gives us its
# outputs. This gives us backwards compatibility with pre-flake consumers.
# All of the real action is in flake.nix.
(import
(
let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in
fetchTarball {
url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
sha256 = lock.nodes.flake-compat.locked.narHash;
}
)
{ src = ./.; }
).defaultNix.default

0 comments on commit 97840c2

Please sign in to comment.