Merge pull request #1175 from tahoe-lafs/3867.mach-nix

Switch Nix packaging to use mach-nix

Fixes: ticket:3867

.circleci/config.yml

@@ -39,11 +39,13 @@ workflows:
       - "centos-8":
           {}
 
-      - "nixos-19-09":
-          {}
+      - "nixos":
+          name: "NixOS 21.05"
+          nixpkgs: "21.05"
 
-      - "nixos-21-05":
-          {}
+      - "nixos":
+          name: "NixOS 21.11"
+          nixpkgs: "21.11"
 
       # Test against PyPy 2.7
       - "pypy27-buster":
@@ -441,20 +443,58 @@ jobs:
         image: "tahoelafsci/fedora:29-py"
         user: "nobody"
 
-  nixos-19-09: &NIXOS
+  nixos:
+    parameters:
+      nixpkgs:
+        description: >-
+          Reference the name of a niv-managed nixpkgs source (see `niv show`
+          and nix/sources.json)
+        type: "string"
+
     docker:
       # Run in a highly Nix-capable environment.
       - <<: *DOCKERHUB_AUTH
-        image: "nixorg/nix:circleci"
+        image: "nixos/nix:2.3.16"
 
     environment:
-      NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz"
-      SOURCE: "nix/"
+      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and
+      # allows us to push to CACHIX_NAME.  We only need this set for
+      # `cachix use` in this step.
+      CACHIX_NAME: "tahoe-lafs-opensource"
 
     steps:
+      - "run":
+          # The nixos/nix image does not include ssh.  Install it so the
+          # `checkout` step will succeed.  We also want cachix for
+          # Nix-friendly caching.
+          name: "Install Basic Dependencies"
+          command: |
+            nix-env \
+              --file https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz \
+              --install \
+              -A openssh cachix bash
+
       - "checkout"
+
+      - run:
+          name: "Cachix setup"
+          # Record the store paths that exist before we did much.  There's no
+          # reason to cache these, they're either in the image or have to be
+          # retrieved before we can use cachix to restore from cache.
+          command: |
+            cachix use "${CACHIX_NAME}"
+            nix path-info --all > /tmp/store-path-pre-build
+
       - "run":
-          name: "Build and Test"
+          # The Nix package doesn't know how to do this part, unfortunately.
+          name: "Generate version"
+          command: |
+            nix-shell \
+              -p 'python3.withPackages (ps: [ ps.setuptools ])' \
+              --run 'python setup.py update_version'
+
+      - "run":
+          name: "Build"
           command: |
             # CircleCI build environment looks like it has a zillion and a
             # half cores.  Don't let Nix autodetect this high core count
@@ -466,17 +506,50 @@ jobs:
             # build a couple simple little dependencies that don't take
             # advantage of multiple cores and we get a little speedup by doing
             # them in parallel.
-            nix-build --cores 3 --max-jobs 2 "$SOURCE"
+            nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>"
 
-  nixos-21-05:
-    <<: *NIXOS
+      - "run":
+          name: "Test"
+          command: |
+            # Let it go somewhat wild for the test suite itself
+            nix-build --cores 8 --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" tests.nix
 
-    environment:
-      # Note this doesn't look more similar to the 19.09 NIX_PATH URL because
-      # there was some internal shuffling by the NixOS project about how they
-      # publish stable revisions.
-      NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/d32b07e6df276d78e3640eb43882b80c9b2b3459.tar.gz"
-      SOURCE: "nix/py3.nix"
+      - run:
+          # Send any new store objects to cachix.
+          name: "Push to Cachix"
+          when: "always"
+          command: |
+            # Cribbed from
+            # https://circleci.com/blog/managing-secrets-when-you-have-pull-requests-from-outside-contributors/
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then
+              # I'm sure you're thinking "CIRCLE_PR_NUMBER must just be the
+              # number of the PR being built".  Sorry, dear reader, you have
+              # guessed poorly.  It is also conditionally set based on whether
+              # this is a PR from a fork or not.
+              #
+              # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
+              echo "Skipping Cachix push for forked PR."
+            else
+              # If this *isn't* a build from a fork then we have the Cachix
+              # write key in our environment and we can push any new objects
+              # to Cachix.
+              #
+              # To decide what to push, we inspect the list of store objects
+              # that existed before and after we did most of our work.  Any
+              # that are new after the work is probably a useful thing to have
+              # around so push it to the cache.  We exclude all derivation
+              # objects (.drv files) because they're cheap to reconstruct and
+              # by the time you know their cache key you've already done all
+              # the work anyway.
+              #
+              # This shell expression for finding the objects and pushing them
+              # was from the Cachix docs:
+              #
+              # https://docs.cachix.org/continuous-integration-setup/circleci.html
+              #
+              # but they seem to have removed it now.
+              bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME"
+            fi
 
   typechecks:
     docker:

default.nix

@@ -0,0 +1,115 @@
+let
+  # sources.nix contains information about which versions of some of our
+  # dependencies we should use.  since we use it to pin nixpkgs and the PyPI
+  # package database, roughly all the rest of our dependencies are *also*
+  # pinned - indirectly.
+  #
+  # sources.nix is managed using a tool called `niv`.  as an example, to
+  # update to the most recent version of nixpkgs from the 21.11 maintenance
+  # release, in the top-level tahoe-lafs checkout directory you run:
+  #
+  #   niv update nixpkgs-21.11
+  #
+  # or, to update the PyPI package database -- which is necessary to make any
+  # newly released packages visible -- you likewise run:
+  #
+  #   niv update pypi-deps-db
+  #
+  # niv also supports chosing a specific revision, following a different
+  # branch, etc.  find complete documentation for the tool at
+  # https://github.com/nmattia/niv
+  sources = import nix/sources.nix;
+in
+{
+  pkgsVersion ? "nixpkgs-21.11" # a string which chooses a nixpkgs from the
+                                # niv-managed sources data
+
+, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself
+
+, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use
+                                  # for dependency resolution
+
+, pythonVersion ? "python37" # a string choosing the python derivation from
+                             # nixpkgs to target
+
+, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
+                           # the dependencies of which the resulting package
+                           # will also depend on.  Include all of the runtime
+                           # extras by default because the incremental cost of
+                           # including them is a lot smaller than the cost of
+                           # re-building the whole thing to add them.
+
+, mach-nix ? import sources.mach-nix { # the mach-nix package to use to build
+                                       # the tahoe-lafs package
+    inherit pkgs pypiData;
+    python = pythonVersion;
+}
+}:
+# The project name, version, and most other metadata are automatically
+# extracted from the source.  Some requirements are not properly extracted
+# and those cases are handled below.  The version can only be extracted if
+# `setup.py update_version` has been run (this is not at all ideal but it
+# seems difficult to fix) - so for now just be sure to run that first.
+mach-nix.buildPythonPackage rec {
+  # Define the location of the Tahoe-LAFS source to be packaged.  Clean up all
+  # as many of the non-source files (eg the `.git` directory, `~` backup
+  # files, nix's own `result` symlink, etc) as possible to avoid needing to
+  # re-build when files that make no difference to the package have changed.
+  src = pkgs.lib.cleanSource ./.;
+
+  # Select whichever package extras were requested.
+  inherit extras;
+
+  # Define some extra requirements that mach-nix does not automatically detect
+  # from inspection of the source.  We typically don't need to put version
+  # constraints on any of these requirements.  The pypi-deps-db we're
+  # operating with makes dependency resolution deterministic so as long as it
+  # works once it will always work.  It could be that in the future we update
+  # pypi-deps-db and an incompatibility arises - in which case it would make
+  # sense to apply some version constraints here.
+  requirementsExtra = ''
+    # mach-nix does not yet support pyproject.toml which means it misses any
+    # build-time requirements of our dependencies which are declared in such a
+    # file.  Tell it about them here.
+    setuptools_rust
+
+    # mach-nix does not yet parse environment markers correctly.  It misses
+    # all of our requirements which have an environment marker.  Duplicate them
+    # here.
+    foolscap
+    eliot
+    pyrsistent
+  '';
+
+  # Specify where mach-nix should find packages for our Python dependencies.
+  # There are some reasonable defaults so we only need to specify certain
+  # packages where the default configuration runs into some issue.
+  providers = {
+    # Through zfec 1.5.5 the wheel has an incorrect runtime dependency
+    # declared on argparse, not available for recent versions of Python 3.
+    # Force mach-nix to use the sdist instead.  This allows us to apply a
+    # patch that removes the offending declaration.
+    zfec = "sdist";
+  };
+
+  # Define certain overrides to the way Python dependencies are built.
+  _ = {
+    # Apply the argparse declaration fix to zfec sdist.
+    zfec.patches = with pkgs; [
+      (fetchpatch {
+        name = "fix-argparse.patch";
+        url = "https://github.com/tahoe-lafs/zfec/commit/c3e736a72cccf44b8e1fb7d6c276400204c6bc1e.patch";
+        sha256 = "1md9i2fx1ya7mgcj9j01z58hs3q9pj4ch5is5b5kq4v86cf6x33x";
+      })
+    ];
+
+    # Remove a click-default-group patch for a test suite problem which no
+    # longer applies because the project apparently no longer has a test suite
+    # in its source distribution.
+    click-default-group.patches = [];
+  };
+
+  passthru.meta.mach-nix = {
+    inherit providers _;
+  };
+}