Merge f5e1af0 into 6480e0c

.circleci/config.yml

@@ -39,11 +39,11 @@ workflows:
       - "centos-8":
           {}
 
-      - "nixos-19-09":
-          {}
+      - "nixos":
+          nixpkgs: "21.05"
 
-      - "nixos-21-05":
-          {}
+      - "nixos":
+          nixpkgs: "21.11"
 
       # Test against PyPy 2.7
       - "pypy27-buster":
@@ -441,20 +441,58 @@ jobs:
         image: "tahoelafsci/fedora:29-py"
         user: "nobody"
 
-  nixos-19-09: &NIXOS
+  nixos:
+    parameters:
+      nixpkgs:
+        description: >-
+          Reference the name of a niv-managed nixpkgs source (see `niv show`
+          and nix/sources.json)
+        type: "string"
+
     docker:
       # Run in a highly Nix-capable environment.
       - <<: *DOCKERHUB_AUTH
-        image: "nixorg/nix:circleci"
+        image: "nixos/nix:2.3.16"
 
     environment:
-      NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz"
-      SOURCE: "nix/"
+      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and
+      # allows us to push to CACHIX_NAME.  We only need this set for
+      # `cachix use` in this step.
+      CACHIX_NAME: "tahoe-lafs-opensource"
 
     steps:
+      - "run":
+          # The nixos/nix image does not include ssh.  Install it so the
+          # `checkout` step will succeed.  We also want cachix for
+          # Nix-friendly caching.
+          name: "Install Basic Dependencies"
+          command: |
+            nix-env \
+              --file https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz \
+              --install \
+              -A openssh cachix bash
+
       - "checkout"
+
+      - run:
+          name: "Cachix setup"
+          # Record the store paths that exist before we did much.  There's no
+          # reason to cache these, they're either in the image or have to be
+          # retrieved before we can use cachix to restore from cache.
+          command: |
+            cachix use "${CACHIX_NAME}"
+            nix path-info --all > /tmp/store-path-pre-build
+
       - "run":
-          name: "Build and Test"
+          # The Nix package doesn't know how to do this part, unfortunately.
+          name: "Generate version"
+          command: |
+            nix-shell \
+              -p 'python3.withPackages (ps: [ ps.setuptools ])' \
+              --run 'python setup.py update_version'
+
+      - "run":
+          name: "Build"
           command: |
             # CircleCI build environment looks like it has a zillion and a
             # half cores.  Don't let Nix autodetect this high core count
@@ -466,17 +504,33 @@ jobs:
             # build a couple simple little dependencies that don't take
             # advantage of multiple cores and we get a little speedup by doing
             # them in parallel.
-            nix-build --cores 3 --max-jobs 2 "$SOURCE"
+            nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>"
 
-  nixos-21-05:
-    <<: *NIXOS
+      - "run":
+          name: "Test"
+          command: |
+            # Let it go somewhat wild for the test suite itself
+            nix-build --cores 8 --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" tests.nix
 
-    environment:
-      # Note this doesn't look more similar to the 19.09 NIX_PATH URL because
-      # there was some internal shuffling by the NixOS project about how they
-      # publish stable revisions.
-      NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/d32b07e6df276d78e3640eb43882b80c9b2b3459.tar.gz"
-      SOURCE: "nix/py3.nix"
+      - run:
+          # Send any new store objects to cachix.
+          name: "Push to Cachix"
+          when: "always"
+          command: |
+            # Cribbed from
+            # https://circleci.com/blog/managing-secrets-when-you-have-pull-requests-from-outside-contributors/
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then
+              # I'm sure you're thinking "CIRCLE_PR_NUMBER must just be the
+              # number of the PR being built".  Sorry, dear reader, you have
+              # guessed poorly.  It is also conditionally set based on whether
+              # this is a PR from a fork or not.
+              #
+              # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
+              echo "Skipping Cachix push for forked PR."
+            else
+              # https://docs.cachix.org/continuous-integration-setup/circleci.html
+              bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME"
+            fi
 
   typechecks:
     docker:

default.nix

@@ -0,0 +1,69 @@
+let
+  sources = import nix/sources.nix;
+in
+{ pkgsVersion ? "nixpkgs-21.11"
+, pkgs ? import sources.${pkgsVersion} { }
+, pypiData ? sources.pypi-deps-db
+, pythonVersion ? "python37"
+, mach-nix ? import sources.mach-nix {
+    inherit pkgs pypiData;
+    python = pythonVersion;
+  }
+}:
+# The project name, version, and most other metadata are automatically
+# extracted from the source.  Some requirements are not properly extracted
+# and those cases are handled below.  The version can only be extracted if
+# `setup.py update_version` has been run (this is not at all ideal but it
+# seems difficult to fix) - so for now just be sure to run that first.
+mach-nix.buildPythonPackage {
+  # Define the location of the Tahoe-LAFS source to be packaged.  Clean up all
+  # as many of the non-source files (eg the `.git` directory, `~` backup
+  # files, nix's own `result` symlink, etc) as possible to avoid needing to
+  # re-build when files that make no difference to the package have changed.
+  src = pkgs.lib.cleanSource ./.;
+
+  # Define some extra requirements that mach-nix does not automatically detect
+  # from inspection of the source.  We typically don't need to put version
+  # constraints on any of these requirements.  The pypi-deps-db we're
+  # operating with makes dependency resolution deterministic so as long as it
+  # works once it will always work.  It could be that in the future we update
+  # pypi-deps-db and an incompatibility arises - in which case it would make
+  # sense to apply some version constraints here.
+  requirementsExtra = ''
+    # mach-nix does not yet support pyproject.toml which means it misses any
+    # build-time requirements of our dependencies which are declared in such a
+    # file.  Tell it about them here.
+    setuptools_rust
+
+    # mach-nix does not yet parse environment markers correctly.  It misses
+    # all of our requirements which have an environment marker.  Duplicate them
+    # here.
+    foolscap
+    eliot
+    pyrsistent
+  '';
+
+  # Specify where mach-nix should find packages for our Python dependencies.
+  # There are some reasonable defaults so we only need to specify certain
+  # packages where the default configuration runs into some issue.
+  providers = {
+    # Through zfec 1.5.5 the wheel has an incorrect runtime dependency
+    # declared on argparse, not available for recent versions of Python 3.
+    # Force mach-nix to use the sdist instead.  This allows us to apply a
+    # patch that removes the offending declaration.
+    zfec = "sdist";
+  };
+
+  # Define certain overrides to the way Python dependencies are built.
+  _ = {
+    # Apply the argparse declaration fix to zfec sdist.
+    zfec.patches = with pkgs; [
+      (fetchpatch {
+        name = "fix-argparse.patch";
+        url = "https://github.com/tahoe-lafs/zfec/commit/c3e736a72cccf44b8e1fb7d6c276400204c6bc1e.patch";
+        sha256 = "1md9i2fx1ya7mgcj9j01z58hs3q9pj4ch5is5b5kq4v86cf6x33x";
+      })
+    ];
+  };
+
+}