Build static binaries on Linux

CHANGELOG.md

@@ -36,6 +36,7 @@
   - Add `--id` and `--search` options to `open` command to open given zettel ID or search page respectively  (#317)
   - Use current directory as the default value for `-d` (a la. git) (#389)
   - Adapt `neuron new` for title IDs (#408)
+- Static binaries
 
 ## 0.6.0.0
 

ci.nix

@@ -1,6 +1,7 @@
 { system ? builtins.currentSystem }:
 let
-  pkgs = import (import ./nixpkgs.nix) { inherit system; };
+  sources = import ./nix/sources.nix {};
+  pkgs = import sources.nixpkgs { inherit system; };
 in
 pkgs.recurseIntoAttrs {
   # Build both default.nix and shell.nix such that both derivations are

default.nix

@@ -1 +1,5 @@
-args@{ ... }:(import ./project.nix args).neuron
+args@{...}:
+let 
+  sources = import nix/sources.nix {};
+  pkgs = import sources.nixpkgs args;
+in (import ./project.nix { pkgs = pkgs; } ).neuron

neuron/neuron.cabal

@@ -1,7 +1,7 @@
 cabal-version: 2.4
 name: neuron
 -- This version must be in sync with what's in Default.dhall
-version: 0.7.3.0
+version: 0.9.0.0
 license: AGPL-3.0-only
 copyright: 2020 Sridhar Ratnakumar
 maintainer: srid@srid.ca

neuron/test/Neuron/VersionSpec.hs

@@ -23,15 +23,15 @@ spec = do
         isLesserOrEqual = shouldNotSatisfy
     it "simple versions" $ do
       -- If the user requires 0.4, and we are "older than" than that, fail (aka. isGreater)
-      "0.8" `isGreater` olderThan
-      "0.7" `isLesserOrEqual` olderThan -- This is current version
+      "1.0" `isGreater` olderThan
+      "0.9" `isLesserOrEqual` olderThan -- This is current version
       "0.4" `isLesserOrEqual` olderThan
     it "full versions" $ do
-      "0.8.1.2" `isGreater` olderThan
-      "0.7.15" `isGreater` olderThan
-      "0.7.12.8" `isGreater` olderThan
-      "0.7.0.0" `isLesserOrEqual` olderThan -- This is current version
+      "1.0.1.2" `isGreater` olderThan
+      "0.9.15" `isGreater` olderThan
+      "0.9.12.8" `isGreater` olderThan
+      "0.9.0.0" `isLesserOrEqual` olderThan -- This is current version
       "0.6.1.0" `isLesserOrEqual` olderThan
     it "within same major version" $ do
-      "0.7.12.8" `isGreater` olderThan
-      "0.7.0.0" `isLesserOrEqual` olderThan -- This is current version
+      "0.9.12.8" `isGreater` olderThan
+      "0.9.0.0" `isLesserOrEqual` olderThan -- This is current version

nix/sources.json

@@ -0,0 +1,38 @@
+{
+    "niv": {
+        "branch": "master",
+        "description": "Easy dependency management for Nix projects",
+        "homepage": "https://github.com/nmattia/niv",
+        "owner": "nmattia",
+        "repo": "niv",
+        "rev": "29ddaaf4e099c3ac0647f5b652469dfc79cd3b53",
+        "sha256": "1va6myp07gkspgxfch8z3rs9nyvys6jmgzkys6a2c4j09qxp1bs0",
+        "type": "tarball",
+        "url": "https://github.com/nmattia/niv/archive/29ddaaf4e099c3ac0647f5b652469dfc79cd3b53.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixpkgs": {
+        "branch": "master",
+        "description": "Nix Packages collection",
+        "homepage": null,
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "990af061bc84d7410a989ce0ce53d165ef51eae2",
+        "sha256": "0wmsvmgcfsq6102avnd7jgv0gaq3ics29449iwafyn7k4ysmji26",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/990af061bc84d7410a989ce0ce53d165ef51eae2.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixpkgs-static": {
+        "branch": "static",
+        "description": "Nix Packages collection",
+        "homepage": null,
+        "owner": "srid",
+        "repo": "nixpkgs",
+        "rev": "312f5dc940b1a2c627e8cce4adc192cfa3e730db",
+        "sha256": "1x1h6j43wp2fgzjlv0nf8h5syvpdp3nhp8xb85hxzdz8k7hkhi4s",
+        "type": "tarball",
+        "url": "https://github.com/srid/nixpkgs/archive/312f5dc940b1a2c627e8cce4adc192cfa3e730db.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    }
+}

nix/sources.nix

@@ -0,0 +1,171 @@
+# This file has been generated by Niv.
+
+let
+
+  #
+  # The fetchers. fetch_<type> fetches specs of type <type>.
+  #
+
+  fetch_file = pkgs: name: spec:
+    let
+      name' = sanitizeName name + "-src";
+    in
+      if spec.builtin or true then
+        builtins_fetchurl { inherit (spec) url sha256; name = name'; }
+      else
+        pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
+
+  fetch_tarball = pkgs: name: spec:
+    let
+      name' = sanitizeName name + "-src";
+    in
+      if spec.builtin or true then
+        builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
+      else
+        pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
+
+  fetch_git = name: spec:
+    let
+      ref =
+        if spec ? ref then spec.ref else
+          if spec ? branch then "refs/heads/${spec.branch}" else
+            if spec ? tag then "refs/tags/${spec.tag}" else
+              abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
+    in
+      builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; };
+
+  fetch_local = spec: spec.path;
+
+  fetch_builtin-tarball = name: throw
+    ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
+        $ niv modify ${name} -a type=tarball -a builtin=true'';
+
+  fetch_builtin-url = name: throw
+    ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
+        $ niv modify ${name} -a type=file -a builtin=true'';
+
+  #
+  # Various helpers
+  #
+
+  # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
+  sanitizeName = name:
+    (
+      concatMapStrings (s: if builtins.isList s then "-" else s)
+        (
+          builtins.split "[^[:alnum:]+._?=-]+"
+            ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
+        )
+    );
+
+  # The set of packages used when specs are fetched using non-builtins.
+  mkPkgs = sources: system:
+    let
+      sourcesNixpkgs =
+        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
+      hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
+      hasThisAsNixpkgsPath = <nixpkgs> == ./.;
+    in
+      if builtins.hasAttr "nixpkgs" sources
+      then sourcesNixpkgs
+      else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
+        import <nixpkgs> {}
+      else
+        abort
+          ''
+            Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
+            add a package called "nixpkgs" to your sources.json.
+          '';
+
+  # The actual fetching function.
+  fetch = pkgs: name: spec:
+
+    if ! builtins.hasAttr "type" spec then
+      abort "ERROR: niv spec ${name} does not have a 'type' attribute"
+    else if spec.type == "file" then fetch_file pkgs name spec
+    else if spec.type == "tarball" then fetch_tarball pkgs name spec
+    else if spec.type == "git" then fetch_git name spec
+    else if spec.type == "local" then fetch_local spec
+    else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
+    else if spec.type == "builtin-url" then fetch_builtin-url name
+    else
+      abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
+
+  # If the environment variable NIV_OVERRIDE_${name} is set, then use
+  # the path directly as opposed to the fetched source.
+  replace = name: drv:
+    let
+      saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
+      ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
+    in
+      if ersatz == "" then drv else ersatz;
+
+  # Ports of functions for older nix versions
+
+  # a Nix version of mapAttrs if the built-in doesn't exist
+  mapAttrs = builtins.mapAttrs or (
+    f: set: with builtins;
+    listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
+  );
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
+  range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
+  stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
+  stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
+  concatMapStrings = f: list: concatStrings (map f list);
+  concatStrings = builtins.concatStringsSep "";
+
+  # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
+  optionalAttrs = cond: as: if cond then as else {};
+
+  # fetchTarball version that is compatible between all the versions of Nix
+  builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
+    let
+      inherit (builtins) lessThan nixVersion fetchTarball;
+    in
+      if lessThan nixVersion "1.12" then
+        fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
+      else
+        fetchTarball attrs;
+
+  # fetchurl version that is compatible between all the versions of Nix
+  builtins_fetchurl = { url, name ? null, sha256 }@attrs:
+    let
+      inherit (builtins) lessThan nixVersion fetchurl;
+    in
+      if lessThan nixVersion "1.12" then
+        fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
+      else
+        fetchurl attrs;
+
+  # Create the final "sources" from the config
+  mkSources = config:
+    mapAttrs (
+      name: spec:
+        if builtins.hasAttr "outPath" spec
+        then abort
+          "The values in sources.json should not have an 'outPath' attribute"
+        else
+          spec // { outPath = replace name (fetch config.pkgs name spec); }
+    ) config.sources;
+
+  # The "config" used by the fetchers
+  mkConfig =
+    { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
+    , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
+    , system ? builtins.currentSystem
+    , pkgs ? mkPkgs sources system
+    }: rec {
+      # The sources, i.e. the attribute set of spec name to spec
+      inherit sources;
+
+      # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
+      inherit pkgs;
+    };
+
+in
+mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }

project.nix

@@ -3,19 +3,19 @@ let
     url = "https://github.com/hercules-ci/gitignore/archive/c4662e6.tar.gz";
     sha256 = "1npnx0h6bd0d7ql93ka7azhj40zgjp815fw2r6smg8ch9p7mzdlx";
   };
+  sources = import nix/sources.nix {};
+  nixpkgs = import sources.nixpkgs {};
 in {
-  system ? builtins.currentSystem,
-  pkgs ? import (import ./nixpkgs.nix) { inherit system; },
-  # Cabal project name
-  name ? "neuron",
-  compiler ? pkgs.haskellPackages,
+  pkgs ? nixpkgs,
+  neuronFlags ? [],
+  disableHsLuaTests ? false,
   withHoogle ? false,
   ...
 }:
 
 let
   inherit (pkgs.haskell.lib)
-    overrideCabal doJailbreak dontCheck justStaticExecutables;
+    overrideCabal doJailbreak dontCheck justStaticExecutables appendConfigureFlags;
 
   inherit (import (gitignoreSrc) { inherit (pkgs) lib; }) gitignoreSource;
 
@@ -25,6 +25,7 @@ let
 
   sources = {
     neuron = gitignoreSource ./neuron;
+    # TODO: Switch to using `niv` exclusively.
     rib = thunkOrPath "rib";
     reflex-dom-pandoc = thunkOrPath "reflex-dom-pandoc";
   };
@@ -34,7 +35,7 @@ let
     cp $src/src-bash/neuron-search $out/bin/neuron-search
     chmod +x $out/bin/neuron-search
     wrapProgram $out/bin/neuron-search --prefix 'PATH' ':' ${
-      with pkgs;
+      with nixpkgs;
       lib.makeBinPath [ fzf ripgrep gawk bat findutils envsubst ]
     }
     PATH=$PATH:$out/bin
@@ -56,7 +57,11 @@ let
     # Jailbreak pandoc to work with newer skylighting
     pandoc = doJailbreak (dontCheck super.pandoc);
 
-    neuron = (justStaticExecutables
+    # Test fails on pkgsMusl
+    # https://github.com/hslua/hslua/issues/67
+    hslua = if disableHsLuaTests then (dontCheck super.hslua) else super.hslua;
+
+    neuron = appendConfigureFlags ((justStaticExecutables
       (overrideCabal (self.callCabal2nix "neuron" sources.neuron { })
         wrapSearchScript)).overrideDerivation (drv: {
           # Avoid transitive runtime dependency on the whole GHC distribution due to
@@ -85,10 +90,10 @@ let
             remove-references-to -t ${self.js-dgtable} $out/bin/neuron
             remove-references-to -t ${self.js-flot} $out/bin/neuron
           '';
-        });
+        })) neuronFlags;
   };
 
-  haskellPackages = compiler.override { overrides = haskellOverrides; };
+  haskellPackages = pkgs.haskellPackages.override { overrides = haskellOverrides; };
 
   nixShellSearchScript = pkgs.stdenv.mkDerivation {
     name = "neuron-search";

static.nix

@@ -0,0 +1,21 @@
+args@{...}:
+let 
+  sources = import nix/sources.nix {};
+  nixpkgs = import sources.nixpkgs-static args;
+  pkgs = nixpkgs.pkgsMusl;
+in 
+  (import ./project.nix { 
+    inherit pkgs;
+    disableHsLuaTests = true; 
+    neuronFlags = [
+      "--ghc-option=-optl=-static"
+      # Disabling shared as workaround. But - https://github.com/nh2/static-haskell-nix/issues/99#issuecomment-665400600
+      # TODO: Patch ghc bootstrap binary to use ncurses6, which might also obviate the nixpkgs revert.
+      "--disable-shared"
+      "--extra-lib-dirs=${pkgs.gmp6.override { withStatic = true; }}/lib"
+      "--extra-lib-dirs=${pkgs.zlib.static}/lib"
+      "--extra-lib-dirs=${pkgs.libffi.overrideAttrs (old: { dontDisableStatic = true; })}/lib"
+      "--extra-lib-dirs=${pkgs.ncurses.override { enableStatic = true; }}/lib"
+    ];
+  }).neuron
+