This page contains basic and advanced examples for using mach-nix inside a nix expression
- Import mach-nix
- mkPython / mkPythonShell
- buildPythonPackage / buildPythonApplication
- Simplified overrides ('_' argument)
- Overrides (overrides_pre / overrides_post)
- Tensorflow
- PyTorch
- JupyterLab
- Docker
- R and Python
- Raspberry PI / aarch64 SD Image
every mach-nix expression should begin like this:
let
mach-nix = import (builtins.fetchGit {
url = "https://github.com/DavHau/mach-nix/";
ref = "refs/tags/3.0.0";
}) {
# optionally bring your own nixpkgs
# pkgs = import <nixpkgs> {};
# optionally specify the python version
# python = "python38";
# optionally update pypi data revision from https://github.com/DavHau/pypi-deps-db
# pypiDataRev = "some_revision";
# pypiDataSha256 = "some_sha256";
};
in
...mach-nix.mkPython { # replace with mkPythonShell if shell is wanted
requirements = builtins.readFile ./requirements.txt;
}packagesExtra accepts:
- python packages built via
mach-nix.buildPythonPackage. - R Packages from
nixpkgs.rPackages(see R example further down) - python package source trees as paths or derivations
- URLs pointing to a tarball archives containing a python source tree
...
mach-nix.mkPython {
requirements = builtins.readFile ./requirements.txt;
packagesExtra = [
"https://github.com/psf/requests/tarball/2a7832b5b06d" # from tarball url
./some/local/project # from local path
mach-nix.buildPythonPackage { ... }; # from package
];
}Alternatively, if requirements are not needed, packagesExtra can be passed directly to mkPython
...
mach-nix.mkPython [
"https://github.com/psf/requests/tarball/2a7832b5b06d" # from tarball url
./some/local/project # from local path
mach-nix.buildPythonPackage { ... }; # from package
]These functions can be used to manually build individual python modules or applications. Those can either be used directly, or fed as packagesExtra of mkPython.
Whenever requirements are not explicitly specified, they will be extracted automatically from the packages setup.py/setup.cfg. The same goes for the name and version.
...
mach-nix.buildPythonPackage /python-project-path...
mach-nix.buildPythonPackage "https://github.com/psf/requests/tarball/2a7832b5b06d"...
mach-nix.buildPythonPackage {
src = "https://github.com/psf/requests/tarball/2a7832b5b06d";
extras = "socks";
}Use requirementsExtra in case the auto detected requirements are incomplete
...
mach-nix.buildPythonPackage {
src = "https://github.com/psf/requests/tarball/2a7832b5b06d";
requirementsExtra = "pytest";
}...
mach-nix.buildPythonPackage {
src = builtins.fetchGit{
url = "https://github.com/user/projectname";
ref = "master";
# rev = "put_commit_hash_here";
};
}Use this if automatic requirements extraction doesn't work at all.
...
mach-nix.buildPythonPackage {
src = "https://github.com/psf/requests/tarball/2a7832b5b06d";
requirements = ''
# list of requirements
'';
}with mach-nix.nixpkgs;
mach-nix.mkPython {
requirements = "some requirements";
_.{package}.buildInputs = [...]; # replace buildInputs
_.{package}.buildInputs.add = [...]; # add buildInputs
_.{package}.buildInputs.mod = # modify buildInputs
oldInputs: filter (inp: ...) oldInputs;
_.{package}.patches = [...]; # replace patches
_.{package}.patches.add = [...]; # add patches
...
}
For example the package web2ldap depends on another python package ldap0 which fails to build because of missing dependencies.
...
with mach-nix.nixpkgs;
mach-nix.mkPython {
requirements = "web2ldap";
# add missing dependencies to ldap0
_.ldap0.buildInputs.add = [ openldap.dev cyrus_sasl.dev ];
}imagecodecs is available via wheel, but if one wants to build it from source, dependencies will be missing since there is no nixpkgs candidate available.
poetry2nix luckily maintains overrides for this package. They can be included into the mach-nix build like this.
...
mach-nix.mkPython rec {
requirements = ''
# bunch of other requirements
imagecodecs
'';
providers = {
_default = "sdist";
};
# Import overrides from poetry2nix
# Caution! Use poetry2nix overrides only in `overrides_post`, not `overrides_pre`.
overridesPost = [
(
import (builtins.fetchurl {
url = "https://raw.githubusercontent.com/nix-community/poetry2nix/1cfaa4084d651d73af137866622e3d0699851008/overrides.nix";
}) { pkgs = mach-nix.nixpkgs; }
)
];
}Tensorflow from pypi does not provide any hardware optimization support. To get a SSE/AVX/FMA enabled version, set the provider for tensorflow to nixpkgs.
...
mach-nix.mkPython {
requirements = ''
# bunch of other requirements
tensorflow
'';
# force tensorflow to be taken from nixpkgs
providers.tensorflow = "nixpkgs";
}This only works if the restrictions in requirements.txt allow for the tensorflow version from nixpkgs.
Install recent tensorflow via wheel
...
mach-nix.mkPython {
requirements = ''
# bunch of other requirements
tensorflow == 2.2.0rc4
'';
# no need to specify provider settings since wheel is the default anyways
}Recent pytorch version, Build dependencies from source
...
mach-nix.mkPython rec {
requirements = ''
# bunch of other requirements
torch == 1.5.0
'';
providers = {
# disallow wheels by default
_default = "nixpkgs,sdist";
# allow wheels only for torch
torch = "wheel";
};
}...
let
pyEnv = mach-nix.mkPython rec {
requirements = ''
jupyterlab
geopandas
pyproj
pygeos
shapely>=1.7.0
'';
providers.shapely = "sdist,nixpkgs";
};
in
mkShell rec {
buildInputs = [
pyEnv
] ;
shellHook = ''
jupyter lab --notebook-dir=~/
'';
}Docker images can be built by using mkDockerImage instead of mkPython. It accepts the same arguments.
Assuming the following expression under ./jupyter-docker.nix:
...
let
image = mach-nix.mkDockerImage {
requirements = ''
jupyterlab
# add more packages here
'';
};
in
# The following overrides a call to nixpkgs.dockerTools.buildImage.
# Find more buildImage examples here: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/docker/examples.nix
image.override (oldAttrs: {
name = "jupyterlab";
config.Cmd = [ "jupyter" "lab" "--notebook-dir=/mnt" "--allow-root" "--ip=0.0.0.0" ];
})Execute the build like:
nix-build ./jupyter-docker.nix -o ./docker-image
Afterwards, load the docker image:
docker load < ./docker-image
Start the jupyterlab container:
docker run --rm -it -p 8888:8888 -v $HOME:/mnt jupyterlab
The following is an example for a Python environment mixed with R packages.
R packages can be added via packagesExtra.
If mach-nix finds R packages inside packagesExtra, it will automatically include rpy2 and add the selected R packages to its buildInputs.
To get a list of available R packages, execute: echo "builtins.attrNames(import <nixpkgs> {}).rPackages" | nix repl
...
mach-nix.mkPython {
requirements = ''
# some python requirements
'';
packagesExtra = with mach-nix.rPackages; [
data_table
];
}This example builds an aarch64 sd image via emulator. For this to work, binfmt support for aarch64 must be installed first. (On NixOS simply set boot.binfmt.emulatedSystems = [ "aarch64-linux" ])
For the SD-image, create a configuration.nix file which adds the mach-nix tool and some default python packages to the system environment.
configuration.nix:
{ config, lib, pkgs, ... }:
let
machNix = import (builtins.fetchGit {
url = "https://github.com/DavHau/mach-nix/";
ref = "refs/tags/put_version_here";
}) { inherit pkgs; };
defaultPythonEnv = machNix.mkPython {
requirements = ''
cryptography
'';
providers.cffi = "nixpkgs";
};
in {
imports = [
<nixpkgs/nixos/modules/installer/cd-dvd/sd-image-aarch64.nix>
];
environment.systemPackages = [ defaultPythonEnv machNix.mach-nix ];
sdImage.compressImage = false; # speeds up the build
}with the following default.nix:
with import <nixpkgs/nixos> {
system = "aarch64-linux";
};
config.system.build.sdImageExecute:
NIXOS_CONFIG=$PWD/configuration.nix nix build -f default.nixOr to select a specific channel:
NIXOS_CONFIG=$PWD/configuration.nix nix build -f default.nix -I nixpkgs=channel:nixos-20.03