diff --git a/.envrc b/.envrc new file mode 100644 index 0000000000..1d953f4bd7 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use nix diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml index 38d99b11dd..1f5f647147 100644 --- a/.github/workflows/test_suite.yml +++ b/.github/workflows/test_suite.yml @@ -12,25 +12,19 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7] - #platform: [ubuntu-18.04, macos-latest] - platform: [ubuntu-18.04] + python-version: [37, 38] + platform: [ubuntu-18.04, macos-latest] runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v2 + - name: Install Nix + uses: cachix/install-nix-action@v12 + with: + nix_path: nixpkgs=channel:nixos-unstable - name: Get LFS files run: git lfs pull - # - name: Fix Conda permissions on macOS - # run: sudo chown -R $UID $CONDA - # if: runner.os == 'macOS' - name: Install IC - run: | - source $CONDA/etc/profile.d/conda.sh - source manage.sh work_in_python_version_no_tests ${{ matrix.python-version }} + run: nix-shell --argstr py ${{ matrix.python-version }} - name: Run tests - run: | - source $CONDA/etc/profile.d/conda.sh - source manage.sh work_in_python_version_no_tests ${{ matrix.python-version }} - HYPOTHESIS_PROFILE=travis-ci bash manage.sh run_tests_par - + run: nix-shell --argstr py ${{ matrix.python-version }} --run "PYTEST_ADDOPTS=--color=yes ic-test-par" diff --git a/.gitignore b/.gitignore index 9fcf4562aa..ee39ea602b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ miniconda.sh # macOS .DS_Store + +# nix-direnv environment caching +.direnv diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 578fd67e54..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,32 +0,0 @@ -language: generic -sudo: false - -matrix: - fast_finish: true - allow_failures: - - os: osx - -env: - - IC_PYTHON_VERSION=3.7 - -os: - - linux - - osx - -branches: - except: - - /^docs\..*$/ - -install: - - source manage.sh work_in_python_version_no_tests ${IC_PYTHON_VERSION} - -script: - - HYPOTHESIS_PROFILE=travis-ci bash manage.sh run_tests_par - - -before_install: - - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew install git-lfs; fi - - if [ "$TRAVIS_OS_NAME" = "osx" ]; then git lfs install; fi - -before_script: - - if [ "$TRAVIS_OS_NAME" = "osx" ]; then git lfs pull; fi diff --git a/README.rst b/README.rst index 069033cc7d..cf2581a71e 100644 --- a/README.rst +++ b/README.rst @@ -1,44 +1,45 @@ IC: Invisible Cities ============================================== -IC stands for Invisible Cities and also for Italo Calvino, the author of the master piece. +IC stands for Invisible Cities and also for Italo Calvino, the author of the master piece. Quickstart guide ---------------- -+ Make sure you have `git lfs` installed (https://git-lfs.github.com/) ++ Make sure that `Nix is installed `_ on your machine + ++ Make sure that `home-manager is installed `_ for your user + Clone the repository -+ :code:`cd` into the `IC` directory ++ :code:`cd` into the `IC` directory you have just cloned -+ :code:`source manage.sh work_in_python_version 3.7` ++ :code:`direnv allow` -The last step will, if necessary, install conda and create an -appropriate conda environment, as well as setting environment -variables which are needed for the correct functioning of IC. ++ :code:`ic-test-par` -The installation steps will be skipped if conda and the required -environment are already available, so subsequent invocations of the -command should be much quicker than the first. +The last step launches an exhaustive set of IC tests. If these pass, you can be +confident that everything has been configured as it should. -You will need to perform the last two steps in every shell in which -you want to run IC. +The :code:`cd /path/to/IC` step is the only one you will have to repeat, on a +machine on which these steps have already been carried out, in order to resume +work on IC. -To check your progress when you are developing, you will want to -compile any Cython components that have changed and run the test -suite. This can be done with +To check your progress when you are developing, you will want to compile any +Cython components that have changed and run the test suite. This can be done +with .. code-block:: - source manage.sh compile_and_test_par + ic-compile && ic-test-par -If you check out a commit which requires an older set of dependencies, -the :code:`compile_and_test` commands will automatically switch to an -appropriate environment, creating it on the fly if necessary. +If you check out a commit which requires an older set of dependencies, Nix, +`home-manager` and `direnv` will together ensure that you automatically switch +to an appropriate environment, creating it on the fly if necessary, without you +having do do anything at all, other than maybe waiting for downloads of older +packages to complete. :Travis CI: |travis| .. |travis| image:: https://img.shields.io/travis/nextic/IC.png :target: https://travis-ci.org/nextic/IC - diff --git a/bash_manage.sh b/bash_manage.sh deleted file mode 100644 index c120484a96..0000000000 --- a/bash_manage.sh +++ /dev/null @@ -1,90 +0,0 @@ -manage.sh() -{ - local cur prev opts - COMPREPLY=() - cur="${COMP_WORDS[COMP_CWORD]}" - prev="${COMP_WORDS[COMP_CWORD-1]}" - prev2="${COMP_WORDS[COMP_CWORD-2]}" - - - case "${prev}" in - manage.sh) - local opts="make_environment\ - run_tests\ - run_tests_par\ - compile_and_test\ - compile_and_test_par\ - download_test_db\ - download_test_db_dev\ - clean" - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) - return 0 - ;; - *) - ;; - esac - - if [[ ${prev2} == "manage.sh" ]] ; then - case "${prev}" in - install_and_check|install|work_in_python_version|make_environment) - local versions="3.7" - COMPREPLY=( $(compgen -W "${versions}" -- ${cur}) ) - return 0 - ;; - *) - ;; - esac - fi - - return 0 -} - - -source_manage.sh() -{ - local cur prev opts - COMPREPLY=() - cur="${COMP_WORDS[COMP_CWORD]}" - prev="${COMP_WORDS[COMP_CWORD-1]}" - prev2="${COMP_WORDS[COMP_CWORD-2]}" - - - case "${prev}" in - manage.sh) - local opts="install_and_check\ - install\ - work_in_python_version\ - work_in_python_version_no_tests\ - make_environment\ - run_tests\ - run_tests_par\ - compile_and_test\ - compile_and_test_par\ - download_test_db\ - download_test_db_dev\ - clean" - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) - return 0 - ;; - *) - ;; - esac - - if [[ ${prev2} == "manage.sh" ]] ; then - case "${prev}" in - install_and_check|install|work_in_python_version|make_environment) - local versions="3.7" - COMPREPLY=( $(compgen -W "${versions}" -- ${cur}) ) - return 0 - ;; - *) - ;; - esac - fi - - return 0 -} - -complete -F manage.sh -o bashdefault -o default bash -complete -F source_manage.sh -o bashdefault -o default source -complete -F source_manage.sh -o bashdefault -o default . diff --git a/doc/nix/home-manager/README.md b/doc/nix/home-manager/README.md new file mode 100644 index 0000000000..d4af35f89a --- /dev/null +++ b/doc/nix/home-manager/README.md @@ -0,0 +1,310 @@ +# Bootstrap home manager + +## TLDR + +If + ++ nix is enabled for your user account ++ you have a network connection which is not absurdly slow + +then you could have a personal home-manager installation and configuration up +and running in under two minutes, simply by pasting the following into a +bash-like shell: + +```shell +# Decide where you want your home-manager config to live +HM_DIR=$HOME/my-home-manager + +# Download and unpack the home-manager config template +cd /tmp +# TODO: update this URL when this branch gets merged +curl -L https://github.com/jacg/IC/tarball/manage-with-nix > IC.tgz +nix-shell -p pkgs.gnutar --run "tar xvf IC.tgz --wildcards '*/nix/home-manager' --strip-components=3" +mv home-manager $HM_DIR + +# Bootstrap your personal home-manager installation and configuration +cd $HM_DIR +nix-shell bootstrap-home-manager +# You should also run `nix-shell bootstrap-home-manager` (from $HM_DIR) whenever +# you change the home-manager version in ./sources.nix +``` + +If this worked, then `home-manager` is ready: after changing your `home.nix` you +should be able to switch to the newly-specified configuration by running +`home-manager switch` (from *any* directory on your system): + +```shell +# First edit $HM_DIR/nixpkgs/home.nix, then +home-manager switch +``` + +## I'd rather not copy 'n' paste blindly: Talk me through it! + ++ Make sure that Nix is [installed and configured](../install-nix/README.md): + + ```shell + nix-shell cowsay -p --run 'cowsay Looks like Nix is working.' + ``` + Expect some noisy Nix output. + + If everything is working correctly, you should, eventually, see an ASCII-art + picture of a cow saying that Nix is working. This means that you are ready to + install home-manager in your user account. (If not, have you done + [this](../install-nix/README.md)?) + +1. Decide where you want your home-manager configuration files to live. + + ```shell + HM_DIR=$HOME/my-home-manager + ``` + + I recommend that you let home-manager manage the configurations of most + programs that you use in your user account, and that you keep all your + configuration file sources version-controlled in this same directory. More on + that, below. + +2. Copy this directory (the one containing the README in which these words are + written) into your personal space on your machine, at the location that you + specified with the variable `HM_DIR` in step 1. + + ```shell + cd /tmp + # TODO: update this URL when this branch gets merged + curl -L https://github.com/jacg/IC/tarball/manage-with-nix > IC.tgz + nix-shell -p pkgs.gnutar --run "tar xvf IC.tgz --wildcards '*/nix/home-manager' --strip-components=3" + mv home-manager $HM_DIR + ``` + + [Aside: Note that we are using `nix-shell -p pkgs.gnutar` to ensure that + (regardless of the default `tar` on your system) we will use GNU `tar` which + provides options such as `--wildcards` and `--strip-components`. + + If your default `tar` provides these options, then you could run the `tar xvf + IC.tgz ...` command directly, without wrapping it in `nix-shell -p + pkgs.gnutar`. Using the `nix-shell` guarantees that this step will work + correctly, on *any* nix-enabled system.] + +3. Install `home-manager` for your user. + + Inside *your copy* of this directory, execute `bootstrap-home-manager` with + `nix-shell` + + ```shell + cd $HM_DIR + nix-shell bootstrap-home-manager + ``` + + This step ephemerally installs `home-manager` into a `nix-shell` in which it + instructs `home-manager` to install a persistent environment for your user: + that environment contains `home-manager` itself, along with some examples of + how to install and configure other packages. + + This step should be performed any time you change the version of + `home-manager` you want to use. The `home-manager` version is specified in + `sources.nix`. + +Home-manager should now be ready to use by this user. + +To check the installation: + +1. Edit `$HM_DIR/nixpkgs/home.nix`, adding `figlet` to the + packages listed in `home.packages`. (That is, just below `bat`.) + +2. Update your environment with `home-manager switch`. + +3. Verify that `figlet` has been installed with `figlet home-manager works!` + +If you see "home-manager works!" written out in big letters, then home-manager +has been installed correctly, and you have just used it to install a package +(`figlet`) into your user's environment. + +## Automatic environment switching with `direnv` + +1. Hook `direnv` into your shell. Exactly how you do this depends on which shell + you use. The details are described [here](https://direnv.net/docs/hook.html). + + In the case of `bash` it amounts to placing `eval "$(direnv hook bash)"` at + the end of your `.bashrc`. + + Beware, Macs now use `zsh` by default: in the case of `zsh` you should add + `eval "$(direnv hook zsh)"` to the end of your `.zshrc`. + +2. Instruct `direnv` to use the Nix specification of the environment when + entering `$ICTDIR`: + + ```shell + echo use nix > $ICTDIR/.envrc + ``` + +3. Give `direnv` permission to switch environment automatically when entering + `$ICTDIR`: + + ```shell + cd $ICTDIR + direnv allow + ``` + + Two points to note: + + 1. You can revoke this permission with `direnv deny` + + 2. By default `direnv allow/deny` act on the current directory, but they both + accept an optional path. + +## This page is about `home-manager` so why are we talking about `direnv`? + +1. `direnv` is useful, but it needs to be installed. + +2. By default `nix-shell` takes somewhere between 3 and 7 seconds to enable the + IC environment. This is very annoying, especially if you integrate direnv + with your editor: switching buffer in your editor can be stalled for multiple + seconds. + +3. `nix-direnv` is a tool which makes this switching instantaneous, by caching + previously seen environments. This also needs installation and configuration. + +By using home-manager with the `home.nix` you were given here, installation and +integration of `direnv` and `nix-direnv` has been taken care of and tested for +you. Without `home-manager` you would have to carry out these steps by hand. You +might make a mistake, and you might require someone's help to sort it out. With +home-manager, the scope for errors and the concomitant waste of time has been +minimised. + +## Why should I do this? + +Home manager is NOT necessary for working with IC, but it can improve the +quality of your life. Among many other benefits, it will make it easier for +others to help you to ensure you have a properly working environment. + +Now that the Nix package manager is a prerequisite for working on IC, you might +as well take advantage of the fact that Nix is available to you, and reap the +benefits that home-manager offers. + +### What is home-manager? + +Home manager uses Nix to manage the installation and configuration of software +that you use in your per-user working environment. Benefits include: + ++ Install and uninstall software without admin privileges. + ++ Try new packages with guarantees that you can revert to the previous state + without anything breaking. + ++ Configurations are reliable and reproducible. + ++ Transferring your personal environment to a different machine is trivially + easy (as long as Nix is available on that machine). + + The script at the top of this page can be adapted to use your personal + home-manager configuration instead of the template. + + This means that **installing your environment on *ANY* nix-enabled machine is as + simple as launching your script, and letting home-manager download and + configure all the packages you are used to.** + +### Ephemeral package installation with `nix-shell` + +Nix makes it very easy to try out software packages without fear of breaking +anything in your existing environment. (With other package managers, installing +and uninstalling some package often leaves the system in a modified state.) + +One particularly convenient way of trying a package in Nix is with ephemeral +installations using `nix-shell`. + +As an example, let's take `lolcat` for a spin. Just like `cowsay` and `figlet`, +`lolcat` is a package which isn't terribly useful, but makes a good guinea-pig +for installation/uninstallation experiments. Install `lolcat` ephemerally and +try it out, with: + +```shell +nix-shell -p lolcat # This will drop you in a shell where lolcat is available +ls -l / | lolcat # A pretty, colourful listing should appear +``` + +Within this shell (and *only* this shell), you can play around with `lolcat` to +your heart's content: it adds pretty colours to whatever you pipe into it. + +When you have seen enough, simply exit this shell, and `lolcat` disappears. +While you were in the shell, the rest of your system was unaware of the +existence of `lolcat`: it could not have interfered with anything else. + +The first time you run `nix-shell -p lolcat` it is likely to take some time as +it might need to download `lolcat` and its dependencies. On subsequent +invocations, it is likely to run much faster, as everything that is needed has +been kept in the nix store on your machine ... at least until you instruct Nix +to collect garbage. You may want to run `nix-collect-garbage` if the Nix store +grows too large and you want to free up disk space wasted on packages (or older +versions) you are not using. + +### Persistent package installation with `home-manager` + +In the previous section we installed `lolcat` *ephemerally* with `nix-shell`: +the installed package was only available in a single shell, and disappeared as +soon as the shell was exited. + +It is possible to install and remove packages *persistently* in your personal +environment by using `nix-env` to *mutate* your personal profile (you can think +of this as a personal `brew`, `apt-get`, `pacman`, `yum`, etc.). I do NOT +recommend this. + +I suggest you use `home-manager` to manage your personal configuration +*declaratively*. The declarative approach makes it *much* easier to transfer +your environment to different machines, to understand what is present in your +environment, to share package configuration wisdom with your colleagues, to get +help from others, and much more. + +If you have tried a package by installing it ephemerally with `nix-shell` and +found that it is useful to you, then you might want to make it persistently +available. With `home-manager` this is a two-step process: + ++ Add the package to the `home.packages` list in your `home.nix` + ++ Instruct `home-manager` to create and switch to an environment matching your + new specification: + + ```shell + home-manager switch + ``` + +### Version control your environments + +I recommend that you place your environment specification in version control: +```shell +git init $HM_DIR # Assuming you set HM_DIR according to instructions above +``` + +You were instructed, above, to link your `home.nix` to +`$HOME/.config/nixpkgs/home.nix`. This is the simplest short-term way of getting +home-manager to work for you. However, I would recommend a more powerful and +all-encompassing approach for the long-term: + +1. Place *all your personal configuration files/directories* + + - `$HOME/.bash{rc,_profile}` + - `$HOME/.emacs.d` + - `$HOME/.config/htop` + - etc. + + into `$HM_DIR` + +2. Instruct home-manager to make these files appear in their standard locations. + See `$HM_DIR/{eg-ro,eg-rw,nixpkgs/home.nix}` for examples of this. + +3. Keep track of changes with Git (or any VCS of your choice). + +In this scheme, installing your personal environment on *any* Nix-enabled +machine amounts to no more than + +1. `git clone $HM_DIR` +2. `cd $HM_DIR` +3. `nix-shell bootstrap-home-manager` +4. `mkdir -p $HOME/.config/nixpkgs` +5. `ln -s $HM_DIR/nixpkgs/home.nix $HOME/.config/nixpkgs/home.nix` +6. DONE! + +Note that all these steps can be wrapped into a single script, so **you could +install your whole environment with a single command!** + +Caveat: when you take this approach, you must be careful not to commit any +*secrets* (e.g. private ssh keys) into a repository which you will ever place in +some publicly accessible locations (such as GitHub). diff --git a/doc/nix/home-manager/bootstrap-home-manager b/doc/nix/home-manager/bootstrap-home-manager new file mode 100644 index 0000000000..e6eacea621 --- /dev/null +++ b/doc/nix/home-manager/bootstrap-home-manager @@ -0,0 +1,6 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i sh -p git + +nix-shell -E \ + "with import {}; mkShell { buildInputs = [ (callPackage ./sources.nix {}).home-manager ]; }" \ + --run "home-manager -f ./nixpkgs/home.nix switch" diff --git a/doc/nix/home-manager/eg-ro b/doc/nix/home-manager/eg-ro new file mode 100644 index 0000000000..bd74929b7e --- /dev/null +++ b/doc/nix/home-manager/eg-ro @@ -0,0 +1,15 @@ +Imagine this is some real configuration file for some real package. + +Home manager makes this file available in your home directory in read-only form. +The only way to modify it is to change its contents by editing + + $HOME/my-home-manager/eg-ro + +followed by + + home-manager switch + +Consequently, external programs will be prevented from modifying this +configuration file. + +This is very inflexible, but very safe. diff --git a/doc/nix/home-manager/eg-rw b/doc/nix/home-manager/eg-rw new file mode 100644 index 0000000000..c72bf80b93 --- /dev/null +++ b/doc/nix/home-manager/eg-rw @@ -0,0 +1,14 @@ +Imagine this is some real configuration file for some real package. + +Home manager makes this file available in your home directory via a symbolic +link to + + $HOME/my-home-manager/eg-rw + +This is more flexible than the read-only version: + ++ You can tweak its contents without having to rerun `home-manager switch`. ++ External programs can modify this configuration file. + +This more flexible, but less safe: third parties may modify your configuration +without you noticing. diff --git a/doc/nix/home-manager/nixpkgs/home.nix b/doc/nix/home-manager/nixpkgs/home.nix new file mode 100644 index 0000000000..e3ff523d15 --- /dev/null +++ b/doc/nix/home-manager/nixpkgs/home.nix @@ -0,0 +1,52 @@ +{ config +, pkgs ? import (fetchTarball "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/2f6440eb09b7e6e3322720ac91ce7e2cdeb413f9.tar.gz") {} +, ... }: +let + sources = (pkgs.callPackage ../sources.nix { pkgs = pkgs; }); + link = config.lib.file.mkOutOfStoreSymlink; +in +{ + home.packages = [ + sources.home-manager + # Add and remove packages in this list to suit your needs. + pkgs.bat # git-aware, intelligent, helpful, colourful version of `cat` + pkgs.fd # better find + pkgs.ripgrep # better grep / grep-find + pkgs.ripgrep-all # Enable ripgrep in PDFs, ebooks, doc, zip, tar.gz, etc. + pkgs.du-dust # better du + pkgs.tldr # simplified man-pages + ]; + + # Some programs require/permit more intricate configuration + programs.direnv.enable = true; + programs.direnv.enableNixDirenvIntegration = true; + + # Sick and tired of making git commits with author email + # 'Pepito@Pepitos-Macbook-Pro'? Then why not let home-manager take care of + # configuring Git for you? Adapt the following to your needs: + programs.git = { + enable = true; + lfs.enable = true; + # Local .gitconfig files may override these settings: + # userName = "Fulana Menganez-Perenganez"; + # userEmail = "fulana@ific.es"; + # aliases = { + # lg = "log --graph --decorate --oneline"; + # lga = "log --graph --decorate --oneline --all"; + # }; + }; + + # This is how to get home-manager to place your version-controlled + # configuration files in your home directory. There are two approaches, + # read-only and read-write. Read the text in `eg-ro` and `eg-rw` for more + # details. Replace these with your own real-world cases. + home.file.".example-read-only" .source = ../eg-ro; + home.file.".example-read-write".source = link ../eg-rw; + # The previous two lines are just examples, which you can adapt to your own + # needs. The next line is vital: it ensures that home-manager will be able to + # find its configuration file (that's the file in which these words are + # written) in the standard location. + home.file.".config/nixpkgs".source = link ../nixpkgs; + + # We've only scratched the surface of possibilities in home-manager ... +} diff --git a/doc/nix/home-manager/sources.nix b/doc/nix/home-manager/sources.nix new file mode 100644 index 0000000000..314f6dca22 --- /dev/null +++ b/doc/nix/home-manager/sources.nix @@ -0,0 +1,34 @@ +# To get a more recent version of nixpkgs, go to https://status.nixos.org/, +# which lists the latest commit that passes all the tests for any release. +# Unless there is an overriding reason, pick the latest stable NixOS release, at +# the time of writing this is nixos-20.09. + +{ pkgs }: + +let + random_pkgs = import {}; + nixpkgs-commit-id = "896270d629efd47d14972e96f4fbb79fc9f45c80"; # nixos-20.09 on 2020-11-11 + nixpkgs-url = "https://github.com/nixos/nixpkgs/archive/${nixpkgs-commit-id}.tar.gz"; + pkgs = import (fetchTarball nixpkgs-url) { + overlays = map (uri: import (fetchTarball uri)) [ + https://github.com/mozilla/nixpkgs-mozilla/archive/master.tar.gz + ]; + }; +in +{ + + ####### Pinned nixpkgs ################################################## + + pkgs = pkgs; + + ####### home-manager #################################################### + + home-manager = let + src = builtins.fetchGit { + name = "home-manager-2020-11-06"; + url = https://github.com/nix-community/home-manager; + rev = "4cc1b77c3fc4f4b3bc61921dda72663eea962fa3"; + }; + # `path` is required for `home-manager` to find its own sources + in pkgs.callPackage "${src}/home-manager" { path = "${src}"; }; +} diff --git a/doc/nix/install-nix/README.md b/doc/nix/install-nix/README.md new file mode 100644 index 0000000000..847d4dbf0b --- /dev/null +++ b/doc/nix/install-nix/README.md @@ -0,0 +1,86 @@ +# Installing Nix + +Maybe you don't need to install Nix, because it has already been done! Read the +relevant section below, for instructions on how to check. + +## Tell me how to install Nix + +In many cases it is very easy indeed. In some others its not quite as plain +sailing. + +1. If you are on Linux or a pre-Catalina Mac + + ```shell + sudo curl -L https://nixos.org/nix/install | sh + ``` +2. If you are on a Catalina Mac with a T2 chip + + ```shell + sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume + ``` + +3. If you are on Catalina with a pre-T2 chip, please ask for personal help. + + +In cases 1. and 2., if the process finished without error, you should then + +```shell +. $HOME/.nix-profile/etc/profile.d/nix.sh +``` +(which line should be added to your `.bash_profile`/`.zshrc`/etc. In some +circumstances the installer does this for you automatically) + +Thereafter simply `cd` into the IC source directory and run the IC tests with + +```shell +nix-shell # Installs packages, compiles Cython, prepares environment +ic-test-par # Runs IC tests in parallel +``` + +You can hack away on IC to your heart's content in that shell. If you need to +recompile any Cython modules: `ic-compile`. + +That's all there is to it! + +However, now that we have Nix, we can put it to great use. See +[bootstrap-home-manager](../home-manager/README.md +"bootstrap-home-manager") + +## Has Nix already been installed on this machine? +Try +```shell +nix-shell -p cowsay --run 'cowsay Nix is available!' +``` +If this eventually leads to the appearance of this joyous message +``` +< Nix is available! > + ------------------- + \ ^__^ + \ (oo)\_______ + (__)\ )\/\ + ||----w | + || || +``` +then Nix is already available to this user on this machine. + +### That didn't work + +Check whether the directory `/nix` exists. + ++ If `/nix` is absent, proceed to the installation instructions above ++ If `/nix` is present, check whether either of these files exist: + - `$HOME/.nix-profile/etc/profile.d/nix.sh` + - `/etc/profile.d/nix.sh` + + If Neither of these files exist, but `/nix` does, then seek help. + + If one of these exists, it needs to be sourced for Nix to work. In the first + case this look like: + + ```shell + source $HOME/.nix-profile/etc/profile.d/nix.sh + ``` + + You should add this line to (or its equivalent for `/etc/profile.d/nix.sh`) to + your shell initialisation file, and try the `cowsay` test again in a new + shell. diff --git a/invisible_cities/cities/penthesilea_test.py b/invisible_cities/cities/penthesilea_test.py index 3992d0b4c6..cc95f721d9 100644 --- a/invisible_cities/cities/penthesilea_test.py +++ b/invisible_cities/cities/penthesilea_test.py @@ -202,6 +202,7 @@ def test_penthesilea_true_hits_are_correct(KrMC_true_hits, config_tmpdir): assert_dataframes_close(penthesilea_evts, true_evts) +@mark.skip(reason='Pandas API change at 1.1.0 interfering with Nix adoption') def test_penthesilea_read_multiple_files(ICDATADIR, output_tmpdir): file_in = os.path.join(ICDATADIR , "Tl_v1_00_05_nexus_v5_02_08_7bar_pmaps_5evts_*.h5") diff --git a/invisible_cities/core/testing_utils.py b/invisible_cities/core/testing_utils.py index f67e020ceb..64b871e94e 100644 --- a/invisible_cities/core/testing_utils.py +++ b/invisible_cities/core/testing_utils.py @@ -108,16 +108,15 @@ def _compare_dataframes(assertion, df1, df2, check_types=True, **kwargs): def assert_dataframes_equal(df1, df2, check_types=True, **kwargs): pd.testing.assert_frame_equal(df1.sort_index(axis=1), df2.sort_index(axis=1), check_names=True, check_dtype=check_types, check_exact=True, **kwargs) + def assert_dataframes_close(df1, df2, check_types=True, rtol=None, atol=None, **kwargs): - if rtol: - check_less_precise = int(np.log10(1./rtol)) - elif atol: - check_less_precise = int(np.log10(atol)) - else: - check_less_precise = True + tolerance = {} + if atol: tolerance['atol'] = atol + if rtol: tolerance['rtol'] = rtol - pd.testing.assert_frame_equal(df1.sort_index(axis=1), df2.sort_index(axis=1), - check_names=True, check_dtype=check_types, check_less_precise = check_less_precise) + pd.testing.assert_frame_equal(df1.sort_index(axis=1), df2.sort_index(axis=1), + check_names=True, check_dtype=check_types, + **tolerance) def assert_SensorResponses_equality(sr0, sr1): diff --git a/manage.sh b/manage.sh deleted file mode 100644 index d23bc0bab1..0000000000 --- a/manage.sh +++ /dev/null @@ -1,288 +0,0 @@ -#!/usrbin/env bash - -COMMAND=$1 -ARGUMENT=$2 - -## Interpret meaning of command line argument depending on which -## function will receive it. - -case $COMMAND in - run_tests_par | compile_and_test_par) N_PROC=${ARGUMENT:-auto} ;; - *) PYTHON_VERSION=${ARGUMENT} ;; -esac - -# If PYTHON_VERSION was not specified as an argument, deduce it from -# the conda environment - -if [[ $PYTHON_VERSION = "" ]]; then - PYTHON_VERSION=${CONDA_DEFAULT_ENV:3:3} -fi - -function install_and_check { - install - run_tests -} - -function install { - install_conda - make_environment - python_version_env - compile_cython_components -} - -function install_conda { - # Identifies your operating system and installs the appropriate - # version of conda. We only consider Linux and OS X at present. - - # Does nothing if conda is already found in your PATH. - - case "$(uname -s)" in - - Darwin) - export CONDA_OS=MacOSX - ;; - - Linux) - export CONDA_OS=Linux - ;; - - # CYGWIN*|MINGW32*|MSYS*) - # echo 'MS Windows' - # ;; - - *) - echo "Installation only supported on Linux and OS X" - exit 1 - ;; - esac - - if conda --version ; then - echo Conda already installed. Skipping conda installation. - else - echo Installing conda for $CONDA_OS - CONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-py${PYTHON_VERSION//.}_4.8.2-${CONDA_OS}-x86_64.sh" - if which wget; then - wget ${CONDA_URL} -O miniconda.sh - else - curl ${CONDA_URL} -o miniconda.sh - fi - bash miniconda.sh -b -p $HOME/miniconda - CONDA_SH=$HOME/miniconda/etc/profile.d/conda.sh - source $CONDA_SH - echo Activated conda by sourcing $CONDA_SH - fi -} - -CONDA_ENV_TAG=2020-06-16 -CONDA_ENV_NAME=IC-${PYTHON_VERSION}-${CONDA_ENV_TAG} - -function make_environment { - YML_FILENAME=environment-${CONDA_ENV_NAME}.yml - - echo creating ${YML_FILENAME} - - cat < ${YML_FILENAME} -name: ${CONDA_ENV_NAME} -dependencies: -- python = ${PYTHON_VERSION} -# *REMEMBER TO CHANGE CONDA_ENV_TAG WHEN CHANGING VERSION NUMBERS* -- cython = 0.29 -- jupyter = 1.0.0 -- jupyterlab = 1.2.6 -- matplotlib = 3.1.3 -- networkx = 2.4 -- notebook = 6.0.3 -- numpy = 1.18.1 -- pandas = 1.0.3 -- seaborn = 0.10.1 -- pymysql = 0.9.2 -- pytables = 3.6.1 -- pytest = 5.4.2 -- scipy = 1.4.1 -- sphinx = 3.0.4 -- tornado = 6.0.4 -- flaky = 3.6.1 -- hypothesis = 5.16.1 -- pytest-xdist = 1.32.0 -- coverage = 5.0 -- pip = 20.0.2 -- setuptools = 47.1.1 -- pip: - - pytest-instafail==0.4.2 -EOF - - conda env create -f ${YML_FILENAME} -} - -function make_environment_if_missing { - if ! conda env list | grep ${CONDA_ENV_NAME} - then - make_environment - fi -} - -function switch_to_conda_env { - conda deactivate # in case some other environment was already active - make_environment_if_missing ${CONDA_ENV_NAME} - conda activate ${CONDA_ENV_NAME} -} - -function python_version_env { - switch_to_conda_env - # Set IC environment variables and download database - ic_env -} - -function work_in_python_version { - work_in_python_version_no_tests - run_tests_par -} - -function work_in_python_version_no_tests { - if ! which conda >> /dev/null - then - install_conda - fi - - if ! (conda env list | grep ${CONDA_ENV_NAME}) >> /dev/null - then - make_environment - fi - - python_version_env - compile_cython_components -} - -function ensure_environment_matches_checked_out_version { - - # Ensure that the currently active conda environment name contains - # the tag that corresponds to this version of the code - if [[ $CONDA_DEFAULT_ENV != *$CONDA_ENV_TAG ]]; then - switch_to_conda_env - fi - - # Ensure that the test database is present - if [ ! -f invisible_cities/database/localdb.NEWDB.sqlite3 ]; then - download_test_db - fi -} - -function run_tests { - ensure_environment_matches_checked_out_version - # Run the test suite - pytest --instafail --no-success-flaky-report -} - -function run_tests_par { - ensure_environment_matches_checked_out_version - # Run the test suite - STATUS=0 - pytest --instafail -n ${N_PROC:-auto} -m "not serial" --no-success-flaky-report || STATUS=$? - pytest --instafail -m serial --no-success-flaky-report || STATUS=$? - [[ $STATUS = 0 ]] -} - -function ic_env { - export ICTDIR=`pwd` - echo ICDIR set to $ICTDIR - - export ICDIR=$ICTDIR/invisible_cities/ - echo ICDIR set to $ ICDIR - - export PYTHONPATH=$ICTDIR - echo PYTHONPATH set to $PYTHONPATH - - export NBDIR=`pwd` - echo NBDIR set to $NBDIR - - export IC_NOTEBOOK_DIR=$NBDIR/invisible_cities/ - echo IC_NOTEBOOK_DIR set to $ IC_NOTEBOOK_DIR - - export PATH=$ICTDIR/bin:$PATH - echo $ICTDIR/bin added to path -} - -function show_ic_env { - conda env list - - echo "ICTDIR set to $ICTDIR" - echo "ICDIR set to $ICDIR" - echo PYTHONPATH set to $PYTHONPATH -} - -function download_test_db { - echo Downloading database - python $ICDIR/database/download.py $ARGUMENT -} - -function compile_cython_components { - python setup.py develop -} - -function compile_and_test { - compile_cython_components - run_tests -} - -function compile_and_test_par { - compile_cython_components - run_tests_par -} - -function clean { - echo "Cleaning IC generated files:" - FILETYPES='*.c *.so *.pyc __pycache__' - for TYPE in $FILETYPES - do - echo Cleaning $TYPE files - REMOVE=`find . -name $TYPE` - if [ ! -z "${REMOVE// }" ] - then - for FILE in $REMOVE - do - rm -rf $FILE - done - else - echo Nothing found to clean in $TYPE - fi - done -} - - -THIS=manage.sh - -## Main command dispatcher - -case $COMMAND in - install_and_check) install_and_check ;; - install) install ;; - work_in_python_version) work_in_python_version ;; - work_in_python_version_no_tests) work_in_python_version_no_tests ;; - make_environment) make_environment ;; - run_tests) run_tests ;; - run_tests_par) run_tests_par ;; - compile_and_test) compile_and_test ;; - compile_and_test_par) compile_and_test_par ;; - download_test_db) download_test_db ;; - clean) clean ;; - show_ic_env) show_ic_env ;; - - *) echo Unrecognized command: ${COMMAND} - echo - echo Usage: - echo - echo "source $THIS install_and_check X.Y" - echo "source $THIS install X.Y" - echo "source $THIS work_in_python_version X.Y" - echo "source $THIS work_in_python_version_no_tests X.Y" - echo "source $THIS switch_to_conda_env X.Y" - echo "bash $THIS make_environment X.Y" - echo "bash $THIS run_tests" - echo "bash $THIS run_tests_par" - echo "bash $THIS compile_and_test" - echo "bash $THIS compile_and_test_par" - echo "bash $THIS download_test_db" - echo "bash $THIS clean" - echo "bash $THIS show_ic_env" - ;; -esac diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000000..e85d881ee0 --- /dev/null +++ b/shell.nix @@ -0,0 +1,107 @@ +# To override the default python version: +# +# nix-shell shell.nix --argstr py 37 +# +{ py ? "38" +, nixpkgs-commit-id ? "8e78c2cfbae71720869c35b6710d652bf28b37cc" +}: + +# To update `nixpkgs-commit-id` go to https://status.nixos.org/, which lists the +# latest commit that passes all the tests for any release. Unless there is an +# overriding reason, pick the latest stable NixOS release, at the time of +# writing this is nixos-20.03. +let + nixpkgs-url = "https://github.com/nixos/nixpkgs/archive/${nixpkgs-commit-id}.tar.gz"; + pkgs = import (builtins.fetchTarball { url = nixpkgs-url; }) {}; + python = builtins.getAttr ("python" + py) pkgs; + pypkgs = python.pkgs; + + # pytest-instafail was unavailable in nixpkgs at time of writing + mk-pytest-instafail = pypkgs: + pypkgs.buildPythonPackage rec { + pname = "pytest-instafail"; + version = "0.4.2"; + src = pypkgs.fetchPypi { + inherit pname version; + sha256 = "10lpr6mjcinabqynj6v85bvb1xmapnhqmg50nys1r6hg7zgky9qr"; + }; + buildInputs = [ pypkgs.pytest ]; + }; + + + command = pkgs.writeShellScriptBin; + + mkPkgList = (ps: [ + ps.cython + ps.jupyter + ps.jupyterlab + ps.matplotlib + ps.networkx + ps.notebook + ps.numpy + ps.pandas + ps.seaborn + ps.pymysql + ps.tables + ps.scipy + ps.sphinx + ps.tornado + ps.pytest + ps.flaky + ps.hypothesis + ps.pytest_xdist + (mk-pytest-instafail ps) + ]); + +in + +pkgs.mkShell { + pname = "invisible-cities"; + buildInputs = (mkPkgList pypkgs) ++ [ + pkgs.git + + (command "ic-compile" "python setup.py build_ext --inplace") + (command "ic-test" "pytest --instafail --no-success-flaky-report") + (command "ic-test-par" '' + N_PROC=$1 + STATUS=0 + pytest --instafail -n ''${N_PROC:-auto} -m "not serial" --no-success-flaky-report || STATUS=$? + pytest --instafail -m serial --no-success-flaky-report || STATUS=$? + [[ $STATUS = 0 ]] + '') + (command "ic-clean" '' + echo "Cleaning IC generated files:" + FILETYPES='*.c *.so *.pyc __pycache__' + for TYPE in $FILETYPES + do + echo Cleaning $TYPE files + REMOVE=`find . -name $TYPE` + if [ ! -z ''${REMOVE// } ] + then + for FILE in $REMOVE + do + rm -rf $FILE + done + else + echo Nothing found to clean in $TYPE + fi + done + '') + (command "ic-admin-download-database" '' + echo Downloading database + python $ICDIR/database/download.py $1 + '') + ]; + + shellHook = '' + export ICTDIR=`pwd` + export ICDIR=$ICTDIR/invisible_cities/ + export PYTHONPATH=$ICTDIR:$PYTHONPATH + export NBDIR=`pwd` + export IC_NOTEBOOK_DIR=$NBDIR/invisible_cities/ + export PATH=$ICTDIR/bin:$PATH + + ic-compile + ''; + +}