Docker and co (draft)

.github/workflows/ci-linux.yml

@@ -30,7 +30,7 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip nox
+        python -m pip install --upgrade pip nox pdm
 
     - name: Run tests
       run: |

.gitlab-ci.yml

@@ -1,8 +1,89 @@
+stages:
+  - pixi
+  - image
+  - lint
+  - test
+  - report
+  - test-release
+  - doc
+
 variables:
   CODECOV_TOKEN: 4ebf45ff-904a-4a74-aa03-d9fba26cc145
+  COVERAGE_DIR: .coverage_$CI_COMMIT_SHA
+  DOCKER_IMAGE_PATH: registry.heptapod.net:443/fluiddyn/fluidfft/ci/default
+
+image: $DOCKER_IMAGE_PATH:stable
+
+# ugly workaround https://gitlab.com/gitlab-org/gitlab/-/issues/370052#note_1207556577
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
+      when: never
+    - if: $CI_COMMIT_BRANCH
+
+
+# pixi-test:
+#   stage: pixi
+#   image: $DOCKER_IMAGE_PATH:pixi
+#   script:
+#     - pixi info
+#     # - rm -rf ../transonic
+#     # - hg clone https://foss.heptapod.net/fluiddyn/transonic ../transonic
+#     - pixi run install-editable
+#     - pixi run pytest -v
+
+
+# Build an image for the other tasks; this should be a scheduled job, as
+# it is quite unnecessary to run on every invocation.
+image:build:
+  stage: image
+  tags:
+    - container-registry-push
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+    - if: '$CI_BUILD_IMAGES == "1"'
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [ "" ]
+  before_script:
+    - ""
+  script:
+    - |
+      cat > /kaniko/.docker/config.json <<EOF
+      {
+        "auths": {
+          "$CI_REGISTRY": {
+            "username": "$CI_REGISTRY_USER",
+            "password": "$CI_REGISTRY_PASSWORD"
+          }
+        }
+      }
+      EOF
+    - >
+      /kaniko/executor --context $CI_PROJECT_DIR
+      --dockerfile $CI_PROJECT_DIR/docker/Dockerfile
+      --single-snapshot
+      --cleanup
+      --destination $DOCKER_IMAGE_PATH:stable
+
+
+validate_code:
+  stage: lint
+  needs:
+    - job: "image:build"
+      optional: true
+  script:
+    - echo "CI_COMMIT_HG_BRANCH $CI_COMMIT_HG_BRANCH"
+    - echo "CI_COMMIT_BRANCH $CI_COMMIT_BRANCH"
+    - nox -s validate_code
+
 
 tests:
-  image: fluiddyn/python3-stable:lastest
+  stage: test
+  needs:
+    - job: "image:build"
+      optional: true
   script:
     - cp site.cfg.default site.cfg
     - python -m venv venv
@@ -12,9 +93,37 @@ tests:
 
 
 tests_pythran_mpi:
-  image: fluiddyn/python3-stable:lastest
+  stage: test
+  needs:
+    - job: "image:build"
+      optional: true
   script:
     - python -m venv venv
     - source venv/bin/activate
     - pip install -U pip nox
     - nox -s tests-full
+
+
+pages:
+  stage: doc
+  needs:
+    - job: "image:build"
+      optional: true
+  variables:
+    FLUIDFFT_TRANSONIC_BACKEND: "python"
+  script:
+    - nox -s doc
+    - mkdir -p public/$CI_COMMIT_REF_NAME
+    - rsync -rvc --delete doc/_build/html/* public/$CI_COMMIT_REF_NAME/
+    # This directory can become too large leading to error.
+    # It can be purged with the botton "Clear runner caches"
+    # in https://foss.heptapod.net/fluiddyn/fluidsim/-/pipelines
+    - ls public
+    - echo "CI_COMMIT_REF_NAME="$CI_COMMIT_REF_NAME
+    - echo See https://fluiddyn.pages.heptapod.net/fluidsim/$CI_COMMIT_REF_NAME
+  artifacts:
+    name: "$CI_COMMIT_REF_NAME"
+    paths:
+      - public
+    expire_in: 5 days
+  when: always

.hgignore

@@ -52,6 +52,7 @@ site.cfg
 .coverage
 .tox
 .nox
+.venv
 .cache
 .mypy_cache
 **/__pythran__/*

Makefile

@@ -1,7 +1,10 @@
 .PHONY: clean cleanall cleanmako cleancython develop build_ext_inplace list-sessions requirements
 
-develop:
-	pip install -v -e .[dev] | grep -v link
+develop: sync
+	pdm run pip install -e . --no-deps --no-build-isolation -v
+
+sync:
+	pdm sync --clean --no-self
 
 clean:
 	rm -rf build
@@ -21,7 +24,7 @@ cleanmako:
 cleanall: clean cleanso cleanmako cleancython cleanpythran
 
 black:
-	black -l 82 src *.py
+	pdm run black
 
 tests:
 	pytest -s src
@@ -56,9 +59,10 @@ list-sessions:
 	@nox --version 2>/dev/null || pip install nox
 	@nox -l
 
-requirements: 'pip-compile(main)' 'pip-compile(doc)' 'pip-compile(test)' 'pip-compile(dev)'
+lock:
+	pdm lock -G :all
 
 # Catch-all target: route all unknown targets to nox sessions
 %:
 	@nox --version 2>/dev/null || pip install nox
-	@nox -s $@
+	@nox -s $@

doc/examples/compute_grad.py

@@ -4,7 +4,7 @@
 nx = ny = 100
 lx = ly = 2 * np.pi
 
-oper = OperatorsPseudoSpectral2D(nx, ny, lx, ly, fft='fft2d.with_fftw2d')
+oper = OperatorsPseudoSpectral2D(nx, ny, lx, ly, fft="fft2d.with_fftw2d")
 
 u = np.sin(oper.XX + oper.YY)
 u_fft = oper.fft(u)

docker/Dockerfile

@@ -0,0 +1,65 @@
+FROM python:3.9
+LABEL Pierre Augier <pierre.augier@univ-grenoble-alpes.fr>
+
+RUN apt-get update
+
+RUN apt-get install -y --no-install-recommends \
+  libfftw3-dev libfftw3-mpi-dev \
+  openmpi-bin libhdf5-openmpi-dev libopenblas-dev
+RUN apt-get install -y --no-install-recommends \
+  gfortran clang
+# also include utilities for debugging
+RUN apt-get install -y --no-install-recommends \
+  emacs vim \
+  meld less \
+  # python3-pip is for Mercurial (hg-git, hg-evolve)
+  mercurial python3-pip git
+RUN apt-get install -y --no-install-recommends \
+  rsync \
+  libgl1 xvfb xauth libgl1-mesa-dev \
+  libxkbcommon-x11-0 \
+  swig sudo graphviz
+
+RUN rm -rf /var/lib/apt/lists/*
+
+RUN groupadd -g 1000 appuser && useradd -m -r -u 1000 -g appuser -s /bin/bash appuser -s /bin/bash && usermod -a -G sudo appuser
+RUN echo appuser:appuser | chpasswd
+USER appuser
+ARG HOME=/home/appuser
+RUN mkdir -p $HOME/opt
+WORKDIR $HOME/opt
+RUN echo $USER $HOME $PWD && whoami
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN $(hg debuginstall -T '{pythonexe}') -m pip install hg-evolve hg-git --no-cache-dir --user
+
+COPY --chown=appuser:appuser docker/hgrc $HOME/.hgrc
+COPY --chown=appuser:appuser docker/pythranrc $HOME/.pythranrc
+
+RUN mkdir -p $HOME/.local/include
+RUN mkdir -p $HOME/.local/lib
+RUN ln -s /usr/include/fftw* $HOME/.local/include
+RUN ln -s /usr/lib/x86_64-linux-gnu/libfftw3* $HOME/.local/lib
+
+ENV LD_LIBRARY_PATH=$HOME/.local/lib
+ENV PATH=$HOME/.local/bin:$PATH
+ENV CPATH=$HOME/.local/include:$CPATH
+
+RUN mkdir -p $HOME/.config/matplotlib
+RUN echo 'backend      : agg' > $HOME/.config/matplotlib/matplotlibrc
+
+RUN wget https://foss.heptapod.net/fluiddyn/fluidfft/raw/branch/default/site.cfg.files/site.cfg.docker -O ~/.fluidfft-site.cfg
+
+RUN wget https://foss.heptapod.net/fluiddyn/fluidfft/raw/branch/default/doc/install/install_p3dfft.sh -O ./install_p3dfft.sh
+RUN chmod +x install_p3dfft.sh
+RUN export FCFLAGS="-w -fallow-argument-mismatch -O2" && \
+    export FFLAGS="-w -fallow-argument-mismatch -O2" && \
+    ./install_p3dfft.sh
+
+RUN wget https://foss.heptapod.net/fluiddyn/fluidfft/raw/branch/default/doc/install/install_pfft.sh -O ./install_pfft.sh
+RUN chmod +x install_pfft.sh
+RUN ./install_pfft.sh
+
+RUN python -m pip install --upgrade pip
+RUN python -m pip install --upgrade pdm nox

docker/Makefile

@@ -0,0 +1,49 @@
+image := image-fluidfft
+tag := $(shell date -I'date'| tr -d "[:punct:]")
+
+define STR_HELP
+This makefile can be used to build images and start containers.
+
+Examples
+--------
+$ make start  # starts Docker service
+$ make pull   # pulls a Docker image fron Docker hub
+$ make build  # by default builds python3-stable image
+$ make run
+$ make build image=python3-stable
+$ make cleanall  # clean (unnecessary, not all) containers and images
+
+endef
+
+export STR_HELP
+
+.PHONY: help
+
+help:
+	@echo "$$STR_HELP"
+
+start:
+	systemctl start docker
+
+build:
+	docker build -f Dockerfile -t fluiddyn/$(image) ..
+	docker tag fluiddyn/$(image) fluiddyn/$(image):$(tag)
+
+list:
+	@printf "\nImages: "
+	docker images
+	@printf "\nContainers: "
+	docker ps
+
+cleancontainers:
+	@echo "Clean exited containers."
+	for cont in $$(docker ps -a | awk 'NR>1{print $$1}'); do docker stop $$cont; docker rm $$cont; done
+
+cleanimages:
+	@echo "Clean dangling images with no tag."
+	for img in $$(docker images -qf "dangling=true"); do docker rmi -f $$img; done
+
+cleanall: cleancontainers cleanimages cleanmako
+
+run:
+	docker run --name $(image) --restart always -it fluiddyn/$(image) bash

docker/hgrc

@@ -0,0 +1,34 @@
+# example user config (see 'hg help config' for more info)
+[ui]
+# name and email, e.g.
+# username = Jane Doe <jdoe@example.com>
+username = appuser
+editor = emacs -nw -Q
+
+# We recommend enabling tweakdefaults to get slight improvements to
+# the UI over time. Make sure to set HGPLAIN in the environment when
+# writing scripts!
+tweakdefaults = True
+
+[extensions]
+# uncomment these lines to enable some popular extensions
+# (see 'hg help extensions' for more info)
+#
+churn =
+shelve =
+hgext.extdiff =
+rebase =
+absorb =
+evolve =
+topic =
+hggit =
+
+[extdiff]
+cmd.meld =
+
+[trusted]
+users = root
+
+[alias]
+lg = log -G
+up = up -v

docker/pythranrc

@@ -0,0 +1,7 @@
+[pythran]
+complex_hook = True
+
+[compiler]
+CXX=clang++
+CC=clang
+blas=openblas