Merge pull request #31 from ftorradeflot/feature_test_env

Feature test environment
rucio · Feb 29, 2024 · dc1395d · dc1395d
2 parents 6421814 + f4e0a00
commit dc1395d
Show file tree

Hide file tree

Showing 25 changed files with 507 additions and 160 deletions.
diff --git a/.github/actions/build-ext/action.yml b/.github/actions/build-ext/action.yml
@@ -0,0 +1,10 @@
+name: "Build"
+runs:
+  using: "composite"
+  steps:
+    - name: Build the extension
+      run: |
+        set -eux
+        python -m pip install .
+      shell: bash
+
diff --git a/.github/actions/post-test/action.yml b/.github/actions/post-test/action.yml
@@ -0,0 +1,25 @@
+name: "Post Test"
+runs:
+  using: "composite"
+  steps:
+    - name: Install check-manifest
+      run: python -m pip install -U check-manifest
+      shell: bash
+    - name: Install Node (for Playwright)
+      uses: actions/setup-node@v1
+      with:
+        node-version: '16.x'
+    - name: Install Playwright
+      run: npx playwright install
+      shell: bash
+    - name: extension checks
+      run: |
+        jupyter server extension list 2>&1 | grep -ie "rucio_jupyterlab.*OK"
+        jupyter labextension list 2>&1 | grep -ie "rucio-jupyterlab.*OK"
+      shell: bash
+    - name: Browser check
+      run: python -m jupyterlab.browser_check
+      shell: bash
+    - name: Check Manifest.in
+      run: check-manifest -v
+      shell: bash
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
@@ -0,0 +1,40 @@
+name: "Setup"
+runs:
+  using: "composite"
+  steps:
+    - name: Install node
+      uses: actions/setup-node@v1
+      with:
+        node-version: '14.x'
+    - name: Install Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+        architecture: 'x64'
+    - name: Setup pip cache
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: pip-3.9-${{ hashFiles('package.json') }}
+        restore-keys: |
+          pip-3.9-
+          pip-
+    - name: Get yarn cache directory path
+      id: yarn-cache-dir-path
+      run: echo "::set-output name=dir::$(yarn cache dir)"
+      shell: bash
+    - name: Setup yarn cache
+      uses: actions/cache@v2
+      id: yarn-cache
+      with:
+        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+        key: yarn-${{ hashFiles('**/yarn.lock') }}
+        restore-keys: |
+          yarn-
+    - name: Install Python dependencies
+      run: python -m pip install -r requirements.txt
+      shell: bash
+    - name: Install JS dependencies
+      run: jlpm
+      shell: bash
+
diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml
@@ -0,0 +1,16 @@
+name: "Test"
+runs:
+  using: "composite"
+  steps:
+    - name: Run ESLint
+      run: jlpm run eslint:check
+      shell: bash
+    - name: Run Jest
+      run: jlpm jest
+      shell: bash
+    - name: Run Pytest
+      run: pytest rucio_jupyterlab/tests/
+      shell: bash
+    - name: Run Pylint
+      run: pylint --fail-under=8.5 rucio_jupyterlab/
+      shell: bash
diff --git a/.github/workflows/build-and-publish-tagged.yml b/.github/workflows/build-and-publish-tagged.yml
@@ -9,61 +9,13 @@ jobs:
     name: Build and Publish to PyPI
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-      - name: Install node
-        uses: actions/setup-node@v1
-        with:
-          node-version: '14.x'
-      - name: Install Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.8'
-          architecture: 'x64'
-      - name: Setup pip cache
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: pip-3.8-${{ hashFiles('package.json') }}
-          restore-keys: |
-            pip-3.8-
-            pip-
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-      - name: Setup yarn cache
-        uses: actions/cache@v2
-        id: yarn-cache
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            yarn-
-      - name: Install Python dependencies
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup
+      - uses: ./.github/actions/test
+      - uses: ./.github/actions/build-ext
+      - uses: ./.github/actions/post-test
+      - name: Build sdist
         run: |
-          python -m pip install -U jupyterlab~=3.1 check-manifest
-          python -m pip install -r requirements.txt
-      - name: Install JS dependencies
-        run: jlpm
-      - name: Run ESLint
-        run: jlpm run eslint:check
-      - name: Run Jest
-        run: jlpm jest
-      - name: Run Pytest
-        run: pytest rucio_jupyterlab/tests/
-      - name: Run Pylint
-        run: pylint --fail-under=8.5 rucio_jupyterlab/
-      - name: Build the extension
-        run: |
-          set -eux
-          python -m pip install .
-          jupyter server extension list 2>&1 | grep -ie "rucio_jupyterlab.*OK"
-          jupyter labextension list 2>&1 | grep -ie "rucio-jupyterlab.*OK"
-          python -m pip uninstall jupyter_server_terminals
-          python -m jupyterlab.browser_check
-
-          check-manifest -v
-
           pip install build
           python -m build --sdist
       - name: Publish distribution to PyPI

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -11,62 +11,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-      - name: Install node
-        uses: actions/setup-node@v1
-        with:
-          node-version: '14.x'
-      - name: Install Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.8'
-          architecture: 'x64'
-      - name: Setup pip cache
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: pip-3.8-${{ hashFiles('package.json') }}
-          restore-keys: |
-            pip-3.8-
-            pip-
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-      - name: Setup yarn cache
-        uses: actions/cache@v2
-        id: yarn-cache
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            yarn-
-      - name: Install Python dependencies
-        run: |
-          python -m pip install -U jupyterlab~=3.1 check-manifest
-          python -m pip install -r requirements.txt
-      - name: Install JS dependencies
-        run: jlpm
-      - name: Run ESLint
-        run: jlpm run eslint:check
-      - name: Run Jest
-        run: jlpm jest
-      - name: Run Pytest
-        run: pytest rucio_jupyterlab/tests/
-      - name: Run Pylint
-        run: pylint --fail-under=8.5 rucio_jupyterlab/
-      - name: Install Node (for Playwright)
-        uses: actions/setup-node@v1
-        with:
-          node-version: '16.x'
-      - name: Install Playwright
-        run: npx playwright install 
-      - name: Build the extension
-        run: |
-          set -eux
-          python -m pip install .
-          jupyter server extension list 2>&1 | grep -ie "rucio_jupyterlab.*OK"
-          jupyter labextension list 2>&1 | grep -ie "rucio-jupyterlab.*OK"
-          python -m jupyterlab.browser_check
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup
+      - uses: ./.github/actions/test
+      - uses: ./.github/actions/build-ext
+      - uses: ./.github/actions/post-test
 
-          check-manifest -v
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "rucio"]
+	path = docker/test_env/rucio
+	url = git@github.com:rucio/rucio.git
+[submodule "docker/test_env/rucio"]
+	path = docker/test_env/rucio
+	url = https://github.com/rucio/rucio.git
diff --git a/Dockerfile b/Dockerfile
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -27,7 +27,7 @@ global-exclude .git
 global-exclude .ipynb_checkpoints
 
 exclude Dockerfile
-exclude docker/*
+recursive-exclude docker *
 exclude jest.config.js
 exclude requirements.txt
 exclude rucio_jupyterlab/tests/*

diff --git a/docker/configure.sh b/docker/configure.sh
diff --git a/docker/container/Dockerfile b/docker/container/Dockerfile
@@ -0,0 +1,40 @@
+FROM jupyter/docker-stacks-foundation:python-3.9
+LABEL maintainer="Muhammad Aditya Hilmy <mhilmy@hey.com>, Francesc Torradeflot <torradeflot@pic.es>"
+
+ENV CONTAINER_PURPOSE="test"
+ARG NB_USER="jovyan"
+ARG NB_UID="1000"
+ARG NB_GID="100"
+
+USER $NB_UID
+
+RUN mamba install -y -n base -c conda-forge \
+		python-gfal2 \
+		nodejs=14 \
+		pip \
+    && mamba clean --all -f -y 
+
+USER root
+
+COPY . /rucio-jupyterlab
+WORKDIR /rucio-jupyterlab
+
+RUN fix-permissions /rucio-jupyterlab \
+    && sed -i -e 's/\r$/\n/' /rucio-jupyterlab/docker/container/configure.sh
+
+COPY docker/container/jupyter_server_config.py $HOME/.jupyter/jupyter_server_config.py
+RUN fix-permissions $HOME/.jupyter
+RUN mkdir -p /opt/rucio/etc \
+	&& chown -R ${NB_UID}:${NB_GID} /opt/rucio/etc
+
+USER $NB_UID
+
+RUN /rucio-jupyterlab/docker/container/install_extension.sh
+
+ENV JUPYTER_ENABLE_LAB=yes
+
+WORKDIR $HOME
+ENTRYPOINT ["/rucio-jupyterlab/docker/container/docker-entrypoint.sh"]
+CMD ["/rucio-jupyterlab/docker/container/configure.sh", "/rucio-jupyterlab/docker/container/start_notebook.sh"]
+
+USER ${NB_UID}
diff --git a/docker/container/README.md b/docker/container/README.md
@@ -0,0 +1,43 @@
+# Rucio jupyterlab extension docker image
+
+## Building the image
+
+### Base image: jupyter/docker-stacks-foundation
+
+We rely on the `docker-stacks-foundation` image from the
+[Jupyter Docker Stacks repository](https://github.com/jupyter/docker-stacks/tree/main).
+
+It is an image containing a minimal conda/mamba environment, we want to use this image to make
+sure the extension works in a "clean" environment.
+
+### Building a custom base image
+
+In our case we want to set the python version to 3.9, and there's no image publicly available with this version.
+
+So we clone the [Jupyter Docker Stacks repository](https://github.com/jupyter/docker-stacks/tree/main)
+
+	git clone git@github.com:jupyter/docker-stacks.git jupyter-docker-stacks
+	cd jupyter-docker-stacks/images/docker-stacks-foundation/
+
+Tune the Dockerfile to install python=3.9, and build/tag the image that will be used as the basis
+for our `rucio-jupyterlab` image.
+
+	docker build . -t jupyter/docker-stacks-foundation:python-3.9
+
+### Building the `rucio-jupyterlab` docker image
+
+From the root folder of the project, run
+
+	docker build . -t rucio-jupyterlab -f docker/container/Dockerfile
+
+## Using the image
+
+### Standalone
+
+Check the instructions in [the main README](../../README.md) to see how to use the image
+standalone, that is without the rest of the Rucio development environment
+
+### Inside Rucio's development environment
+
+Check the instructions on [how to use the test environment](../test_env/README.md) to 
+see how to use this image together with the Rucio development environment.