From fd21b6e6c6c0468e699e8c0e87b149434396ff6b Mon Sep 17 00:00:00 2001 From: Leo Thomas Date: Mon, 26 Feb 2024 13:11:10 -0500 Subject: [PATCH 1/9] Add dockerfile and docker-compose file for running a conda env within a docker container --- Dockerfile | 26 ++++++++++++++++++++++++++ docker-compose.yml | 16 ++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..07dfd3a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +# FROM condaforge/miniforge3:latest + +# WORKDIR /model + +# COPY . . + +# RUN mamba install -y -n base --file environment.yml + +# #RUN mamba activate claymodel +# #RUN echo "mamba activate ${ENV_NAME}" >> ~/.bash_profile + +# ENTRYPOINT [ "bash", "-l", "-c" ] + +# CMD ["jupyter-lab", "--ip", "0.0.0.0", "--port", "8888", "--allow-root"] + +FROM mambaorg/micromamba:1.5.6 + +WORKDIR /model + +COPY --chown=$MAMBA_USER:$MAMBA_USER . . + +RUN micromamba create -y -n claymodel --file environment.yml && \ + micromamba clean --all --yes + +ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"] +CMD ["jupyter-lab", "--ip", "0.0.0.0", "--port", "8888", "--allow-root"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5ebceb5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: '3.8' + +services: + claymodel: + build: + context: . + dockerfile: Dockerfile + args: + - platform=linux/amd64 # Specify the target platform for build + platform: linux/amd64 # Specify the target platform for run + ports: + - "8888:8888" # Expose port 8888 on the host and bind it to port 8888 in the container + volumes: + - .:/model # Mount the current directory to /app in the container + environment: + - ENV_NAME=claymodel # Ensure the claymodel conda environment is activated \ No newline at end of file From c62fa329f84437c34b0a6001b23e0f16c16ce3fd Mon Sep 17 00:00:00 2001 From: Leo Thomas Date: Mon, 26 Feb 2024 16:44:43 -0500 Subject: [PATCH 2/9] Remove unused code from Dockerfile --- Dockerfile | 15 --------------- docker-compose.yml | 2 +- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 07dfd3a..fcb3eef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,3 @@ -# FROM condaforge/miniforge3:latest - -# WORKDIR /model - -# COPY . . - -# RUN mamba install -y -n base --file environment.yml - -# #RUN mamba activate claymodel -# #RUN echo "mamba activate ${ENV_NAME}" >> ~/.bash_profile - -# ENTRYPOINT [ "bash", "-l", "-c" ] - -# CMD ["jupyter-lab", "--ip", "0.0.0.0", "--port", "8888", "--allow-root"] - FROM mambaorg/micromamba:1.5.6 WORKDIR /model diff --git a/docker-compose.yml b/docker-compose.yml index 5ebceb5..a9726ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: dockerfile: Dockerfile args: - platform=linux/amd64 # Specify the target platform for build - platform: linux/amd64 # Specify the target platform for run + platform: linux/amd64 # Specify the target platform for run ports: - "8888:8888" # Expose port 8888 on the host and bind it to port 8888 in the container volumes: From a92b264033061e2a57bd9672a1365d7610034ec5 Mon Sep 17 00:00:00 2001 From: Leo Thomas Date: Wed, 28 Feb 2024 15:14:13 -0500 Subject: [PATCH 3/9] Regenerate conda-lock.yml without pytorch-cuda --- .dockerignore | 6 ++++++ README.md | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bb9e77d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +* +!environment.yml +!conda-lock.yml +!**/*.py +!**/*.ipynb +!**/*.sh diff --git a/README.md b/README.md index 1e2d52d..f754e46 100644 --- a/README.md +++ b/README.md @@ -95,3 +95,28 @@ python -m http.server --directory _build/html ``` There is a GitHub Action on `./github/workflows/deploy-docs.yml` that builds the site and pushes it to GitHub Pages. +Additionally the project root will be mounted as volume to the running container so any modifications made to the code base locally will be immediately reflected within the docker container. + +### Custom commands: + +This default command can be overridden with a custom command: +```bash +docker-compose run claymodel {custom command} +``` +or using docker directly: +```bash +docker run --rm -it -v $(pwd):/model -p 8888:8888 -e ENV_NAME=claymodel --platform linux/amd64 claymodel:latest {custom command} +``` + +For example, the `bash` command can be used to access an interactive bash session within the running docker container, with the `micromamba` environment already activated: +(with docker-compose) +```bash +# with docker-compose +docker-compose run claymdel bash +(claymodel) mambauser@f04261284e87:/model$ +``` +```bash +# with straight docker +docker run --rm -it -v $(pwd):/model -p 8888:8888 -e ENV_NAME=claymodel --platform linux/amd64 claymodel:latest bash +(claymodel) mambauser@f04261284e87:/model$ +``` From ad896717c2aee2da243efd07a809283878ff60ac Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Feb 2024 19:57:53 +0000 Subject: [PATCH 4/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Dockerfile | 2 +- README.md | 9 +++++++-- docker-compose.yml | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index fcb3eef..cc4ff09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM mambaorg/micromamba:1.5.6 +FROM mambaorg/micromamba:1.5.6 WORKDIR /model diff --git a/README.md b/README.md index f754e46..d7c5542 100644 --- a/README.md +++ b/README.md @@ -80,16 +80,20 @@ More options can be found using `python trainer.py fit --help`, or at the Our Documentation uses [Jupyter Book](https://jupyterbook.org/intro.html). Install it with: +In order to avoid installing dependencies directly onto the machine, a Dockerized version of the Clay model has been provided. To run it using [docker compose](https://docs.docker.com/compose/) run the following command from the repo root: ```bash pip install -U jupyter-book ``` Then build it with: +Alternatively, you can build and run the Docker image using docker directly with: ```bash jupyter-book build docs/ ``` You can preview the site locally with: +In both cases, the default command will be run, which instantiates a [jupyter-lab](https://jupyterlab.readthedocs.io/en/stable/getting_started/starting.html) instance, from which the documentation notebooks can be run. The jupyter notebook will be accssible on the exposed port (`8888`). +### Note: accessing the jupyter-lab instance in the browser requires an authentication token that can be found in the container logs. Look for the following lines: ```bash python -m http.server --directory _build/html ``` @@ -99,17 +103,18 @@ Additionally the project root will be mounted as volume to the running container ### Custom commands: -This default command can be overridden with a custom command: +This default command can be overridden with a custom command: ```bash docker-compose run claymodel {custom command} ``` -or using docker directly: +or using docker directly: ```bash docker run --rm -it -v $(pwd):/model -p 8888:8888 -e ENV_NAME=claymodel --platform linux/amd64 claymodel:latest {custom command} ``` For example, the `bash` command can be used to access an interactive bash session within the running docker container, with the `micromamba` environment already activated: (with docker-compose) +For example, the `bash` command can be used to access an interactive bash session within the running docker container, with the `micromamba` environment already activated: ```bash # with docker-compose docker-compose run claymdel bash diff --git a/docker-compose.yml b/docker-compose.yml index a9726ce..9ac0201 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,12 +2,12 @@ version: '3.8' services: claymodel: - build: + build: context: . dockerfile: Dockerfile - args: + args: - platform=linux/amd64 # Specify the target platform for build - platform: linux/amd64 # Specify the target platform for run + platform: linux/amd64 # Specify the target platform for run ports: - "8888:8888" # Expose port 8888 on the host and bind it to port 8888 in the container volumes: From 2b1497ea2e3d6d8266945ad685ba5c2c24dce94b Mon Sep 17 00:00:00 2001 From: Leo Thomas Date: Mon, 11 Mar 2024 16:07:47 -0700 Subject: [PATCH 5/9] Regenerate conda-lock.yml --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d7c5542..4dc8744 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ For example, the `bash` command can be used to access an interactive bash sessio For example, the `bash` command can be used to access an interactive bash session within the running docker container, with the `micromamba` environment already activated: ```bash # with docker-compose -docker-compose run claymdel bash +docker-compose run claymodel bash (claymodel) mambauser@f04261284e87:/model$ ``` ```bash From 93d73047628e291b5b8c41652add523943895a17 Mon Sep 17 00:00:00 2001 From: Leo Thomas Date: Mon, 11 Mar 2024 16:08:45 -0700 Subject: [PATCH 6/9] Update readme --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 4dc8744..1b3508b 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,27 @@ Finally, double-check that the libraries have been installed. mamba list +### Advanced + +This is for those who want full reproducibility of the virtual environment. +Create a virtual environment with just Python and conda-lock installed first. + + mamba create --name claymodel python=3.11 conda-lock=2.5.1 + mamba activate claymodel + +Generate a unified [`conda-lock.yml`](https://github.com/conda/conda-lock) file +based on the dependency specification in `environment.yml`. Use only when +creating a new `conda-lock.yml` file or refreshing an existing one. + + conda-lock lock --mamba --file environment.yml --with-cuda=12.0 + +Installing/Updating a virtual environment from a lockile. Use this to sync your +dependencies to the exact versions in the `conda-lock.yml` file. + + conda-lock install --mamba --name claymodel conda-lock.yml + +See also https://conda.github.io/conda-lock/output/#unified-lockfile for more +usage details. ## Usage From ce425e2808ac0ae0642d272fab80f170dbbc080a Mon Sep 17 00:00:00 2001 From: Daniel Wiesmann Date: Thu, 6 Jun 2024 12:18:39 +0100 Subject: [PATCH 7/9] Fixed readme after rebase --- README.md | 52 +++++++++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 1b3508b..2810301 100644 --- a/README.md +++ b/README.md @@ -42,27 +42,6 @@ Finally, double-check that the libraries have been installed. mamba list -### Advanced - -This is for those who want full reproducibility of the virtual environment. -Create a virtual environment with just Python and conda-lock installed first. - - mamba create --name claymodel python=3.11 conda-lock=2.5.1 - mamba activate claymodel - -Generate a unified [`conda-lock.yml`](https://github.com/conda/conda-lock) file -based on the dependency specification in `environment.yml`. Use only when -creating a new `conda-lock.yml` file or refreshing an existing one. - - conda-lock lock --mamba --file environment.yml --with-cuda=12.0 - -Installing/Updating a virtual environment from a lockile. Use this to sync your -dependencies to the exact versions in the `conda-lock.yml` file. - - conda-lock install --mamba --name claymodel conda-lock.yml - -See also https://conda.github.io/conda-lock/output/#unified-lockfile for more -usage details. ## Usage @@ -101,25 +80,46 @@ More options can be found using `python trainer.py fit --help`, or at the Our Documentation uses [Jupyter Book](https://jupyterbook.org/intro.html). Install it with: -In order to avoid installing dependencies directly onto the machine, a Dockerized version of the Clay model has been provided. To run it using [docker compose](https://docs.docker.com/compose/) run the following command from the repo root: ```bash pip install -U jupyter-book ``` Then build it with: -Alternatively, you can build and run the Docker image using docker directly with: ```bash jupyter-book build docs/ ``` You can preview the site locally with: -In both cases, the default command will be run, which instantiates a [jupyter-lab](https://jupyterlab.readthedocs.io/en/stable/getting_started/starting.html) instance, from which the documentation notebooks can be run. The jupyter notebook will be accssible on the exposed port (`8888`). -### Note: accessing the jupyter-lab instance in the browser requires an authentication token that can be found in the container logs. Look for the following lines: ```bash python -m http.server --directory _build/html ``` There is a GitHub Action on `./github/workflows/deploy-docs.yml` that builds the site and pushes it to GitHub Pages. + +## Docker + +In order to avoid installing dependencies directly onto the machine, a Dockerized version of the Clay model has been provided. To run it using [docker compose](https://docs.docker.com/compose/) run the following command from the repo root: +```bash +docker-compose up # using the --build flag to force a rebuild, if necessary +``` + +Alternatively, you can build and run the Docker image using docker directly with: +```bash +docker build . -t claymodel --platform linux/amd64 # build image +docker run --rm -it -v $(pwd):/model -p 8888:8888 -e ENV_NAME=claymodel --platform linux/amd64 claymodel:latest # run container +``` + +In both cases, the default command will be run, which instantiates a [jupyter-lab](https://jupyterlab.readthedocs.io/en/stable/getting_started/starting.html) instance, from which the documentation notebooks can be run. The jupyter notebook will be accssible on the exposed port (`8888`). +### Note: accessing the jupyter-lab instance in the browser requires an authentication token that can be found in the container logs. Look for the following lines: +```bash +model-claymodel-1 | +model-claymodel-1 | To access the server, open this file in a browser: +model-claymodel-1 | file:///home/mambauser/.local/share/jupyter/runtime/jpserver-1-open.html +model-claymodel-1 | Or copy and paste one of these URLs: +model-claymodel-1 | http://3a2045c995b0:8888/lab?token=587bc427e6a84b14bae0f4783567e792cfb3b95e6131f569 +model-claymodel-1 | http://127.0.0.1:8888/lab?token=587bc427e6a84b14bae0f4783567e792cfb3b95e6131f569 +``` + Additionally the project root will be mounted as volume to the running container so any modifications made to the code base locally will be immediately reflected within the docker container. ### Custom commands: @@ -133,8 +133,6 @@ or using docker directly: docker run --rm -it -v $(pwd):/model -p 8888:8888 -e ENV_NAME=claymodel --platform linux/amd64 claymodel:latest {custom command} ``` -For example, the `bash` command can be used to access an interactive bash session within the running docker container, with the `micromamba` environment already activated: -(with docker-compose) For example, the `bash` command can be used to access an interactive bash session within the running docker container, with the `micromamba` environment already activated: ```bash # with docker-compose From d71794c70f633cf4987568235f723b2fbf2d1ca7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:19:40 +0000 Subject: [PATCH 8/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Dockerfile | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index cc4ff09..26b3228 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,4 +8,4 @@ RUN micromamba create -y -n claymodel --file environment.yml && \ micromamba clean --all --yes ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"] -CMD ["jupyter-lab", "--ip", "0.0.0.0", "--port", "8888", "--allow-root"] \ No newline at end of file +CMD ["jupyter-lab", "--ip", "0.0.0.0", "--port", "8888", "--allow-root"] diff --git a/docker-compose.yml b/docker-compose.yml index 9ac0201..baa7828 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,4 +13,4 @@ services: volumes: - .:/model # Mount the current directory to /app in the container environment: - - ENV_NAME=claymodel # Ensure the claymodel conda environment is activated \ No newline at end of file + - ENV_NAME=claymodel # Ensure the claymodel conda environment is activated From a952edce13298d84250b8ab95ae21dd84666b464 Mon Sep 17 00:00:00 2001 From: Daniel Wiesmann Date: Thu, 6 Jun 2024 15:05:45 +0100 Subject: [PATCH 9/9] Updates on docker files --- Dockerfile | 6 +++--- docker-compose.yml | 8 +------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 26b3228..28c5e93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM mambaorg/micromamba:1.5.6 +FROM mambaorg/micromamba:1.5.8 WORKDIR /model COPY --chown=$MAMBA_USER:$MAMBA_USER . . -RUN micromamba create -y -n claymodel --file environment.yml && \ - micromamba clean --all --yes +RUN micromamba create --name claymodel --yes --file conda-lock.yml \ + && micromamba clean --all --yes ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"] CMD ["jupyter-lab", "--ip", "0.0.0.0", "--port", "8888", "--allow-root"] diff --git a/docker-compose.yml b/docker-compose.yml index baa7828..a717878 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,15 +2,9 @@ version: '3.8' services: claymodel: - build: - context: . - dockerfile: Dockerfile - args: - - platform=linux/amd64 # Specify the target platform for build + image: claymodel platform: linux/amd64 # Specify the target platform for run ports: - "8888:8888" # Expose port 8888 on the host and bind it to port 8888 in the container - volumes: - - .:/model # Mount the current directory to /app in the container environment: - ENV_NAME=claymodel # Ensure the claymodel conda environment is activated