From b9eca9d6c2669e0948085cb8e5f1fbce7d819dfd Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Thu, 21 Nov 2024 09:46:00 -0500 Subject: [PATCH 1/2] Dockerfile poetry in dev pip in production * Changes Docker structure so that there is poetry in the development step, and pip in the production step * Cleands up the documentation in the dockerfile * Adds some directories and filenames to .gitignore and .dockerignore * init.sh has shellcheck improvements and installs python packages --- .dockerignore | 2 + .gitignore | 3 ++ Dockerfile | 132 ++++++++++++++++++++++++++----------------------- compose.yml | 4 +- env.example | 2 - init.sh | 13 +++-- pyproject.toml | 9 +++- 7 files changed, 93 insertions(+), 72 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8d19362 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.venv +.env diff --git a/.gitignore b/.gitignore index a0fabef..31d2a3a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ .bash_history .python_history .cache/ +.venv +.local +__pycache__ diff --git a/Dockerfile b/Dockerfile index b5b783b..d6db964 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,82 +1,90 @@ -# PYTHON image -# Use the official Docker Python image because it has the absolute latest bugfix version of Python -# it has the absolute latest system packages -# it’s based on Debian Bookworm (Debian 12), released June 2023 -# Initial Image size is 51MB -# At the end Image size is 156MB +################################################################################ +# BASE # +################################################################################ +FROM python:3.11-slim-bookworm AS base -# I did not recommed using an alpine image because it lacks the package installer pip and the support for installing -# wheel packages, which are both needed for installing applications like Pandas and Numpy. +ARG POETRY_VERSION=1.8.3 +ARG UID=1000 +ARG GID=1000 + +# Where python should look for packages and modules when using import +ENV PYTHONPATH="/app" -# The base layer will contain the dependencies shared by the other layers -FROM python:3.11-slim-bookworm as base +# Ensure the stdout and stderr streams are sent straight to terminal +ENV PYTHONUNBUFFERED=1 -# Allowing the argumenets to be read into the dockerfile. Ex: .env > compose.yml > Dockerfile -ARG POETRY_VERSION -# true = development / false = production -ARG DEV +# Extend the socket timeout. Default would be 15s +ENV PIP_DEFAULT_TIMEOUT=100 + +RUN groupadd -g ${GID} -o app +RUN useradd -m -d /app -u ${UID} -g ${GID} -o -s /bin/bash app + +# RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \ +# vim-tiny -# Set the working directory to /app WORKDIR /app -# Use this page as a reference for python and poetry environment variables: https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUNBUFFERED -# Ensure the stdout and stderr streams are sent straight to terminal, then you can see the output of your application -ENV PYTHONUNBUFFERED=1\ - # Avoid the generation of .pyc files during package install - # Disable pip's cache, then reduce the size of the image - PIP_NO_CACHE_DIR=off \ - # Save runtime because it is not look for updating pip version - PIP_DISABLE_PIP_VERSION_CHECK=on \ - PIP_DEFAULT_TIMEOUT=100 \ - # Disable poetry interaction - POETRY_NO_INTERACTION=1 \ - POETRY_VIRTUALENVS_CREATE=1 \ - POETRY_VIRTUALENVS_IN_PROJECT=1 \ - POETRY_CACHE_DIR=/tmp/poetry_cache +CMD ["tail", "-f", "/dev/null"] + +################################################################################ +# POETRY +################################################################################ +# +# Both BUILD and DEVELOPMENT need poetry +# +FROM base AS poetry RUN pip install poetry==${POETRY_VERSION} -# Install the app. Just copy the files needed to install the dependencies -COPY pyproject.toml poetry.lock README.md ./ +# Ensure that the virtual environment directory is in the project. This path +# will be be `/app/.venv/` +ENV POETRY_VIRTUALENVS_IN_PROJECT=1 -# Poetry cache is used to avoid installing the dependencies every time the code changes, we will keep this folder in development environment and remove it in production -# --no-root, poetry will install only the dependencies avoiding to install the project itself, we will install the project in the final layer -# --without dev to avoid installing dev dependencies, we do not need test and linters in production environment -# --with dev to install dev dependencies, we need test and linters in development environment -# --mount, mount a folder for plugins with poetry cache, this will speed up the process of building the image -RUN if [ {${DEV}} ]; then \ - echo "Installing dev dependencies"; \ - poetry install --no-root --with dev \ - else \ - echo "Skipping dev dependencies"; \ - poetry install --no-root --without dev && rm -rf ${POETRY_CACHE_DIR}; \ - fi - -# Set up our final runtime layer -FROM python:3.11-slim-bookworm as runtime +# Create the virtual environment if it does not already exist +ENV POETRY_VIRTUALENVS_CREATE=1 -ARG UID=1000 -ARG GID=1000 +################################################################################ +# BUILD # +################################################################################ +# +# This step uses poetry to generate a requirements.txt file for PRODUCTION +# +FROM poetry AS build -# Create our users here in the last layer or else it will be lost in the previous discarded layers -# Create a system group named "app_user" with the -r flag -RUN groupadd -g ${GID} -o app -RUN useradd -m -d /app -u ${UID} -g ${GID} -o -s /bin/bash app -RUN mkdir -p /venv && chown ${UID}:${GID} /venv -RUN which pip && sleep 10 +# README.md is needed so that poetry command will work. +COPY pyproject.toml poetry.lock README.md ./ + +RUN poetry export --without dev -f requirements.txt --output requirements.txt -# By adding /venv/bin to the PATH the dependencies in the virtual environment -# are used -ENV VIRTUAL_ENV=/venv \ - PATH="/venv/bin:$PATH" +################################################################################ +# DEVELOPMENT # +################################################################################ +# +# In development we want poetry in the container, so it inherits from the POETRY +# step. This step is the place to install development-only sytem dependencies +# +FROM poetry AS development -COPY --chown=${UID}:${GID} --from=base "/app/.venv" ${VIRTUAL_ENV} +# RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \ +# wget\ -# Switch to the non-root user "user" USER app -WORKDIR /app +################################################################################ +# PRODUCTION # +################################################################################ +FROM base AS production + +# Setting this to 'off' actually turns off the cache. This is set to decrease +# the size of the image. +ENV PIP_NO_CACHE_DIR=off + +# Speed up pip usage by not checking for the version +ENV PIP_DISABLE_PIP_VERSION_CHECK=on COPY --chown=${UID}:${GID} . /app +COPY --chown=${UID}:${GID} --from=build "/app/requirements.txt" /app/requirements.txt -CMD ["tail", "-f", "/dev/null"] +RUN pip install -r /app/requirements.txt + +USER app diff --git a/compose.yml b/compose.yml index 233dd81..7c22b36 100644 --- a/compose.yml +++ b/compose.yml @@ -2,13 +2,13 @@ services: app: build: context: . - target: runtime + target: development dockerfile: Dockerfile args: UID: ${UID:-1000} GID: ${GID:-1000} DEV: ${DEV:-false} - POETRY_VERSION: ${POETRY_VERSION:-1.5.1} + POETRY_VERSION: ${POETRY_VERSION:-1.8.3} env_file: - .env volumes: diff --git a/env.example b/env.example index 067a9a3..adaebf5 100644 --- a/env.example +++ b/env.example @@ -1,4 +1,2 @@ UID=YOUR_UID GID=YOUR_GID -POETRY_VERSION=1.5.1 -DEV=true diff --git a/init.sh b/init.sh index d2e2c7e..2444f63 100755 --- a/init.sh +++ b/init.sh @@ -5,12 +5,15 @@ if [ -f ".env" ]; then else echo "🌎 .env does not exist. Copying .env-example to .env" cp env.example .env - YOUR_UID=`id -u` - YOUR_GID=`id -g` - echo "πŸ™‚ Setting your UID ($YOUR_UID) and GID ($YOUR_UID) in .env" - docker run --rm -v ./.env:/.env alpine echo "$(sed s/YOUR_UID/$YOUR_UID/ .env)" > .env - docker run --rm -v ./.env:/.env alpine echo "$(sed s/YOUR_GID/$YOUR_GID/ .env)" > .env + YOUR_UID=$(id -u) + YOUR_GID=$(id -g) + echo "πŸ™‚ Setting your UID (${YOUR_UID}) and GID (${YOUR_UID}) in .env" + docker run --rm -v ./.env:/.env alpine echo "$(sed s/YOUR_UID/${YOUR_UID}/ .env)" >.env + docker run --rm -v ./.env:/.env alpine echo "$(sed s/YOUR_GID/${YOUR_GID}/ .env)" >.env fi echo "🚒 Build docker images" docker compose build + +echo "πŸ“¦ Build python packages" +docker compose run --rm app poetry install diff --git a/pyproject.toml b/pyproject.toml index 905559b..4fb8ef5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,14 @@ name = "python-starter" version = "0.1.0" description = "Boilerplate code for developing a python app in docker" -authors = ["Monique Rio , Samuel Sciolla , Lianet Sepulveda Torres , "] +authors = [ + "Jayamala Perumal Subramani ", + "Monique Rio ", + "Samuel Sciolla ", + "Lianet Sepulveda Torres ", + "K'ron Spar ", + "Anthony Thomas " + ] readme = "README.md" packages = [{include = "python_starter"}] From 1fa5eb9347d6b43d1322b0c575f382a81804d4fe Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Thu, 21 Nov 2024 14:07:25 -0500 Subject: [PATCH 2/2] start of changed readme file --- README.md | 78 +++++++++++++------------------------------------------ 1 file changed, 18 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 84ac85d..ee3c8ab 100644 --- a/README.md +++ b/README.md @@ -1,75 +1,33 @@ -# python-docker-boilerplate +# python-starter -Boilerplate code for starting a python project with docker and docker-compose +Starter repository for python projects -## How to set up your python environment +## Installation -### Install python +1. Run the `.init.sh` script. This will set up your `.env` file, build the + docker image, and install the python packages. -On mac, + ```bash + ./init.sh + ``` -* You can read this blog to install python in a right way in - python: https://opensource.com/article/19/5/python-3-default-mac - -* **Recommendation**: Install python using brew and pyenv +2. Edit `pyproject.toml` to have the proper name and author(s) of your project. -### Managing python dependencies +3. Rename the `python_starter` directory to the name of your project. -* **Install poetry** - -* On Mac OS, Windows and Linux, - * Install poetry: - * ``curl -sSL https://install.python-poetry.org | python3 -`` - * This way allows poetry and its dependencies to be isolated from your dependencies. I don't recommend to use - * pip to install poetry because poetry and your application dependencies will be installed in the same environment. - * ```poetry init```: - * Use this command to set up your local environment, repository details, and dependencies. - * It will generate a pyproject.toml file with the information you provide. - * Package name [python-starter]: - * Version [0.1.0]: - * Description []: - * Author []: n - * License []: - * Compatible Python versions [^3.11]: - * Would you like to define your main dependencies interactively? (yes/no) [yes]: no - * Would you like to define your development dependencies interactively? (yes/no) no - * ```poetry install```: - * Use this command to automatically install the dependencies specified in the pyproject.toml file. - * It will generate a poetry.lock file with the dependencies and their versions. - * It will create a virtual environment in the home directory, e.g. /Users/user_name/Library/Caches/pypoetry/.. - * ```poetry env use python```: - * Use this command to find the virtual environment directory, created by poetry. - * ```source ~/Library/Caches/pypoetry/virtualenvs/python-starter-0xoBsgdA-py3.11/bin/activate``` - * Use this command to activate the virtual environment. - * ```poetry shell```: - * Use this command to activate the virtual environment. - * ```poetry add pytest```: - * Use this command to add dependencies. - * ```poetry add --dev pytest```: - * Use this command to add development dependencies. - * `` poetry update ``: - * Use this command if you change your .toml file and want to generate a new version the .lock file - -## Set up in a docker environment - -``` -./init.sh -``` - -This will: - -* copy the project folder -* build the docker image -* install the dependencies -* create a container with the application - -## How to run the application +## How to use the starter ``docker compose exec app python --version`` +To run python scripts with poetry installed packages run something like: + +``docker compose exec app poetry run python your_script.py`` ## Tests +``docker compose exec app poetry run pytest`` + ## Background + This repository goes with this documentation: -https://mlit.atlassian.net/wiki/spaces/LD/pages/10092544004/Python+in+LIT +