Merge branch 'poetry-migration'

.dockerignore

@@ -0,0 +1,5 @@
+docs
+logs
+data
+helm
+tests

.gitignore

@@ -1,6 +1,7 @@
 # app.db database for development
 app.db
 logs
+data
 
 # ignore any sensitive configuration file
 conf/*

CHANGELOG.rst

@@ -6,10 +6,18 @@ This document lists all important changes to Quetzal.
 
 Quetzal version numbers follow `semantic versioning <http://semver.org>`_.
 
-Planned (but not implemented yet):
+Planned (but not implemented yet)
+---------------------------------
 
 * Better commit detection with a 3-way merge detection
 
+
+0.6.0 (not released yet)
+------------------------
+
+* Moved the package and dependency management to poetry.
+
+
 0.5.1 (2020-03-05)
 ------------------
 

docker-compose.yaml

@@ -1,4 +1,4 @@
-version: '3'
+version: '3.4'
 services:
   nginx:
     build:
@@ -9,8 +9,8 @@ services:
       - "80:80"
       - "443:443"
     environment:
-      # A comma-separate list of server names
-      SERVER_NAMES: "local.quetz.al,david.quetz.al"
+      # A comma-separated list of server names
+      SERVER_NAMES: "local.quetz.al"
       ADMIN_EMAIL: "admin@quetz.al"
       DISABLE_CERTBOT: "1"
     links:
@@ -46,8 +46,7 @@ services:
     build:
       context: .
       dockerfile: ./docker/app/Dockerfile
-    #entrypoint: flask run --host 0.0.0.0 --port 5000 --no-reload
-    #/entrypoint-server.sh
+      target: development
     environment:
       FLASK_ENV: development
       DB_USERNAME: db_user
@@ -58,12 +57,13 @@ services:
       USE_GUNICORN: 1
       QUETZAL_BACKGROUND_JOBS: 1
       QUETZAL_ADMIN_MAIL: quetzal.api@gmail.com
-      QUETZAL_DATA_STORAGE: GCP
+      ## Set this for Google Cloud Platform
+      # QUETZAL_DATA_STORAGE: GCP
       # When using local storage instead of GCP buckets for data, use this
       # configuration together with the volumes declared later
-#      QUETZAL_DATA_STORAGE: file
-#      QUETZAL_FILE_DATA_DIR: /mnt/data
-#      QUETZAL_FILE_USER_DATA_DIR: /mnt/workspaces
+      QUETZAL_DATA_STORAGE: file
+      QUETZAL_FILE_DATA_DIR: /mnt/data
+      QUETZAL_FILE_USER_DATA_DIR: /mnt/workspaces
       # KOMBU_LOG_CONNECTION: 1  # Useful to debug kombu bug
     volumes:
       # Setup a volume for development: it allows the web container to have the
@@ -75,22 +75,23 @@ services:
       - ./conf:/code/conf:ro
       # When using local storage instead of GCP buckets for data, create two
       # directories and add them here:
-#      - ./data:/mnt/data
-#      - ./workspaces:/mnt/workspaces
+      - ./data/global:/mnt/data
+      - ./data/workspaces:/mnt/workspaces
     depends_on:
       - db
       - rabbitmq
-    ports:
-      # TODO: remove when the nginx service is added
-      - "5000:5000"
-    # Maybe needed to attach with docker-compose run --service-ports web command
-    # stdin_open: true
+    # ports:
+    #   # Remove when the nginx service is added
+    #  - "5000:5000"
+    # # Maybe needed to attach with docker-compose run --service-ports web command
+    # # stdin_open: true
     # tty: true
 
   worker:
     build:
       context: .
       dockerfile: ./docker/app/Dockerfile
+      target: development
     entrypoint: /entrypoint-worker.sh
     environment:
       FLASK_ENV: development
@@ -99,13 +100,17 @@ services:
       DB_HOST: db
       DB_PORT: 5432
       DB_DATABASE: quetzal
+      USE_GUNICORN: 1
       QUETZAL_BACKGROUND_JOBS: 1
-      QUETZAL_DATA_STORAGE: GCP
+      QUETZAL_ADMIN_MAIL: quetzal.api@gmail.com
+      ## Set this for Google Cloud Platform
+      # QUETZAL_DATA_STORAGE: GCP
       # When using local storage instead of GCP buckets for data, use this
       # configuration together with the volumes declared later
-#      QUETZAL_DATA_STORAGE: file
-#      QUETZAL_FILE_DATA_DIR: /mnt/data
-#      QUETZAL_FILE_USER_DATA_DIR: /mnt/workspaces
+      QUETZAL_DATA_STORAGE: file
+      QUETZAL_FILE_DATA_DIR: /mnt/data
+      QUETZAL_FILE_USER_DATA_DIR: /mnt/workspaces
+      # KOMBU_LOG_CONNECTION: 1  # Useful to debug kombu bug
     volumes:
       # Setup a volume for development: it allows the web container to have the
       # latest changes so one does not need to build everytime there is a
@@ -116,34 +121,14 @@ services:
       - ./conf:/code/conf:ro
       # When using local storage instead of GCP buckets for data, create two
       # directories and add them here:
-#      - ./data:/mnt/data
-#      - ./workspaces:/mnt/workspaces
-    depends_on:
-      - db
-      - rabbitmq
-
-  unittests:
-    build:
-      context: .
-      dockerfile: ./docker/app/Dockerfile
-    #entrypoint: pytest -q
-    entrypoint: /bin/true
-    environment:
-      FLASK_ENV: tests
-      DB_USERNAME: db_user
-      DB_PASSWORD: db_password
-      DB_HOST: db
-      DB_PORT: 5432
-      DB_DATABASE: unittests
-    volumes:
-      # Setup a volume for development: it allows the web container to have the
-      # latest changes so one does not need to build everytime there is a
-      # modification.
-      - .:/code
-      # The secret files in the configuration are set as a volume, but don't use
-      # it like this on production, use a secret!
-      - ./conf:/code/conf:ro
+      - ./data/global:/mnt/data
+      - ./data/workspaces:/mnt/workspaces
     depends_on:
       - db
       - rabbitmq
-
+    # ports:
+    #   # Remove when the nginx service is added
+    #  - "5000:5000"
+    # # Maybe needed to attach with docker-compose run --service-ports web command
+    # # stdin_open: true
+    # tty: true

docker/app/Dockerfile

@@ -1,33 +1,120 @@
-FROM python:3.6-slim
+################################################################################
+# Multi-stage poetry python installation based on
+# https://github.com/python-poetry/poetry/issues/1879#issuecomment-59213351
 
-# Python unbuffered so that messages appear immediately
-# TODO: consider moving this to the docker-compose environment
-ENV PYTHONUNBUFFERED 1
+################################################################################
+# Stage 1: python-base
+#          Used as a base Python image with the environment variables set for
+#          Python and poetry
+FROM python:3.7-slim AS python-base
 
-# Prepare application
-RUN mkdir /code
-WORKDIR /code
+ENV \
+    # Python-related environment variables
+    PYTHONUNBUFFERED=1 \
+    # prevents python creating .pyc files
+    PYTHONDONTWRITEBYTECODE=1 \
+    \
+    # pip-related environment variables
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    \
+    # poetry-related environment variables
+    # https://python-poetry.org/docs/configuration/#using-environment-variables
+    POETRY_VERSION=1.0.5 \
+    # make poetry install to this location
+    POETRY_HOME="/opt/poetry" \
+    # make poetry create the virtual environment in the project's root
+    # it gets named `.venv`
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    # do not ask any interactive question
+    POETRY_NO_INTERACTION=1 \
+    \
+    # paths-related environment variables
+    # this is where our requirements + virtual environment will live
+    PYSETUP_PATH="/opt/pysetup" \
+    VENV_PATH="/opt/pysetup/.venv"
 
-# Copy and install requirements
-COPY requirements.txt requirements-dev.txt requirements-test.txt ./
-RUN pip install -r requirements.txt
-# requirements-dev.txt and requirements-test.txt are not installed but
-# left in the image in case you want to install them
+# prepend poetry and venv to path
+ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+################################################################################
 
-# Add application source code
-COPY config.py wsgi.py openapi.yaml init.sh ./
-COPY quetzal ./quetzal
-COPY migrations ./migrations
+################################################################################
+# Stage 2: builder-base
+#          Used to build all dependencies. This step may need some compilation
+#          tools (like build-essential), which can be quite large, but we don't
+#          want to distribute an image with these temporary tools
+FROM python-base AS builder-base
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+        # deps for installing poetry
+        curl \
+        # deps for building python deps
+        build-essential
 
-# Add entrypoints
-COPY docker/app/entrypoint-server.sh docker/app/entrypoint-worker.sh /
+# install poetry - respects $POETRY_VERSION & $POETRY_HOME
+RUN curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python
+
+# copy project requirement files here to ensure they will be cached.
+WORKDIR $PYSETUP_PATH
+#COPY config.py wsgi.py openapi.yaml init.sh README.rst ./
+#COPY quetzal ./quetzal
+#COPY migrations ./migrations
+COPY README.rst poetry.lock pyproject.toml ./
+#RUN poetry export -f requirements.txt > requirements.txt \
+# && poetry export --dev -f requirements.txt > requirements-dev.txt
 
-# Flask configuration
-ENV FLASK_ENV development
+# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
+RUN poetry install --no-dev --no-root
+################################################################################
 
-# Permit version override
-ARG QUETZAL_VERSION
-ENV QUETZAL_VERSION ${QUETZAL_VERSION}
+################################################################################
+# Stage 4: quetzal-base
+FROM python-base AS quetzal-base
+WORKDIR $PYSETUP_PATH
+
+# copy in our built poetry + venv
+COPY --from=builder-base $POETRY_HOME $POETRY_HOME
+COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
+
+# will become mountpoint of our code
+WORKDIR /code
+# Copy all quetzal source code except the quetzal package, so that there is
+# no rebuild for each new modification
+COPY migrations ./migrations
+COPY config.py wsgi.py openapi.yaml init.sh pyproject.toml README.rst ./
+COPY docker/app/entrypoint-server.sh docker/app/entrypoint-worker.sh /
 
 EXPOSE 5000
 ENTRYPOINT /entrypoint-server.sh
+
+################################################################################
+
+################################################################################
+# Stage 5: development
+#          Used as a development environment of the Quetzal application.
+FROM quetzal-base AS development
+ENV FLASK_ENV=development
+
+# Install runtime and dev deps. This should be a bit faster because runtime deps
+# were installed in hte previous stage
+WORKDIR $PYSETUP_PATH
+RUN poetry install --no-root
+
+# will become mountpoint of our code
+WORKDIR /code
+COPY quetzal ./quetzal
+################################################################################
+
+################################################################################
+# Stage 6: production
+#          A leaner version of the development stage. This is the final stage
+#          and it is the image that will be distributed.
+FROM quetzal-base AS production
+ENV FLASK_ENV=production \
+    USE_GUNICORN=1
+
+# will become mountpoint of our code
+WORKDIR /code
+COPY quetzal ./quetzal
+################################################################################

docs/requirements.txt

@@ -0,0 +1,3 @@
+sphinx ~= 2.4.4
+sphinx-rtd-theme ~= 0.4.3
+sphinx-tabs ~= 1.1.13

docs/source/dev_quickstart.rst

@@ -2,12 +2,52 @@
 Quickstart
 ==========
 
-How to get a development environment
+To get a quick development environment follow these steps:
+
+1. Install `poetry <https://python-poetry.org/docs/#installation>`_,
+   `Docker <https://docs.docker.com/install/>`_, and
+   `docker-compose <https://docs.docker.com/compose/install/>`_
+   (usually docker-compose is already included by Docker).
+
+2. Clone a local copy of Quetzal:
+
+   .. code-block:: console
+
+    git clone git@github.com:quetz-al/quetzal.git
+
+3. Create a virtual environment and install Quetzal:
+
+   .. code-block:: console
+
+    cd quetzal
+    poetry install
+
+4. For an environment based on docker-compose, build the Docker images:
+
+   .. code-block:: console
+
+    docker-compose build
+
+5. At this point you can get a local Quetzal server that saves files in a local
+   filesystem. Launch it with:
+
+   .. code-block:: console
+
+    docker-compose up
+
+6. For a server that runs outside the docker-compose environment (for
+   development or testing purposes), modify the ``config.py``
+   file according to your needs (in particular the hostnames to the database or
+   the rabbit queue) and launch a server with:
+
+   .. code-block:: console
+
+    FLASK_ENV=local-tests flask run --host 0.0.0.0 --port 5000
 
 
 =============
 Cloud storage
 =============
 
 Take the development environment to the next step by using cloud storage to
-store data.
+store data... (description on how to do this coming soon).