Merge 12909dc into 2c509e2

HISTORY.rst

@@ -2,18 +2,22 @@
 History / Changelog
 ===================
 
------------------
-HEAD (unreleased)
------------------
+-------
+v0.23.4
+-------
 
 End-User Summary
 ================
 
+- Fixing issue of database query in Clinvar Export feature where too large queries were created.
 - Fixing search feature.
 
 Full Change List
 ================
 
+- Docker image now includes commits to the next tag so the versioneer version display makes sense.
+- Dockerfile entrypoint script uses timeout of 600s now for guniorn workers.
+- Fixing issue of database query in Clinvar Export feature where too large queries were created and postgres ran out of stack memory.
 - Adding more Sentry integrations (redis, celery, sqlalchemy).
 - Fixing search feature.
 

docker/Dockerfile

@@ -2,6 +2,7 @@ FROM python:3.6-buster
 
 ARG app_git_url=https://github.com/bihealth/varfish-server.git
 ARG app_git_tag
+ARG app_git_depth=1
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV CUSTOM_STATIC_DIR /usr/src/app/local-static
@@ -12,7 +13,7 @@ RUN chmod +x /usr/local/bin/wait
 
 # Copy source code into Docker image.
 RUN mkdir -p /usr/src
-RUN git clone --depth 1 --branch $app_git_tag $app_git_url /usr/src/app
+RUN git clone --depth $app_git_depth --branch $app_git_tag $app_git_url /usr/src/app
 
 # Install system dependencies.
 RUN apt-get update && \

docker/build-docker.sh

@@ -5,10 +5,14 @@ BUILD_NO=0
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd $DIR
 
-GIT_DESCRIBE=$(git describe | cut -d - -f 1)
+GIT_DESCRIBE=$(git describe --tags | cut -d - -f 1)
 GIT_TAG=${GIT_TAG-$GIT_DESCRIBE}
 DOCKER_VERSION=$(echo $GIT_TAG | sed -e 's/^v//')-$BUILD_NO
+GIT_DEPTH=$(($(git rev-list HEAD ^$(git describe --abbrev=0 --tags) --count) + 1))
+GIT_URL=https://github.com/bihealth/varfish-server.git
 
 docker build . \
     --build-arg app_git_tag=$GIT_TAG \
+    --build-arg app_git_depth=$GIT_DEPTH \
+    --build-arg app_git_url=$GIT_URL \
     -t bihealth/varfish-server:$DOCKER_VERSION

docker/docker-entrypoint.sh

@@ -28,6 +28,8 @@ set -euo pipefail
 #                      default: 8080
 #   LOG_LEVEL       -- logging verbosity
 #                      default: info
+#   GUNICORN_TIMEOUT -- timeout for gunicorn workers in seconds
+#                       default: 600
 
 APP_DIR=${APP_DIR-/usr/src/app}
 CELERY_QUEUES=${CELERY_QUEUES-default,query,import}
@@ -38,6 +40,7 @@ export PYTHONUNBUFFERED=${PYTHONUNBUFFERED-1}
 HTTP_HOST=${HTTP_HOST-0.0.0.0}
 HTTP_PORT=${HTTP_PORT-8080}
 LOG_LEVEL=${LOG_LEVEL-info}
+GUNICORN_TIMEOUT=${GUNICORN_TIMEOUT-600}
 
 if [[ "$NO_WAIT" -ne 1 ]]; then
   /usr/local/bin/wait
@@ -58,7 +61,12 @@ if [[ "$1" == wsgi ]]; then
   python manage.py migrate
   >&2 echo "VARFISH MIGRATIONS END"
 
-  exec gunicorn --access-logfile - --log-level "$LOG_LEVEL" --bind "$HTTP_HOST:$HTTP_PORT" config.wsgi
+  exec gunicorn \
+    --access-logfile - \
+    --log-level "$LOG_LEVEL" \
+    --bind "$HTTP_HOST:$HTTP_PORT" \
+    --timeout "$GUNICORN_TIMEOUT" \
+    config.wsgi
 elif [[ "$1" == celeryd ]]; then
   cd $APP_DIR
 

variants/queries.py

@@ -1811,6 +1811,11 @@ def run(self, *, case=None, cases=None, project=None):
         return self._query(case_ids)
 
     def _query(self, case_ids: typing.List[int]):
+        def chunks(lst, n=50):
+            """Yield successive n-sized chunks from lst."""
+            for i in range(0, len(lst), n):
+                yield lst[i : i + n]
+
         result_flags = self._query_model(SmallVariantFlags, case_ids)
         result_comments = self._query_model(SmallVariantComment, case_ids)
         result_ratings = self._query_model(AcmgCriteriaRating, case_ids)
@@ -1822,17 +1827,19 @@ def _query(self, case_ids: typing.List[int]):
             )
         )
 
-        keys = ["case_id", "release", "chromosome", "start", "reference", "alternative"]
-        stmt = (
-            select([SmallVariant.sa.id])
-            .select_from(SmallVariant.sa.table)
-            .where(
-                tuple_(*[getattr(SmallVariant.sa, key) for key in keys]).in_(
-                    [[getattr(k, key) for key in keys] for k in variant_keys]
+        small_var_ids = []
+        for variant_keys_chunk in chunks(variant_keys):
+            keys = ["case_id", "release", "chromosome", "start", "reference", "alternative"]
+            stmt = (
+                select([SmallVariant.sa.id])
+                .select_from(SmallVariant.sa.table)
+                .where(
+                    tuple_(*[getattr(SmallVariant.sa, key) for key in keys]).in_(
+                        [[getattr(k, key) for key in keys] for k in variant_keys_chunk]
+                    )
                 )
             )
-        )
-        small_var_ids = [rec.id for rec in self.engine.execute(stmt)]
+            small_var_ids += [rec.id for rec in self.engine.execute(stmt)]
 
         flags_ids = [x["id"] for x in result_flags]
         comments_ids = [x["id"] for x in result_comments]