reworked docker repackaging branch

.gitignore

@@ -107,3 +107,7 @@ ENV/
 
 # vscode garbage
 .vscode
+
+# unix history files
+.bash_history
+.python_history

Dockerfile

@@ -0,0 +1,39 @@
+FROM python:3.6.5-jessie
+ENV PYTHONDONTWRITEBYTECODE 1
+
+MAINTAINER Victor Ng <vng@mozilla.com>
+EXPOSE 8000
+
+# add a non-privileged user for installing and running
+# the application
+RUN groupadd --gid 10001 app && \
+    useradd --uid 10001 --gid 10001 --home /app --create-home app 
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends build-essential gettext curl \
+                                               libopenblas-dev libatlas3-base gfortran && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Upgrade pip
+RUN pip install --upgrade pip==9
+
+# First copy requirements.txt so we can take advantage of docker
+# caching.
+COPY requirements.txt /app/requirements.txt
+COPY prod-requirements.txt /app/prod-requirements.txt
+RUN cat requirements.txt prod-requirements.txt > docker-requirements.txt
+RUN pip install --no-cache-dir -r docker-requirements.txt
+
+COPY . /app
+USER app
+
+
+# Using /bin/bash as the entrypoint works around some volume mount issues on Windows
+# where volume-mounted files do not have execute bits set.
+# https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
+ENTRYPOINT ["/bin/bash", "/app/bin/run"]
+
+# bin/run supports web|web-dev|test options
+CMD ["web"]

Makefile

@@ -1,3 +1,5 @@
+.PHONY: build up tests flake8 ci
+
 all:
 	# PySpark only knows eggs, not wheels
 	python setup.py sdist
@@ -10,6 +12,20 @@ test:
 	python setup.py test
 	flake8 taar tests
 
-# Updating pip hashes is awful
-freeze:
-	bin/hashfreeze
+build:
+	docker-compose build
+
+up:
+	docker-compose up
+
+tests:
+	docker-compose run web tox -etests
+
+flake8:
+	docker-compose run web tox -eflake8
+
+ci:
+	docker-compose run web tox
+
+shell:
+	docker-compose run web bash 

README.md

@@ -115,7 +115,6 @@ Collaborative Recommender ::
     TAAR_ADDON_MAPPING_BUCKET = "telemetry-public-analysis-2"
     TAAR_ADDON_MAPPING_KEY = "telemetry-ml/addon_recommender/addon_mapping.json"
 
-
 Ensemble Recommender ::
 
     TAAR_ENSEMBLE_BUCKET = "telemetry-parquet"
@@ -136,5 +135,3 @@ Similarity Recommender ::
     TAAR_SIMILARITY_BUCKET = "telemetry-parquet"
     TAAR_SIMILARITY_DONOR_KEY = "taar/similarity/donors.json"
     TAAR_SIMILARITY_LRCURVES_KEY = "taar/similarity/lr_curves.json"
-     
-

bin/build

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# create a version.json
+printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
+  "$CIRCLE_SHA1" \
+  "$CIRCLE_TAG" \
+  "$CIRCLE_PROJECT_USERNAME" \
+  "$CIRCLE_PROJECT_REPONAME" \
+  "$CIRCLE_BUILD_URL" \
+> version.json
+
+echo "Building the docker image with the tag app:build"
+docker build -t app:build .

bin/deploy

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# default variables
+: "${CIRCLE_TAG:=latest}"
+
+# Usage: retry MAX CMD...
+# Retry CMD up to MAX times. If it fails MAX times, returns failure.
+# Example: retry 3 docker push "mozilla/telemetry-analysis-service:$TAG"
+function retry() {
+    max=$1
+    shift
+    count=1
+    until "$@"; do
+        count=$((count + 1))
+        if [[ $count -gt $max ]]; then
+            return 1
+        fi
+        echo "$count / $max"
+    done
+    return 0
+}
+
+echo "Logging into Docker hub"
+retry 3 docker login -u="$DOCKER_USER" -p="$DOCKER_PASS"
+
+echo "Tagging app:build with $CIRCLE_TAG"
+docker tag app:build "$DOCKERHUB_REPO:$CIRCLE_TAG" ||
+  (echo "Couldn't tag app:build as $DOCKERHUB_REPO:$CIRCLE_TAG" && false)
+
+echo "Pushing tag $CIRCLE_TAG to $DOCKERHUB_REPO"
+retry 3 docker push "$DOCKERHUB_REPO:$CIRCLE_TAG" ||
+  (echo "Couldn't push $DOCKERHUB_REPO:$CIRCLE_TAG" && false)
+
+echo "Pushed $DOCKERHUB_REPO:$TAG"

bin/pipstrap.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+"""A small script that can act as a trust root for installing pip 8
+
+Embed this in your project, and your VCS checkout is all you have to trust. In
+a post-peep era, this lets you claw your way to a hash-checking version of pip,
+with which you can install the rest of your dependencies safely. All it assumes
+is Python 2.7 or better and *some* version of pip already installed. If
+anything goes wrong, it will exit with a non-zero status code.
+
+"""
+# This is here so embedded copies are MIT-compliant:
+# Copyright (c) 2016 Erik Rose
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+from __future__ import print_function
+from hashlib import sha256
+from os.path import join
+from pipes import quote
+from shutil import rmtree
+from subprocess import check_output
+from sys import exit
+from tempfile import mkdtemp
+try:
+    from urllib2 import build_opener, HTTPHandler, HTTPSHandler
+except ImportError:
+    from urllib.request import build_opener, HTTPHandler, HTTPSHandler
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse  # 3.4
+
+
+PACKAGES = [
+    # Pip has no dependencies, as it vendors everything:
+    ('https://pypi.python.org/packages/source/p/pip/pip-8.0.2.tar.gz',
+     '46f4bd0d8dfd51125a554568d646fe4200a3c2c6c36b9f2d06d2212148439521'),
+    # This version of setuptools has only optional dependencies:
+    ('https://pypi.python.org/packages/source/s/setuptools/'
+     'setuptools-19.4.tar.gz',
+     '214bf29933f47cf25e6faa569f710731728a07a19cae91ea64f826051f68a8cf'),
+    # We require Python 2.7 or later because we don't support wheel's
+    # conditional dep on argparse. This version of wheel has no other
+    # dependencies:
+    ('https://pypi.python.org/packages/source/w/wheel/wheel-0.26.0.tar.gz',
+     'eaad353805c180a47545a256e6508835b65a8e830ba1093ed8162f19a50a530c')
+]
+
+
+class HashError(Exception):
+    def __str__(self):
+        url, path, actual, expected = self.args
+        return ('{url} did not match the expected hash {expected}. Instead, '
+                'it was {actual}. The file (left at {path}) may have been '
+                'tampered with.'.format(**locals()))
+
+
+def hashed_download(url, temp, digest):
+    """Download ``url`` to ``temp``, make sure it has the SHA-256 ``digest``,
+    and return its path."""
+    # Based on pip 1.4.1's URLOpener but with cert verification removed
+    def opener():
+        opener = build_opener(HTTPSHandler())
+        # Strip out HTTPHandler to prevent MITM spoof:
+        for handler in opener.handlers:
+            if isinstance(handler, HTTPHandler):
+                opener.handlers.remove(handler)
+        return opener
+
+    def read_chunks(response, chunk_size):
+        while True:
+            chunk = response.read(chunk_size)
+            if not chunk:
+                break
+            yield chunk
+
+    response = opener().open(url)
+    path = join(temp, urlparse(url).path.split('/')[-1])
+    actual_hash = sha256()
+    with open(path, 'wb') as file:
+        for chunk in read_chunks(response, 4096):
+            file.write(chunk)
+            actual_hash.update(chunk)
+
+    actual_digest = actual_hash.hexdigest()
+    if actual_digest != digest:
+        raise HashError(url, path, actual_digest, digest)
+    return path
+
+
+def main():
+    temp = mkdtemp(prefix='pipstrap-')
+    try:
+        downloads = [hashed_download(url, temp, digest)
+                     for url, digest in PACKAGES]
+        check_output('pip install --no-index --no-deps -U ' +
+                     ' '.join(quote(d) for d in downloads),
+                     shell=True)
+    except HashError as exc:
+        print(exc)
+    except Exception:
+        rmtree(temp)
+        raise
+    else:
+        rmtree(temp)
+        return 0
+    return 1
+
+
+if __name__ == '__main__':
+    exit(main())

bin/run

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# default variables
+: "${PORT:=8000}"
+: "${SLEEP:=1}"
+: "${THREADS:=8}"
+: "${TRIES:=60}"
+: "${WORKERS:=4}"
+
+usage() {
+  echo "usage: bin/run web|web-dev|test"
+  exit 1
+}
+
+wait_for() {
+  tries=0
+  echo "Waiting for $1 to listen on $2..."
+  while true; do
+    [[ $tries -lt $TRIES ]] || return
+    (echo > /dev/tcp/$1/$2) >/dev/null 2>&1
+    result=
+    [[ $? -eq 0 ]] && return
+    sleep $SLEEP
+    tries=$((tries + 1))
+  done
+}
+
+[ $# -lt 1 ] && usage
+
+case $1 in
+  web)
+    exec newrelic-admin run-program gunicorn taar.flask_app:app -b 0.0.0.0:${PORT} --workers ${WORKERS} --threads ${THREADS} --access-logfile -
+    ;;
+  web-dev)
+    exec python taar/flask_app.py --host=0.0.0.0 --port=${PORT}
+    ;;
+  test)
+    coverage erase
+    tox
+    coverage report -m
+    if [[ ! -z ${CI+check} ]]; then
+      # submit coverage
+      coverage xml
+      env
+      bash <(curl -s https://codecov.io/bash) -s /tmp
+    fi
+    ;;
+  *)
+    exec "$@"
+    ;;
+esac

bin/test

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# default variables
+export DEVELOPMENT=1
+
+# pass CI env vars into docker containers for codecov submission
+[ ! -z ${CI+check} ] && \
+    echo "Getting Codecov environment variables" && \
+    export CI_ENV=`bash <(curl -s https://codecov.io/env)`
+
+# run docker compose with the given environment variables
+docker-compose run -e DEVELOPMENT $CI_ENV web tox -etests

docker-compose.yml

@@ -0,0 +1,39 @@
+version: "2"
+services:
+  web:
+    environment:
+      - PYTHONPATH=/app
+
+      # TAAR service configuration
+      - TAAR_API_PLUGIN=taar.plugin
+      - TAAR_ITEM_MATRIX_BUCKET=telemetry-public-analysis-2
+      - TAAR_ITEM_MATRIX_KEY=telemetry-ml/addon_recommender/item_matrix.json
+      - TAAR_ADDON_MAPPING_BUCKET=telemetry-public-analysis-2
+      - TAAR_ADDON_MAPPING_KEY=telemetry-ml/addon_recommender/addon_mapping.json
+      - TAAR_ENSEMBLE_BUCKET=telemetry-parquet
+      - TAAR_ENSEMBLE_KEY=taar/ensemble/ensemble_weight.json
+      - TAAR_WHITELIST_BUCKET=telemetry-parquet
+      - TAAR_WHITELIST_KEY=telemetry-ml/addon_recommender/only_guids_top_200.json
+      - TAAR_LOCALE_BUCKET=telemetry-parquet
+      - TAAR_LOCALE_KEY=taar/locale/top10_dict.json
+      - TAAR_SIMILARITY_BUCKET=telemetry-parquet
+      - TAAR_SIMILARITY_DONOR_KEY=taar/similarity/donors.json
+      - TAAR_SIMILARITY_LRCURVES_KEY=taar/similarity/lr_curves.json
+
+      # These are 'common' TAAR/TAARlite configuration
+      - TAAR_MAX_RESULTS=10
+
+      # These are TAAR specific and ignored by TAARlite
+      - DYNAMO_REGION=us-west-2
+      - DYNAMO_TABLE_NAME=taar_addon_data_20180206
+
+      # Pickup the AWS secret and access key from host env
+      - AWS_SECRET_ACCESS_KEY
+      - AWS_ACCESS_KEY_ID
+    build: .
+    ports:
+      - "8000:8000"
+    volumes:
+      - .:/app
+    command:
+      "web-dev"

prod-requirements.txt

@@ -1 +1,4 @@
+blinker==1.4
 certifi==2018.10.15
+newrelic==4.6.0.106
+gunicorn==19.9.0