Merge 60baa91 into b15a6be

API.md

@@ -0,0 +1,94 @@
+API documentation
+
+# Get addon recommendations
+
+Allow the Authenticated User to update their details.
+
+**URL** : `/v1/api/recommendations/<hashed_id>/`
+
+**Method** : `POST`
+
+**Auth required** : NO
+
+**Permissions required** : None
+
+**Data constraints**
+
+```json
+{
+    "options": {"promoted": [
+                                ["[1 to 30 chars]", Some Number],
+                                ["[1 to 30 chars]", Some Number],
+                            ]
+               }
+}
+```
+
+Note that the only valid key for the top level JSON is `options`.
+
+`options` is always a dictionary of optional values.  
+
+To denote no optional data - it is perfectly valid for the JSON data
+to have no `options` key, or even simpler - not have POST data at all.
+
+Each item in the promoted addon GUID list is accompanied by an
+integer weight.  Any weight is greater than a TAAR recommended addon
+GUID.
+
+**Data examples**
+
+Partial data is allowed.
+
+```json
+{
+    "options": {"promoted": [
+                                ["guid1", 10],
+                                ["guid2", 5],
+                            ]
+               }
+}
+```
+
+
+## Success Responses
+
+**Condition** : Data provided is valid
+
+**Code** : `200 OK`
+
+**Content example** : Response will reflect a list of addon GUID suggestions.
+
+```json
+{
+    "results": ["taar-guid1", "taar-guid2", "taar-guid3"],
+    "result_info": [],
+}
+```
+
+## Error Response
+
+**Condition** : If provided data is invalid, e.g. options object is not a dictionary.
+
+**Code** : `400 BAD REQUEST`
+
+**Content example** :
+
+```json
+{
+    "invalid_option": [
+        "Please provide a dictionary with a `promoted` key mapped to a list of promoted addon GUIDs",
+    ]
+}
+```
+
+## Notes
+
+* Endpoint will ignore irrelevant and read-only data such as parameters that
+  don't exist, or fields.
+* Endpoint will try to fail gracefully and return an empty list in the
+  results key if no suggestions can be made.
+* The only condition when the endpoint should return an error code if
+  the options data is malformed.
+
+
+

Dockerfile

@@ -0,0 +1,39 @@
+FROM python:3.6.5-jessie
+ENV PYTHONDONTWRITEBYTECODE 1
+
+MAINTAINER Victor Ng <vng@mozilla.com>
+EXPOSE 8000
+
+# add a non-privileged user for installing and running
+# the application
+RUN groupadd --gid 10001 app && \
+    useradd --uid 10001 --gid 10001 --home /app --create-home app 
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends build-essential gettext curl \
+                                               libopenblas-dev libatlas3-base gfortran && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Upgrade pip
+RUN pip install --upgrade pip==9
+
+# First copy requirements.txt so we can take advantage of docker
+# caching.
+COPY requirements.txt /app/requirements.txt
+COPY prod-requirements.txt /app/prod-requirements.txt
+RUN cat requirements.txt prod-requirements.txt > docker-requirements.txt
+RUN pip install --no-cache-dir -r docker-requirements.txt
+
+COPY . /app
+USER app
+
+
+# Using /bin/bash as the entrypoint works around some volume mount issues on Windows
+# where volume-mounted files do not have execute bits set.
+# https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
+ENTRYPOINT ["/bin/bash", "/app/bin/run"]
+
+# bin/run supports web|web-dev|test options
+CMD ["web"]

Makefile

@@ -1,3 +1,5 @@
+.PHONY: build up tests flake8 ci
+
 all:
 	# PySpark only knows eggs, not wheels
 	python setup.py sdist
@@ -10,6 +12,17 @@ test:
 	python setup.py test
 	flake8 taar tests
 
-# Updating pip hashes is awful
-freeze:
-	bin/hashfreeze
+build:
+	docker-compose build
+
+up:
+	docker-compose up
+
+tests:
+	docker-compose run web tox -etests
+
+flake8:
+	docker-compose run web tox -eflake8
+
+ci:
+	docker-compose run web tox

bin/build

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# create a version.json
+printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
+  "$CIRCLE_SHA1" \
+  "$CIRCLE_TAG" \
+  "$CIRCLE_PROJECT_USERNAME" \
+  "$CIRCLE_PROJECT_REPONAME" \
+  "$CIRCLE_BUILD_URL" \
+> version.json
+
+echo "Building the docker image with the tag app:build"
+docker build -t app:build .

bin/deploy

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# default variables
+: "${CIRCLE_TAG:=latest}"
+
+# Usage: retry MAX CMD...
+# Retry CMD up to MAX times. If it fails MAX times, returns failure.
+# Example: retry 3 docker push "mozilla/telemetry-analysis-service:$TAG"
+function retry() {
+    max=$1
+    shift
+    count=1
+    until "$@"; do
+        count=$((count + 1))
+        if [[ $count -gt $max ]]; then
+            return 1
+        fi
+        echo "$count / $max"
+    done
+    return 0
+}
+
+echo "Logging into Docker hub"
+retry 3 docker login -u="$DOCKER_USER" -p="$DOCKER_PASS"
+
+echo "Tagging app:build with $CIRCLE_TAG"
+docker tag app:build "$DOCKERHUB_REPO:$CIRCLE_TAG" ||
+  (echo "Couldn't tag app:build as $DOCKERHUB_REPO:$CIRCLE_TAG" && false)
+
+echo "Pushing tag $CIRCLE_TAG to $DOCKERHUB_REPO"
+retry 3 docker push "$DOCKERHUB_REPO:$CIRCLE_TAG" ||
+  (echo "Couldn't push $DOCKERHUB_REPO:$CIRCLE_TAG" && false)
+
+echo "Pushed $DOCKERHUB_REPO:$TAG"

bin/pipstrap.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+"""A small script that can act as a trust root for installing pip 8
+
+Embed this in your project, and your VCS checkout is all you have to trust. In
+a post-peep era, this lets you claw your way to a hash-checking version of pip,
+with which you can install the rest of your dependencies safely. All it assumes
+is Python 2.7 or better and *some* version of pip already installed. If
+anything goes wrong, it will exit with a non-zero status code.
+
+"""
+# This is here so embedded copies are MIT-compliant:
+# Copyright (c) 2016 Erik Rose
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+from __future__ import print_function
+from hashlib import sha256
+from os.path import join
+from pipes import quote
+from shutil import rmtree
+from subprocess import check_output
+from sys import exit
+from tempfile import mkdtemp
+try:
+    from urllib2 import build_opener, HTTPHandler, HTTPSHandler
+except ImportError:
+    from urllib.request import build_opener, HTTPHandler, HTTPSHandler
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse  # 3.4
+
+
+PACKAGES = [
+    # Pip has no dependencies, as it vendors everything:
+    ('https://pypi.python.org/packages/source/p/pip/pip-8.0.2.tar.gz',
+     '46f4bd0d8dfd51125a554568d646fe4200a3c2c6c36b9f2d06d2212148439521'),
+    # This version of setuptools has only optional dependencies:
+    ('https://pypi.python.org/packages/source/s/setuptools/'
+     'setuptools-19.4.tar.gz',
+     '214bf29933f47cf25e6faa569f710731728a07a19cae91ea64f826051f68a8cf'),
+    # We require Python 2.7 or later because we don't support wheel's
+    # conditional dep on argparse. This version of wheel has no other
+    # dependencies:
+    ('https://pypi.python.org/packages/source/w/wheel/wheel-0.26.0.tar.gz',
+     'eaad353805c180a47545a256e6508835b65a8e830ba1093ed8162f19a50a530c')
+]
+
+
+class HashError(Exception):
+    def __str__(self):
+        url, path, actual, expected = self.args
+        return ('{url} did not match the expected hash {expected}. Instead, '
+                'it was {actual}. The file (left at {path}) may have been '
+                'tampered with.'.format(**locals()))
+
+
+def hashed_download(url, temp, digest):
+    """Download ``url`` to ``temp``, make sure it has the SHA-256 ``digest``,
+    and return its path."""
+    # Based on pip 1.4.1's URLOpener but with cert verification removed
+    def opener():
+        opener = build_opener(HTTPSHandler())
+        # Strip out HTTPHandler to prevent MITM spoof:
+        for handler in opener.handlers:
+            if isinstance(handler, HTTPHandler):
+                opener.handlers.remove(handler)
+        return opener
+
+    def read_chunks(response, chunk_size):
+        while True:
+            chunk = response.read(chunk_size)
+            if not chunk:
+                break
+            yield chunk
+
+    response = opener().open(url)
+    path = join(temp, urlparse(url).path.split('/')[-1])
+    actual_hash = sha256()
+    with open(path, 'wb') as file:
+        for chunk in read_chunks(response, 4096):
+            file.write(chunk)
+            actual_hash.update(chunk)
+
+    actual_digest = actual_hash.hexdigest()
+    if actual_digest != digest:
+        raise HashError(url, path, actual_digest, digest)
+    return path
+
+
+def main():
+    temp = mkdtemp(prefix='pipstrap-')
+    try:
+        downloads = [hashed_download(url, temp, digest)
+                     for url, digest in PACKAGES]
+        check_output('pip install --no-index --no-deps -U ' +
+                     ' '.join(quote(d) for d in downloads),
+                     shell=True)
+    except HashError as exc:
+        print(exc)
+    except Exception:
+        rmtree(temp)
+        raise
+    else:
+        rmtree(temp)
+        return 0
+    return 1
+
+
+if __name__ == '__main__':
+    exit(main())

bin/run

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# default variables
+: "${PORT:=8000}"
+: "${SLEEP:=1}"
+: "${THREADS:=8}"
+: "${TRIES:=60}"
+: "${WORKERS:=4}"
+
+usage() {
+  echo "usage: bin/run web|web-dev|test"
+  exit 1
+}
+
+wait_for() {
+  tries=0
+  echo "Waiting for $1 to listen on $2..."
+  while true; do
+    [[ $tries -lt $TRIES ]] || return
+    (echo > /dev/tcp/$1/$2) >/dev/null 2>&1
+    result=
+    [[ $? -eq 0 ]] && return
+    sleep $SLEEP
+    tries=$((tries + 1))
+  done
+}
+
+[ $# -lt 1 ] && usage
+
+case $1 in
+  web)
+    exec newrelic-admin run-program gunicorn taar_api.app:app -b 0.0.0.0:${PORT} --workers ${WORKERS} --threads ${THREADS} --access-logfile -
+    ;;
+  web-dev)
+    exec python taar_api/app.py --host=0.0.0.0 --port=${PORT}
+    ;;
+  test)
+    coverage erase
+    tox
+    coverage report -m
+    if [[ ! -z ${CI+check} ]]; then
+      # submit coverage
+      coverage xml
+      env
+      bash <(curl -s https://codecov.io/bash) -s /tmp
+    fi
+    ;;
+  *)
+    exec "$@"
+    ;;
+esac