Merge pull request #5 from kbase/service-skeleton

Service skeleton

.travis.yml

@@ -0,0 +1,20 @@
+dist: trusty
+sudo: required
+language: python
+python:
+  - 3.6
+services:
+  - docker
+# env:
+#   global:
+
+before_install:
+  - sudo apt-get -qq update
+  - pip install coveralls
+
+install:
+  - pip install -r requirements.txt
+  - pip install -r dev-requirements.txt
+
+script:
+  - make test

Makefile

@@ -0,0 +1,17 @@
+install:
+	pip install -r requirements.txt
+
+build-docs:
+	-rm -r docs
+	-rm -r docsource/internal_apis
+	mkdir -p docs
+	sphinx-apidoc --separate -o docsource/internal_apis src
+
+test:
+	# flake8 feeds
+	pytest --verbose test --cov feeds
+
+start:
+	gunicorn --worker-class gevent --timeout 300 --workers 10 --bind :5000 feeds.server:app
+
+.PHONY: test

README.md

@@ -1,2 +1,17 @@
-# feeds
+# Feeds
 A service to manage event feeds that will get served to KBase users.
+
+## Install
+```
+make
+```
+
+## Start the server
+```
+make start
+```
+
+## Run tests
+```
+make test
+```

deploy.cfg.example

@@ -0,0 +1,6 @@
+[feeds]
+redis-host=localhost
+redis-port=6379
+redis-user=
+redis-pw=
+auth-url=https://ci.kbase.us/services/auth

dev-requirements.txt

@@ -0,0 +1,4 @@
+coverage==4.5.1
+pytest-cov==2.6.0
+flake8==3.5.0
+pytest==3.8.2

feeds/activity/base.py

@@ -0,0 +1,6 @@
+class BaseActivity(object):
+    """
+    Common parent class for Activity and Notification.
+    Activity will be done later. But a Notification is an Activity.
+    """
+    pass

feeds/activity/notification.py

@@ -0,0 +1,62 @@
+from .base import BaseActivity
+import uuid
+import json
+from datetime import datetime
+from ..util import epoch_ms
+from .. import verbs
+
+class Notification(BaseActivity):
+    def __init__(self, actor, verb, note_object, source, target=None, context={}):
+        """
+        A notification is roughly of this form:
+            actor, verb, object, (target)
+        (with target optional)
+        for example, If I share a narrative (workspace) with another user, that
+        would be the overall activity:
+            wjriehl shared narrative XYZ with you.
+        or, broken down:
+            actor: wjriehl
+            verb: share
+            object: narrative xyz
+            target: you (another user)
+
+        :param actor: user id of the actor (or 'kbase').
+        :param verb: type of note, uses standard activity streams verbs, plus some others.
+            This is either a string or a Verb. A MissingVerbError will be raised if it's a string
+            and not in the list.
+        :param note_object: object of the note. Should be a string. Examples:
+            a Narrative name
+            a workspace id
+            a group name
+        :param source: source service for the note. String.
+        :param target: target of the note. Optional. Should be a user id or group id if present.
+        :param context: freeform context of the note. key-value pairs.
+
+        TODO:
+            * decide on global ids for admin use
+            * validate actor = real kbase id (or special)
+            * validate type is valid
+            * validate object is valid
+            * validate target is valid
+            * validate context fits
+        """
+        self.actor = actor
+        self.verb = verbs.translate_verb(verb)
+        self.object = note_object
+        self.source = source
+        self.target = target
+        self.context = context
+        self.time = epoch_ms()  # int timestamp down to millisecond
+
+    def _validate(self):
+        """
+        Validates whether the notification fields are accurate. Should be called before sending a new notification to storage.
+        """
+        self.validate_actor(self.actor)
+
+    def validate_actor(self):
+        """
+        TODO: add group validation. only users are actors for now.
+        TODO: migrate to base class for users
+        """
+        pass

feeds/auth.py

@@ -0,0 +1,112 @@
+"""
+This module handles authentication management. This mainly means:
+* validating auth tokens
+* validating user ids
+"""
+
+from .config import get_config
+import requests
+import json
+from .exceptions import (
+    InvalidTokenError,
+    TokenLookupError
+)
+from .util import epoch_ms
+from cachetools import (
+    Cache,
+    TTLCache
+)
+AUTH_URL = get_config().auth_url
+AUTH_API_PATH = '/api/V2/'
+CACHE_EXPIRE_TIME = 300  # seconds
+
+class TokenCache(TTLCache):
+    """
+    Extends the TTLCache to handle KBase auth tokens.
+    So they have a base expiration of 5 minutes,
+    but expire sooner if the token itself expires.
+    """
+    def __getitem__(self, key, cache_getitem=Cache.__getitem__):
+        token = super(TokenCache, self).__getitem__(key, cache_getitem=cache_getitem)
+        if token.get('expires', 0) < epoch_ms():
+            return self.__missing__(key)
+        else:
+            return token
+
+__token_cache = TokenCache(1000, CACHE_EXPIRE_TIME)
+
+def validate_service_token(token):
+    """
+    Validates a service token. If valid, and of type Service, returns the token name.
+    If invalid, raises an InvalidTokenError. If any other errors occur, raises
+    a TokenLookupError.
+
+    TODO: I know this is going to be rife with issues. The name of the token doesn't have
+    to be the service. But as long as it's a Service token, then it came from in KBase, so
+    everything should be ok.
+    TODO: Add 'source' to PUT notification endpoint.
+    """
+    token = __fetch_token(token)
+    if token.get('type') == 'Service':
+        return token.get('name')
+    else:
+        raise InvalidTokenError("Token is not a Service token!")
+
+def validate_user_token(token):
+    """
+    Validates a user auth token.
+    If valid, does nothing. If invalid, raises an InvalidTokenError.
+    """
+    __fetch_token(token)
+
+def validate_user_id(user_id):
+    return validate_user_ids([user_id])
+
+def validate_user_ids(user_ids):
+    """
+    Validates whether users are real or not.
+    Returns the parsed response from the server, as a dict. Each
+    key is a user that exists, each value is their user name.
+    Raises an HTTPError if something bad happens.
+    """
+    r = __auth_request('users?list={}'.format(','.join(user_ids)))
+    return json.loads(r.content)
+
+def __fetch_token(token):
+    """
+    Returns token info from the auth server. Caches it locally for a while.
+    If the token is invalid or there's any other auth problems, either
+    an InvalidTokenError or TokenLookupError gets raised.
+    """
+    fetched = __token_cache.get(token)
+    if fetched:
+        return fetched
+    else:
+        try:
+            r = __auth_request('token', token)
+            token_info = json.loads(r.content)
+            __token_cache[token] = token_info
+            return token_info
+        except requests.HTTPError as e:
+            _handle_errors(e)
+
+def __auth_request(path, token):
+    """
+    Makes a request of the auth server after cramming the token in a header.
+    Only makes GET requests, since that's all we should need.
+    """
+    headers = {'Authorization', token}
+    r = requests.get(AUTH_URL + AUTH_API_PATH + path, headers=headers)
+    # the requests that fail based on the token (401, 403) get returned for the
+    # calling function to turn into an informative error
+    # others - 404, 500 - get raised
+    r.raise_for_status()
+    return r
+
+def _handle_errors(err):
+    if err.response.status_code == 401:
+        err_content = json.loads(err.response.content)
+        err_msg = err_content.get('error', {}).get('apperror', 'Invalid token')
+        raise InvalidTokenError(msg=err_msg, http_error=err)
+    else:
+        raise TokenLookupError(http_error=err)

feeds/config.py

@@ -0,0 +1,97 @@
+import os
+import configparser
+from .exceptions import ConfigError
+import logging
+
+DEFAULT_CONFIG_PATH = "deploy.cfg"
+ENV_CONFIG_PATH = "FEEDS_CONFIG"
+ENV_CONFIG_BACKUP = "KB_DEPLOYMENT_CONFIG"
+ENV_AUTH_TOKEN = "AUTH_TOKEN"
+
+INI_SECTION = "feeds"
+DB_HOST = "redis-host"
+DB_HOST_PORT = "redis-port"
+DB_USER = "redis-user"
+DB_PW = "redis-pw"
+AUTH_URL = "auth-url"
+
+class FeedsConfig(object):
+    """
+    Loads a config set from the root deploy.cfg file. This should be in ini format.
+
+    Keys of note are:
+
+    redis-host
+    redis-port
+    redis-user
+    redis-pw
+    auth-url
+    """
+
+    def __init__(self):
+        # Look for the file. ENV_CONFIG_PATH > ENV_CONFIG_BACKUP > DEFAULT_CONFIG_PATH
+        self.auth_token = os.environ.get(ENV_AUTH_TOKEN)
+        if self.auth_token is None:
+            raise RuntimeError("The AUTH_TOKEN environment variable must be set!")
+        config_file = self._find_config_path()
+        cfg = self._load_config(config_file)
+        if not cfg.has_section(INI_SECTION):
+            raise ConfigError("Error parsing config file: section {} not found!".format(INI_SECTION))
+        self.redis_host = self._get_line(cfg, DB_HOST)
+        self.redis_port = self._get_line(cfg, DB_HOST_PORT)
+        self.redis_user = self._get_line(cfg, DB_USER, required=False)
+        self.redis_pw = self._get_line(cfg, DB_PW, required=False)
+        self.auth_url = self._get_line(cfg, AUTH_URL)
+
+    def _find_config_path(self):
+        """
+        A little helper to test whether a given file path, or one given by an environment variable, exists.
+        """
+        for env in [ENV_CONFIG_PATH, ENV_CONFIG_BACKUP]:
+            env_path = os.environ.get(env)
+            if env_path:
+                if not os.path.isfile(env_path):
+                    raise ConfigError(
+                        "Environment variable {} is set to {}, "
+                        "which is not a config file.".format(ENV_CONFIG_PATH, env_path)
+                    )
+                else:
+                    return env_path
+        if not os.path.isfile(DEFAULT_CONFIG_PATH):
+            raise ConfigError(
+                "Unable to find config file - can't start server. Either set the {} or {} "
+                "environment variable to a path, or copy 'deploy.cfg.example' to "
+                "'deploy.cfg'".format(ENV_CONFIG_PATH, ENV_CONFIG_BACKUP)
+            )
+        return DEFAULT_CONFIG_PATH
+
+    def _load_config(self, cfg_file):
+        config = configparser.ConfigParser()
+        with open(cfg_file, "r") as cfg:
+            try:
+                config.read_file(cfg)
+            except configparser.Error as e:
+                raise ConfigError("Error parsing config file {}: {}".format(cfg_file, e))
+        return config
+
+    def _get_line(self, config, key, required=True):
+        """
+        A little wrapper that raises a ConfigError if a required key isn't present.
+        """
+        val = None
+        try:
+            val = config.get(INI_SECTION, key)
+        except configparser.NoOptionError:
+            if required:
+                raise ConfigError("Required option {} not found in config".format(key))
+        if not val and required:
+            raise ConfigError("Required option {} has no value!".format(key))
+        return val
+
+__config = None
+
+def get_config():
+    global __config
+    if not __config:
+        __config = FeedsConfig()
+    return __config

feeds/exceptions.py

@@ -0,0 +1,40 @@
+from requests import HTTPError
+
+class ConfigError(Exception):
+    """
+    Raised when there's a problem with the service configuration.
+    """
+    pass
+
+class MissingVerbError(Exception):
+    """
+    Raised when trying to convert from string -> registered verb, but the string's wrong.
+    """
+    pass
+
+class InvalidTokenError(Exception):
+    """
+    Raised when finding out that a user or service auth token is invalid.
+    Wraps HTTPError.
+    """
+    def __init__(self, msg=None, http_error=None):
+        if msg is None:
+            msg = "Invalid token."
+        super(InvalidTokenError, self).__init__(msg)
+        self.http_error = http_error
+
+class TokenLookupError(Exception):
+    """
+    Raised when having problems looking up an auth token. Wraps HTTPError.
+    """
+    def __init__(self, msg=None, http_error=None):
+        if msg is None:
+            msg = "Unable to look up token information."
+        super(TokenLookupError, self).__init__(msg)
+        self.http_error = http_error
+
+class InvalidActorError(Exception):
+    """
+    Raised when an actor doesn't exist in the system as either a user or Group.
+    """
+    pass