From b5f0001b69c190a1b31d0def0666e64cde6e8064 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Tue, 9 Oct 2018 11:24:34 -0700 Subject: [PATCH 01/20] Initial commit with some boilerplate --- .travis.yml | 19 +++++++++++++++++++ Makefile | 9 +++++++++ README.md | 4 +++- dev-requirements.txt | 3 +++ feeds/__init__.py | 0 feeds/activity/__init__.py | 0 feeds/activity/base.py | 6 ++++++ feeds/activity/notification.py | 16 ++++++++++++++++ feeds/server.py | 23 +++++++++++++++++++++++ feeds/util.py | 16 ++++++++++++++++ requirements.txt | 2 ++ tests/__init__.py | 0 tests/test_notification.py | 1 + tests/test_server.py | 1 + tests/test_util.py | 4 ++++ tox.ini | 6 ++++++ 16 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 .travis.yml create mode 100644 Makefile create mode 100644 dev-requirements.txt create mode 100644 feeds/__init__.py create mode 100644 feeds/activity/__init__.py create mode 100644 feeds/activity/base.py create mode 100644 feeds/activity/notification.py create mode 100644 feeds/server.py create mode 100644 feeds/util.py create mode 100644 requirements.txt create mode 100644 tests/__init__.py create mode 100644 tests/test_notification.py create mode 100644 tests/test_server.py create mode 100644 tests/test_util.py create mode 100644 tox.ini diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..4f9cdea --- /dev/null +++ b/.travis.yml @@ -0,0 +1,19 @@ +dist: trusty +sudo: required +language: python +python: + - "3.7" +services: + - docker +# env: +# global: + +before_install: + - sudo apt-get -qq update + - pip install coveralls + +install: + - pip install requirements.txt + +script: + - make test \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1416aa3 --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +build-docs: + -rm -r docs + -rm -r docsource/internal_apis + mkdir -p docs + sphinx-apidoc --separate -o docsource/internal_apis src + +test: + flake8 src + pytest --verbose tests --cov feeds \ No newline at end of file diff --git a/README.md b/README.md index 89c6086..eff40c1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,4 @@ -# feeds +# Feeds A service to manage event feeds that will get served to KBase users. + +docker run --name feeds-redis -v feeds_data:/data -p 6379:6379 -d redis redis-server --appendonly yes \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..bbe634f --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,3 @@ +coverage==4.5.1 +pytest-cov==2.6.0 +flake8==3.5.0 diff --git a/feeds/__init__.py b/feeds/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/feeds/activity/__init__.py b/feeds/activity/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/feeds/activity/base.py b/feeds/activity/base.py new file mode 100644 index 0000000..d32f229 --- /dev/null +++ b/feeds/activity/base.py @@ -0,0 +1,6 @@ +class BaseActivity(object): + """ + Common parent class for Activity and Notification. + Activity will be done later. But a Notification is an Activity. + """ + pass diff --git a/feeds/activity/notification.py b/feeds/activity/notification.py new file mode 100644 index 0000000..c6991e3 --- /dev/null +++ b/feeds/activity/notification.py @@ -0,0 +1,16 @@ +from .base import BaseActivity + +class Notification(BaseActivity): + def __init__(self, actor, note_type, note_object, target=None, content={}): + self.actor = actor + self.note_type = note_type + self.object = note_object + self.target = target + self.content = content + + @property + def serialization_id(self): + + + def serialize(self): + return \ No newline at end of file diff --git a/feeds/server.py b/feeds/server.py new file mode 100644 index 0000000..43e563c --- /dev/null +++ b/feeds/server.py @@ -0,0 +1,23 @@ +import os +import json +from flask import ( + Flask, + request +) +app = Flask(__name__) + +@app.route('/notifications/', methods=['GET']) +def get_notifications(): + # dummy code below + max_notes = request.args.get('n', default=10, type=int) + rev_sort = request.args.get('rev', default=0, type=int) + rev_sort = False if rev_sort==0 else True + level_filter = request.args.get('f', default=None, type=str) + include_seen = request.args.get('seen', default=0, type=int) + include_seen = False if include_seen==0 else True + return json.dumps({ + "max_notes": max_notes, + "rev_sort": rev_sort, + "level_filter": level_filter, + "include_seen": include_seen + }) diff --git a/feeds/util.py b/feeds/util.py new file mode 100644 index 0000000..5901c2e --- /dev/null +++ b/feeds/util.py @@ -0,0 +1,16 @@ +import requests + +def check_user_id(user_id): + """ + Test to see if a user id is real. + Returns True if so, False if not. + """ + pass + +def check_user_ids(id_list): + """ + Test to see if user ids are all real. + Returns True if so, a list of invalid names if not. + """ + pass + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d672df1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +flask==1.0.2 +requests==2.19.1 \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_notification.py b/tests/test_notification.py new file mode 100644 index 0000000..bb377e8 --- /dev/null +++ b/tests/test_notification.py @@ -0,0 +1 @@ +import pytest \ No newline at end of file diff --git a/tests/test_server.py b/tests/test_server.py new file mode 100644 index 0000000..bb377e8 --- /dev/null +++ b/tests/test_server.py @@ -0,0 +1 @@ +import pytest \ No newline at end of file diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 0000000..1386e48 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,4 @@ +import pytest + +def test_stuff(): + assert True \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..dfc81fd --- /dev/null +++ b/tox.ini @@ -0,0 +1,6 @@ +[flake8] +max-line-length = 100 +exclude = + test/*, +putty-ignore = + feeds/*/__init__.py : F401,E126 From 268c4dcc9bd9406dd44d5b525d58b00e75e0710c Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Tue, 9 Oct 2018 11:29:10 -0700 Subject: [PATCH 02/20] change travis python version --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4f9cdea..d86fbd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ dist: trusty sudo: required language: python python: - - "3.7" + - 3.6 services: - docker # env: From 6a26997fe370b1d7f3512940205f4094da077e00 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Tue, 9 Oct 2018 11:30:58 -0700 Subject: [PATCH 03/20] more travis updates --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d86fbd8..a1359a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,8 @@ before_install: - pip install coveralls install: - - pip install requirements.txt + - pip install -r requirements.txt + - pip install -r dev-requirements.txt script: - make test \ No newline at end of file From 81f764d1a5dfb926b7959c386a92fb5347cb213c Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Tue, 9 Oct 2018 11:52:14 -0700 Subject: [PATCH 04/20] add basic notification and tests --- feeds/activity/notification.py | 13 +++++++++++-- tests/test_notification.py | 15 ++++++++++++++- tests/util.py | 13 +++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 tests/util.py diff --git a/feeds/activity/notification.py b/feeds/activity/notification.py index c6991e3..dd0dabd 100644 --- a/feeds/activity/notification.py +++ b/feeds/activity/notification.py @@ -1,16 +1,25 @@ from .base import BaseActivity +import uuid class Notification(BaseActivity): def __init__(self, actor, note_type, note_object, target=None, content={}): + """ + :param actor: user id of the actor (or kbase or global) + :param note_type: type of note, uses standard activity streams verbs + :param note_object: object of the note + :param target: target of the note + :param content: freeform content of the note + """ self.actor = actor self.note_type = note_type self.object = note_object self.target = target self.content = content + self._id = uuid.uuid4() @property - def serialization_id(self): - + def id(self): + return self._id def serialize(self): return \ No newline at end of file diff --git a/tests/test_notification.py b/tests/test_notification.py index bb377e8..0217c5f 100644 --- a/tests/test_notification.py +++ b/tests/test_notification.py @@ -1 +1,14 @@ -import pytest \ No newline at end of file +import pytest +from feeds.activity.notification import Notification +import uuid +from .util import validate_uuid + + +def test_basic_notification(): + n = Notification('foo', 'bar', 'baz') + assert n.actor == 'foo' + assert n.note_type == 'bar' + assert n.object == 'baz' + assert n.content == {} + assert n.target == None + assert validate_uuid(n.id) \ No newline at end of file diff --git a/tests/util.py b/tests/util.py new file mode 100644 index 0000000..e082f5f --- /dev/null +++ b/tests/util.py @@ -0,0 +1,13 @@ +import uuid + +def validate_uuid(test_uuid): + """ + test_uuid should be a UUID object + """ + test_str = str(test_uuid) + try: + val = uuid.UUID(test_str, version=4) + except ValueError: + return False + + return test_str == str(val) \ No newline at end of file From 0e8657606f3ff029eaf530d241c06492780e6d89 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Tue, 9 Oct 2018 11:52:44 -0700 Subject: [PATCH 05/20] rename tests -> test --- {tests => test}/__init__.py | 0 {tests => test}/test_notification.py | 0 {tests => test}/test_server.py | 0 {tests => test}/test_util.py | 0 {tests => test}/util.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename {tests => test}/__init__.py (100%) rename {tests => test}/test_notification.py (100%) rename {tests => test}/test_server.py (100%) rename {tests => test}/test_util.py (100%) rename {tests => test}/util.py (100%) diff --git a/tests/__init__.py b/test/__init__.py similarity index 100% rename from tests/__init__.py rename to test/__init__.py diff --git a/tests/test_notification.py b/test/test_notification.py similarity index 100% rename from tests/test_notification.py rename to test/test_notification.py diff --git a/tests/test_server.py b/test/test_server.py similarity index 100% rename from tests/test_server.py rename to test/test_server.py diff --git a/tests/test_util.py b/test/test_util.py similarity index 100% rename from tests/test_util.py rename to test/test_util.py diff --git a/tests/util.py b/test/util.py similarity index 100% rename from tests/util.py rename to test/util.py From a9ecdc5c5e3bde0d6d37fb617b70bf26d35083da Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Thu, 11 Oct 2018 18:54:58 -0700 Subject: [PATCH 06/20] changes. still in flight. --- Makefile | 4 +-- dev-requirements.txt | 1 + feeds/__init__.py | 2 ++ feeds/activity/base.py | 2 +- feeds/activity/notification.py | 63 ++++++++++++++++++++++++++++------ test/test_notification.py | 16 ++++----- test/test_util.py | 2 -- 7 files changed, 66 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 1416aa3..3f04ac9 100644 --- a/Makefile +++ b/Makefile @@ -5,5 +5,5 @@ build-docs: sphinx-apidoc --separate -o docsource/internal_apis src test: - flake8 src - pytest --verbose tests --cov feeds \ No newline at end of file + flake8 feeds + pytest --verbose test --cov feeds \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt index bbe634f..524d7d2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,4 @@ coverage==4.5.1 pytest-cov==2.6.0 flake8==3.5.0 +pytest==3.8.2 \ No newline at end of file diff --git a/feeds/__init__.py b/feeds/__init__.py index e69de29..acfe99c 100644 --- a/feeds/__init__.py +++ b/feeds/__init__.py @@ -0,0 +1,2 @@ +import os +os.environ['CELERY_CONFIG_MODULE'] = 'feeds.celery_config' \ No newline at end of file diff --git a/feeds/activity/base.py b/feeds/activity/base.py index d32f229..135e1b9 100644 --- a/feeds/activity/base.py +++ b/feeds/activity/base.py @@ -3,4 +3,4 @@ class BaseActivity(object): Common parent class for Activity and Notification. Activity will be done later. But a Notification is an Activity. """ - pass + pass \ No newline at end of file diff --git a/feeds/activity/notification.py b/feeds/activity/notification.py index dd0dabd..165a9cb 100644 --- a/feeds/activity/notification.py +++ b/feeds/activity/notification.py @@ -1,25 +1,66 @@ from .base import BaseActivity import uuid +import json +from datetime import datetime class Notification(BaseActivity): - def __init__(self, actor, note_type, note_object, target=None, content={}): + serialize_token = "|" + + def __init__(self, actor, verb, note_object, target=None, context={}): """ + A notification is roughly of this form - actor, verb, object, (target) + (with target optional) + for example, If I share a narrative (workspace) + :param actor: user id of the actor (or kbase or global) - :param note_type: type of note, uses standard activity streams verbs + :param verb: type of note, uses standard activity streams verbs, plus some others :param note_object: object of the note :param target: target of the note - :param content: freeform content of the note + :param context: freeform context of the note + + TODO: + * validate actor = real kbase id (or special) + * validate type is valid + * validate object is valid + * validate target is valid + * validate context fits """ self.actor = actor - self.note_type = note_type + self.verb = verb self.object = note_object self.target = target - self.content = content - self._id = uuid.uuid4() + self.context = context + self.time = int(datetime.utcnow().timestamp()*1000) # int timestamp down to millisecond + + def serialize_storage(self): + """ + Serializes this notification for storage + """ + return "{}|{}|{}|{}|{}|{}".format(self._id, self.actor, self.verb, self.object, self.target, json.dumps(self.context)) + + def serialize_transport(self): + """ + Returns a dict form of this for transport (maybe some __ function?) + """ + return { + "id": self._id, + "actor": self.actor, + "type": self.verb, + "object": self.object, + "target": self.target, + "context": self.context + } + + @classmethod + def deserialize_storage(cls, storage_str): + """ + :param storage_str: string + """ + split_str = storage_str.split('|', 5) + return cls(*split_str) + + + + - @property - def id(self): - return self._id - def serialize(self): - return \ No newline at end of file diff --git a/test/test_notification.py b/test/test_notification.py index 0217c5f..454b23b 100644 --- a/test/test_notification.py +++ b/test/test_notification.py @@ -3,12 +3,12 @@ import uuid from .util import validate_uuid - def test_basic_notification(): - n = Notification('foo', 'bar', 'baz') - assert n.actor == 'foo' - assert n.note_type == 'bar' - assert n.object == 'baz' - assert n.content == {} - assert n.target == None - assert validate_uuid(n.id) \ No newline at end of file + assert True + # n = Notification('foo', 'bar', 'baz') + # assert n.actor == 'foo' + # assert n.note_type == 'bar' + # assert n.object == 'baz' + # assert n.content == {} + # assert n.target == None + # assert validate_uuid(n.id) \ No newline at end of file diff --git a/test/test_util.py b/test/test_util.py index 1386e48..a1ace31 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,4 +1,2 @@ import pytest -def test_stuff(): - assert True \ No newline at end of file From 3274e32d7bfb68f0d4ea86053fd0acb719f3b2ec Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Thu, 11 Oct 2018 18:55:31 -0700 Subject: [PATCH 07/20] add verbs --- feeds/verbs.py | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 feeds/verbs.py diff --git a/feeds/verbs.py b/feeds/verbs.py new file mode 100644 index 0000000..4224e3a --- /dev/null +++ b/feeds/verbs.py @@ -0,0 +1,99 @@ +_verb_register = dict() + +def register(verb): + if not issubclass(verb, Verb): + raise TypeError("Can only register Verb subclasses") + if verb.id is None: + raise ValueError("A verb must have an id") + elif str(verb.id) in _verb_register: + raise ValueError("The verb id '{}' is already taken by {}".format(verb.id, _verb_register[str(verb.id)].infinitive)) + if verb.infinitive is None: + raise ValueError("A verb must have an infinitive form") + elif verb.infinitive.lower() in _verb_register: + raise ValueError("The verb '{}' is already registered!".format(verb.infinitive)) + if verb.past_tense is None: + raise ValueError("A verb must have a past tense form") + elif verb.past_tense.lower() in _verb_register: + raise ValueError("The verb '{}' is already registered!".format(verb.past_tense)) + + _verb_register.update({ + str(verb.id): verb, + verb.infinitive.lower(): verb, + verb.past_tense.lower(): verb + }) + +def get_verb(key): + # if they're both None, fail. + # otherwise, look it up. + assert key is not None + key = str(key) + if key.lower() in _verb_register: + return _verb_register[key]() + else: + raise ValueError('Verb "{}" not found.'.format(key)) + +class Verb(object): + id = None + infinitive = None + past_tense = None + + def __str__(self): + return self.infinitive + + def serialize(self): + return self.id + +class Invite(Verb): + id = 1 + infinitive = "invite" + past_tense = "invited" + +class Accept(Verb): + id = 2 + infinitive = "accept" + past_tense = "accepted" + +class Reject(Verb): + id = 3 + infinitive = "reject" + past_tense = "rejected" + +class Share(Verb): + id = 4 + infinitive = "share" + past_tense = "shared" + +class Unshare(Verb): + id = 5 + infinitive = "unshare" + past_tense = "unshared" + +class Join(Verb): + id = 6 + infinitive = "join" + past_tense = "joined" + +class Leave(Verb): + id = 7 + infinitive = "leave" + past_tense = "left" + +class Request(Verb): + id = 8 + infinitive = "request" + past_tense = "requested" + +class Update(Verb): + id = 9 + infinitive = "update" + past_tense = "updated" + +register(Invite) +register(Accept) +register(Reject) +register(Share) +register(Unshare) +register(Join) +register(Leave) +register(Request) +register(Update) From a3bc25291d6ef2f13070c97beb40390d9ea3543a Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Thu, 11 Oct 2018 18:55:51 -0700 Subject: [PATCH 08/20] add tests for verbs --- test/test_verbs.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 test/test_verbs.py diff --git a/test/test_verbs.py b/test/test_verbs.py new file mode 100644 index 0000000..d799f68 --- /dev/null +++ b/test/test_verbs.py @@ -0,0 +1,87 @@ +import pytest +from feeds import verbs + +def test_register_verb(): + class TestVerb(verbs.Verb): + id=666 + infinitive="test" + past_tense="tested" + verbs.register(TestVerb) + assert '666' in verbs._verb_register + assert verbs._verb_register['666'] == TestVerb + assert 'test' in verbs._verb_register + assert verbs._verb_register['test'] == TestVerb + assert 'tested' in verbs._verb_register + assert verbs._verb_register['tested'] == TestVerb + +def test_register_verb_fail(): + with pytest.raises(TypeError) as e: + verbs.register(str) + assert "Can only register Verb subclasses" in str(e.value) + + class CopyId(verbs.Verb): + id=1 + infinitive='fail' + past_tense='fail' + with pytest.raises(ValueError) as e: + verbs.register(CopyId) + assert "The verb id '1' is already taken" in str(e.value) + + class CopyInf(verbs.Verb): + id=1000 + infinitive='invite' + past_tense='fail' + with pytest.raises(ValueError) as e: + verbs.register(CopyInf) + assert "The verb 'invite' is already registered!" in str(e.value) + + class CopyPT(verbs.Verb): + id=1000 + infinitive='fail' + past_tense='invited' + with pytest.raises(ValueError) as e: + verbs.register(CopyPT) + assert "The verb 'invited' is already registered!" in str(e.value) + + class NoId(verbs.Verb): + infinitive='fail' + past_tense='fail' + with pytest.raises(ValueError) as e: + verbs.register(NoId) + assert "A verb must have an id" in str(e.value) + + class NoInf(verbs.Verb): + id=1000 + past_tense='fail' + with pytest.raises(ValueError) as e: + verbs.register(NoInf) + assert "A verb must have an infinitive form" in str(e.value) + + class NoPT(verbs.Verb): + id=1000 + infinitive='fail' + with pytest.raises(ValueError) as e: + verbs.register(NoPT) + assert "A verb must have a past tense form" in str(e.value) + +def test_get_verb(): + v = verbs.get_verb(1) + v2 = verbs.get_verb('invite') + v3 = verbs.get_verb('invited') + assert v.__class__ == v2.__class__ == v3.__class__ + assert v.id == 1 + assert v.infinitive == 'invite' + assert v.past_tense == 'invited' + assert str(v) == 'invite' + +def test_get_verb_fail(): + with pytest.raises(ValueError) as e: + verbs.get_verb('fail') + assert 'Verb "fail" not found' in str(e.value) + + with pytest.raises(AssertionError) as e: + verbs.get_verb(None) + +def test_serialize(): + v = verbs.get_verb('invite') + assert v.serialize() == 1 \ No newline at end of file From 2aefd7b5871045f60d4b712868e01f62178e7aa6 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Mon, 15 Oct 2018 16:34:00 -0700 Subject: [PATCH 09/20] add basic server and verbs description --- feeds/server.py | 9 +++++++++ feeds/verbs.py | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/feeds/server.py b/feeds/server.py index 43e563c..1c807fd 100644 --- a/feeds/server.py +++ b/feeds/server.py @@ -1,3 +1,4 @@ + import os import json from flask import ( @@ -8,6 +9,12 @@ @app.route('/notifications/', methods=['GET']) def get_notifications(): + """ + General flow should be: + 1. validate/authenticate user + 2. make user feed object + 3. query user feed for most recent, based on params + """ # dummy code below max_notes = request.args.get('n', default=10, type=int) rev_sort = request.args.get('rev', default=0, type=int) @@ -21,3 +28,5 @@ def get_notifications(): "level_filter": level_filter, "include_seen": include_seen }) + + diff --git a/feeds/verbs.py b/feeds/verbs.py index 4224e3a..d20b7c5 100644 --- a/feeds/verbs.py +++ b/feeds/verbs.py @@ -3,14 +3,17 @@ def register(verb): if not issubclass(verb, Verb): raise TypeError("Can only register Verb subclasses") + if verb.id is None: raise ValueError("A verb must have an id") elif str(verb.id) in _verb_register: raise ValueError("The verb id '{}' is already taken by {}".format(verb.id, _verb_register[str(verb.id)].infinitive)) + if verb.infinitive is None: raise ValueError("A verb must have an infinitive form") elif verb.infinitive.lower() in _verb_register: raise ValueError("The verb '{}' is already registered!".format(verb.infinitive)) + if verb.past_tense is None: raise ValueError("A verb must have a past tense form") elif verb.past_tense.lower() in _verb_register: @@ -26,6 +29,7 @@ def get_verb(key): # if they're both None, fail. # otherwise, look it up. assert key is not None + key = str(key) if key.lower() in _verb_register: return _verb_register[key]() From fdd3ef599816347116e3b3c918266a907bd13c83 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Mon, 15 Oct 2018 18:10:51 -0700 Subject: [PATCH 10/20] add server skeleton --- README.md | 4 +- feeds/server.py | 110 +++++++++++++++++++++++++++++++++++++----------- feeds/util.py | 3 ++ 3 files changed, 90 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index eff40c1..39e4340 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,2 @@ # Feeds -A service to manage event feeds that will get served to KBase users. - -docker run --name feeds-redis -v feeds_data:/data -p 6379:6379 -d redis redis-server --appendonly yes \ No newline at end of file +A service to manage event feeds that will get served to KBase users. \ No newline at end of file diff --git a/feeds/server.py b/feeds/server.py index 1c807fd..94caed7 100644 --- a/feeds/server.py +++ b/feeds/server.py @@ -1,32 +1,94 @@ - import os import json +import flask from flask import ( Flask, request ) -app = Flask(__name__) - -@app.route('/notifications/', methods=['GET']) -def get_notifications(): - """ - General flow should be: - 1. validate/authenticate user - 2. make user feed object - 3. query user feed for most recent, based on params - """ - # dummy code below - max_notes = request.args.get('n', default=10, type=int) - rev_sort = request.args.get('rev', default=0, type=int) - rev_sort = False if rev_sort==0 else True - level_filter = request.args.get('f', default=None, type=str) - include_seen = request.args.get('seen', default=0, type=int) - include_seen = False if include_seen==0 else True - return json.dumps({ - "max_notes": max_notes, - "rev_sort": rev_sort, - "level_filter": level_filter, - "include_seen": include_seen - }) +from flask.logging import default_handler +from .util import epoch_ms +import logging + +VERSION = "0.0.1" + +def _initialize_logging(): + root = logging.getLogger() + root.addHandler(default_handler) + root.setLevel('INFO') + +def _log(msg, *args): + logging.getLogger(__name__).info(msg, *args) + +def create_app(test_config=None): + _initialize_logging() + + app = Flask(__name__, instance_relative_config=True) + if test_config is None: + app.config.from_pyfile('config.py', silent=True) + else: + app.config.from_mapping(test_config) + + @app.route('/', methods=['GET']) + def root(): + return flask.jsonify({ + "service": "Notification Feeds Service", + "version": VERSION, + "servertime": epoch_ms() + }) + + @app.route('/api/V1/notifications/', methods=['GET']) + def get_notifications(): + """ + General flow should be: + 1. validate/authenticate user + 2. make user feed object + 3. query user feed for most recent, based on params + """ + # dummy code below + max_notes = request.args.get('n', default=10, type=int) + rev_sort = request.args.get('rev', default=0, type=int) + rev_sort = False if rev_sort==0 else True + level_filter = request.args.get('f', default=None, type=str) + include_seen = request.args.get('seen', default=0, type=int) + include_seen = False if include_seen==0 else True + return json.dumps({ + "max_notes": max_notes, + "rev_sort": rev_sort, + "level_filter": level_filter, + "include_seen": include_seen + }) + + @app.route('/api/V1/notification/', methods=['GET']) + def get_single_notification(note_id): + raise NotImplementedError() + + @app.route('/api/V1/notifications/unsee/', methods=['POST']) + def mark_notifications_unseen(): + """Form data should have a list of notification ids to mark as unseen""" + raise NotImplementedError() + + @app.route('/api/V1/notifications/see/', methods=['POST']) + def mark_notifications_seen(): + """Form data should have a list of notifications to mark as seen""" + raise NotImplementedError() + + @app.route('/api/V1/notification/', methods=['PUT']) + def add_notification(): + """ + Adds a new notification for other users to see. + Form data requires the following: + * `actor` - a user or org id. + * `type` - one of the type keywords (see below, TBD (as of 10/8)) + * `target` - optional, a user or org id. - always receives this notification + * `object` - object of the notice. For invitations, the group to be invited to. For narratives, the narrative UPA. + * `level` - alert, error, warning, or request. + * `content` - optional, content of the notification, otherwise it'll be autogenerated from the info above. + * `global` - true or false. If true, gets added to the global notification feed and everyone gets a copy. + + This also requires a service token as an Authorization header. Once validated, will be used + as the Source of the notification, and used in logic to determine which feeds get notified. + """ + raise NotImplementedError() + return app diff --git a/feeds/util.py b/feeds/util.py index 5901c2e..a6f1f0c 100644 --- a/feeds/util.py +++ b/feeds/util.py @@ -1,4 +1,5 @@ import requests +import time def check_user_id(user_id): """ @@ -14,3 +15,5 @@ def check_user_ids(id_list): """ pass +def epoch_ms(): + return int(time.time()*1000) \ No newline at end of file From d1db39b34ffb0574e9141fc9cbe607874c532785 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Tue, 16 Oct 2018 16:17:29 -0700 Subject: [PATCH 11/20] add config and some tests for it. --- Makefile | 7 +++- README.md | 17 +++++++++- deploy.cfg.example | 6 ++++ feeds/__init__.py | 4 ++- feeds/config.py | 82 +++++++++++++++++++++++++++++++++++++++++++++ feeds/exceptions.py | 5 +++ feeds/server.py | 10 ++++-- requirements.txt | 4 ++- test/test_config.py | 55 ++++++++++++++++++++++++++++++ 9 files changed, 184 insertions(+), 6 deletions(-) create mode 100644 deploy.cfg.example create mode 100644 feeds/config.py create mode 100644 feeds/exceptions.py create mode 100644 test/test_config.py diff --git a/Makefile b/Makefile index 3f04ac9..3fe3ff4 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ +install: + pip install -r requirements.txt + build-docs: -rm -r docs -rm -r docsource/internal_apis @@ -6,4 +9,6 @@ build-docs: test: flake8 feeds - pytest --verbose test --cov feeds \ No newline at end of file + pytest --verbose test --cov feeds + +start: all \ No newline at end of file diff --git a/README.md b/README.md index 39e4340..e0c96b5 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,17 @@ # Feeds -A service to manage event feeds that will get served to KBase users. \ No newline at end of file +A service to manage event feeds that will get served to KBase users. + +## Install +``` +make +``` + +## Start the server +``` +make start +``` + +## Run tests +``` +make test +``` diff --git a/deploy.cfg.example b/deploy.cfg.example new file mode 100644 index 0000000..75f8542 --- /dev/null +++ b/deploy.cfg.example @@ -0,0 +1,6 @@ +[feeds] +redis-host=localhost +redis-port=6379 +redis-user= +redis-pw= +auth-url=https://ci.kbase.us/services/auth \ No newline at end of file diff --git a/feeds/__init__.py b/feeds/__init__.py index acfe99c..c8afb6e 100644 --- a/feeds/__init__.py +++ b/feeds/__init__.py @@ -1,2 +1,4 @@ import os -os.environ['CELERY_CONFIG_MODULE'] = 'feeds.celery_config' \ No newline at end of file +from .server import create_app + +app = create_app() \ No newline at end of file diff --git a/feeds/config.py b/feeds/config.py new file mode 100644 index 0000000..e6f0338 --- /dev/null +++ b/feeds/config.py @@ -0,0 +1,82 @@ +import os +import configparser +from .exceptions import ConfigError +import logging + +DEFAULT_CONFIG_PATH = "deploy.cfg" +ENV_CONFIG_PATH = "FEEDS_CONFIG" +ENV_CONFIG_BACKUP = "KB_DEPLOYMENT_CONFIG" +INI_SECTION = "feeds" +DB_HOST = "redis-host" +DB_HOST_PORT = "redis-port" +DB_USER = "redis-user" +DB_PW = "redis-pw" +AUTH_URL = "auth-url" + + +class FeedsConfig(object): + """ + Loads a config set from the root deploy.cfg file. This should be in ini format. + + Keys of note are: + + redis-host + redis-port + redis-user + redis-pw + auth-url + """ + + def __init__(self): + # Look for the file. ENV_CONFIG_PATH > ENV_CONFIG_BACKUP > DEFAULT_CONFIG_PATH + config_file = self._find_config_path() + cfg = self._load_config(config_file) + if not cfg.has_section(INI_SECTION): + raise ConfigError("Error parsing config file: section {} not found!".format(INI_SECTION)) + self.redis_host = self._get_line(cfg, DB_HOST) + self.redis_port = self._get_line(cfg, DB_HOST_PORT) + self.redis_user = self._get_line(cfg, DB_USER, required=False) + self.redis_pw = self._get_line(cfg, DB_PW, required=False) + self.auth_url = self._get_line(cfg, AUTH_URL) + + def _find_config_path(self): + """ + A little helper to test whether a given file path, or one given by an environment variable, exists. + """ + for env in [ENV_CONFIG_PATH, ENV_CONFIG_BACKUP]: + env_path = os.environ.get(env) + if env_path: + if not os.path.isfile(env_path): + raise ConfigError("Environment variable {} is set to {}, which is not a config file.".format(ENV_CONFIG_PATH, env_path)) + else: + return env_path + if not os.path.isfile(DEFAULT_CONFIG_PATH): + raise ConfigError( + "Unable to find config file - can't start server. Either set the {} or {} " + "environment variable to a path, or copy 'deploy.cfg.example' to " + "'deploy.cfg'".format(ENV_CONFIG_PATH, ENV_CONFIG_BACKUP) + ) + return DEFAULT_CONFIG_PATH + + def _load_config(self, cfg_file): + config = configparser.ConfigParser() + with open(cfg_file, "r") as cfg: + try: + config.read_file(cfg) + except configparser.Error as e: + raise ConfigError("Error parsing config file {}: {}".format(cfg_file, e)) + return config + + def _get_line(self, config, key, required=True): + """ + A little wrapper that raises a ConfigError if a required key isn't present. + """ + val = None + try: + val = config.get(INI_SECTION, key) + except configparser.NoOptionError: + if required: + raise ConfigError("Required option {} not found in config".format(key)) + if not val and required: + raise ConfigError("Required option {} has no value!".format(key)) + return val \ No newline at end of file diff --git a/feeds/exceptions.py b/feeds/exceptions.py new file mode 100644 index 0000000..196cab0 --- /dev/null +++ b/feeds/exceptions.py @@ -0,0 +1,5 @@ +class ConfigError(Exception): + """ + Raised when there's a problem with the service configuration. + """ + pass diff --git a/feeds/server.py b/feeds/server.py index 94caed7..c0a5245 100644 --- a/feeds/server.py +++ b/feeds/server.py @@ -7,6 +7,7 @@ ) from flask.logging import default_handler from .util import epoch_ms +from .config import FeedsConfig import logging VERSION = "0.0.1" @@ -16,11 +17,17 @@ def _initialize_logging(): root.addHandler(default_handler) root.setLevel('INFO') +def _initialize_config(): + # TODO - include config for: + # * database access + return FeedsConfig() + def _log(msg, *args): logging.getLogger(__name__).info(msg, *args) def create_app(test_config=None): _initialize_logging() + _initialize_config() app = Flask(__name__, instance_relative_config=True) if test_config is None: @@ -90,5 +97,4 @@ def add_notification(): """ raise NotImplementedError() - return app - + return app \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d672df1..4792e0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ flask==1.0.2 -requests==2.19.1 \ No newline at end of file +requests==2.19.1 +gunicorn==19.9.0 +gevent==1.3.7 \ No newline at end of file diff --git a/test/test_config.py b/test/test_config.py new file mode 100644 index 0000000..9bb62e3 --- /dev/null +++ b/test/test_config.py @@ -0,0 +1,55 @@ +import pytest +from feeds import config +from feeds.exceptions import ConfigError +from unittest import mock +from pathlib import Path +import os + +# TODO - more error checking + +def write_test_cfg(path, cfg_lines): + if os.path.exists(path): + raise ValueError("Not gonna overwrite some existing file with this test stuff!") + with open(path, "w") as f: + f.write("\n".join(cfg_lines)) + +def test_config_from_env_ok(): + cfg_lines = [ + '[feeds]', + 'redis-host=foo', + 'redis-port=bar', + 'auth-url=baz' + ] + cfg_path = "fake_test_config_delete_me.cfg" + write_test_cfg(cfg_path, cfg_lines) + + os.environ['FEEDS_CONFIG'] = cfg_path + cfg = config.FeedsConfig() + assert cfg.auth_url == 'baz' + assert cfg.redis_host == 'foo' + assert cfg.redis_port == 'bar' + + del os.environ['FEEDS_CONFIG'] + os.environ['KB_DEPLOYMENT_CONFIG'] = cfg_path + cfg = config.FeedsConfig() + assert cfg.auth_url == 'baz' + assert cfg.redis_host == 'foo' + assert cfg.redis_port == 'bar' + + del os.environ['KB_DEPLOYMENT_CONFIG'] + os.remove(cfg_path) + +def test_config_from_env_errors(): + cfg_lines = [ + '[not-feeds]', + 'redis-host=foo' + ] + cfg_path = "fake_test_config_delete_me.cfg" + write_test_cfg(cfg_path, cfg_lines) + os.environ['FEEDS_CONFIG'] = cfg_path + with pytest.raises(ConfigError) as e: + config.FeedsConfig() + assert "Error parsing config file: section feeds not found!" in str(e.value) + + del os.environ['FEEDS_CONFIG'] + os.remove(cfg_path) \ No newline at end of file From 4d5e62e33a0893eaec8eb1cf397b035d5a2c3897 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Tue, 16 Oct 2018 16:18:44 -0700 Subject: [PATCH 12/20] clear out test/util.py --- test/util.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/test/util.py b/test/util.py index e082f5f..e69de29 100644 --- a/test/util.py +++ b/test/util.py @@ -1,13 +0,0 @@ -import uuid - -def validate_uuid(test_uuid): - """ - test_uuid should be a UUID object - """ - test_str = str(test_uuid) - try: - val = uuid.UUID(test_str, version=4) - except ValueError: - return False - - return test_str == str(val) \ No newline at end of file From a061f2db39bebec2668df183e0f1f9c2a9dd6761 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Tue, 16 Oct 2018 16:39:26 -0700 Subject: [PATCH 13/20] cleanup --- feeds/util.py | 2 +- test/test_notification.py | 1 - test/test_util.py | 7 +++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/feeds/util.py b/feeds/util.py index a6f1f0c..a4a0a2f 100644 --- a/feeds/util.py +++ b/feeds/util.py @@ -16,4 +16,4 @@ def check_user_ids(id_list): pass def epoch_ms(): - return int(time.time()*1000) \ No newline at end of file + return int(round(time.time()*1000)) \ No newline at end of file diff --git a/test/test_notification.py b/test/test_notification.py index 454b23b..415cff3 100644 --- a/test/test_notification.py +++ b/test/test_notification.py @@ -1,7 +1,6 @@ import pytest from feeds.activity.notification import Notification import uuid -from .util import validate_uuid def test_basic_notification(): assert True diff --git a/test/test_util.py b/test/test_util.py index a1ace31..206bc3a 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,2 +1,9 @@ import pytest +from feeds.util import epoch_ms +def test_epoch_ms(): + ''' + Just make sure it's an int with 13 digits. Be more rigorous later. At least, before year 2287. + ''' + t = epoch_ms() + assert len(str(t)) == 13 \ No newline at end of file From f688c88d436be35649189c9657cf27b4312e0bc9 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Tue, 16 Oct 2018 16:40:36 -0700 Subject: [PATCH 14/20] added startup to makefile --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3fe3ff4..c425ef6 100644 --- a/Makefile +++ b/Makefile @@ -11,4 +11,5 @@ test: flake8 feeds pytest --verbose test --cov feeds -start: all \ No newline at end of file +start: + gunicorn --worker-class gevent --timeout 300 --workers 10 --bind :5000 feeds:app \ No newline at end of file From b93efc94d3a02675aa963006c56499565339cfb4 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Wed, 17 Oct 2018 10:24:53 -0700 Subject: [PATCH 15/20] fix word-wrapping --- Makefile | 6 ++++-- feeds/config.py | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c425ef6..12a96c2 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,10 @@ build-docs: sphinx-apidoc --separate -o docsource/internal_apis src test: - flake8 feeds + # flake8 feeds pytest --verbose test --cov feeds start: - gunicorn --worker-class gevent --timeout 300 --workers 10 --bind :5000 feeds:app \ No newline at end of file + gunicorn --worker-class gevent --timeout 300 --workers 10 --bind :5000 feeds:app + +.PHONY: test \ No newline at end of file diff --git a/feeds/config.py b/feeds/config.py index e6f0338..87ef532 100644 --- a/feeds/config.py +++ b/feeds/config.py @@ -47,7 +47,10 @@ def _find_config_path(self): env_path = os.environ.get(env) if env_path: if not os.path.isfile(env_path): - raise ConfigError("Environment variable {} is set to {}, which is not a config file.".format(ENV_CONFIG_PATH, env_path)) + raise ConfigError( + "Environment variable {} is set to {}, " + "which is not a config file.".format(ENV_CONFIG_PATH, env_path) + ) else: return env_path if not os.path.isfile(DEFAULT_CONFIG_PATH): From f51b771129ca2c0a94f55986b72f600572684e40 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Wed, 17 Oct 2018 10:52:07 -0700 Subject: [PATCH 16/20] Change entrypoint. --- Makefile | 2 +- feeds/__init__.py | 4 ---- feeds/server.py | 4 +++- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 12a96c2..df3bf04 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,6 @@ test: pytest --verbose test --cov feeds start: - gunicorn --worker-class gevent --timeout 300 --workers 10 --bind :5000 feeds:app + gunicorn --worker-class gevent --timeout 300 --workers 10 --bind :5000 feeds.server:app .PHONY: test \ No newline at end of file diff --git a/feeds/__init__.py b/feeds/__init__.py index c8afb6e..e69de29 100644 --- a/feeds/__init__.py +++ b/feeds/__init__.py @@ -1,4 +0,0 @@ -import os -from .server import create_app - -app = create_app() \ No newline at end of file diff --git a/feeds/server.py b/feeds/server.py index c0a5245..e01cd5c 100644 --- a/feeds/server.py +++ b/feeds/server.py @@ -97,4 +97,6 @@ def add_notification(): """ raise NotImplementedError() - return app \ No newline at end of file + return app + +app = create_app() \ No newline at end of file From 4b036fd8fa2186cbc0963759bcf2234396be2484 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Wed, 17 Oct 2018 17:49:33 -0700 Subject: [PATCH 17/20] more text and MissingVerbError --- feeds/activity/notification.py | 40 ++++++++++++---------------------- feeds/exceptions.py | 6 +++++ feeds/util.py | 4 ++-- feeds/verbs.py | 4 +++- test/test_verbs.py | 3 ++- 5 files changed, 27 insertions(+), 30 deletions(-) diff --git a/feeds/activity/notification.py b/feeds/activity/notification.py index 165a9cb..de3aded 100644 --- a/feeds/activity/notification.py +++ b/feeds/activity/notification.py @@ -2,18 +2,26 @@ import uuid import json from datetime import datetime +from ..util import epoch_ms +from .. import verbs class Notification(BaseActivity): - serialize_token = "|" - def __init__(self, actor, verb, note_object, target=None, context={}): """ - A notification is roughly of this form - actor, verb, object, (target) + A notification is roughly of this form: + actor, verb, object, (target) (with target optional) - for example, If I share a narrative (workspace) + for example, If I share a narrative (workspace) with another user, that + would be the overall activity: + wjriehl shared narrative XYZ with you. + or, broken down: + actor: wjriehl + verb: share + object: narrative xyz + target: you (another user) :param actor: user id of the actor (or kbase or global) - :param verb: type of note, uses standard activity streams verbs, plus some others + :param verb: type of note, uses standard activity streams verbs, plus some others. This is either a string or a Verb. :param note_object: object of the note :param target: target of the note :param context: freeform context of the note @@ -30,13 +38,7 @@ def __init__(self, actor, verb, note_object, target=None, context={}): self.object = note_object self.target = target self.context = context - self.time = int(datetime.utcnow().timestamp()*1000) # int timestamp down to millisecond - - def serialize_storage(self): - """ - Serializes this notification for storage - """ - return "{}|{}|{}|{}|{}|{}".format(self._id, self.actor, self.verb, self.object, self.target, json.dumps(self.context)) + self.time = epoch_ms() # int timestamp down to millisecond def serialize_transport(self): """ @@ -50,17 +52,3 @@ def serialize_transport(self): "target": self.target, "context": self.context } - - @classmethod - def deserialize_storage(cls, storage_str): - """ - :param storage_str: string - """ - split_str = storage_str.split('|', 5) - return cls(*split_str) - - - - - - diff --git a/feeds/exceptions.py b/feeds/exceptions.py index 196cab0..75ac61d 100644 --- a/feeds/exceptions.py +++ b/feeds/exceptions.py @@ -3,3 +3,9 @@ class ConfigError(Exception): Raised when there's a problem with the service configuration. """ pass + +class MissingVerbError(Exception): + """ + Raised when trying to convert from string -> registered verb, but the string's wrong. + """ + pass \ No newline at end of file diff --git a/feeds/util.py b/feeds/util.py index a4a0a2f..bbc7c9d 100644 --- a/feeds/util.py +++ b/feeds/util.py @@ -1,5 +1,5 @@ import requests -import time +from datetime import datetime def check_user_id(user_id): """ @@ -16,4 +16,4 @@ def check_user_ids(id_list): pass def epoch_ms(): - return int(round(time.time()*1000)) \ No newline at end of file + return int(datetime.utcnow().timestamp()*1000) \ No newline at end of file diff --git a/feeds/verbs.py b/feeds/verbs.py index d20b7c5..01029a9 100644 --- a/feeds/verbs.py +++ b/feeds/verbs.py @@ -1,3 +1,5 @@ +from .exceptions import MissingVerbError + _verb_register = dict() def register(verb): @@ -34,7 +36,7 @@ def get_verb(key): if key.lower() in _verb_register: return _verb_register[key]() else: - raise ValueError('Verb "{}" not found.'.format(key)) + raise MissingVerbError('Verb "{}" not found.'.format(key)) class Verb(object): id = None diff --git a/test/test_verbs.py b/test/test_verbs.py index d799f68..a135535 100644 --- a/test/test_verbs.py +++ b/test/test_verbs.py @@ -1,5 +1,6 @@ import pytest from feeds import verbs +from feeds.exceptions import MissingVerbError def test_register_verb(): class TestVerb(verbs.Verb): @@ -75,7 +76,7 @@ def test_get_verb(): assert str(v) == 'invite' def test_get_verb_fail(): - with pytest.raises(ValueError) as e: + with pytest.raises(MissingVerbError) as e: verbs.get_verb('fail') assert 'Verb "fail" not found' in str(e.value) From 9f72629e4409461a1f075ad3d3bb28fa62cd461e Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Wed, 17 Oct 2018 21:04:24 -0700 Subject: [PATCH 18/20] updates to auth exceptions and tests --- feeds/activity/notification.py | 42 +++++++++------ feeds/auth.py | 99 ++++++++++++++++++++++++++++++++++ feeds/config.py | 16 +++++- feeds/exceptions.py | 29 ++++++++++ feeds/util.py | 15 ------ feeds/verbs.py | 16 ++++++ requirements.txt | 3 +- test/test_config.py | 35 +++++++++++- 8 files changed, 219 insertions(+), 36 deletions(-) create mode 100644 feeds/auth.py diff --git a/feeds/activity/notification.py b/feeds/activity/notification.py index de3aded..e5b273e 100644 --- a/feeds/activity/notification.py +++ b/feeds/activity/notification.py @@ -6,7 +6,7 @@ from .. import verbs class Notification(BaseActivity): - def __init__(self, actor, verb, note_object, target=None, context={}): + def __init__(self, actor, verb, note_object, source, target=None, context={}): """ A notification is roughly of this form: actor, verb, object, (target) @@ -20,13 +20,20 @@ def __init__(self, actor, verb, note_object, target=None, context={}): object: narrative xyz target: you (another user) - :param actor: user id of the actor (or kbase or global) - :param verb: type of note, uses standard activity streams verbs, plus some others. This is either a string or a Verb. - :param note_object: object of the note - :param target: target of the note - :param context: freeform context of the note + :param actor: user id of the actor (or 'kbase'). + :param verb: type of note, uses standard activity streams verbs, plus some others. + This is either a string or a Verb. A MissingVerbError will be raised if it's a string + and not in the list. + :param note_object: object of the note. Should be a string. Examples: + a Narrative name + a workspace id + a group name + :param source: source service for the note. String. + :param target: target of the note. Optional. Should be a user id or group id if present. + :param context: freeform context of the note. key-value pairs. TODO: + * decide on global ids for admin use * validate actor = real kbase id (or special) * validate type is valid * validate object is valid @@ -34,21 +41,22 @@ def __init__(self, actor, verb, note_object, target=None, context={}): * validate context fits """ self.actor = actor - self.verb = verb + self.verb = verbs.translate_verb(verb) self.object = note_object + self.source = source self.target = target self.context = context self.time = epoch_ms() # int timestamp down to millisecond - def serialize_transport(self): + def _validate(self): """ - Returns a dict form of this for transport (maybe some __ function?) + Validates whether the notification fields are accurate. Should be called before sending a new notification to storage. """ - return { - "id": self._id, - "actor": self.actor, - "type": self.verb, - "object": self.object, - "target": self.target, - "context": self.context - } + self.validate_actor(self.actor) + + def validate_actor(self): + """ + TODO: add group validation. only users are actors for now. + TODO: migrate to base class for users + """ + pass \ No newline at end of file diff --git a/feeds/auth.py b/feeds/auth.py new file mode 100644 index 0000000..172076e --- /dev/null +++ b/feeds/auth.py @@ -0,0 +1,99 @@ +""" +This module handles authentication management. This mainly means: +* validating auth tokens +* validating user ids +""" + +from .config import config +import requests +import json +from .exceptions import ( + InvalidTokenError, + TokenLookupError +) +from .util import epoch_ms +from cachetools import ( + Cache, + TTLCache +) +AUTH_URL = config.auth_url +AUTH_API_PATH = '/api/V2/' +CACHE_EXPIRE_TIME = 300 # seconds + +class TokenCache(TTLCache): + """ + Extends the TTLCache to handle KBase auth tokens. + So they have a base expiration of 5 minutes, + but expire sooner if the token itself expires. + """ + def __getitem__(self, key, cache_getitem=Cache.__getitem__): + token = super(TokenCache, self).__getitem__(key, cache_getitem=cache_getitem) + if token.get('expires', 0) < epoch_ms(): + return self.__missing__(key) + else: + return token + +__token_cache = TokenCache(1000, CACHE_EXPIRE_TIME) + +def validate_service_token(token): + """ + Validates a service token. If valid, and of type Service, returns the token name. + If invalid, raises an InvalidTokenError. If any other errors occur, raises + a TokenLookupError. + + TODO: I know this is going to be rife with issues. The name of the token doesn't have + to be the service. But as long as it's a Service token, then it came from in KBase, so + everything should be ok. + TODO: Add 'source' to PUT notification endpoint. + """ + token = __fetch_token(token) + if token.get('type') == 'Service': + return token.get('name') + else: + raise InvalidTokenError("Token is not a Service token!") + +def valid_user_token(token): + """ + Validates a user auth token. + If valid, does nothing. If invalid, raises an InvalidTokenError. + """ + __fetch_token(token) + +def __fetch_token(token): + """ + Returns token info from the auth server. Caches it locally for a while. + If the token is invalid or there's any other auth problems, either + an InvalidTokenError or TokenLookupError gets raised. + """ + fetched = __token_cache.get(token) + if fetched: + return fetched + else: + try: + r = __auth_request('token', token) + token_info = json.loads(r.content) + __token_cache[token] = token_info + return token_info + except requests.HTTPError as e: + _handle_errors(e) + +def __auth_request(path, token): + """ + Makes a request of the auth server after cramming the token in a header. + Only makes GET requests, since that's all we should need. + """ + headers = {'Authorization', token} + r = requests.get(AUTH_URL + AUTH_API_PATH + path, headers=headers) + # the requests that fail based on the token (401, 403) get returned for the + # calling function to turn into an informative error + # others - 404, 500 - get raised + r.raise_for_status() + return r + +def _handle_errors(err): + if err.response.status_code == 401: + err_content = json.loads(err.response.content) + err_msg = err_content.get('error', {}).get('apperror', 'Invalid token') + raise InvalidTokenError(msg=err_msg, http_error=err) + else: + raise TokenLookupError(http_error=err) diff --git a/feeds/config.py b/feeds/config.py index 87ef532..97528eb 100644 --- a/feeds/config.py +++ b/feeds/config.py @@ -6,6 +6,8 @@ DEFAULT_CONFIG_PATH = "deploy.cfg" ENV_CONFIG_PATH = "FEEDS_CONFIG" ENV_CONFIG_BACKUP = "KB_DEPLOYMENT_CONFIG" +ENV_AUTH_TOKEN = "AUTH_TOKEN" + INI_SECTION = "feeds" DB_HOST = "redis-host" DB_HOST_PORT = "redis-port" @@ -13,7 +15,6 @@ DB_PW = "redis-pw" AUTH_URL = "auth-url" - class FeedsConfig(object): """ Loads a config set from the root deploy.cfg file. This should be in ini format. @@ -29,6 +30,9 @@ class FeedsConfig(object): def __init__(self): # Look for the file. ENV_CONFIG_PATH > ENV_CONFIG_BACKUP > DEFAULT_CONFIG_PATH + self.auth_token = os.environ.get(ENV_AUTH_TOKEN) + if self.auth_token is None: + raise RuntimeError("The AUTH_TOKEN environment variable must be set!") config_file = self._find_config_path() cfg = self._load_config(config_file) if not cfg.has_section(INI_SECTION): @@ -82,4 +86,12 @@ def _get_line(self, config, key, required=True): raise ConfigError("Required option {} not found in config".format(key)) if not val and required: raise ConfigError("Required option {} has no value!".format(key)) - return val \ No newline at end of file + return val + +__config = None + +def get_config(): + global __config + if not __config: + __config = FeedsConfig() + return __config \ No newline at end of file diff --git a/feeds/exceptions.py b/feeds/exceptions.py index 75ac61d..80df8fe 100644 --- a/feeds/exceptions.py +++ b/feeds/exceptions.py @@ -1,3 +1,5 @@ +from requests import HTTPError + class ConfigError(Exception): """ Raised when there's a problem with the service configuration. @@ -8,4 +10,31 @@ class MissingVerbError(Exception): """ Raised when trying to convert from string -> registered verb, but the string's wrong. """ + pass + +class InvalidTokenError(Exception): + """ + Raised when finding out that a user or service auth token is invalid. + Wraps HTTPError. + """ + def __init__(self, msg=None, http_error=None): + if msg is None: + msg = "Invalid token." + super(InvalidTokenError, self).__init__(msg) + self.http_error = http_error + +class TokenLookupError(Exception): + """ + Raised when having problems looking up an auth token. Wraps HTTPError. + """ + def __init__(self, msg=None, http_error=None): + if msg is None: + msg = "Unable to look up token information." + super(TokenLookupError, self).__init__(msg) + self.http_error = http_error + +class InvalidActorError(Exception): + """ + Raised when an actor doesn't exist in the system as either a user or Group. + """ pass \ No newline at end of file diff --git a/feeds/util.py b/feeds/util.py index bbc7c9d..f58cfa2 100644 --- a/feeds/util.py +++ b/feeds/util.py @@ -1,19 +1,4 @@ -import requests from datetime import datetime -def check_user_id(user_id): - """ - Test to see if a user id is real. - Returns True if so, False if not. - """ - pass - -def check_user_ids(id_list): - """ - Test to see if user ids are all real. - Returns True if so, a list of invalid names if not. - """ - pass - def epoch_ms(): return int(datetime.utcnow().timestamp()*1000) \ No newline at end of file diff --git a/feeds/verbs.py b/feeds/verbs.py index 01029a9..9deda5c 100644 --- a/feeds/verbs.py +++ b/feeds/verbs.py @@ -27,6 +27,22 @@ def register(verb): verb.past_tense.lower(): verb }) +def translate_verb(verb): + """ + Translates a given verb into a verb object. + 4 cases - + - if it's a string, return get_verb (let the MissingVerbError rise) + - if it's a verb, but not registered, raise a MissingVerbError + - if it's a verb that's registered, return it + - if it's not a Verb or a str, raise a TypeError + """ + if isinstance(verb, str): + return get_verb(verb) + elif issubclass(verb, Verb): + return get_verb(verb.infinitive) + else: + raise TypeError("Must be either a subclass of Verb or a string.") + def get_verb(key): # if they're both None, fail. # otherwise, look it up. diff --git a/requirements.txt b/requirements.txt index 4792e0e..b7ccd54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ flask==1.0.2 requests==2.19.1 gunicorn==19.9.0 -gevent==1.3.7 \ No newline at end of file +gevent==1.3.7 +cachetools=2.1.0 \ No newline at end of file diff --git a/test/test_config.py b/test/test_config.py index 9bb62e3..9c7c581 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -7,6 +7,8 @@ # TODO - more error checking +FAKE_AUTH_TOKEN = "I'm an auth token!" + def write_test_cfg(path, cfg_lines): if os.path.exists(path): raise ValueError("Not gonna overwrite some existing file with this test stuff!") @@ -23,6 +25,8 @@ def test_config_from_env_ok(): cfg_path = "fake_test_config_delete_me.cfg" write_test_cfg(cfg_path, cfg_lines) + os.environ['AUTH_TOKEN'] = FAKE_AUTH_TOKEN + os.environ['FEEDS_CONFIG'] = cfg_path cfg = config.FeedsConfig() assert cfg.auth_url == 'baz' @@ -37,9 +41,11 @@ def test_config_from_env_ok(): assert cfg.redis_port == 'bar' del os.environ['KB_DEPLOYMENT_CONFIG'] + del os.environ['AUTH_TOKEN'] os.remove(cfg_path) def test_config_from_env_errors(): + os.environ['AUTH_TOKEN'] = FAKE_AUTH_TOKEN cfg_lines = [ '[not-feeds]', 'redis-host=foo' @@ -51,5 +57,32 @@ def test_config_from_env_errors(): config.FeedsConfig() assert "Error parsing config file: section feeds not found!" in str(e.value) + del os.environ['AUTH_TOKEN'] + del os.environ['FEEDS_CONFIG'] + os.remove(cfg_path) + +def test_config_from_env_no_auth(): + with pytest.raises(RuntimeError) as e: + config.FeedsConfig() + assert "The AUTH_TOKEN environment variable must be set!" in str(e.value) + +def test_get_config(): + cfg_lines = [ + '[feeds]', + 'redis-host=foo', + 'redis-port=bar', + 'auth-url=baz' + ] + cfg_path = "fake_test_config_delete_me.cfg" + write_test_cfg(cfg_path, cfg_lines) + os.environ['FEEDS_CONFIG'] = cfg_path + os.environ['AUTH_TOKEN'] = FAKE_AUTH_TOKEN + + cfg = config.get_config() + assert cfg.redis_host == 'foo' + assert cfg.redis_port == 'bar' + assert cfg.auth_url == 'baz' + assert cfg.auth_token == FAKE_AUTH_TOKEN + os.remove("fake_test_config_delete_me.cfg") del os.environ['FEEDS_CONFIG'] - os.remove(cfg_path) \ No newline at end of file + del os.environ['AUTH_TOKEN'] \ No newline at end of file From 36e15d48f99c9d71202db72e80d01b2f665cd9d4 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Wed, 17 Oct 2018 21:13:23 -0700 Subject: [PATCH 19/20] added user lookup --- feeds/auth.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/feeds/auth.py b/feeds/auth.py index 172076e..ea968b9 100644 --- a/feeds/auth.py +++ b/feeds/auth.py @@ -4,7 +4,7 @@ * validating user ids """ -from .config import config +from .config import get_config import requests import json from .exceptions import ( @@ -16,7 +16,7 @@ Cache, TTLCache ) -AUTH_URL = config.auth_url +AUTH_URL = get_config().auth_url AUTH_API_PATH = '/api/V2/' CACHE_EXPIRE_TIME = 300 # seconds @@ -52,13 +52,26 @@ def validate_service_token(token): else: raise InvalidTokenError("Token is not a Service token!") -def valid_user_token(token): +def validate_user_token(token): """ Validates a user auth token. If valid, does nothing. If invalid, raises an InvalidTokenError. """ __fetch_token(token) +def validate_user_id(user_id): + return validate_user_ids([user_id]) + +def validate_user_ids(user_ids): + """ + Validates whether users are real or not. + Returns the parsed response from the server, as a dict. Each + key is a user that exists, each value is their user name. + Raises an HTTPError if something bad happens. + """ + r = __auth_request('users?list={}'.format(','.join(user_ids))) + return json.loads(r.content) + def __fetch_token(token): """ Returns token info from the auth server. Caches it locally for a while. From 2eec41947cc99f259b1f45fa0a242ecceb215c45 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Wed, 17 Oct 2018 21:23:04 -0700 Subject: [PATCH 20/20] fix typo --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b7ccd54..c45ca79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ flask==1.0.2 requests==2.19.1 gunicorn==19.9.0 gevent==1.3.7 -cachetools=2.1.0 \ No newline at end of file +cachetools==2.1.0 \ No newline at end of file