From 9d4bd0c719ed19774f022ae4436d68bb111a307f Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Tue, 23 Oct 2018 11:38:55 -0700 Subject: [PATCH 1/4] basic mongo stuff --- deploy.cfg.example | 20 +++++----- feeds/config.py | 45 ++++++++++------------- feeds/storage/mongodb/__init__.py | 0 feeds/storage/mongodb/activity_storage.py | 14 +++++++ feeds/storage/mongodb/connection.py | 18 +++++++++ feeds/storage/mongodb/timeline_storage.py | 13 +++++++ test/test_config.py | 19 +++++----- 7 files changed, 84 insertions(+), 45 deletions(-) create mode 100644 feeds/storage/mongodb/__init__.py create mode 100644 feeds/storage/mongodb/activity_storage.py create mode 100644 feeds/storage/mongodb/connection.py create mode 100644 feeds/storage/mongodb/timeline_storage.py diff --git a/deploy.cfg.example b/deploy.cfg.example index eb4f7ae..095e991 100644 --- a/deploy.cfg.example +++ b/deploy.cfg.example @@ -1,15 +1,15 @@ [feeds] -# Redis info -redis-host=localhost -redis-port=6379 -redis-user= -redis-pw= +# DB info +# db-engine - allowed values = redis, mongodb +db-engine=mongodb +db-host=localhost +db-port=6379 +db-user= +db-pw= -# MongoDB Info -mongo-host= -mongo-db= -mongo-user= -mongo-pw= auth-url=https://ci.kbase.us/services/auth + +# admins are allowed to use their auth tokens to create global notifications. +# examples would be notices about KBase downtime or events. admins=wjriehl,scanon,kkeller \ No newline at end of file diff --git a/feeds/config.py b/feeds/config.py index e1f5132..ad48abc 100644 --- a/feeds/config.py +++ b/feeds/config.py @@ -8,14 +8,16 @@ ENV_AUTH_TOKEN = "AUTH_TOKEN" INI_SECTION = "feeds" -DB_HOST = "redis-host" -DB_HOST_PORT = "redis-port" -DB_USER = "redis-user" -DB_PW = "redis-pw" -DB_DB = "redis-db" -AUTH_URL = "auth-url" -ADMIN_LIST = "admins" -GLOBAL_FEED = "global-feed" + +KEY_DB_HOST = "db-host" +KEY_DB_PORT = "db-port" +KEY_DB_USER = "db-user" +KEY_DB_PW = "db-pw" +KEY_DB_NAME = "db-name" +KEY_DB_ENGINE = "db-engine" +KEY_AUTH_URL = "auth-url" +KEY_ADMIN_LIST = "admins" +KEY_GLOBAL_FEED = "global-feed" class FeedsConfig(object): @@ -23,14 +25,6 @@ class FeedsConfig(object): Loads a config set from the root deploy.cfg file. This should be in ini format. Keys of note are: - - redis-host - redis-port - redis-user - redis-pw - auth-url - global-feed - name of the feed to represent a global user, or - notifications that everyone should see. """ def __init__(self): @@ -44,16 +38,15 @@ def __init__(self): raise ConfigError( "Error parsing config file: section {} not found!".format(INI_SECTION) ) - self.redis_host = self._get_line(cfg, DB_HOST) - self.redis_port = self._get_line(cfg, DB_HOST_PORT) - self.redis_user = self._get_line(cfg, DB_USER, required=False) - self.redis_pw = self._get_line(cfg, DB_PW, required=False) - self.redis_db = self._get_line(cfg, DB_DB, required=False) - if self.redis_db is None: - self.redis_db = 0 - self.global_feed = self._get_line(cfg, GLOBAL_FEED) - self.auth_url = self._get_line(cfg, AUTH_URL) - self.admins = self._get_line(cfg, ADMIN_LIST).split(",") + self.db_engine = self._get_line(cfg, KEY_DB_ENGINE) + self.db_host = self._get_line(cfg, KEY_DB_HOST) + self.db_port = self._get_line(cfg, KEY_DB_PORT) + self.db_user = self._get_line(cfg, KEY_DB_USER, required=False) + self.db_pw = self._get_line(cfg, KEY_DB_PW, required=False) + self.db_name = self._get_line(cfg, KEY_DB_NAME, required=False) + self.global_feed = self._get_line(cfg, KEY_GLOBAL_FEED) + self.auth_url = self._get_line(cfg, KEY_AUTH_URL) + self.admins = self._get_line(cfg, KEY_ADMIN_LIST).split(",") def _find_config_path(self): """ diff --git a/feeds/storage/mongodb/__init__.py b/feeds/storage/mongodb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/feeds/storage/mongodb/activity_storage.py b/feeds/storage/mongodb/activity_storage.py new file mode 100644 index 0000000..3d45807 --- /dev/null +++ b/feeds/storage/mongodb/activity_storage.py @@ -0,0 +1,14 @@ +from ..base import ActivityStorage +from .connection import get_mongo_connection + + +class MongoActivityStorage(ActivityStorage): + def add_to_storage(self, activity): + mongo = get_mongo_connection() + mongo.do_stuff + + def get_from_storage(self, activity_ids): + raise NotImplementedError() + + def remove_from_storage(self, activity_ids): + raise NotImplementedError() diff --git a/feeds/storage/mongodb/connection.py b/feeds/storage/mongodb/connection.py new file mode 100644 index 0000000..74caf84 --- /dev/null +++ b/feeds/storage/mongodb/connection.py @@ -0,0 +1,18 @@ +from pymongo import MongoClient +from feeds.config import get_config + +_connection = None + + +def get_mongo_connection(): + global _connection + if _connection is None: + _connection = _build_mongo_connection() + return _connection + + +def _build_mongo_connection(): + cfg = get_config() + new_conn = MongoClient(host=cfg.db_host, port=cfg.db_port) + # TODO: other indexing and config stuff goes here as needed. + return new_conn diff --git a/feeds/storage/mongodb/timeline_storage.py b/feeds/storage/mongodb/timeline_storage.py new file mode 100644 index 0000000..edb40c4 --- /dev/null +++ b/feeds/storage/mongodb/timeline_storage.py @@ -0,0 +1,13 @@ +from ..base import TimelineStorage +# from .connection import get_mongo_connection + + +class MongoTimelineStorage(TimelineStorage): + def add_to_timeline(self, activity): + raise NotImplementedError() + + def get_timeline(self): + raise NotImplementedError() + + def remove_from_timeline(self, activity_ids): + raise NotImplementedError() diff --git a/test/test_config.py b/test/test_config.py index 1e661cf..fad8401 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -12,8 +12,9 @@ GOOD_CONFIG = [ '[feeds]', - 'redis-host=foo', - 'redis-port=bar', + 'db-engine=redis', + 'db-host=foo', + 'db-port=bar', 'auth-url=baz', 'global-feed=global', 'admins=admin1,admin2,admin3' @@ -44,15 +45,15 @@ def test_config_from_env_ok(dummy_config, dummy_auth_token): os.environ['FEEDS_CONFIG'] = cfg_path cfg = config.FeedsConfig() assert cfg.auth_url == 'baz' - assert cfg.redis_host == 'foo' - assert cfg.redis_port == 'bar' + assert cfg.db_host == 'foo' + assert cfg.db_port == 'bar' del os.environ['FEEDS_CONFIG'] os.environ['KB_DEPLOYMENT_CONFIG'] = cfg_path cfg = config.FeedsConfig() assert cfg.auth_url == 'baz' - assert cfg.redis_host == 'foo' - assert cfg.redis_port == 'bar' + assert cfg.db_host == 'foo' + assert cfg.db_port == 'bar' del os.environ['KB_DEPLOYMENT_CONFIG'] @@ -60,7 +61,7 @@ def test_config_from_env_ok(dummy_config, dummy_auth_token): def test_config_from_env_errors(dummy_config, dummy_auth_token): cfg_lines = [ '[not-feeds]', - 'redis-host=foo' + 'db-host=foo' ] cfg_path = dummy_config(cfg_lines) @@ -81,8 +82,8 @@ def test_get_config(dummy_config, dummy_auth_token): os.environ['FEEDS_CONFIG'] = cfg_path cfg = config.get_config() - assert cfg.redis_host == 'foo' - assert cfg.redis_port == 'bar' + assert cfg.db_host == 'foo' + assert cfg.db_port == 'bar' assert cfg.auth_url == 'baz' assert cfg.auth_token == FAKE_AUTH_TOKEN del os.environ['FEEDS_CONFIG'] \ No newline at end of file From f6d8e5f23ac0c7ee98d70d760f4e4e247f4821ac Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Tue, 23 Oct 2018 12:55:19 -0700 Subject: [PATCH 2/4] add notes to example config, change config keys for redis connection --- .travis.yml | 2 -- deploy.cfg.example | 14 +++++++++++--- feeds/storage/redis/connection.py | 8 ++++---- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index bba81da..45c6e70 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,6 @@ python: - 3.6 services: - docker -# env: -# global: before_install: - sudo apt-get -qq update diff --git a/deploy.cfg.example b/deploy.cfg.example index 095e991..a7b7b8c 100644 --- a/deploy.cfg.example +++ b/deploy.cfg.example @@ -1,15 +1,23 @@ [feeds] # DB info -# db-engine - allowed values = redis, mongodb +# db-engine - allowed values = redis, mongodb. Others will raise an error on startup. db-engine=mongodb + +# db-name - name of the database to use. default = "feeds". +db-name=feeds + +# Other db info. The usual - host, port, user, and password. You know the drill. db-host=localhost db-port=6379 db-user= db-pw= - auth-url=https://ci.kbase.us/services/auth # admins are allowed to use their auth tokens to create global notifications. # examples would be notices about KBase downtime or events. -admins=wjriehl,scanon,kkeller \ No newline at end of file +admins=wjriehl,scanon,kkeller,mmdrake + +# fake user name for the global feed. Should be something that's not a valid +# user name. +global-feed=_global_ diff --git a/feeds/storage/redis/connection.py b/feeds/storage/redis/connection.py index 286d2bc..abdc73f 100644 --- a/feeds/storage/redis/connection.py +++ b/feeds/storage/redis/connection.py @@ -22,10 +22,10 @@ def setup_redis(): ''' config = get_config() pool = redis.ConnectionPool( - host=config.redis_host, - port=config.redis_port, - password=config.redis_pw, - db=config.redis_db + host=config.db_host, + port=config.db_port, + password=config.db_pw, + db=config.db_name # decode_responses=config.get('decode_responses', True), # # connection options From a8be5581b7a1befef6284301237a6209f1f82bf4 Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Tue, 23 Oct 2018 18:10:11 -0700 Subject: [PATCH 3/4] Caught up with redis -> mongodb --- feeds/activity/notification.py | 19 ++++++++ feeds/config.py | 11 ++++- feeds/exceptions.py | 12 +++++ feeds/feeds/notification/notification_feed.py | 14 +++--- feeds/logger.py | 10 +++++ feeds/managers/notification_manager.py | 14 +++--- feeds/server.py | 21 ++++----- feeds/storage/base.py | 7 +-- feeds/storage/mongodb/activity_storage.py | 45 ++++++++++++++++--- feeds/storage/mongodb/connection.py | 42 ++++++++++++++--- feeds/storage/mongodb/timeline_storage.py | 30 +++++++++++-- feeds/storage/redis/timeline_storage.py | 4 -- 12 files changed, 177 insertions(+), 52 deletions(-) create mode 100644 feeds/logger.py diff --git a/feeds/activity/notification.py b/feeds/activity/notification.py index 8db906f..4a0ea0e 100644 --- a/feeds/activity/notification.py +++ b/feeds/activity/notification.py @@ -114,3 +114,22 @@ def deserialize(cls, serial): deserial.time = struct['m'] deserial.id = struct['i'] return deserial + + @classmethod + def from_dict(cls, serial): + """ + Returns a new Notification from a serialized dictionary (e.g. used in Mongo) + """ + assert serial + deserial = cls( + serial['actor'], + str(serial['verb']), + serial['object'], + serial['source'], + level=str(serial['level']), + target=serial.get('target'), + context=serial.get('context') + ) + deserial.time = serial['created'] + deserial.id = serial['act_id'] + return deserial \ No newline at end of file diff --git a/feeds/config.py b/feeds/config.py index ad48abc..c1a108b 100644 --- a/feeds/config.py +++ b/feeds/config.py @@ -18,7 +18,7 @@ KEY_AUTH_URL = "auth-url" KEY_ADMIN_LIST = "admins" KEY_GLOBAL_FEED = "global-feed" - +KEY_DEBUG = "debug" class FeedsConfig(object): """ @@ -41,12 +41,21 @@ def __init__(self): self.db_engine = self._get_line(cfg, KEY_DB_ENGINE) self.db_host = self._get_line(cfg, KEY_DB_HOST) self.db_port = self._get_line(cfg, KEY_DB_PORT) + try: + self.db_port = int(self.db_port) + except: + raise ConfigError("{} must be an int! Got {}".format(KEY_DB_PORT, self.db_port)) self.db_user = self._get_line(cfg, KEY_DB_USER, required=False) self.db_pw = self._get_line(cfg, KEY_DB_PW, required=False) self.db_name = self._get_line(cfg, KEY_DB_NAME, required=False) self.global_feed = self._get_line(cfg, KEY_GLOBAL_FEED) self.auth_url = self._get_line(cfg, KEY_AUTH_URL) self.admins = self._get_line(cfg, KEY_ADMIN_LIST).split(",") + self.debug = self._get_line(cfg, KEY_DEBUG, required=False) + if not self.debug or self.debug.lower() != "true": + self.debug = False + else: + self.debug = True def _find_config_path(self): """ diff --git a/feeds/exceptions.py b/feeds/exceptions.py index 747a14b..a163e1d 100644 --- a/feeds/exceptions.py +++ b/feeds/exceptions.py @@ -69,3 +69,15 @@ class MissingLevelError(Exception): Raised if looking for a Notification Level that doesn't exist. """ pass + +class ActivityStorageError(Exception): + """ + Raised if an activity is failed to be stored in a database. + """ + pass + +class ActivityRetrievalError(Exception): + """ + Raised if the service fails to retrieve an activity from a database. + """ + pass \ No newline at end of file diff --git a/feeds/feeds/notification/notification_feed.py b/feeds/feeds/notification/notification_feed.py index b5da75d..89e70ac 100644 --- a/feeds/feeds/notification/notification_feed.py +++ b/feeds/feeds/notification/notification_feed.py @@ -1,7 +1,7 @@ from ..base import BaseFeed from feeds.activity.notification import Notification -from feeds.storage.redis.activity_storage import RedisActivityStorage -from feeds.storage.redis.timeline_storage import RedisTimelineStorage +from feeds.storage.mongodb.activity_storage import MongoActivityStorage +from feeds.storage.mongodb.timeline_storage import MongoTimelineStorage from cachetools import TTLCache import logging @@ -9,8 +9,8 @@ class NotificationFeed(BaseFeed): def __init__(self, user_id): self.user_id = user_id - self.timeline_storage = RedisTimelineStorage(self.user_id) - self.activity_storage = RedisActivityStorage() + self.timeline_storage = MongoTimelineStorage(self.user_id) + self.activity_storage = MongoActivityStorage() self.timeline = None self.cache = TTLCache(1000, 600) @@ -41,10 +41,8 @@ def get_activities(self, count=10): # 4. Return them. if count < 1 or not isinstance(count, int): raise ValueError('Count must be an integer > 0') - self._update_timeline() - note_ids = self.timeline - serial_notes = self.activity_storage.get_from_storage(note_ids) - note_list = [Notification.deserialize(note) for note in serial_notes] + serial_notes = self.timeline_storage.get_timeline(count=count) + note_list = [Notification.from_dict(note) for note in serial_notes] return note_list def mark_activities(self, activity_ids, seen=False): diff --git a/feeds/logger.py b/feeds/logger.py new file mode 100644 index 0000000..4d9ed5c --- /dev/null +++ b/feeds/logger.py @@ -0,0 +1,10 @@ +import logging + +def get_log(name): + return logging.getLogger(__name__) + +def log(name, msg, *args, level=logging.INFO): + logging.getLogger(__name__).log(level, msg, *args) + +def log_error(name, error): + log(name, "Exception: " + str(error) + error, level=logging.ERROR) diff --git a/feeds/managers/notification_manager.py b/feeds/managers/notification_manager.py index a69a7f7..31a9c70 100644 --- a/feeds/managers/notification_manager.py +++ b/feeds/managers/notification_manager.py @@ -7,7 +7,7 @@ """ from .base import BaseManager -from ..storage.redis.activity_storage import RedisActivityStorage +from ..storage.mongodb.activity_storage import MongoActivityStorage from ..feeds.notification.notification_feed import NotificationFeed from feeds.config import get_config @@ -25,12 +25,12 @@ def add_notification(self, note): note.validate() # any errors get raised to be caught by the server. target_users = self.get_target_users(note) # add the notification to the database. - activity_storage = RedisActivityStorage() - activity_storage.add_to_storage(note) - for user in target_users: - # add the notification to the appropriate users' feeds. - feed = NotificationFeed(user) - feed.add_notification(note) + activity_storage = MongoActivityStorage() + activity_storage.add_to_storage(note, target_users) + # for user in target_users: + # # add the notification to the appropriate users' feeds. + # feed = NotificationFeed(user) + # feed.add_notification(note) def get_target_users(self, note): """ diff --git a/feeds/server.py b/feeds/server.py index fd371c0..73fd9a1 100644 --- a/feeds/server.py +++ b/feeds/server.py @@ -8,7 +8,7 @@ from http.client import responses from flask.logging import default_handler from .util import epoch_ms -from .config import FeedsConfig +from .config import get_config from .auth import ( validate_service_token, validate_user_token @@ -32,13 +32,6 @@ def _initialize_logging(): root.addHandler(default_handler) root.setLevel('INFO') - -def _initialize_config(): - # TODO - include config for: - # * database access - return FeedsConfig() - - def _log(msg, *args, level=logging.INFO): logging.getLogger(__name__).log(level, msg, *args) @@ -96,7 +89,7 @@ def _get_notification_params(params): def create_app(test_config=None): _initialize_logging() - _initialize_config() + cfg = get_config() app = Flask(__name__, instance_relative_config=True) if test_config is None: @@ -106,6 +99,7 @@ def create_app(test_config=None): @app.before_request def preprocess_request(): + _log('%s %s', request.method, request.path) pass @app.after_request @@ -187,10 +181,11 @@ def add_notification(): Once validated, will be used as the Source of the notification, and used in logic to determine which feeds get notified. """ - token = _get_auth_token(request) - service = validate_service_token(token) # can also be an admin user - if not service: - raise InvalidTokenError("Token must come from a service, not a user!") + if not cfg.debug: + token = _get_auth_token(request) + service = validate_service_token(token) # can also be an admin user + if not service: + raise InvalidTokenError("Token must come from a service, not a user!") params = _get_notification_params(json.loads(request.get_data())) # create a Notification from params. new_note = Notification( diff --git a/feeds/storage/base.py b/feeds/storage/base.py index 4567a5b..1bccd96 100644 --- a/feeds/storage/base.py +++ b/feeds/storage/base.py @@ -24,13 +24,14 @@ def remove_from_storage(self, activity_ids): class TimelineStorage(BaseStorage): - def __init__(self): - pass + def __init__(self, user_id): + assert user_id + self.user_id = user_id def add_to_timeline(self, activity): raise NotImplementedError() - def get_timeline(self): + def get_timeline(self, count=10): raise NotImplementedError() def remove_from_timeline(self, activity_ids): diff --git a/feeds/storage/mongodb/activity_storage.py b/feeds/storage/mongodb/activity_storage.py index 3d45807..41a854a 100644 --- a/feeds/storage/mongodb/activity_storage.py +++ b/feeds/storage/mongodb/activity_storage.py @@ -1,14 +1,47 @@ +from typing import List from ..base import ActivityStorage -from .connection import get_mongo_connection - +from .connection import get_feeds_collection +from feeds.exceptions import ( + ActivityStorageError, + ActivityRetrievalError +) class MongoActivityStorage(ActivityStorage): - def add_to_storage(self, activity): - mongo = get_mongo_connection() - mongo.do_stuff + def add_to_storage(self, activity, target_users: List[str]): + """ + Adds a single activity to the MongoDB. + Returns None if successful. + Raises an ActivityStorageError if it fails. + """ + coll = get_feeds_collection() + act_doc = { + "act_id": activity.id, + "actor": activity.actor, + "verb": activity.verb.id, + "object": activity.object, + "target": activity.target, + "source": activity.source, + "level": activity.level.id, + "users": target_users, + "unseen": target_users, + "created": activity.time, + "context": activity.context + } + try: + coll.insert_one(act_doc) + except e: + raise ActivityStorageError("Failed to store activity: " + str(e)) def get_from_storage(self, activity_ids): - raise NotImplementedError() + pass def remove_from_storage(self, activity_ids): raise NotImplementedError() + + def change_seen_mark(self, act_id: str, user: str, seen: bool): + """ + :param act_id: activity id + :user: user id + :seen: whether or not it's been seen. Boolean. + """ + raise NotImplementedError() \ No newline at end of file diff --git a/feeds/storage/mongodb/connection.py b/feeds/storage/mongodb/connection.py index 74caf84..edd68f6 100644 --- a/feeds/storage/mongodb/connection.py +++ b/feeds/storage/mongodb/connection.py @@ -1,18 +1,46 @@ -from pymongo import MongoClient +import pymongo +from pymongo import ( + MongoClient +) from feeds.config import get_config +import feeds.logger as log _connection = None +_COL_NOTIFICATIONS = "notifications" +_INDEXES = [ + [("created", pymongo.DESCENDING)] +] + + +def get_feeds_collection(): + conn = get_mongo_connection() + return conn.get_collection(_COL_NOTIFICATIONS) + def get_mongo_connection(): global _connection if _connection is None: - _connection = _build_mongo_connection() + _connection = FeedsMongoConnection() return _connection -def _build_mongo_connection(): - cfg = get_config() - new_conn = MongoClient(host=cfg.db_host, port=cfg.db_port) - # TODO: other indexing and config stuff goes here as needed. - return new_conn +class FeedsMongoConnection(object): + def __init__(self): + self.cfg = get_config() + log.log(__name__, "opening MongoDB connection {} {}".format(self.cfg.db_host, self.cfg.db_port)) + self.conn = MongoClient(host=self.cfg.db_host, port=self.cfg.db_port) + self.db = self.conn[self.cfg.db_name] + self._setup_indexes() + self._setup_schema() + + def get_collection(self, collection_name): + return self.db[collection_name] + + def _setup_indexes(self): + coll = self.get_collection(_COL_NOTIFICATIONS) + for index in _INDEXES: + coll.create_index(index) + + def _setup_schema(self): + pass diff --git a/feeds/storage/mongodb/timeline_storage.py b/feeds/storage/mongodb/timeline_storage.py index edb40c4..e4a53a8 100644 --- a/feeds/storage/mongodb/timeline_storage.py +++ b/feeds/storage/mongodb/timeline_storage.py @@ -1,13 +1,37 @@ +from pymongo import ( + ASCENDING, + DESCENDING +) from ..base import TimelineStorage -# from .connection import get_mongo_connection +from .connection import get_feeds_collection class MongoTimelineStorage(TimelineStorage): def add_to_timeline(self, activity): raise NotImplementedError() - def get_timeline(self): - raise NotImplementedError() + def get_timeline(self, count=10, include_seen=False, level=None, verb=None, sort=None): + """ + :param count: int > 0 + :param include_seen: boolean + :param level: Level or None + :param verb: Verb or None + """ + # TODO: add filtering + # TODO: input validation + coll = get_feeds_collection() + query = { + "users": [self.user_id] + } + if not include_seen: + query['unseen'] = [self.user_id] + if level is not None: + query['level'] = level.id + if verb is not None: + query['verb'] = verb.id + timeline = coll.find(query).sort("created", DESCENDING) + serial_notes = [note for note in timeline] + return serial_notes def remove_from_timeline(self, activity_ids): raise NotImplementedError() diff --git a/feeds/storage/redis/timeline_storage.py b/feeds/storage/redis/timeline_storage.py index a059bf0..1885c2c 100644 --- a/feeds/storage/redis/timeline_storage.py +++ b/feeds/storage/redis/timeline_storage.py @@ -8,10 +8,6 @@ class RedisTimelineStorage(TimelineStorage): # TODO: CACHING!! - def __init__(self, user_id): - assert user_id - self.user_id = user_id - def get_timeline(self, count=10): """ Gets the user's timeline of activity ids. From 2721833217e15f00d80131af34262d505449a06f Mon Sep 17 00:00:00 2001 From: Bill Riehl Date: Wed, 24 Oct 2018 11:39:53 -0700 Subject: [PATCH 4/4] fix config tests, fix flake8 errors --- feeds/activity/notification.py | 2 +- feeds/config.py | 3 ++- feeds/exceptions.py | 4 +++- feeds/logger.py | 3 +++ feeds/managers/notification_manager.py | 5 ----- feeds/server.py | 1 + feeds/storage/mongodb/activity_storage.py | 9 +++++---- feeds/storage/mongodb/connection.py | 4 +++- feeds/storage/mongodb/timeline_storage.py | 7 ++----- test/test_config.py | 8 ++++---- 10 files changed, 24 insertions(+), 22 deletions(-) diff --git a/feeds/activity/notification.py b/feeds/activity/notification.py index 4a0ea0e..7b6a545 100644 --- a/feeds/activity/notification.py +++ b/feeds/activity/notification.py @@ -132,4 +132,4 @@ def from_dict(cls, serial): ) deserial.time = serial['created'] deserial.id = serial['act_id'] - return deserial \ No newline at end of file + return deserial diff --git a/feeds/config.py b/feeds/config.py index c1a108b..88e6062 100644 --- a/feeds/config.py +++ b/feeds/config.py @@ -20,6 +20,7 @@ KEY_GLOBAL_FEED = "global-feed" KEY_DEBUG = "debug" + class FeedsConfig(object): """ Loads a config set from the root deploy.cfg file. This should be in ini format. @@ -43,7 +44,7 @@ def __init__(self): self.db_port = self._get_line(cfg, KEY_DB_PORT) try: self.db_port = int(self.db_port) - except: + except ValueError: raise ConfigError("{} must be an int! Got {}".format(KEY_DB_PORT, self.db_port)) self.db_user = self._get_line(cfg, KEY_DB_USER, required=False) self.db_pw = self._get_line(cfg, KEY_DB_PW, required=False) diff --git a/feeds/exceptions.py b/feeds/exceptions.py index a163e1d..284dfea 100644 --- a/feeds/exceptions.py +++ b/feeds/exceptions.py @@ -70,14 +70,16 @@ class MissingLevelError(Exception): """ pass + class ActivityStorageError(Exception): """ Raised if an activity is failed to be stored in a database. """ pass + class ActivityRetrievalError(Exception): """ Raised if the service fails to retrieve an activity from a database. """ - pass \ No newline at end of file + pass diff --git a/feeds/logger.py b/feeds/logger.py index 4d9ed5c..1fdd3ec 100644 --- a/feeds/logger.py +++ b/feeds/logger.py @@ -1,10 +1,13 @@ import logging + def get_log(name): return logging.getLogger(__name__) + def log(name, msg, *args, level=logging.INFO): logging.getLogger(__name__).log(level, msg, *args) + def log_error(name, error): log(name, "Exception: " + str(error) + error, level=logging.ERROR) diff --git a/feeds/managers/notification_manager.py b/feeds/managers/notification_manager.py index 31a9c70..bb077da 100644 --- a/feeds/managers/notification_manager.py +++ b/feeds/managers/notification_manager.py @@ -8,7 +8,6 @@ from .base import BaseManager from ..storage.mongodb.activity_storage import MongoActivityStorage -from ..feeds.notification.notification_feed import NotificationFeed from feeds.config import get_config @@ -27,10 +26,6 @@ def add_notification(self, note): # add the notification to the database. activity_storage = MongoActivityStorage() activity_storage.add_to_storage(note, target_users) - # for user in target_users: - # # add the notification to the appropriate users' feeds. - # feed = NotificationFeed(user) - # feed.add_notification(note) def get_target_users(self, note): """ diff --git a/feeds/server.py b/feeds/server.py index 73fd9a1..04f20d2 100644 --- a/feeds/server.py +++ b/feeds/server.py @@ -32,6 +32,7 @@ def _initialize_logging(): root.addHandler(default_handler) root.setLevel('INFO') + def _log(msg, *args, level=logging.INFO): logging.getLogger(__name__).log(level, msg, *args) diff --git a/feeds/storage/mongodb/activity_storage.py b/feeds/storage/mongodb/activity_storage.py index 41a854a..38c9a17 100644 --- a/feeds/storage/mongodb/activity_storage.py +++ b/feeds/storage/mongodb/activity_storage.py @@ -2,9 +2,10 @@ from ..base import ActivityStorage from .connection import get_feeds_collection from feeds.exceptions import ( - ActivityStorageError, - ActivityRetrievalError + ActivityStorageError ) +from pymongo import PyMongoError + class MongoActivityStorage(ActivityStorage): def add_to_storage(self, activity, target_users: List[str]): @@ -29,7 +30,7 @@ def add_to_storage(self, activity, target_users: List[str]): } try: coll.insert_one(act_doc) - except e: + except PyMongoError as e: raise ActivityStorageError("Failed to store activity: " + str(e)) def get_from_storage(self, activity_ids): @@ -44,4 +45,4 @@ def change_seen_mark(self, act_id: str, user: str, seen: bool): :user: user id :seen: whether or not it's been seen. Boolean. """ - raise NotImplementedError() \ No newline at end of file + raise NotImplementedError() diff --git a/feeds/storage/mongodb/connection.py b/feeds/storage/mongodb/connection.py index edd68f6..b03bdde 100644 --- a/feeds/storage/mongodb/connection.py +++ b/feeds/storage/mongodb/connection.py @@ -28,7 +28,9 @@ def get_mongo_connection(): class FeedsMongoConnection(object): def __init__(self): self.cfg = get_config() - log.log(__name__, "opening MongoDB connection {} {}".format(self.cfg.db_host, self.cfg.db_port)) + log.log(__name__, "opening MongoDB connection {} {}".format( + self.cfg.db_host, self.cfg.db_port) + ) self.conn = MongoClient(host=self.cfg.db_host, port=self.cfg.db_port) self.db = self.conn[self.cfg.db_name] self._setup_indexes() diff --git a/feeds/storage/mongodb/timeline_storage.py b/feeds/storage/mongodb/timeline_storage.py index e4a53a8..876b794 100644 --- a/feeds/storage/mongodb/timeline_storage.py +++ b/feeds/storage/mongodb/timeline_storage.py @@ -1,7 +1,4 @@ -from pymongo import ( - ASCENDING, - DESCENDING -) +import pymongo from ..base import TimelineStorage from .connection import get_feeds_collection @@ -29,7 +26,7 @@ def get_timeline(self, count=10, include_seen=False, level=None, verb=None, sort query['level'] = level.id if verb is not None: query['verb'] = verb.id - timeline = coll.find(query).sort("created", DESCENDING) + timeline = coll.find(query).sort("created", pymongo.DESCENDING) serial_notes = [note for note in timeline] return serial_notes diff --git a/test/test_config.py b/test/test_config.py index fad8401..e5b38c8 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -14,7 +14,7 @@ '[feeds]', 'db-engine=redis', 'db-host=foo', - 'db-port=bar', + 'db-port=5', 'auth-url=baz', 'global-feed=global', 'admins=admin1,admin2,admin3' @@ -46,14 +46,14 @@ def test_config_from_env_ok(dummy_config, dummy_auth_token): cfg = config.FeedsConfig() assert cfg.auth_url == 'baz' assert cfg.db_host == 'foo' - assert cfg.db_port == 'bar' + assert cfg.db_port == 5 del os.environ['FEEDS_CONFIG'] os.environ['KB_DEPLOYMENT_CONFIG'] = cfg_path cfg = config.FeedsConfig() assert cfg.auth_url == 'baz' assert cfg.db_host == 'foo' - assert cfg.db_port == 'bar' + assert cfg.db_port == 5 del os.environ['KB_DEPLOYMENT_CONFIG'] @@ -83,7 +83,7 @@ def test_get_config(dummy_config, dummy_auth_token): cfg = config.get_config() assert cfg.db_host == 'foo' - assert cfg.db_port == 'bar' + assert cfg.db_port == 5 assert cfg.auth_url == 'baz' assert cfg.auth_token == FAKE_AUTH_TOKEN del os.environ['FEEDS_CONFIG'] \ No newline at end of file