From 54d6fd360e043d0e477f29ac1074b41aa5928080 Mon Sep 17 00:00:00 2001 From: Pinkesh Badjatiya Date: Thu, 30 Jun 2016 18:01:51 +0530 Subject: [PATCH 01/10] More kafka cleanup --- listenstore/listenstore/cli.py | 9 +-------- manage.py | 5 ----- webserver/templates/index/contribute.html | 2 +- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/listenstore/listenstore/cli.py b/listenstore/listenstore/cli.py index 262b94e889..590202dd01 100644 --- a/listenstore/listenstore/cli.py +++ b/listenstore/listenstore/cli.py @@ -4,7 +4,6 @@ from .utils import argparse_factory, parse_args_and_config from .listenstore import PostgresListenStore -from .kafkaconsumer import KafkaConsumer class Command(object): @@ -19,7 +18,6 @@ def __init__(self): You have no logger, databases, or config in __init__. """ self.opt_parser = argparse_factory(self.desc) self._listenStore = None - self._kafkaConsumer = None # NB: only sets level after our Command starts running @@ -53,6 +51,7 @@ def start(self): finally: self.cleanup() + def cleanup(self): return @@ -67,11 +66,5 @@ def listen_store(self): """ pass - @property - def kafka_consumer(self): - if self._kafkaConsumer is None: - self._kafkaConsumer = KafkaConsumer(self.config) - return self._kafkaConsumer - def renice(self, increment): os.nice(increment) diff --git a/manage.py b/manage.py index e339203b87..8b63ca2acb 100644 --- a/manage.py +++ b/manage.py @@ -23,11 +23,6 @@ def runserver(host, port, debug): schedule_jobs(app) app.run(host=host, port=port, debug=debug) -@cli.command() -def init_kafka(archive, force): - """Initializes kafka""" - - print("Done!") @cli.command() @click.option("--force", "-f", is_flag=True, help="Drop existing database and user.") diff --git a/webserver/templates/index/contribute.html b/webserver/templates/index/contribute.html index d4326d176c..214bcf5c0b 100644 --- a/webserver/templates/index/contribute.html +++ b/webserver/templates/index/contribute.html @@ -23,7 +23,7 @@

Developers

ListenBrainz is in its infancy and we need a lot of help to implement more features and to debug the existing - features. If you feel like helping out and have experience with Python, Postgres, Kafka and/or Cassandra, + features. If you feel like helping out and have experience with Python, Postgres and Redis, we'd love some help.

From 680d0a770a523cea8cd82ae5547f97286416f0d4 Mon Sep 17 00:00:00 2001 From: Pinkesh Badjatiya Date: Thu, 30 Jun 2016 18:53:41 +0530 Subject: [PATCH 02/10] Update tests for Postgres --- .../listenstore/tests/test_listenstore.py | 25 ------------------- listenstore/tests/test_listenstore.py | 19 +++++++------- listenstore/tests/util.py | 5 ++-- 3 files changed, 11 insertions(+), 38 deletions(-) delete mode 100644 listenstore/listenstore/tests/test_listenstore.py diff --git a/listenstore/listenstore/tests/test_listenstore.py b/listenstore/listenstore/tests/test_listenstore.py deleted file mode 100644 index 2cc8bf8ff5..0000000000 --- a/listenstore/listenstore/tests/test_listenstore.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding=utf-8 -from __future__ import division, absolute_import, print_function, unicode_literals -import unittest -from datetime import date -from .. import listenstore - - -class RangeTestCase(unittest.TestCase): - - def testDateRange(self): - self.assertEqual(listenstore.daterange(date(2015, 8, 12), 'day'), (2015, 8, 12)) - self.assertEqual(listenstore.daterange(date(2015, 8, 12), 'month'), (2015, 8)) - self.assertEqual(listenstore.daterange(date(2015, 8, 12), 'year'), (2015,)) - - def testDateRanges(self): - max_date = date(2015, 9, 6) - min_date = date(2014, 6, 1) - - expected = [(2015, 9), (2015, 8), (2015, 7), (2015, 6), (2015, 5), (2015, 4), - (2015, 3), (2015, 2), (2015, 1), (2014, 12), (2014, 11), (2014, 10), - (2014, 9), (2014, 8), (2014, 7), (2014, 6)] - - self.assertEqual(list(listenstore.dateranges(listenstore.date_to_id(min_date), - listenstore.date_to_id(max_date), 'month', 'asc')), - expected) diff --git a/listenstore/tests/test_listenstore.py b/listenstore/tests/test_listenstore.py index fbe4cc4860..b8157efd17 100644 --- a/listenstore/tests/test_listenstore.py +++ b/listenstore/tests/test_listenstore.py @@ -4,27 +4,26 @@ import logging from datetime import date, datetime from .util import generate_data -from listenstore.listenstore import ListenStore +from listenstore.listenstore import PostgresListenStore -# TODO: update for postgres class TestListenStore(unittest2.TestCase): @classmethod - @unittest2.skip("We don't have Cassandra on Jenkins server") + # @unittest2.skip("We don't have Postgres on Jenkins server") def setUpClass(self): self.log = logging.getLogger(__name__) - conf = {"replication_factor": 1, - "cassandra_keyspace": "listenbrainz_integration_test", - "cassandra_server": "localhost:9092"} - self.logstore = ListenStore(conf) + conf = { + "SQLALCHEMY_DATABASE_URI": "postgresql://listenbrainz@/listenbrainz" + } + self.logstore = PostgresListenStore(conf) self._create_test_data() @classmethod def _create_test_data(self): self.log.info("Inserting test data...") test_data = generate_data(datetime(2015, 9, 3, 0, 0, 0), 1000) - self.logstore.insert_batch(test_data) + self.logstore.insert_postgresql(test_data) self.log.info("Test data inserted") @classmethod @@ -32,7 +31,7 @@ def tearDownClass(self): #self.logstore.drop_schema() self.logstore = None - @unittest2.skip("We don't have Cassandra on Jenkins server") + # @unittest2.skip("We don't have Postgres on Jenkins server") def test_fetch_listens(self): - listens = self.logstore.fetch_listens(uid="test", limit=10) + listens = self.logstore.fetch_listens(user_id="test", limit=10) self.assertEqual(len(list(listens)), 10) diff --git a/listenstore/tests/util.py b/listenstore/tests/util.py index 3bbf23f9db..e828495f2a 100644 --- a/listenstore/tests/util.py +++ b/listenstore/tests/util.py @@ -7,13 +7,12 @@ def generate_data(from_date, num_records): test_data = [] - current_date = from_date - artist_msid = str(uuid.uuid4()) + for i in range(1, num_records): item = Listen(uid="test", timestamp=current_date, artist_msid=artist_msid, - track_msid=str(uuid.uuid4())) + recording_msid=str(uuid.uuid4())) test_data.append(item) current_date += timedelta(seconds=1) return test_data From 102519bfac471106be2ccb664165bae80d0cc966 Mon Sep 17 00:00:00 2001 From: Pinkesh Badjatiya Date: Thu, 30 Jun 2016 18:57:49 +0530 Subject: [PATCH 03/10] Fix arg name --- listenstore/tests/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/listenstore/tests/util.py b/listenstore/tests/util.py index e828495f2a..1e14d88b03 100644 --- a/listenstore/tests/util.py +++ b/listenstore/tests/util.py @@ -11,7 +11,7 @@ def generate_data(from_date, num_records): artist_msid = str(uuid.uuid4()) for i in range(1, num_records): - item = Listen(uid="test", timestamp=current_date, artist_msid=artist_msid, + item = Listen(user_id="test", timestamp=current_date, artist_msid=artist_msid, recording_msid=str(uuid.uuid4())) test_data.append(item) current_date += timedelta(seconds=1) From b59b87fc6e0cb07a4068e112cce493fd461c4734 Mon Sep 17 00:00:00 2001 From: Pinkesh Badjatiya Date: Thu, 30 Jun 2016 19:03:14 +0530 Subject: [PATCH 04/10] Update DB name --- listenstore/tests/test_listenstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/listenstore/tests/test_listenstore.py b/listenstore/tests/test_listenstore.py index b8157efd17..a09184bfae 100644 --- a/listenstore/tests/test_listenstore.py +++ b/listenstore/tests/test_listenstore.py @@ -14,7 +14,7 @@ class TestListenStore(unittest2.TestCase): def setUpClass(self): self.log = logging.getLogger(__name__) conf = { - "SQLALCHEMY_DATABASE_URI": "postgresql://listenbrainz@/listenbrainz" + "SQLALCHEMY_DATABASE_URI": "postgresql://lb_test@/lb_test" } self.logstore = PostgresListenStore(conf) self._create_test_data() From 9e4af7515c7e806bce242de9f207fca70663c210 Mon Sep 17 00:00:00 2001 From: Pinkesh Badjatiya Date: Fri, 1 Jul 2016 22:47:00 +0530 Subject: [PATCH 05/10] Move DB connection out of flask app --- db/__init__.py | 13 ++++++---- db/user.py | 55 +++++++++++++++++++++++-------------------- webserver/__init__.py | 2 +- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/db/__init__.py b/db/__init__.py index cb076d5ee9..043172797f 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -1,21 +1,24 @@ -from flask_sqlalchemy import SQLAlchemy +from sqlalchemy import create_engine +from sqlalchemy.pool import NullPool # This value must be incremented after schema changes on replicated tables! SCHEMA_VERSION = 1 -db = SQLAlchemy() +engine = None -def init_db_connection(app): +def init_db_connection(connect_str): """Initializes database connection using the specified Flask app. Configuration file must contain `SQLALCHEMY_DATABASE_URI` key. See https://pythonhosted.org/Flask-SQLAlchemy/config.html#configuration-keys for more info. """ - db.init_app(app) + global engine + engine = create_engine(connect_str, poolclass=NullPool) def run_sql_script(sql_file_path): with open(sql_file_path) as sql: - db.session.connection().execute(sql.read()) + with engine.connect() as connection: + connection.execute(sql.read()) diff --git a/db/user.py b/db/user.py index e694140018..da20e846c0 100644 --- a/db/user.py +++ b/db/user.py @@ -1,44 +1,47 @@ -from db import db +import db import uuid - +import sqlalchemy def create(musicbrainz_id): - result = db.session.execute("""INSERT INTO "user" (musicbrainz_id, auth_token) - VALUES (:mb_id, :token) - RETURNING id""", - {"mb_id": musicbrainz_id, "token": str(uuid.uuid4())}) - db.session.commit() - return result.fetchone()["id"] + with db.engine.connect() as connection: + result = connection.execute(sqlalchemy.text("""INSERT INTO "user" (musicbrainz_id, auth_token) + VALUES (:mb_id, :token) + RETURNING id""" ), + {"mb_id": musicbrainz_id, "token": str(uuid.uuid4())}) + return result.fetchone()["id"] def get(id): """Get user with a specified ID (integer).""" - result = db.session.execute("""SELECT id, created, musicbrainz_id, auth_token - FROM "user" - WHERE id = :id""", - {"id": id}) - row = result.fetchone() - return dict(row) if row else None + with db.engine.connect() as connection: + result = connection.execute(sqlalchemy.text("""SELECT id, created, musicbrainz_id, auth_token + FROM "user" + WHERE id = :id"""), + {"id": id}) + row = result.fetchone() + return dict(row) if row else None def get_by_mb_id(musicbrainz_id): """Get user with a specified MusicBrainz ID.""" - result = db.session.execute("""SELECT id, created, musicbrainz_id, auth_token - FROM "user" - WHERE LOWER(musicbrainz_id) = LOWER(:mb_id)""", - {"mb_id": musicbrainz_id}) - row = result.fetchone() - return dict(row) if row else None + with db.engine.connect() as connection: + result = connection.execute(sqlalchemy.text("""SELECT id, created, musicbrainz_id, auth_token + FROM "user" + WHERE LOWER(musicbrainz_id) = LOWER(:mb_id)"""), + {"mb_id": musicbrainz_id}) + row = result.fetchone() + return dict(row) if row else None def get_by_token(token): """Get user from an auth token""" - result = db.session.execute("""SELECT id, created, musicbrainz_id - FROM "user" - WHERE auth_token = :auth_token""", - {"auth_token": token}) - row = result.fetchone() - return dict(row) if row else None + with db.engine.connect() as connection: + result = connection.execute(sqlalchemy.text("""SELECT id, created, musicbrainz_id + FROM "user" + WHERE auth_token = :auth_token"""), + {"auth_token": token}) + row = result.fetchone() + return dict(row) if row else None def get_or_create(musicbrainz_id): diff --git a/webserver/__init__.py b/webserver/__init__.py index 8fc041536e..477fe7483c 100644 --- a/webserver/__init__.py +++ b/webserver/__init__.py @@ -33,7 +33,7 @@ def create_app(): # Database connection import db - db.init_db_connection(app) + db.init_db_connection(app.config['SQLALCHEMY_DATABASE_URI']) from webserver.external import messybrainz messybrainz.init_db_connection(app.config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI']) From 1b950f3dc7f239354d18454ca653eafce84f264e Mon Sep 17 00:00:00 2001 From: Pinkesh Badjatiya Date: Fri, 1 Jul 2016 23:12:31 +0530 Subject: [PATCH 06/10] Sample test --- db/test/test_user.py | 10 ++++ db/testing.py | 44 ++++++++++++++++ listenstore/tests/test_listenstore.py | 73 ++++++++++++++------------- 3 files changed, 91 insertions(+), 36 deletions(-) create mode 100644 db/test/test_user.py create mode 100644 db/testing.py diff --git a/db/test/test_user.py b/db/test/test_user.py new file mode 100644 index 0000000000..45faafe00d --- /dev/null +++ b/db/test/test_user.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +from db.testing import DatabaseTestCase +import db.user + + +class UserTestCase(DatabaseTestCase): + + def test_create(self): + user_id = db.user.create("izzy_cheezy") + self.assertIsNotNone(db.user.get(user_id)) diff --git a/db/testing.py b/db/testing.py new file mode 100644 index 0000000000..ab469fc684 --- /dev/null +++ b/db/testing.py @@ -0,0 +1,44 @@ +import db +import db.data +import unittest +import json +import os + +# Configuration +import sys +sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")) +import config + +ADMIN_SQL_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'admin', 'sql') +TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_data') + + +class DatabaseTestCase(unittest.TestCase): + + def setUp(self): + db.init_db_connection(config['TEST_SQLALCHEMY_DATABASE_URI']) + self.reset_db() + + def tearDown(self): + pass + + def reset_db(self): + self.drop_tables() + self.init_db() + + def init_db(self): + db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql')) + db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql')) + db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql')) + db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql')) + db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql')) + + def drop_tables(self): + with db.engine.connect() as connection: + connection.execute('DROP TABLE IF EXISTS "user" CASCADE') + connection.execute('DROP TABLE IF EXISTS listen CASCADE') + + def load_data_files(self, mbid): + """ Get the data files from the disk """ + # return os.path.join(TEST_DATA_PATH, mbid + '.json') + return diff --git a/listenstore/tests/test_listenstore.py b/listenstore/tests/test_listenstore.py index a09184bfae..12e7ae731a 100644 --- a/listenstore/tests/test_listenstore.py +++ b/listenstore/tests/test_listenstore.py @@ -1,37 +1,38 @@ # coding=utf-8 -from __future__ import division, absolute_import, print_function, unicode_literals -import unittest2 -import logging -from datetime import date, datetime -from .util import generate_data -from listenstore.listenstore import PostgresListenStore - - -class TestListenStore(unittest2.TestCase): - - @classmethod - # @unittest2.skip("We don't have Postgres on Jenkins server") - def setUpClass(self): - self.log = logging.getLogger(__name__) - conf = { - "SQLALCHEMY_DATABASE_URI": "postgresql://lb_test@/lb_test" - } - self.logstore = PostgresListenStore(conf) - self._create_test_data() - - @classmethod - def _create_test_data(self): - self.log.info("Inserting test data...") - test_data = generate_data(datetime(2015, 9, 3, 0, 0, 0), 1000) - self.logstore.insert_postgresql(test_data) - self.log.info("Test data inserted") - - @classmethod - def tearDownClass(self): - #self.logstore.drop_schema() - self.logstore = None - - # @unittest2.skip("We don't have Postgres on Jenkins server") - def test_fetch_listens(self): - listens = self.logstore.fetch_listens(user_id="test", limit=10) - self.assertEqual(len(list(listens)), 10) +# from __future__ import division, absolute_import, print_function, unicode_literals +# from db.testing import DatabaseTestCase +# import unittest2 +# import logging +# from datetime import date, datetime +# from .util import generate_data +# from listenstore.listenstore import PostgresListenStore +# +# # +# # class TestListenStore(DatabaseTestCase): +# # +# # @classmethod +# # # @unittest2.skip("We don't have Postgres on Jenkins server") +# # def setUpClass(self): +# # self.log = logging.getLogger(__name__) +# # conf = { +# # "SQLALCHEMY_DATABASE_URI": "postgresql://lb_test@/lb_test" +# # } +# # self.logstore = PostgresListenStore(conf) +# # self._create_test_data() +# # +# # @classmethod +# # def _create_test_data(self): +# # self.log.info("Inserting test data...") +# # test_data = generate_data(datetime(2015, 9, 3, 0, 0, 0), 1000) +# # self.logstore.insert_postgresql(test_data) +# # self.log.info("Test data inserted") +# # +# # @classmethod +# # def tearDownClass(self): +# # #self.logstore.drop_schema() +# # self.logstore = None +# # +# # # @unittest2.skip("We don't have Postgres on Jenkins server") +# # def test_fetch_listens(self): +# # listens = self.logstore.fetch_listens(user_id="test", limit=10) +# # self.assertEqual(len(list(listens)), 10) From 52d9cd89d1d7a7e62851b8960d157181c4ba0e0c Mon Sep 17 00:00:00 2001 From: Pinkesh Badjatiya Date: Fri, 1 Jul 2016 23:34:50 +0530 Subject: [PATCH 07/10] Success!! --- db/test/__init__.py | 0 db/testing.py | 4 +--- 2 files changed, 1 insertion(+), 3 deletions(-) create mode 100644 db/test/__init__.py diff --git a/db/test/__init__.py b/db/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/db/testing.py b/db/testing.py index ab469fc684..8ec8a0773d 100644 --- a/db/testing.py +++ b/db/testing.py @@ -1,5 +1,4 @@ import db -import db.data import unittest import json import os @@ -16,7 +15,7 @@ class DatabaseTestCase(unittest.TestCase): def setUp(self): - db.init_db_connection(config['TEST_SQLALCHEMY_DATABASE_URI']) + db.init_db_connection(config.TEST_SQLALCHEMY_DATABASE_URI) self.reset_db() def tearDown(self): @@ -27,7 +26,6 @@ def reset_db(self): self.init_db() def init_db(self): - db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql')) db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql')) db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql')) db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql')) From 44ce37ac4b8eab95b2a262c76bad7db637fcafd5 Mon Sep 17 00:00:00 2001 From: Pinkesh Badjatiya Date: Sat, 2 Jul 2016 01:44:41 +0530 Subject: [PATCH 08/10] Update listen insert/fetch tests for postgres --- db/testing.py | 7 ++- listenstore/listenstore/listenstore.py | 19 +++---- listenstore/tests/test_listenstore.py | 71 ++++++++++++-------------- listenstore/tests/util.py | 12 +++-- 4 files changed, 57 insertions(+), 52 deletions(-) diff --git a/db/testing.py b/db/testing.py index 8ec8a0773d..b29e50536b 100644 --- a/db/testing.py +++ b/db/testing.py @@ -2,6 +2,7 @@ import unittest import json import os +from flask import Flask # Configuration import sys @@ -15,6 +16,8 @@ class DatabaseTestCase(unittest.TestCase): def setUp(self): + self.app = Flask(__name__) + self.app.config.from_object(config) db.init_db_connection(config.TEST_SQLALCHEMY_DATABASE_URI) self.reset_db() @@ -36,7 +39,7 @@ def drop_tables(self): connection.execute('DROP TABLE IF EXISTS "user" CASCADE') connection.execute('DROP TABLE IF EXISTS listen CASCADE') - def load_data_files(self, mbid): + def load_data_files(self): """ Get the data files from the disk """ - # return os.path.join(TEST_DATA_PATH, mbid + '.json') + # return os.path.join(TEST_DATA_PATH, file_name) return diff --git a/listenstore/listenstore/listenstore.py b/listenstore/listenstore/listenstore.py index 37ad45d50e..da33661136 100644 --- a/listenstore/listenstore/listenstore.py +++ b/listenstore/listenstore/listenstore.py @@ -96,9 +96,10 @@ def insert_postgresql(self, listens): except Exception, e: # Log errors self.log.error(e) - def execute(self, connection, query, params=None): - res = connection.execute(query, params) - return res.fetchall() + def execute(self, query, params={}): + with self.engine.connect() as connection: + res = connection.execute(query, params) + return res.fetchall() def fetch_listens_from_storage(self, user_id, from_id, to_id, limit, order, precision): query = """ SELECT id, user_id, extract(epoch from ts), artist_msid, album_msid, recording_msid, raw_data """ + \ @@ -106,12 +107,12 @@ def fetch_listens_from_storage(self, user_id, from_id, to_id, limit, order, prec """ AND extract(epoch from ts) > %(from_id)s AND extract(epoch from ts) < %(to_id)s """ + \ """ ORDER BY extract(epoch from ts) """ + ORDER_TEXT[order] + """ LIMIT %(limit)s""" params = { - 'user_id' : user_id, - 'from_id' : from_id, - 'to_id' : to_id, - 'limit' : limit + 'user_id': user_id, + 'from_id': from_id, + 'to_id': to_id, + 'limit': limit } - with self.engine.connect() as connection: - results = self.execute(connection, query, params) + + results = self.execute(query, params) for row in results: yield self.convert_row(row) diff --git a/listenstore/tests/test_listenstore.py b/listenstore/tests/test_listenstore.py index 12e7ae731a..248b3e3cf0 100644 --- a/listenstore/tests/test_listenstore.py +++ b/listenstore/tests/test_listenstore.py @@ -1,38 +1,35 @@ # coding=utf-8 -# from __future__ import division, absolute_import, print_function, unicode_literals -# from db.testing import DatabaseTestCase -# import unittest2 -# import logging -# from datetime import date, datetime -# from .util import generate_data -# from listenstore.listenstore import PostgresListenStore -# -# # -# # class TestListenStore(DatabaseTestCase): -# # -# # @classmethod -# # # @unittest2.skip("We don't have Postgres on Jenkins server") -# # def setUpClass(self): -# # self.log = logging.getLogger(__name__) -# # conf = { -# # "SQLALCHEMY_DATABASE_URI": "postgresql://lb_test@/lb_test" -# # } -# # self.logstore = PostgresListenStore(conf) -# # self._create_test_data() -# # -# # @classmethod -# # def _create_test_data(self): -# # self.log.info("Inserting test data...") -# # test_data = generate_data(datetime(2015, 9, 3, 0, 0, 0), 1000) -# # self.logstore.insert_postgresql(test_data) -# # self.log.info("Test data inserted") -# # -# # @classmethod -# # def tearDownClass(self): -# # #self.logstore.drop_schema() -# # self.logstore = None -# # -# # # @unittest2.skip("We don't have Postgres on Jenkins server") -# # def test_fetch_listens(self): -# # listens = self.logstore.fetch_listens(user_id="test", limit=10) -# # self.assertEqual(len(list(listens)), 10) +from __future__ import division, absolute_import, print_function, unicode_literals +from db.testing import DatabaseTestCase +import logging +from datetime import datetime +from .util import generate_data, to_epoch +from listenstore.listenstore import PostgresListenStore + + +class TestListenStore(DatabaseTestCase): + + def setUp(self): + super(TestListenStore, self).setUp() + self.log = logging.getLogger(__name__) + # conf = { + # "SQLALCHEMY_DATABASE_URI": "postgresql://lb_test@/lb_test" + # } + self.logstore = PostgresListenStore(self.app.config) + self._create_test_data() + + def tearDown(self): + # self.logstore.drop_schema() + self.logstore = None + + def _create_test_data(self): + date = datetime(2015, 9, 3, 0, 0, 0) + self.log.info("Inserting test data...") + test_data = generate_data(date, 100) + self.logstore.insert_postgresql(test_data) + self.log.info("Test data inserted") + + def test_fetch_listens(self): + date = datetime(2015, 9, 3, 0, 0, 0) + listens = self.logstore.fetch_listens(user_id="test", from_id=to_epoch(date), limit=10) + self.assertEquals(len(list(listens)), 10) diff --git a/listenstore/tests/util.py b/listenstore/tests/util.py index 1e14d88b03..4f02ca8fb5 100644 --- a/listenstore/tests/util.py +++ b/listenstore/tests/util.py @@ -1,18 +1,22 @@ # coding=utf-8 from __future__ import division, absolute_import, print_function, unicode_literals -from datetime import timedelta +from datetime import datetime, timedelta from listenstore.listen import Listen import uuid def generate_data(from_date, num_records): test_data = [] - current_date = from_date + current_date = to_epoch(from_date) artist_msid = str(uuid.uuid4()) - for i in range(1, num_records): + for i in range(num_records): + current_date += 1 # Add one second item = Listen(user_id="test", timestamp=current_date, artist_msid=artist_msid, recording_msid=str(uuid.uuid4())) test_data.append(item) - current_date += timedelta(seconds=1) return test_data + + +def to_epoch(date): + return (date - datetime.utcfromtimestamp(0)).total_seconds() From 3c6fced20d0403b5a04b1ebe30bd1a5ef1c55401 Mon Sep 17 00:00:00 2001 From: Pinkesh Badjatiya Date: Sat, 2 Jul 2016 01:59:50 +0530 Subject: [PATCH 09/10] Remove flask_app + Update PostgresListenStore Databse URI --- db/testing.py | 4 +--- listenstore/tests/test_listenstore.py | 7 ++----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/db/testing.py b/db/testing.py index b29e50536b..b003c0821c 100644 --- a/db/testing.py +++ b/db/testing.py @@ -2,7 +2,6 @@ import unittest import json import os -from flask import Flask # Configuration import sys @@ -16,8 +15,7 @@ class DatabaseTestCase(unittest.TestCase): def setUp(self): - self.app = Flask(__name__) - self.app.config.from_object(config) + self.config = config db.init_db_connection(config.TEST_SQLALCHEMY_DATABASE_URI) self.reset_db() diff --git a/listenstore/tests/test_listenstore.py b/listenstore/tests/test_listenstore.py index 248b3e3cf0..f0e2b7d2a5 100644 --- a/listenstore/tests/test_listenstore.py +++ b/listenstore/tests/test_listenstore.py @@ -4,7 +4,7 @@ import logging from datetime import datetime from .util import generate_data, to_epoch -from listenstore.listenstore import PostgresListenStore +from webserver.postgres_connection import init_postgres_connection class TestListenStore(DatabaseTestCase): @@ -12,10 +12,7 @@ class TestListenStore(DatabaseTestCase): def setUp(self): super(TestListenStore, self).setUp() self.log = logging.getLogger(__name__) - # conf = { - # "SQLALCHEMY_DATABASE_URI": "postgresql://lb_test@/lb_test" - # } - self.logstore = PostgresListenStore(self.app.config) + self.logstore = init_postgres_connection(self.config.TEST_SQLALCHEMY_DATABASE_URI) self._create_test_data() def tearDown(self): From 00f4914d7efd942c1bb299a6ce6f66ec5d465184 Mon Sep 17 00:00:00 2001 From: Pinkesh Badjatiya Date: Mon, 4 Jul 2016 22:30:17 +0530 Subject: [PATCH 10/10] Fix init_test_db() in manage.py --- config.py.sample | 3 ++- manage.py | 18 +++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/config.py.sample b/config.py.sample index b3d98f93b1..6ae6263fb5 100644 --- a/config.py.sample +++ b/config.py.sample @@ -10,7 +10,8 @@ SQLALCHEMY_DATABASE_URI = "postgresql://listenbrainz:listenbrainz@db:5432/listen MESSYBRAINZ_SQLALCHEMY_DATABASE_URI = "postgresql://messybrainz:messybrainz@db:5432/messybrainz" # Database for testing -TEST_SQLALCHEMY_DATABASE_URI = "postgresql://lb_test@/lb_test" +TEST_SQLALCHEMY_DATABASE_URI = "postgresql://lb_test:lb_test@db:5432/lb_test" + # Other postgres configuration options # Oldest listens which can be stored in the database, in days. diff --git a/manage.py b/manage.py index 8b63ca2acb..cf6fa6a46f 100644 --- a/manage.py +++ b/manage.py @@ -38,7 +38,7 @@ def init_db(force, skip_create): uri = urlsplit(create_app().config['SQLALCHEMY_DATABASE_URI']) if force: - exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + + exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' -h ' + uri.hostname + ' -p ' + str(uri.port) + ' < ' + os.path.join(ADMIN_SQL_DIR, 'drop_db.sql'), shell=True) @@ -84,29 +84,33 @@ def init_test_db(force=False): `PG_CONNECT_TEST` variable must be defined in the config file. """ + + uri = urlsplit(create_app().config['TEST_SQLALCHEMY_DATABASE_URI']) if force: - exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' < ' + + exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + + ' -h ' + uri.hostname + ' -p ' + str(uri.port) + ' < ' + os.path.join(ADMIN_SQL_DIR, 'drop_test_db.sql'), shell=True) if exit_code != 0: raise Exception('Failed to drop existing database and user! Exit code: %i' % exit_code) print('Creating database and user for testing...') - exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' < ' + + exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + + ' -h ' + uri.hostname + ' -p ' + str(uri.port) + ' < ' + os.path.join(ADMIN_SQL_DIR, 'create_test_db.sql'), shell=True) if exit_code != 0: raise Exception('Failed to create new database and user! Exit code: %i' % exit_code) - exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' -d ab_test < ' + + exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' -d lb_test ' + + ' -h ' + uri.hostname + ' -p ' + str(uri.port) + ' < ' + os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'), shell=True) if exit_code != 0: raise Exception('Failed to create database extensions! Exit code: %i' % exit_code) - db.init_db_connection(config.PG_CONNECT_TEST) + db.init_db_connection(config.TEST_SQLALCHEMY_DATABASE_URI) - db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_types.sql')) db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql')) db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql')) db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql')) @@ -134,7 +138,7 @@ def init_msb_db(force, skip_create): uri = urlsplit(create_app().config['MESSYBRAINZ_SQLALCHEMY_DATABASE_URI']) if force: - exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + + exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' -h ' + uri.hostname + ' -p ' + str(uri.port) + ' < ' + os.path.join(MSB_ADMIN_SQL_DIR, 'drop_db.sql'), shell=True)