-
-
Notifications
You must be signed in to change notification settings - Fork 210
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #87 from pinkeshbadjatiya/fix_tests_cleanup
Cleanup kafka bits & Fix tests setup
- Loading branch information
Showing
14 changed files
with
145 additions
and
120 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,21 +1,24 @@ | ||
from flask_sqlalchemy import SQLAlchemy | ||
from sqlalchemy import create_engine | ||
from sqlalchemy.pool import NullPool | ||
|
||
# This value must be incremented after schema changes on replicated tables! | ||
SCHEMA_VERSION = 1 | ||
|
||
db = SQLAlchemy() | ||
engine = None | ||
|
||
|
||
def init_db_connection(app): | ||
def init_db_connection(connect_str): | ||
"""Initializes database connection using the specified Flask app. | ||
Configuration file must contain `SQLALCHEMY_DATABASE_URI` key. See | ||
https://pythonhosted.org/Flask-SQLAlchemy/config.html#configuration-keys | ||
for more info. | ||
""" | ||
db.init_app(app) | ||
global engine | ||
engine = create_engine(connect_str, poolclass=NullPool) | ||
|
||
|
||
def run_sql_script(sql_file_path): | ||
with open(sql_file_path) as sql: | ||
db.session.connection().execute(sql.read()) | ||
with engine.connect() as connection: | ||
connection.execute(sql.read()) |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
# -*- coding: utf-8 -*- | ||
from db.testing import DatabaseTestCase | ||
import db.user | ||
|
||
|
||
class UserTestCase(DatabaseTestCase): | ||
|
||
def test_create(self): | ||
user_id = db.user.create("izzy_cheezy") | ||
self.assertIsNotNone(db.user.get(user_id)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
import db | ||
import unittest | ||
import json | ||
import os | ||
|
||
# Configuration | ||
import sys | ||
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")) | ||
import config | ||
|
||
ADMIN_SQL_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'admin', 'sql') | ||
TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'test_data') | ||
|
||
|
||
class DatabaseTestCase(unittest.TestCase): | ||
|
||
def setUp(self): | ||
self.config = config | ||
db.init_db_connection(config.TEST_SQLALCHEMY_DATABASE_URI) | ||
self.reset_db() | ||
|
||
def tearDown(self): | ||
pass | ||
|
||
def reset_db(self): | ||
self.drop_tables() | ||
self.init_db() | ||
|
||
def init_db(self): | ||
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql')) | ||
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql')) | ||
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql')) | ||
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql')) | ||
|
||
def drop_tables(self): | ||
with db.engine.connect() as connection: | ||
connection.execute('DROP TABLE IF EXISTS "user" CASCADE') | ||
connection.execute('DROP TABLE IF EXISTS listen CASCADE') | ||
|
||
def load_data_files(self): | ||
""" Get the data files from the disk """ | ||
# return os.path.join(TEST_DATA_PATH, file_name) | ||
return |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,38 +1,32 @@ | ||
# coding=utf-8 | ||
from __future__ import division, absolute_import, print_function, unicode_literals | ||
import unittest2 | ||
from db.testing import DatabaseTestCase | ||
import logging | ||
from datetime import date, datetime | ||
from .util import generate_data | ||
from listenstore.listenstore import ListenStore | ||
from datetime import datetime | ||
from .util import generate_data, to_epoch | ||
from webserver.postgres_connection import init_postgres_connection | ||
|
||
# TODO: update for postgres | ||
|
||
class TestListenStore(unittest2.TestCase): | ||
class TestListenStore(DatabaseTestCase): | ||
|
||
@classmethod | ||
@unittest2.skip("We don't have Cassandra on Jenkins server") | ||
def setUpClass(self): | ||
def setUp(self): | ||
super(TestListenStore, self).setUp() | ||
self.log = logging.getLogger(__name__) | ||
conf = {"replication_factor": 1, | ||
"cassandra_keyspace": "listenbrainz_integration_test", | ||
"cassandra_server": "localhost:9092"} | ||
self.logstore = ListenStore(conf) | ||
self.logstore = init_postgres_connection(self.config.TEST_SQLALCHEMY_DATABASE_URI) | ||
self._create_test_data() | ||
|
||
@classmethod | ||
def tearDown(self): | ||
# self.logstore.drop_schema() | ||
self.logstore = None | ||
|
||
def _create_test_data(self): | ||
date = datetime(2015, 9, 3, 0, 0, 0) | ||
self.log.info("Inserting test data...") | ||
test_data = generate_data(datetime(2015, 9, 3, 0, 0, 0), 1000) | ||
self.logstore.insert_batch(test_data) | ||
test_data = generate_data(date, 100) | ||
self.logstore.insert_postgresql(test_data) | ||
self.log.info("Test data inserted") | ||
|
||
@classmethod | ||
def tearDownClass(self): | ||
#self.logstore.drop_schema() | ||
self.logstore = None | ||
|
||
@unittest2.skip("We don't have Cassandra on Jenkins server") | ||
def test_fetch_listens(self): | ||
listens = self.logstore.fetch_listens(uid="test", limit=10) | ||
self.assertEqual(len(list(listens)), 10) | ||
date = datetime(2015, 9, 3, 0, 0, 0) | ||
listens = self.logstore.fetch_listens(user_id="test", from_id=to_epoch(date), limit=10) | ||
self.assertEquals(len(list(listens)), 10) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,19 +1,22 @@ | ||
# coding=utf-8 | ||
from __future__ import division, absolute_import, print_function, unicode_literals | ||
from datetime import timedelta | ||
from datetime import datetime, timedelta | ||
from listenstore.listen import Listen | ||
import uuid | ||
|
||
|
||
def generate_data(from_date, num_records): | ||
test_data = [] | ||
|
||
current_date = from_date | ||
|
||
current_date = to_epoch(from_date) | ||
artist_msid = str(uuid.uuid4()) | ||
for i in range(1, num_records): | ||
item = Listen(uid="test", timestamp=current_date, artist_msid=artist_msid, | ||
track_msid=str(uuid.uuid4())) | ||
|
||
for i in range(num_records): | ||
current_date += 1 # Add one second | ||
item = Listen(user_id="test", timestamp=current_date, artist_msid=artist_msid, | ||
recording_msid=str(uuid.uuid4())) | ||
test_data.append(item) | ||
current_date += timedelta(seconds=1) | ||
return test_data | ||
|
||
|
||
def to_epoch(date): | ||
return (date - datetime.utcfromtimestamp(0)).total_seconds() |
Oops, something went wrong.