From 34ca568b6112e0b0d60f8e19afb9425c45d9e537 Mon Sep 17 00:00:00 2001 From: msaltnet Date: Sat, 4 Nov 2023 23:46:49 +0900 Subject: [PATCH] can store binance candle data --- smtm/database.py | 33 +++++- tests/database_test.py | 255 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 274 insertions(+), 14 deletions(-) diff --git a/smtm/database.py b/smtm/database.py index a006d48..d4a3b68 100644 --- a/smtm/database.py +++ b/smtm/database.py @@ -26,6 +26,10 @@ def __del__(self): self.conn.close() def create_table(self): + self._create_upbit_table() + self._create_binance_table() + + def _create_upbit_table(self): """테이블 생성 id TEXT 고유 식별자 period(S)-date_time e.g. 60S-YYYY-MM-DD HH:MM:SS period INT 캔들의 기간(초), 분봉 - 60 @@ -44,17 +48,38 @@ def create_table(self): ) self.conn.commit() - def query(self, start, end, market, period=60): + def _create_binance_table(self): + """테이블 생성 + id TEXT 고유 식별자 period(S)-date_time e.g. 60S-YYYY-MM-DD HH:MM:SS + period INT 캔들의 기간(초), 분봉 - 60 + recovered INT 복구된 데이터인지여부 + market TEXT 거래 시장 종류 BTC + date_time DATETIME 정보의 기준 시간, 'YYYY-MM-DD HH:MM:SS' 형식의 sql datetime format + opening_price FLOAT 시작 거래 가격 + high_price FLOAT 최고 거래 가격 + low_price FLOAT 최저 거래 가격 + closing_price FLOAT 마지막 거래 가격 + acc_price FLOAT 단위 시간내 누적 거래 금액 + acc_volume FLOAT 단위 시간내 누적 거래 양 + """ + self.cursor.execute( + """CREATE TABLE IF NOT EXISTS binance (id TEXT PRIMARY KEY, period INT, recovered INT, market TEXT, date_time DATETIME, opening_price FLOAT, high_price FLOAT, low_price FLOAT, closing_price FLOAT, acc_price FLOAT, acc_volume FLOAT)""" + ) + self.conn.commit() + + def query(self, start, end, market, period=60, is_upbit=True): """데이터 조회""" + table = "upbit" if is_upbit is True else "binance" self.cursor.execute( - "SELECT period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume FROM upbit WHERE market = ? AND period = ? AND date_time >= ? AND date_time < ? ORDER BY datetime(date_time) ASC", + f"SELECT id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume FROM {table} WHERE market = ? AND period = ? AND date_time >= ? AND date_time < ? ORDER BY datetime(date_time) ASC", (market, period, start, end), ) return self.cursor.fetchall() - def update(self, data, period=60): + def update(self, data, period=60, is_upbit=True): """데이터베이스 데이터 추가 또는 업데이트""" + table = "upbit" if is_upbit is True else "binance" tuple_list = [] for item in data: recovered = item["recovered"] if "recovered" in item else 0 @@ -76,7 +101,7 @@ def update(self, data, period=60): self.logger.info(f"Updated: {len(tuple_list)}") self.cursor.executemany( - "REPLACE INTO upbit(id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + f"REPLACE INTO {table} (id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", tuple_list, ) self.conn.commit() diff --git a/tests/database_test.py b/tests/database_test.py index 1ea5fe4..ca912ba 100644 --- a/tests/database_test.py +++ b/tests/database_test.py @@ -14,7 +14,7 @@ def tearDown(self): def test_constructor_make_connection_correctly(self, mock_connect): dummy_connection = MagicMock() mock_connect.return_value = dummy_connection - db = Database() + Database() mock_connect.assert_called_once_with("smtm.db", check_same_thread=False, timeout=30.0) dummy_connection.cursor.assert_called_once() @@ -23,22 +23,30 @@ def test_create_table_should_execute_and_commit_correct_statement(self): db.cursor = MagicMock() db.conn = MagicMock() db.create_table() - db.cursor.execute.assert_called_once_with( - "CREATE TABLE IF NOT EXISTS upbit (id TEXT PRIMARY KEY, period INT, recovered INT, market TEXT, date_time DATETIME, opening_price FLOAT, high_price FLOAT, low_price FLOAT, closing_price FLOAT, acc_price FLOAT, acc_volume FLOAT)" + self.assertEqual(db.cursor.execute.call_count, 2) + self.assertEqual(db.conn.commit.call_count, 2) + self.assertEqual( + db.cursor.execute.call_args_list[0][0][0], + "CREATE TABLE IF NOT EXISTS upbit (id TEXT PRIMARY KEY, period INT, recovered INT, market TEXT, date_time DATETIME, opening_price FLOAT, high_price FLOAT, low_price FLOAT, closing_price FLOAT, acc_price FLOAT, acc_volume FLOAT)", ) - db.conn.commit.assert_called_once() + self.assertEqual( + db.cursor.execute.call_args_list[1][0][0], + "CREATE TABLE IF NOT EXISTS binance (id TEXT PRIMARY KEY, period INT, recovered INT, market TEXT, date_time DATETIME, opening_price FLOAT, high_price FLOAT, low_price FLOAT, closing_price FLOAT, acc_price FLOAT, acc_volume FLOAT)", + ) + - def test_query_should_execute_and_commit_correct_statement(self): +class DatabaseUpbitTests(unittest.TestCase): + def test_query_should_execute_and_commit_correct_statement_with_upbit_table(self): db = Database() db.cursor = MagicMock() - db.query("start_date", "end_date", "mango_market") + db.query("start_date", "end_date", "mango_market", period=60, is_upbit=True) db.cursor.execute.assert_called_once_with( - "SELECT period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume FROM upbit WHERE market = ? AND period = ? AND date_time >= ? AND date_time < ? ORDER BY datetime(date_time) ASC", + "SELECT id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume FROM upbit WHERE market = ? AND period = ? AND date_time >= ? AND date_time < ? ORDER BY datetime(date_time) ASC", ("mango_market", 60, "start_date", "end_date"), ) db.cursor.fetchall.assert_called_once() - def test_update_should_execute_and_commit_correct_statement(self): + def test_update_should_execute_and_commit_correct_statement_with_upbit_table(self): db = Database() dummy_data = [ { @@ -76,7 +84,7 @@ def test_update_should_execute_and_commit_correct_statement(self): db.cursor = MagicMock() db.conn = MagicMock() - db.update(dummy_data) + db.update(dummy_data, period=60, is_upbit=True) expected_tuple_list = [ ( "60S-2020-03-10T22:52:00", @@ -120,7 +128,234 @@ def test_update_should_execute_and_commit_correct_statement(self): ] db.cursor.executemany.assert_called_once_with( - "REPLACE INTO upbit(id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "REPLACE INTO upbit (id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", expected_tuple_list, ) db.conn.commit.assert_called_once() + + +class DatabaseBinanceTests(unittest.TestCase): + def test_query_should_execute_and_commit_correct_statement_with_binance_table(self): + db = Database() + db.cursor = MagicMock() + db.query("start_date", "end_date", "mango_market", period=60, is_upbit=False) + db.cursor.execute.assert_called_once_with( + "SELECT id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume FROM binance WHERE market = ? AND period = ? AND date_time >= ? AND date_time < ? ORDER BY datetime(date_time) ASC", + ("mango_market", 60, "start_date", "end_date"), + ) + db.cursor.fetchall.assert_called_once() + + def test_update_should_execute_and_commit_correct_statement_with_binance_table(self): + db = Database() + dummy_data = [ + { + "market": "mango", + "date_time": "2020-03-10T22:52:00", + "opening_price": 9777000.00000000, + "high_price": 9778000.00000000, + "low_price": 9763000.00000000, + "closing_price": 9778000.00000000, + "acc_price": 11277224.71063000, + "acc_volume": 1.15377852, + "recovered": 1, + }, + { + "market": "mango", + "date_time": "2020-03-10T22:53:00", + "opening_price": 8777000.00000000, + "high_price": 8778000.00000000, + "low_price": 8763000.00000000, + "closing_price": 8778000.00000000, + "acc_price": 11277224.71063000, + "acc_volume": 1.15377852, + }, + { + "market": "mango", + "date_time": "2020-03-10T22:54:00", + "opening_price": 7777000.00000000, + "high_price": 7778000.00000000, + "low_price": 7763000.00000000, + "closing_price": 7778000.00000000, + "acc_price": 11277224.71063000, + "acc_volume": 1.15377852, + }, + ] + + db.cursor = MagicMock() + db.conn = MagicMock() + db.update(dummy_data, period=60, is_upbit=False) + expected_tuple_list = [ + ( + "60S-2020-03-10T22:52:00", + 60, + 1, + "mango", + "2020-03-10T22:52:00", + 9777000.0, + 9778000.0, + 9763000.0, + 9778000.0, + 11277224.71063, + 1.15377852, + ), + ( + "60S-2020-03-10T22:53:00", + 60, + 0, + "mango", + "2020-03-10T22:53:00", + 8777000.0, + 8778000.0, + 8763000.0, + 8778000.0, + 11277224.71063, + 1.15377852, + ), + ( + "60S-2020-03-10T22:54:00", + 60, + 0, + "mango", + "2020-03-10T22:54:00", + 7777000.0, + 7778000.0, + 7763000.0, + 7778000.0, + 11277224.71063, + 1.15377852, + ), + ] + + db.cursor.executemany.assert_called_once_with( + "REPLACE INTO binance (id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + expected_tuple_list, + ) + db.conn.commit.assert_called_once() + + +class DatabaseInMemoryTests(unittest.TestCase): + def test_update_and_query_should_execute_and_commit_correct_statement_with_upbit_table(self): + # create in-memory database + db = Database(":memory:") + data = db.query( + "2020-03-10T22:52:00", "2020-03-10T22:53:00", "mango", period=60, is_upbit=True + ) + self.assertEqual(data, []) + dummy_data = [ + { + "id": "60S-2020-03-10T22:52:00", + "period": 60, + "recovered": 0, + "market": "mango", + "date_time": "2020-03-10T22:52:00", + "opening_price": 9777000.00000000, + "high_price": 9778000.00000000, + "low_price": 9763000.00000000, + "closing_price": 9778000.00000000, + "acc_price": 11277224.71063000, + "acc_volume": 1.15377852, + "recovered": 1, + }, + { + "id": "60S-2020-03-10T22:53:00", + "period": 60, + "recovered": 0, + "market": "mango", + "date_time": "2020-03-10T22:53:00", + "opening_price": 8777000.00000000, + "high_price": 8778000.00000000, + "low_price": 8763000.00000000, + "closing_price": 8778000.00000000, + "acc_price": 11277224.71063000, + "acc_volume": 1.15377852, + }, + ] + db.update(dummy_data, period=60, is_upbit=True) + data = db.query( + "2020-03-10T22:52:00", "2020-03-10T22:54:00", "mango", period=60, is_upbit=True + ) + self.assertEqual(data[0], dummy_data[0]) + self.assertEqual(data[1], dummy_data[1]) + update_dummy_data = [ + { + "id": "60S-2020-03-10T22:53:00", + "period": 60, + "recovered": 0, + "market": "mango", + "date_time": "2020-03-10T22:53:00", + "opening_price": 7777000.00000000, + "high_price": 8778000.00000000, + "low_price": 8763000.00000000, + "closing_price": 8778000.00000000, + "acc_price": 21277224.71063000, + "acc_volume": 19.15377852, + }, + ] + db.update(update_dummy_data, period=60, is_upbit=True) + data = db.query( + "2020-03-10T22:53:00", "2020-03-10T22:54:00", "mango", period=60, is_upbit=True + ) + self.assertEqual(data, update_dummy_data) + + def test_update_should_execute_and_commit_correct_statement_with_binance_table(self): + # create in-memory database + db = Database(":memory:") + data = db.query( + "2020-03-10T22:52:00", "2020-03-10T22:53:00", "mango", period=60, is_upbit=False + ) + self.assertEqual(data, []) + dummy_data = [ + { + "id": "60S-2020-03-10T22:52:00", + "period": 60, + "recovered": 0, + "market": "mango", + "date_time": "2020-03-10T22:52:00", + "opening_price": 9777000.00000000, + "high_price": 9778000.00000000, + "low_price": 9763000.00000000, + "closing_price": 9778000.00000000, + "acc_price": 11277224.71063000, + "acc_volume": 1.15377852, + "recovered": 1, + }, + { + "id": "60S-2020-03-10T22:53:00", + "period": 60, + "recovered": 0, + "market": "mango", + "date_time": "2020-03-10T22:53:00", + "opening_price": 8777000.00000000, + "high_price": 8778000.00000000, + "low_price": 8763000.00000000, + "closing_price": 8778000.00000000, + "acc_price": 11277224.71063000, + "acc_volume": 1.15377852, + }, + ] + db.update(dummy_data, period=60, is_upbit=False) + data = db.query( + "2020-03-10T22:52:00", "2020-03-10T22:54:00", "mango", period=60, is_upbit=False + ) + self.assertEqual(data[0], dummy_data[0]) + self.assertEqual(data[1], dummy_data[1]) + update_dummy_data = [ + { + "id": "60S-2020-03-10T22:53:00", + "period": 60, + "recovered": 0, + "market": "mango", + "date_time": "2020-03-10T22:53:00", + "opening_price": 7777000.00000000, + "high_price": 8778000.00000000, + "low_price": 8763000.00000000, + "closing_price": 8778000.00000000, + "acc_price": 21277224.71063000, + "acc_volume": 19.15377852, + }, + ] + db.update(update_dummy_data, period=60, is_upbit=False) + data = db.query( + "2020-03-10T22:53:00", "2020-03-10T22:54:00", "mango", period=60, is_upbit=False + ) + self.assertEqual(data, update_dummy_data)