Skip to content

Commit

Permalink
can store binance candle data
Browse files Browse the repository at this point in the history
  • Loading branch information
@msaltnet
msaltnet committed Nov 4, 2023
1 parent e8f490e commit 34ca568
Show file tree
Hide file tree
Showing 2 changed files with 274 additions and 14 deletions.
33 changes: 29 additions & 4 deletions smtm/database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ def __del__(self):
self.conn.close()

def create_table(self):
self._create_upbit_table()
self._create_binance_table()

def _create_upbit_table(self):
"""테이블 생성
id TEXT 고유 식별자 period(S)-date_time e.g. 60S-YYYY-MM-DD HH:MM:SS
period INT 캔들의 기간(초), 분봉 - 60
Expand All @@ -44,17 +48,38 @@ def create_table(self):
)
self.conn.commit()

def query(self, start, end, market, period=60):
def _create_binance_table(self):
"""테이블 생성
id TEXT 고유 식별자 period(S)-date_time e.g. 60S-YYYY-MM-DD HH:MM:SS
period INT 캔들의 기간(초), 분봉 - 60
recovered INT 복구된 데이터인지여부
market TEXT 거래 시장 종류 BTC
date_time DATETIME 정보의 기준 시간, 'YYYY-MM-DD HH:MM:SS' 형식의 sql datetime format
opening_price FLOAT 시작 거래 가격
high_price FLOAT 최고 거래 가격
low_price FLOAT 최저 거래 가격
closing_price FLOAT 마지막 거래 가격
acc_price FLOAT 단위 시간내 누적 거래 금액
acc_volume FLOAT 단위 시간내 누적 거래 양
"""
self.cursor.execute(
"""CREATE TABLE IF NOT EXISTS binance (id TEXT PRIMARY KEY, period INT, recovered INT, market TEXT, date_time DATETIME, opening_price FLOAT, high_price FLOAT, low_price FLOAT, closing_price FLOAT, acc_price FLOAT, acc_volume FLOAT)"""
)
self.conn.commit()

def query(self, start, end, market, period=60, is_upbit=True):
"""데이터 조회"""
table = "upbit" if is_upbit is True else "binance"

self.cursor.execute(
"SELECT period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume FROM upbit WHERE market = ? AND period = ? AND date_time >= ? AND date_time < ? ORDER BY datetime(date_time) ASC",
f"SELECT id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume FROM {table} WHERE market = ? AND period = ? AND date_time >= ? AND date_time < ? ORDER BY datetime(date_time) ASC",
(market, period, start, end),
)
return self.cursor.fetchall()

def update(self, data, period=60):
def update(self, data, period=60, is_upbit=True):
"""데이터베이스 데이터 추가 또는 업데이트"""
table = "upbit" if is_upbit is True else "binance"
tuple_list = []
for item in data:
recovered = item["recovered"] if "recovered" in item else 0
Expand All @@ -76,7 +101,7 @@ def update(self, data, period=60):

self.logger.info(f"Updated: {len(tuple_list)}")
self.cursor.executemany(
"REPLACE INTO upbit(id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
f"REPLACE INTO {table} (id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
tuple_list,
)
self.conn.commit()
255 changes: 245 additions & 10 deletions tests/database_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def tearDown(self):
def test_constructor_make_connection_correctly(self, mock_connect):
dummy_connection = MagicMock()
mock_connect.return_value = dummy_connection
db = Database()
Database()
mock_connect.assert_called_once_with("smtm.db", check_same_thread=False, timeout=30.0)
dummy_connection.cursor.assert_called_once()

Expand All @@ -23,22 +23,30 @@ def test_create_table_should_execute_and_commit_correct_statement(self):
db.cursor = MagicMock()
db.conn = MagicMock()
db.create_table()
db.cursor.execute.assert_called_once_with(
"CREATE TABLE IF NOT EXISTS upbit (id TEXT PRIMARY KEY, period INT, recovered INT, market TEXT, date_time DATETIME, opening_price FLOAT, high_price FLOAT, low_price FLOAT, closing_price FLOAT, acc_price FLOAT, acc_volume FLOAT)"
self.assertEqual(db.cursor.execute.call_count, 2)
self.assertEqual(db.conn.commit.call_count, 2)
self.assertEqual(
db.cursor.execute.call_args_list[0][0][0],
"CREATE TABLE IF NOT EXISTS upbit (id TEXT PRIMARY KEY, period INT, recovered INT, market TEXT, date_time DATETIME, opening_price FLOAT, high_price FLOAT, low_price FLOAT, closing_price FLOAT, acc_price FLOAT, acc_volume FLOAT)",
)
db.conn.commit.assert_called_once()
self.assertEqual(
db.cursor.execute.call_args_list[1][0][0],
"CREATE TABLE IF NOT EXISTS binance (id TEXT PRIMARY KEY, period INT, recovered INT, market TEXT, date_time DATETIME, opening_price FLOAT, high_price FLOAT, low_price FLOAT, closing_price FLOAT, acc_price FLOAT, acc_volume FLOAT)",
)


def test_query_should_execute_and_commit_correct_statement(self):
class DatabaseUpbitTests(unittest.TestCase):
def test_query_should_execute_and_commit_correct_statement_with_upbit_table(self):
db = Database()
db.cursor = MagicMock()
db.query("start_date", "end_date", "mango_market")
db.query("start_date", "end_date", "mango_market", period=60, is_upbit=True)
db.cursor.execute.assert_called_once_with(
"SELECT period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume FROM upbit WHERE market = ? AND period = ? AND date_time >= ? AND date_time < ? ORDER BY datetime(date_time) ASC",
"SELECT id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume FROM upbit WHERE market = ? AND period = ? AND date_time >= ? AND date_time < ? ORDER BY datetime(date_time) ASC",
("mango_market", 60, "start_date", "end_date"),
)
db.cursor.fetchall.assert_called_once()

def test_update_should_execute_and_commit_correct_statement(self):
def test_update_should_execute_and_commit_correct_statement_with_upbit_table(self):
db = Database()
dummy_data = [
{
Expand Down Expand Up @@ -76,7 +84,7 @@ def test_update_should_execute_and_commit_correct_statement(self):

db.cursor = MagicMock()
db.conn = MagicMock()
db.update(dummy_data)
db.update(dummy_data, period=60, is_upbit=True)
expected_tuple_list = [
(
"60S-2020-03-10T22:52:00",
Expand Down Expand Up @@ -120,7 +128,234 @@ def test_update_should_execute_and_commit_correct_statement(self):
]

db.cursor.executemany.assert_called_once_with(
"REPLACE INTO upbit(id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"REPLACE INTO upbit (id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
expected_tuple_list,
)
db.conn.commit.assert_called_once()


class DatabaseBinanceTests(unittest.TestCase):
def test_query_should_execute_and_commit_correct_statement_with_binance_table(self):
db = Database()
db.cursor = MagicMock()
db.query("start_date", "end_date", "mango_market", period=60, is_upbit=False)
db.cursor.execute.assert_called_once_with(
"SELECT id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume FROM binance WHERE market = ? AND period = ? AND date_time >= ? AND date_time < ? ORDER BY datetime(date_time) ASC",
("mango_market", 60, "start_date", "end_date"),
)
db.cursor.fetchall.assert_called_once()

def test_update_should_execute_and_commit_correct_statement_with_binance_table(self):
db = Database()
dummy_data = [
{
"market": "mango",
"date_time": "2020-03-10T22:52:00",
"opening_price": 9777000.00000000,
"high_price": 9778000.00000000,
"low_price": 9763000.00000000,
"closing_price": 9778000.00000000,
"acc_price": 11277224.71063000,
"acc_volume": 1.15377852,
"recovered": 1,
},
{
"market": "mango",
"date_time": "2020-03-10T22:53:00",
"opening_price": 8777000.00000000,
"high_price": 8778000.00000000,
"low_price": 8763000.00000000,
"closing_price": 8778000.00000000,
"acc_price": 11277224.71063000,
"acc_volume": 1.15377852,
},
{
"market": "mango",
"date_time": "2020-03-10T22:54:00",
"opening_price": 7777000.00000000,
"high_price": 7778000.00000000,
"low_price": 7763000.00000000,
"closing_price": 7778000.00000000,
"acc_price": 11277224.71063000,
"acc_volume": 1.15377852,
},
]

db.cursor = MagicMock()
db.conn = MagicMock()
db.update(dummy_data, period=60, is_upbit=False)
expected_tuple_list = [
(
"60S-2020-03-10T22:52:00",
60,
1,
"mango",
"2020-03-10T22:52:00",
9777000.0,
9778000.0,
9763000.0,
9778000.0,
11277224.71063,
1.15377852,
),
(
"60S-2020-03-10T22:53:00",
60,
0,
"mango",
"2020-03-10T22:53:00",
8777000.0,
8778000.0,
8763000.0,
8778000.0,
11277224.71063,
1.15377852,
),
(
"60S-2020-03-10T22:54:00",
60,
0,
"mango",
"2020-03-10T22:54:00",
7777000.0,
7778000.0,
7763000.0,
7778000.0,
11277224.71063,
1.15377852,
),
]

db.cursor.executemany.assert_called_once_with(
"REPLACE INTO binance (id, period, recovered, market, date_time, opening_price, high_price, low_price, closing_price, acc_price, acc_volume) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
expected_tuple_list,
)
db.conn.commit.assert_called_once()


class DatabaseInMemoryTests(unittest.TestCase):
def test_update_and_query_should_execute_and_commit_correct_statement_with_upbit_table(self):
# create in-memory database
db = Database(":memory:")
data = db.query(
"2020-03-10T22:52:00", "2020-03-10T22:53:00", "mango", period=60, is_upbit=True
)
self.assertEqual(data, [])
dummy_data = [
{
"id": "60S-2020-03-10T22:52:00",
"period": 60,
"recovered": 0,
"market": "mango",
"date_time": "2020-03-10T22:52:00",
"opening_price": 9777000.00000000,
"high_price": 9778000.00000000,
"low_price": 9763000.00000000,
"closing_price": 9778000.00000000,
"acc_price": 11277224.71063000,
"acc_volume": 1.15377852,
"recovered": 1,
},
{
"id": "60S-2020-03-10T22:53:00",
"period": 60,
"recovered": 0,
"market": "mango",
"date_time": "2020-03-10T22:53:00",
"opening_price": 8777000.00000000,
"high_price": 8778000.00000000,
"low_price": 8763000.00000000,
"closing_price": 8778000.00000000,
"acc_price": 11277224.71063000,
"acc_volume": 1.15377852,
},
]
db.update(dummy_data, period=60, is_upbit=True)
data = db.query(
"2020-03-10T22:52:00", "2020-03-10T22:54:00", "mango", period=60, is_upbit=True
)
self.assertEqual(data[0], dummy_data[0])
self.assertEqual(data[1], dummy_data[1])
update_dummy_data = [
{
"id": "60S-2020-03-10T22:53:00",
"period": 60,
"recovered": 0,
"market": "mango",
"date_time": "2020-03-10T22:53:00",
"opening_price": 7777000.00000000,
"high_price": 8778000.00000000,
"low_price": 8763000.00000000,
"closing_price": 8778000.00000000,
"acc_price": 21277224.71063000,
"acc_volume": 19.15377852,
},
]
db.update(update_dummy_data, period=60, is_upbit=True)
data = db.query(
"2020-03-10T22:53:00", "2020-03-10T22:54:00", "mango", period=60, is_upbit=True
)
self.assertEqual(data, update_dummy_data)

def test_update_should_execute_and_commit_correct_statement_with_binance_table(self):
# create in-memory database
db = Database(":memory:")
data = db.query(
"2020-03-10T22:52:00", "2020-03-10T22:53:00", "mango", period=60, is_upbit=False
)
self.assertEqual(data, [])
dummy_data = [
{
"id": "60S-2020-03-10T22:52:00",
"period": 60,
"recovered": 0,
"market": "mango",
"date_time": "2020-03-10T22:52:00",
"opening_price": 9777000.00000000,
"high_price": 9778000.00000000,
"low_price": 9763000.00000000,
"closing_price": 9778000.00000000,
"acc_price": 11277224.71063000,
"acc_volume": 1.15377852,
"recovered": 1,
},
{
"id": "60S-2020-03-10T22:53:00",
"period": 60,
"recovered": 0,
"market": "mango",
"date_time": "2020-03-10T22:53:00",
"opening_price": 8777000.00000000,
"high_price": 8778000.00000000,
"low_price": 8763000.00000000,
"closing_price": 8778000.00000000,
"acc_price": 11277224.71063000,
"acc_volume": 1.15377852,
},
]
db.update(dummy_data, period=60, is_upbit=False)
data = db.query(
"2020-03-10T22:52:00", "2020-03-10T22:54:00", "mango", period=60, is_upbit=False
)
self.assertEqual(data[0], dummy_data[0])
self.assertEqual(data[1], dummy_data[1])
update_dummy_data = [
{
"id": "60S-2020-03-10T22:53:00",
"period": 60,
"recovered": 0,
"market": "mango",
"date_time": "2020-03-10T22:53:00",
"opening_price": 7777000.00000000,
"high_price": 8778000.00000000,
"low_price": 8763000.00000000,
"closing_price": 8778000.00000000,
"acc_price": 21277224.71063000,
"acc_volume": 19.15377852,
},
]
db.update(update_dummy_data, period=60, is_upbit=False)
data = db.query(
"2020-03-10T22:53:00", "2020-03-10T22:54:00", "mango", period=60, is_upbit=False
)
self.assertEqual(data, update_dummy_data)

0 comments on commit 34ca568

Please sign in to comment.