From 343c605cf588fa7ef5db04286f006acb3268c893 Mon Sep 17 00:00:00 2001 From: dorin100 Date: Wed, 18 Nov 2020 13:34:25 +0200 Subject: [PATCH] save db tables as csvs for data team --- .github/workflows/sync_test.yaml | 4 +- setup.py | 2 +- sync_tests/write_values_to_db.py | 76 +++++++++++++++++++++++--------- 3 files changed, 60 insertions(+), 22 deletions(-) diff --git a/.github/workflows/sync_test.yaml b/.github/workflows/sync_test.yaml index b7926939b..50b73216d 100644 --- a/.github/workflows/sync_test.yaml +++ b/.github/workflows/sync_test.yaml @@ -48,6 +48,7 @@ jobs: pip install psutil pip install sqlite-utils>=2.18 sqlite-utils --version + pip install pandas - name: run sync test run: | cd cardano-node-tests @@ -64,6 +65,7 @@ jobs: python sync_tests/write_values_to_db.py -e ${{ matrix.env }} git add sync_tests/sync_results.db + git add sync_tests/csv_files git commit -m "added sync values for tag ${{ github.event.inputs.tag_no1 }} ${{ github.event.inputs.tag_no2 }} - ${{ matrix.env }} - ${{ matrix.os }}" git push origin master - name: generate artifacts @@ -71,4 +73,4 @@ jobs: if: always() with: name: node_logs_${{ matrix.env }}_${{ matrix.os }} - path: cardano-node-tests/logfile.log + path: cardano-node-tests/logfile.log \ No newline at end of file diff --git a/setup.py b/setup.py index aa2cf9973..b07191a04 100644 --- a/setup.py +++ b/setup.py @@ -5,5 +5,5 @@ name="cardano-node-tests", packages=find_packages(), use_scm_version=True, - setup_requires=["setuptools_scm", "psutil"], + setup_requires=["setuptools_scm", "psutil", "pandas", "requests"], ) diff --git a/sync_tests/write_values_to_db.py b/sync_tests/write_values_to_db.py index 7e9d45d15..2e429e7da 100644 --- a/sync_tests/write_values_to_db.py +++ b/sync_tests/write_values_to_db.py @@ -1,7 +1,8 @@ -import argparse import sqlite3 -from pathlib import Path from sqlite3 import Error +from pathlib import Path +import argparse +import pandas as pd database_name = r"sync_results.db" results_file_name = r"sync_results.log" @@ -19,6 +20,7 @@ def create_connection(db_file): def add_test_values_into_db(env, test_values): + print(f"Write values into {env} table") current_directory = Path.cwd() database_path = Path(current_directory) / "sync_tests" / database_name print(f"database_path: {database_path}") @@ -26,16 +28,14 @@ def add_test_values_into_db(env, test_values): conn = create_connection(database_path) try: - sql = ( - f" INSERT INTO {env} " - f"(env, tag_no1, tag_no2, pr_no1, pr_no2, cardano_cli_version1, cardano_cli_version2, " - f"cardano_cli_git_rev1, cardano_cli_git_rev2, start_sync_time1, end_sync_time1, start_sync_time2, " - f"end_sync_time2, byron_sync_time_secs1, shelley_sync_time_secs1, shelley_sync_time_secs2, " - f"total_chunks1, total_chunks2, latest_block_no1, latest_block_no2, latest_slot_no1, latest_slot_no2, " - f"start_node_seconds1, start_node_seconds2, platform_system, platform_release, platform_version, " - f"sync_details1) " - f"VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" - ) + sql = f' INSERT INTO {env} ' \ + f'(env, tag_no1, tag_no2, pr_no1, pr_no2, cardano_cli_version1, cardano_cli_version2, ' \ + f'cardano_cli_git_rev1, cardano_cli_git_rev2, start_sync_time1, end_sync_time1, start_sync_time2, ' \ + f'end_sync_time2, byron_sync_time_secs1, shelley_sync_time_secs1, shelley_sync_time_secs2, ' \ + f'total_chunks1, total_chunks2, latest_block_no1, latest_block_no2, latest_slot_no1, latest_slot_no2, ' \ + f'start_node_seconds1, start_node_seconds2, platform_system, platform_release, platform_version, ' \ + f'sync_details1) ' \ + f'VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)' print(f"sql: {sql}") @@ -44,7 +44,42 @@ def add_test_values_into_db(env, test_values): conn.commit() cur.close() except sqlite3.Error as error: - print("!!! ERROR: Failed to insert data into pool_details table:\n", error) + print(f"!!! ERROR: Failed to insert data into {env} table:\n", error) + return False + finally: + if conn: + conn.close() + return True + + +def export_db_tables_to_csv(env): + print(f"Export {env} table into CSV file") + current_directory = Path.cwd() + database_path = Path(current_directory) / "sync_tests" / database_name + csv_files_path = Path(current_directory) / "sync_tests" / "csv_files" + print(f"database_path : {database_path}") + print(f"csv_files_path: {csv_files_path}") + + Path(csv_files_path).mkdir(parents=True, exist_ok=True) + + conn = create_connection(database_path) + try: + sql = f"select * from {env}" + print(f"sql: {sql}") + + cur = conn.cursor() + cur.execute(sql) + + with open(csv_files_path / f"{env}.csv", "w") as csv_file: + df = pd.read_sql(f"select * from {env}", conn) + df.to_csv(csv_file, escapechar="\n", index=False) + + conn.commit() + cur.close() + + print(f"Data exported Successfully into {csv_files_path / f'{env}.csv'}") + except sqlite3.Error as error: + print(f"!!! ERROR: Failed to insert data into {env} table:\n", error) return False finally: if conn: @@ -58,7 +93,8 @@ def main(): current_directory = Path.cwd() print(f"current_directory: {current_directory}") - # sync_test_clean_state.py is creating the "sync_results.log" file that has the test values to be added into the db + # sync_test_clean_state.py is creating the "sync_results.log" file that has the test values + # to be added into the db with open(current_directory / "sync_tests" / results_file_name, "r+") as file: file_values = file.read() print(f"file_values: {file_values}") @@ -68,18 +104,18 @@ def main(): print(f"env: {env}") print(f"test_values: {test_values}") - # add the test values into the local copy of the database (to be pushed into master) + # Add the test values into the local copy of the database (to be pushed into master) add_test_values_into_db(env, test_values) + # Export data into CSV file + export_db_tables_to_csv(env) + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Add sync test values into database\n\n") - parser.add_argument( - "-e", - "--environment", - help="The environment on which to run the tests - testnet, staging or mainnet.", - ) + parser.add_argument("-e", "--environment", + help="The environment on which to run the tests - testnet, staging or mainnet.") args = parser.parse_args()