Merge pull request #246 from input-output-hk/dorin/to_csv

save db tables as csvs for data team

.github/workflows/sync_test.yaml

@@ -48,6 +48,7 @@ jobs:
           pip install psutil
           pip install sqlite-utils>=2.18
           sqlite-utils --version
+          pip install pandas
       - name: run sync test
         run: |
           cd cardano-node-tests
@@ -64,11 +65,12 @@ jobs:
           python sync_tests/write_values_to_db.py -e ${{ matrix.env }}
 
           git add sync_tests/sync_results.db
+          git add sync_tests/csv_files
           git commit -m "added sync values for tag ${{ github.event.inputs.tag_no1 }} ${{ github.event.inputs.tag_no2 }} - ${{ matrix.env }} - ${{ matrix.os }}"
           git push origin master
       - name: generate artifacts
         uses: actions/upload-artifact@v2
         if: always()
         with:
           name: node_logs_${{ matrix.env }}_${{ matrix.os }}
-          path: cardano-node-tests/logfile.log
+          path: cardano-node-tests/logfile.log

setup.py

@@ -5,5 +5,5 @@
     name="cardano-node-tests",
     packages=find_packages(),
     use_scm_version=True,
-    setup_requires=["setuptools_scm", "psutil"],
+    setup_requires=["setuptools_scm", "psutil", "pandas", "requests"],
 )

sync_tests/write_values_to_db.py

@@ -1,7 +1,8 @@
-import argparse
 import sqlite3
-from pathlib import Path
 from sqlite3 import Error
+from pathlib import Path
+import argparse
+import pandas as pd
 
 database_name = r"sync_results.db"
 results_file_name = r"sync_results.log"
@@ -19,23 +20,22 @@ def create_connection(db_file):
 
 
 def add_test_values_into_db(env, test_values):
+    print(f"Write values into {env} table")
     current_directory = Path.cwd()
     database_path = Path(current_directory) / "sync_tests" / database_name
     print(f"database_path: {database_path}")
 
     conn = create_connection(database_path)
 
     try:
-        sql = (
-            f" INSERT INTO {env} "
-            f"(env, tag_no1, tag_no2, pr_no1, pr_no2, cardano_cli_version1, cardano_cli_version2, "
-            f"cardano_cli_git_rev1, cardano_cli_git_rev2, start_sync_time1, end_sync_time1, start_sync_time2, "
-            f"end_sync_time2, byron_sync_time_secs1, shelley_sync_time_secs1, shelley_sync_time_secs2, "
-            f"total_chunks1, total_chunks2, latest_block_no1, latest_block_no2, latest_slot_no1, latest_slot_no2, "
-            f"start_node_seconds1, start_node_seconds2, platform_system, platform_release, platform_version, "
-            f"sync_details1) "
-            f"VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
-        )
+        sql = f' INSERT INTO {env} ' \
+              f'(env, tag_no1, tag_no2, pr_no1, pr_no2, cardano_cli_version1, cardano_cli_version2, ' \
+              f'cardano_cli_git_rev1, cardano_cli_git_rev2, start_sync_time1, end_sync_time1, start_sync_time2, ' \
+              f'end_sync_time2, byron_sync_time_secs1, shelley_sync_time_secs1, shelley_sync_time_secs2, ' \
+              f'total_chunks1, total_chunks2, latest_block_no1, latest_block_no2, latest_slot_no1, latest_slot_no2, ' \
+              f'start_node_seconds1, start_node_seconds2, platform_system, platform_release, platform_version, ' \
+              f'sync_details1) ' \
+              f'VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)'
 
         print(f"sql: {sql}")
 
@@ -44,7 +44,42 @@ def add_test_values_into_db(env, test_values):
         conn.commit()
         cur.close()
     except sqlite3.Error as error:
-        print("!!! ERROR: Failed to insert data into pool_details table:\n", error)
+        print(f"!!! ERROR: Failed to insert data into {env} table:\n", error)
+        return False
+    finally:
+        if conn:
+            conn.close()
+    return True
+
+
+def export_db_tables_to_csv(env):
+    print(f"Export {env} table into CSV file")
+    current_directory = Path.cwd()
+    database_path = Path(current_directory) / "sync_tests" / database_name
+    csv_files_path = Path(current_directory) / "sync_tests" / "csv_files"
+    print(f"database_path : {database_path}")
+    print(f"csv_files_path: {csv_files_path}")
+
+    Path(csv_files_path).mkdir(parents=True, exist_ok=True)
+
+    conn = create_connection(database_path)
+    try:
+        sql = f"select * from {env}"
+        print(f"sql: {sql}")
+
+        cur = conn.cursor()
+        cur.execute(sql)
+
+        with open(csv_files_path / f"{env}.csv", "w") as csv_file:
+            df = pd.read_sql(f"select * from {env}", conn)
+            df.to_csv(csv_file, escapechar="\n", index=False)
+
+        conn.commit()
+        cur.close()
+
+        print(f"Data exported Successfully into {csv_files_path / f'{env}.csv'}")
+    except sqlite3.Error as error:
+        print(f"!!! ERROR: Failed to insert data into {env} table:\n", error)
         return False
     finally:
         if conn:
@@ -58,7 +93,8 @@ def main():
     current_directory = Path.cwd()
     print(f"current_directory: {current_directory}")
 
-    # sync_test_clean_state.py is creating the "sync_results.log" file that has the test values to be added into the db
+    # sync_test_clean_state.py is creating the "sync_results.log" file that has the test values
+    # to be added into the db
     with open(current_directory / "sync_tests" / results_file_name, "r+") as file:
         file_values = file.read()
         print(f"file_values: {file_values}")
@@ -68,18 +104,18 @@ def main():
     print(f"env: {env}")
     print(f"test_values: {test_values}")
 
-    # add the test values into the local copy of the database (to be pushed into master)
+    # Add the test values into the local copy of the database (to be pushed into master)
     add_test_values_into_db(env, test_values)
 
+    # Export data into CSV file
+    export_db_tables_to_csv(env)
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Add sync test values into database\n\n")
 
-    parser.add_argument(
-        "-e",
-        "--environment",
-        help="The environment on which to run the tests - testnet, staging or mainnet.",
-    )
+    parser.add_argument("-e", "--environment",
+                        help="The environment on which to run the tests - testnet, staging or mainnet.")
 
     args = parser.parse_args()