diff --git a/METdbLoad/sql/mv_mysql.sql b/METdbLoad/sql/mv_mysql.sql index ddb990ab..05d4f04a 100644 --- a/METdbLoad/sql/mv_mysql.sql +++ b/METdbLoad/sql/mv_mysql.sql @@ -912,6 +912,7 @@ CREATE TABLE line_data_vl1l2 uvoobar DOUBLE, f_speed_bar DOUBLE DEFAULT -9999, o_speed_bar DOUBLE DEFAULT -9999, + total_dir INT UNSIGNED, dir_me DOUBLE, dir_mae DOUBLE, dir_mse DOUBLE, @@ -953,6 +954,7 @@ CREATE TABLE line_data_val1l2 uvooabar DOUBLE, fa_speed_bar DOUBLE DEFAULT -9999, oa_speed_bar DOUBLE DEFAULT -9999, + total_dir INT UNSIGNED, dira_me DOUBLE, dira_mae DOUBLE, dira_mse DOUBLE, @@ -1590,7 +1592,6 @@ CREATE TABLE line_data_ssvar obs_valid_end DATETIME, alpha DOUBLE, total INT UNSIGNED, - n_bin INT UNSIGNED, bin_i INT UNSIGNED, bin_n INT UNSIGNED, @@ -1602,7 +1603,6 @@ CREATE TABLE line_data_ssvar fobar DOUBLE, ffbar DOUBLE, oobar DOUBLE, - fbar_ncl DOUBLE, fbar_ncu DOUBLE, fstdev DOUBLE, @@ -1714,6 +1714,7 @@ CREATE TABLE line_data_vcnt anom_corr_uncntr DOUBLE DEFAULT -9999, anom_corr_uncntr_bcl DOUBLE DEFAULT -9999, anom_corr_uncntr_bcu DOUBLE DEFAULT -9999, + total_dir INT UNSIGNED, dir_me DOUBLE, dir_me_bcl DOUBLE, dir_me_bcu DOUBLE, diff --git a/METdbLoad/sql/updates/update_for_6_0_beta5.sql b/METdbLoad/sql/updates/update_for_6_0_beta5.sql new file mode 100644 index 00000000..e51cfb0a --- /dev/null +++ b/METdbLoad/sql/updates/update_for_6_0_beta5.sql @@ -0,0 +1,13 @@ +DELIMITER | + + +ALTER TABLE line_data_vcnt + ADD COLUMN total_dir DOUBLE | + +ALTER TABLE line_data_val1l2 + ADD COLUMN total_dir DOUBLE | + +ALTER TABLE line_data_vl1l2 + ADD COLUMN total_dir DOUBLE | + +DELIMITER ; diff --git a/METdbLoad/tests/update_schema_6.0_beta4/test_loading.py b/METdbLoad/tests/update_schema_6.0_beta4/test_loading.py index 192abf11..4d5978db 100644 --- a/METdbLoad/tests/update_schema_6.0_beta4/test_loading.py +++ b/METdbLoad/tests/update_schema_6.0_beta4/test_loading.py @@ -29,13 +29,20 @@ def setup_db(): print(exc) # Create a dataclass of the database information - #TCDiag = make_dataclass("TCDiag", ["total", "index", "diag_src", "diag_val"], frozen=True) - #orig = TCDiag(orig_total, orig_index, orig_diag_src, orig_diag_val) - DBS = make_dataclass("DBS", ["hostname", "username", "password", "dbname"]) - db_settings = DBS(parms['hostname'], parms['username'], parms['password'], parms['dbname']) + DBS = make_dataclass("DBS", ["hostname", "port", "username", "password", "dbname"]) + db_settings = DBS(parms['hostname'], parms['port'], parms['username'], parms['password'], parms['dbname']) # Return the db settings (hostname, username, etc.) - yield db_settings + conn = pymysql.connect( + host=db_settings.hostname, + port=db_settings.port, + user=db_settings.username, + password=db_settings.password, + db=db_settings.dbname, + charset='utf8mb4' + ) + + yield conn def test_ecnt_db_created(setup_db): @@ -45,16 +52,8 @@ def test_ecnt_db_created(setup_db): # found. - conn = pymysql.connect( - host=setup_db.hostname, - user=setup_db.username, - password=setup_db.password, - db=setup_db.dbname, - charset='utf8mb4' - ) - try: - with conn.cursor() as cursor: + with setup_db.cursor() as cursor: # Check that the mv_load_test database was created check_db_exists_query = "show databases;" cursor.execute(check_db_exists_query) @@ -69,23 +68,14 @@ def test_ecnt_db_created(setup_db): finally: - conn.close() + setup_db.close() def test_tables_created(setup_db): # log into the database and verify the ECNT, VCNT, VL1L2, and VAL1L2 tables exist - - conn = pymysql.connect( - host=setup_db.hostname, - user=setup_db.username, - password=setup_db.password, - db=setup_db.dbname, - charset='utf8mb4' - ) - try: - with conn.cursor() as cursor: + with setup_db.cursor() as cursor: # Check that the line_data_ecnt, line_data_vcnt, line_data_vl1l2, and # line_data_val1l2 tables were created cursor.execute(CONST_LOAD_DB_CMD) @@ -102,23 +92,15 @@ def test_tables_created(setup_db): assert 'line_data_val1l2' in list_of_rows finally: - conn.close() + setup_db.close() def test_ecnt_columns(setup_db): # log into the database and verify the ign_conv_oerr and ign_corr_oerr columns are in the # list_data_ecnt database table. - conn = pymysql.connect( - host=setup_db.hostname, - user=setup_db.username, - password=setup_db.password, - db=setup_db.dbname, - charset='utf8mb4' - ) - try: - with conn.cursor() as cursor: + with setup_db.cursor() as cursor: # Check that the line_data_ecnt, line_data_vcnt, line_data_vl1l2, and # line_data_val1l2 tables were created cursor.execute(CONST_LOAD_DB_CMD) @@ -134,7 +116,7 @@ def test_ecnt_columns(setup_db): assert 'ign_corr_oerr' in list_of_rows finally: - conn.close() + setup_db.close() def test_vcnt_columns(setup_db): # log into the database and verify the dir_me, dir_me_bcl, dir_me_bcu, ..., etc. columns are in the @@ -145,16 +127,9 @@ def test_vcnt_columns(setup_db): 'dir_mse', 'dir_mse_bcl', 'dir_mse_bcu', 'dir_rmse', 'dir_rmse_bcl', 'dir_rmse_bcu' ] - conn = pymysql.connect( - host=setup_db.hostname, - user=setup_db.username, - password=setup_db.password, - db=setup_db.dbname, - charset='utf8mb4' - ) try: - with conn.cursor() as cursor: + with setup_db.cursor() as cursor: # Check that the line_data_vcnt expected columns were created cursor.execute(CONST_LOAD_DB_CMD) @@ -169,23 +144,16 @@ def test_vcnt_columns(setup_db): assert expected in list_of_rows finally: - conn.close() + setup_db.close() def test_vl1l2_columns(setup_db): # log into the database and verify the dir_me, dir_mae, and dir_mse columns are in the # list_data_vl1l2 database table. expected_cols = ['dir_me', 'dir_mae', 'dir_mse'] - conn = pymysql.connect( - host=setup_db.hostname, - user=setup_db.username, - password=setup_db.password, - db=setup_db.dbname, - charset='utf8mb4' - ) try: - with conn.cursor() as cursor: + with setup_db.cursor() as cursor: # Check that the line_data_vl1l2 table has the expected columns cursor.execute(CONST_LOAD_DB_CMD) @@ -200,23 +168,16 @@ def test_vl1l2_columns(setup_db): assert expected in list_of_rows finally: - conn.close() + setup_db.close() def test_val1l2_columns(setup_db): # log into the database and verify the dira_me, dira_mae, and dira_mse columns are in the # list_data_val1l2 database table. expected_cols = ['dira_me', 'dira_mae', 'dira_mse'] - conn = pymysql.connect( - host=setup_db.hostname, - user=setup_db.username, - password=setup_db.password, - db=setup_db.dbname, - charset='utf8mb4' - ) try: - with conn.cursor() as cursor: + with setup_db.cursor() as cursor: # Check that the line_data_vl1l2 table has the expected columns cursor.execute(CONST_LOAD_DB_CMD) @@ -231,7 +192,7 @@ def test_val1l2_columns(setup_db): assert expected in list_of_rows finally: - conn.close() + setup_db.close() def test_ecnt_vals(setup_db): @@ -242,16 +203,8 @@ def test_ecnt_vals(setup_db): ign_conv_oerr = "33.41424" ign_corr_oerr = "440.06905" - conn = pymysql.connect( - host=setup_db.hostname, - user=setup_db.username, - password=setup_db.password, - db=setup_db.dbname, - charset='utf8mb4' - ) - try: - with conn.cursor() as cursor: + with setup_db.cursor() as cursor: # Check that the line_data_vl1l2 table has the expected columns cursor.execute(CONST_LOAD_DB_CMD) @@ -266,6 +219,6 @@ def test_ecnt_vals(setup_db): finally: - conn.close() + setup_db.close() diff --git a/METdbLoad/tests/update_schema_6.0_beta4/test_loading.yaml b/METdbLoad/tests/update_schema_6.0_beta4/test_loading.yaml index 532eb3d8..7172865a 100644 --- a/METdbLoad/tests/update_schema_6.0_beta4/test_loading.yaml +++ b/METdbLoad/tests/update_schema_6.0_beta4/test_loading.yaml @@ -1,4 +1,5 @@ -hostname: 'localhost:3306' +hostname: 'localhost' +port: 3306 username: 'mvadmin' password: 'passwordgoeshere' dbname: 'mv_load_test' diff --git a/METdbLoad/tests/update_schema_6.0_beta5/Data/total_dir.tar b/METdbLoad/tests/update_schema_6.0_beta5/Data/total_dir.tar new file mode 100644 index 00000000..30528723 Binary files /dev/null and b/METdbLoad/tests/update_schema_6.0_beta5/Data/total_dir.tar differ diff --git a/METdbLoad/tests/update_schema_6.0_beta5/test_loading.yaml b/METdbLoad/tests/update_schema_6.0_beta5/test_loading.yaml new file mode 100644 index 00000000..d7c79958 --- /dev/null +++ b/METdbLoad/tests/update_schema_6.0_beta5/test_loading.yaml @@ -0,0 +1,7 @@ +hostname: 'localhost' +port: 3306 +username: 'mvadmin' +password: '160GiltVa0D5M' +dbname: 'mv_load_test' +output_dir: './output' + diff --git a/METdbLoad/tests/update_schema_6.0_beta5/test_schema.py b/METdbLoad/tests/update_schema_6.0_beta5/test_schema.py new file mode 100644 index 00000000..4ddbf968 --- /dev/null +++ b/METdbLoad/tests/update_schema_6.0_beta5/test_schema.py @@ -0,0 +1,154 @@ +import pytest +import pymysql +import yaml +from dataclasses import make_dataclass + +####################################################################### +# These tests can only be run on the host where the database is running. +# Pre-condition: +# The data in the accompanying data directory ./Data, should +# already be loaded in the database using the corresponding +# schema: mv_mysql.sql and the appropriate xml specification file. +# This is to avoid having the password visible in the test code. +# + +CONST_LOAD_DB_CMD = "use mv_load_test" + + +@pytest.fixture +def setup_db(): + """ + Read in the config file to retrieve the database login information. + + """ + config_file = 'test_loading.yaml' + with open(config_file, 'r') as stream: + try: + parms: dict = yaml.load(stream, Loader=yaml.FullLoader) + # pathlib.Path(parms['output_dir']).mkdir(parents=True, exist_ok=True) + except yaml.YAMLError as exc: + print(exc) + + # Create a dataclass of the database information + DBS = make_dataclass("DBS", ["hostname", "port", "username", "password", "dbname"]) + db_settings = DBS(parms['hostname'], parms['port'], parms['username'], parms['password'], parms['dbname']) + + # Return the db connection object + conn = pymysql.connect( + host=db_settings.hostname, + port=db_settings.port, + user=db_settings.username, + password=db_settings.password, + db=db_settings.dbname, + charset='utf8mb4' + ) + ## settings (hostname, username, etc.) + yield conn + + +def test_db_created(setup_db): + ''' + Verify that the mv_load_test database was created + Args: + setup_db: db connection object + + Returns: None + + ''' + + # connect to the database and verify the VCNT, VL1L2, and VAL1L2 tables exist + try: + with setup_db.cursor() as cursor: + # Check that the line_data_vcnt, line_data_vl1l2, and + # line_data_val1l2 tables were created + cursor.execute(CONST_LOAD_DB_CMD) + check_db_exists_query = "show databases;" + cursor.execute(check_db_exists_query) + + + # Get all rows + rows = cursor.fetchall() + list_of_rows = [r[0] for r in rows] + assert 'mv_load_test' in list_of_rows + + + finally: + setup_db.close() + +def test_tables_created(setup_db): + + # connect to the database and verify the VCNT, VL1L2, and VAL1L2 tables exist + try: + with setup_db.cursor() as cursor: + # Check that the line_data_vcnt, line_data_vl1l2, and + # line_data_val1l2 tables were created + cursor.execute(CONST_LOAD_DB_CMD) + check_db_exists_query = "show tables;" + cursor.execute(check_db_exists_query) + + + # Get all rows + rows = cursor.fetchall() + list_of_rows = [r[0] for r in rows] + assert 'line_data_vcnt' in list_of_rows + assert 'line_data_vl1l2' in list_of_rows + assert 'line_data_val1l2' in list_of_rows + + finally: + setup_db.close() + + +def test_vl1l2_columns(setup_db): + # log into the database and verify the total_dir column is in the + # list_data_vl1l2 database table. + + try: + with setup_db.cursor() as cursor: + cursor.execute(CONST_LOAD_DB_CMD) + check_columns_exist = "desc line_data_vl1l2;" + cursor.execute(check_columns_exist) + + # Get all rows + rows = cursor.fetchall() + list_of_rows = [r[0] for r in rows] + assert 'total_dir' in list_of_rows + + finally: + setup_db.close() + + +def test_val1l2_columns(setup_db): + # log into the database and verify the total_dir column is in the + # list_data_val1l2 database table. + + try: + with setup_db.cursor() as cursor: + cursor.execute(CONST_LOAD_DB_CMD) + check_columns_exist = "desc line_data_val1l2;" + cursor.execute(check_columns_exist) + + # Get all rows + rows = cursor.fetchall() + list_of_rows = [r[0] for r in rows] + assert 'total_dir' in list_of_rows + + finally: + setup_db.close() + + def test_vcnt_columns(setup_db): + # log into the database and verify the total_dir column is in the + # list_data_vcnt database table. + + try: + with setup_db.cursor() as cursor: + cursor.execute(CONST_LOAD_DB_CMD) + check_columns_exist = "desc line_data_vcnt;" + cursor.execute(check_columns_exist) + + # Get all rows + rows = cursor.fetchall() + list_of_rows = [r[0] for r in rows] + assert 'total_dir' in list_of_rows + + finally: + setup_db.close()