From 7fb4eee77170298e7042d2128cdfad6ae1130595 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Wed, 27 Mar 2024 16:52:58 -0500 Subject: [PATCH 01/24] fix(functions, test): functions added, need to test since it probably doesn't work --- datasimulator/file_handling.py | 20 ++++++++++++++++++++ datasimulator/main.py | 13 ++++++++----- datasimulator/utils.py | 7 +++++++ 3 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 datasimulator/file_handling.py diff --git a/datasimulator/file_handling.py b/datasimulator/file_handling.py new file mode 100644 index 0000000..9501528 --- /dev/null +++ b/datasimulator/file_handling.py @@ -0,0 +1,20 @@ +from utils import attempt +from cdislogging import get_logger +logger = get_logger("data-simulator", log_level="info") + + +def write_node_to_file(outfile, node): + outfile.write(f"{node.name}\t{node.category}\n") + + +def write_to_file_or_log_error(file_path, submission_order): + return True + # file_open_result = attempt(lambda: open(file_path, "w")) + # if not file_open_result["success"]: + # exception = file_open_result["error"] + # logger.error(f"Error trying to open a new file DataImportOrderPath. Error: {exception}") + # else: + # output_file = file_open_result["result"] + # for node in submission_order: + # write_node_to_file(output_file, node) + # logger.info("DataImportOrderPath.txt has been successfully created.") diff --git a/datasimulator/main.py b/datasimulator/main.py index f979011..ab40d74 100755 --- a/datasimulator/main.py +++ b/datasimulator/main.py @@ -5,9 +5,8 @@ from .graph import Graph from .submit_data_utils import submit_test_data - +from file_handling import write_to_file_or_log_error from cdislogging import get_logger - logger = get_logger("data-simulator", log_level="info") @@ -144,9 +143,13 @@ def run_submission_order_generation(graph, data_path, node_name=None): else: submission_order = graph.generate_submission_order() - with open(os.path.join(data_path, "DataImportOrderPath.txt"), "w") as outfile: - for node in submission_order: - outfile.write(node.name + "\t" + node.category + "\n") + file_path = os.path.join(data_path, "DataImportOrderPath.txt") + path_exists = os.path.exists(file_path) + if not path_exists: + logger.error("Cannot create file because path does not exist. Did you create a 'test-data' folder?") + else: + write_to_file_or_log_error(file_path, submission_order) + # python main.py simulate --url https://s3.amazonaws.com/dictionary-artifacts/bhcdictionary/0.4.3/schema.json --path ./data-simulator/sample_test_data --program DEV --project test diff --git a/datasimulator/utils.py b/datasimulator/utils.py index 87e8fbc..a31dab1 100644 --- a/datasimulator/utils.py +++ b/datasimulator/utils.py @@ -6,6 +6,13 @@ from .errors import UserError +def attempt(action): + try: + return {"success": True, "result": action()} + except Exception as e: + return {"success": False, "error": e} + + def is_mixed_type(arr): # An enum is said "mixed type" if the enum items don't all have the same type. The only # exception to this is NoneType, which is allowed in enums regardless of the type of other From c9e39b954e7047613f6097df41ccac0ab5eb33e5 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Wed, 27 Mar 2024 16:53:23 -0500 Subject: [PATCH 02/24] fix(test): trying to add tests but it's complaining --- tests/test_file_handling.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/test_file_handling.py diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py new file mode 100644 index 0000000..800df66 --- /dev/null +++ b/tests/test_file_handling.py @@ -0,0 +1,13 @@ +from datasimulator.graph import Graph +from dictionaryutils import dictionary +from datasimulator.file_handling import write_to_file_or_log_error + + +def test_write_to_file_or_log_error(): + graph = Graph(dictionary, "DEV", "test") + graph.generate_nodes_from_dictionary() + graph.construct_graph_edges() + submission_order = graph.generate_submission_order() + file_path = "test-da/DataImportOrderPath.txt" + output = write_to_file_or_log_error(file_path, submission_order) + assert False From e9b965de80e151f6de7066f9e4f7340d674163e5 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 10:22:02 -0500 Subject: [PATCH 03/24] fix(test, import, .gitignore): working on tests, plus imports, minor add to gitignore --- .gitignore | 1 + datasimulator/file_handling.py | 23 +++++++++++------------ tests/test_file_handling.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 4c5df07..d0c0608 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ sample_test_data/ *.egg-info/ tests/TestData +/.idea/ \ No newline at end of file diff --git a/datasimulator/file_handling.py b/datasimulator/file_handling.py index 9501528..a8b2f54 100644 --- a/datasimulator/file_handling.py +++ b/datasimulator/file_handling.py @@ -1,20 +1,19 @@ -from utils import attempt from cdislogging import get_logger logger = get_logger("data-simulator", log_level="info") - +from .utils import attempt # same as datasimulator.utils def write_node_to_file(outfile, node): outfile.write(f"{node.name}\t{node.category}\n") def write_to_file_or_log_error(file_path, submission_order): - return True - # file_open_result = attempt(lambda: open(file_path, "w")) - # if not file_open_result["success"]: - # exception = file_open_result["error"] - # logger.error(f"Error trying to open a new file DataImportOrderPath. Error: {exception}") - # else: - # output_file = file_open_result["result"] - # for node in submission_order: - # write_node_to_file(output_file, node) - # logger.info("DataImportOrderPath.txt has been successfully created.") + with open(file_path, "w") as output_file: + file_open_result = {} # attempt(lambda: open(file_path, "w")) + if not file_open_result["success"]: + exception = file_open_result["error"] + logger.error(f"Error trying to open a new file DataImportOrderPath. Error: {exception}") + else: + output_file = file_open_result["result"] + for node in submission_order: + write_node_to_file(output_file, node) + logger.info("DataImportOrderPath.txt has been successfully created.") diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py index 800df66..cf526d4 100644 --- a/tests/test_file_handling.py +++ b/tests/test_file_handling.py @@ -1,7 +1,7 @@ from datasimulator.graph import Graph from dictionaryutils import dictionary from datasimulator.file_handling import write_to_file_or_log_error - +from datasimulator.utils import attempt def test_write_to_file_or_log_error(): graph = Graph(dictionary, "DEV", "test") From eb2e178b5dc249d9f79febf42bc99ff3e8634c7c Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 11:47:25 -0500 Subject: [PATCH 04/24] fix(bug): minor fixes in main --- datasimulator/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datasimulator/main.py b/datasimulator/main.py index ab40d74..9069433 100755 --- a/datasimulator/main.py +++ b/datasimulator/main.py @@ -5,7 +5,7 @@ from .graph import Graph from .submit_data_utils import submit_test_data -from file_handling import write_to_file_or_log_error +from .file_handling import write_to_file_or_log_error from cdislogging import get_logger logger = get_logger("data-simulator", log_level="info") @@ -144,7 +144,7 @@ def run_submission_order_generation(graph, data_path, node_name=None): submission_order = graph.generate_submission_order() file_path = os.path.join(data_path, "DataImportOrderPath.txt") - path_exists = os.path.exists(file_path) + path_exists = os.path.exists(data_path) if not path_exists: logger.error("Cannot create file because path does not exist. Did you create a 'test-data' folder?") else: From b251929edf85ae73ec9d7c681755eb716a9c671b Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 11:47:42 -0500 Subject: [PATCH 05/24] fix(function): finalizing file writer --- datasimulator/file_handling.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/datasimulator/file_handling.py b/datasimulator/file_handling.py index a8b2f54..7544016 100644 --- a/datasimulator/file_handling.py +++ b/datasimulator/file_handling.py @@ -1,19 +1,20 @@ +from .utils import attempt # same as datasimulator.utils from cdislogging import get_logger logger = get_logger("data-simulator", log_level="info") -from .utils import attempt # same as datasimulator.utils + def write_node_to_file(outfile, node): outfile.write(f"{node.name}\t{node.category}\n") def write_to_file_or_log_error(file_path, submission_order): - with open(file_path, "w") as output_file: - file_open_result = {} # attempt(lambda: open(file_path, "w")) - if not file_open_result["success"]: - exception = file_open_result["error"] - logger.error(f"Error trying to open a new file DataImportOrderPath. Error: {exception}") - else: - output_file = file_open_result["result"] - for node in submission_order: - write_node_to_file(output_file, node) - logger.info("DataImportOrderPath.txt has been successfully created.") + file_open_result = attempt(lambda: open(file_path, "w")) + if not file_open_result["success"]: + exception = file_open_result["error"] + logger.error(f"Error trying to open a new file DataImportOrderPath. Error: {exception}") + else: + output_file = file_open_result["result"] + for node in submission_order: + write_node_to_file(output_file, node) + logger.info("DataImportOrderPath.txt has been successfully created.") + return file_open_result["success"] From 53f22acbfc4fc92c098af49a6d47feb2d92c5e69 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 11:48:01 -0500 Subject: [PATCH 06/24] fix(test): finalizing test --- tests/test_file_handling.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py index cf526d4..0801103 100644 --- a/tests/test_file_handling.py +++ b/tests/test_file_handling.py @@ -1,13 +1,15 @@ +import os from datasimulator.graph import Graph from dictionaryutils import dictionary from datasimulator.file_handling import write_to_file_or_log_error -from datasimulator.utils import attempt -def test_write_to_file_or_log_error(): + +def test_write_to_file_or_log_error(tmpdir): + """ asking for tmpdir in the parameter provides an empty test directory for testing """ graph = Graph(dictionary, "DEV", "test") graph.generate_nodes_from_dictionary() graph.construct_graph_edges() submission_order = graph.generate_submission_order() - file_path = "test-da/DataImportOrderPath.txt" - output = write_to_file_or_log_error(file_path, submission_order) - assert False + test_path = tmpdir.mkdir("test-data").join("DataImportOrderPath.txt") + succeeded = write_to_file_or_log_error(test_path, submission_order) + assert succeeded and os.path.getsize(test_path) > 0 From 82835900f961c818fa102bbe1d317dbee2d9469c Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 13:43:56 -0500 Subject: [PATCH 07/24] rephrasing --- datasimulator/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasimulator/main.py b/datasimulator/main.py index 9069433..2a01690 100755 --- a/datasimulator/main.py +++ b/datasimulator/main.py @@ -146,7 +146,7 @@ def run_submission_order_generation(graph, data_path, node_name=None): file_path = os.path.join(data_path, "DataImportOrderPath.txt") path_exists = os.path.exists(data_path) if not path_exists: - logger.error("Cannot create file because path does not exist. Did you create a 'test-data' folder?") + logger.error(f"Cannot create file because path does not exist. Here is the path we expect: '{data_path}'") else: write_to_file_or_log_error(file_path, submission_order) From 5bbef2b3c5417bcac2f281ee7680ab4dd854c375 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 13:54:42 -0500 Subject: [PATCH 08/24] Update datasimulator/file_handling.py fix(improvement): generalizing error message Co-authored-by: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> --- datasimulator/file_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasimulator/file_handling.py b/datasimulator/file_handling.py index 7544016..5c52832 100644 --- a/datasimulator/file_handling.py +++ b/datasimulator/file_handling.py @@ -11,7 +11,7 @@ def write_to_file_or_log_error(file_path, submission_order): file_open_result = attempt(lambda: open(file_path, "w")) if not file_open_result["success"]: exception = file_open_result["error"] - logger.error(f"Error trying to open a new file DataImportOrderPath. Error: {exception}") + logger.error(f"Error trying to open a new file '{file_path}'. Error: {exception}") else: output_file = file_open_result["result"] for node in submission_order: From b58c50f636536789d786cea54755fb3743bfcf54 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 13:56:16 -0500 Subject: [PATCH 09/24] Update datasimulator/file_handling.py fix(rename): better function name Co-authored-by: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> --- datasimulator/file_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasimulator/file_handling.py b/datasimulator/file_handling.py index 5c52832..49509c8 100644 --- a/datasimulator/file_handling.py +++ b/datasimulator/file_handling.py @@ -7,7 +7,7 @@ def write_node_to_file(outfile, node): outfile.write(f"{node.name}\t{node.category}\n") -def write_to_file_or_log_error(file_path, submission_order): +def write_submission_order_to_file_or_log_error(file_path, submission_order): file_open_result = attempt(lambda: open(file_path, "w")) if not file_open_result["success"]: exception = file_open_result["error"] From 631e46c48c0bc089609eec458f5596934609c8eb Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 13:57:39 -0500 Subject: [PATCH 10/24] fix(refactor): removing function --- datasimulator/file_handling.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/datasimulator/file_handling.py b/datasimulator/file_handling.py index 7544016..52a6dde 100644 --- a/datasimulator/file_handling.py +++ b/datasimulator/file_handling.py @@ -3,10 +3,6 @@ logger = get_logger("data-simulator", log_level="info") -def write_node_to_file(outfile, node): - outfile.write(f"{node.name}\t{node.category}\n") - - def write_to_file_or_log_error(file_path, submission_order): file_open_result = attempt(lambda: open(file_path, "w")) if not file_open_result["success"]: @@ -15,6 +11,6 @@ def write_to_file_or_log_error(file_path, submission_order): else: output_file = file_open_result["result"] for node in submission_order: - write_node_to_file(output_file, node) + output_file.write(f"{node.name}\t{node.category}\n") logger.info("DataImportOrderPath.txt has been successfully created.") return file_open_result["success"] From ef160317995ab93a24709d82fdf23732573e494b Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 14:00:25 -0500 Subject: [PATCH 11/24] fix(file): closing file --- datasimulator/file_handling.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datasimulator/file_handling.py b/datasimulator/file_handling.py index 52a6dde..e58f80f 100644 --- a/datasimulator/file_handling.py +++ b/datasimulator/file_handling.py @@ -12,5 +12,6 @@ def write_to_file_or_log_error(file_path, submission_order): output_file = file_open_result["result"] for node in submission_order: output_file.write(f"{node.name}\t{node.category}\n") + output_file.close() logger.info("DataImportOrderPath.txt has been successfully created.") return file_open_result["success"] From e6f70e823356d657cb28b04378793d109c27c825 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 14:35:28 -0500 Subject: [PATCH 12/24] fix(refactor): hard coding test --- tests/test_file_handling.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py index 0801103..eca3e35 100644 --- a/tests/test_file_handling.py +++ b/tests/test_file_handling.py @@ -1,15 +1,21 @@ import os -from datasimulator.graph import Graph -from dictionaryutils import dictionary from datasimulator.file_handling import write_to_file_or_log_error +from datasimulator.node import Node def test_write_to_file_or_log_error(tmpdir): """ asking for tmpdir in the parameter provides an empty test directory for testing """ - graph = Graph(dictionary, "DEV", "test") - graph.generate_nodes_from_dictionary() - graph.construct_graph_edges() - submission_order = graph.generate_submission_order() + node_a = Node( + 'first_node', + {"category": "foo", "properties": "bar", "links": "biz", "oneOf": "baz"}, + 'test_project', + False) + node_b = Node( + 'second_node', + {"category": "aoo", "properties": "aar", "links": "aiz", "oneOf": "aaz"}, + 'test_project', + False) + submission_order = [node_a, node_b] test_path = tmpdir.mkdir("test-data").join("DataImportOrderPath.txt") succeeded = write_to_file_or_log_error(test_path, submission_order) assert succeeded and os.path.getsize(test_path) > 0 From 3b290b75147135c0263cb3addc2493c2c2c26c24 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 14:36:44 -0500 Subject: [PATCH 13/24] fix(cleanup): remove unused import --- datasimulator/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datasimulator/utils.py b/datasimulator/utils.py index a31dab1..9dce081 100644 --- a/datasimulator/utils.py +++ b/datasimulator/utils.py @@ -1,4 +1,3 @@ -import os import random import json from functools import reduce From 409adc978626b5f19ac738fc5d4cc313748c505a Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 14:44:36 -0500 Subject: [PATCH 14/24] fix(imports): fixing merge imports --- datasimulator/main.py | 4 ++-- tests/test_file_handling.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/datasimulator/main.py b/datasimulator/main.py index 2a01690..9dd49ca 100755 --- a/datasimulator/main.py +++ b/datasimulator/main.py @@ -5,7 +5,7 @@ from .graph import Graph from .submit_data_utils import submit_test_data -from .file_handling import write_to_file_or_log_error +from .file_handling import write_submission_order_to_file_or_log_error from cdislogging import get_logger logger = get_logger("data-simulator", log_level="info") @@ -148,7 +148,7 @@ def run_submission_order_generation(graph, data_path, node_name=None): if not path_exists: logger.error(f"Cannot create file because path does not exist. Here is the path we expect: '{data_path}'") else: - write_to_file_or_log_error(file_path, submission_order) + write_submission_order_to_file_or_log_error(file_path, submission_order) diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py index eca3e35..30dad33 100644 --- a/tests/test_file_handling.py +++ b/tests/test_file_handling.py @@ -1,5 +1,5 @@ import os -from datasimulator.file_handling import write_to_file_or_log_error +from datasimulator.file_handling import write_submission_order_to_file_or_log_error from datasimulator.node import Node @@ -17,5 +17,5 @@ def test_write_to_file_or_log_error(tmpdir): False) submission_order = [node_a, node_b] test_path = tmpdir.mkdir("test-data").join("DataImportOrderPath.txt") - succeeded = write_to_file_or_log_error(test_path, submission_order) + succeeded = write_submission_order_to_file_or_log_error(test_path, submission_order) assert succeeded and os.path.getsize(test_path) > 0 From 5c2a1b20ff07b737645d3e6cf65e7110ef7953b0 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 15:58:42 -0500 Subject: [PATCH 15/24] fix(refactor): refactor attempt 1, currying --- datasimulator/file_handling.py | 36 +++++++++++++++++++++++----------- datasimulator/main.py | 4 ++-- datasimulator/utils.py | 7 ------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/datasimulator/file_handling.py b/datasimulator/file_handling.py index 4076281..6b5fe94 100644 --- a/datasimulator/file_handling.py +++ b/datasimulator/file_handling.py @@ -1,17 +1,31 @@ -from .utils import attempt # same as datasimulator.utils from cdislogging import get_logger logger = get_logger("data-simulator", log_level="info") -def write_submission_order_to_file_or_log_error(file_path, submission_order): - file_open_result = attempt(lambda: open(file_path, "w")) - if not file_open_result["success"]: - exception = file_open_result["error"] - logger.error(f"Error trying to open a new file '{file_path}'. Error: {exception}") - else: - output_file = file_open_result["result"] +def try_writing_to_file(file_path, action): + """ + Args: + file_path (str): alleged path to file + action (callable): whatever you want to do with the open file + + Returns: result dict in the form + - "success" (bool): whether it succeeded + - "result": action return if successful + - "error": exception if failed + """ + try: + output_file = open(file_path, "w") + result = action(output_file) + output_file.close() + return {"success": True, "result": result} + except Exception as e: + logger.error(f"Error trying to open a new file '{file_path}'. Error: {e}") + return {"success": False, "result": e} + + +def write_submission_order_to_file(submission_order): + def write_node_to_file(output_file): for node in submission_order: output_file.write(f"{node.name}\t{node.category}\n") - output_file.close() - logger.info("DataImportOrderPath.txt has been successfully created.") - return file_open_result["success"] + return write_node_to_file + diff --git a/datasimulator/main.py b/datasimulator/main.py index 9dd49ca..121b98d 100755 --- a/datasimulator/main.py +++ b/datasimulator/main.py @@ -5,7 +5,7 @@ from .graph import Graph from .submit_data_utils import submit_test_data -from .file_handling import write_submission_order_to_file_or_log_error +from .file_handling import write_submission_order_to_file, try_writing_to_file from cdislogging import get_logger logger = get_logger("data-simulator", log_level="info") @@ -148,7 +148,7 @@ def run_submission_order_generation(graph, data_path, node_name=None): if not path_exists: logger.error(f"Cannot create file because path does not exist. Here is the path we expect: '{data_path}'") else: - write_submission_order_to_file_or_log_error(file_path, submission_order) + try_writing_to_file(file_path, write_submission_order_to_file(submission_order)) diff --git a/datasimulator/utils.py b/datasimulator/utils.py index 9dce081..64284d6 100644 --- a/datasimulator/utils.py +++ b/datasimulator/utils.py @@ -5,13 +5,6 @@ from .errors import UserError -def attempt(action): - try: - return {"success": True, "result": action()} - except Exception as e: - return {"success": False, "error": e} - - def is_mixed_type(arr): # An enum is said "mixed type" if the enum items don't all have the same type. The only # exception to this is NoneType, which is allowed in enums regardless of the type of other From 761e2d8de7b322108fd68f5facd6fe84accbeee0 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 15:59:33 -0500 Subject: [PATCH 16/24] fix(refactor): refactor attempt 1, fix test case --- tests/test_file_handling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py index 30dad33..d920c31 100644 --- a/tests/test_file_handling.py +++ b/tests/test_file_handling.py @@ -1,5 +1,5 @@ import os -from datasimulator.file_handling import write_submission_order_to_file_or_log_error +from datasimulator.file_handling import write_submission_order_to_file, try_writing_to_file from datasimulator.node import Node @@ -17,5 +17,5 @@ def test_write_to_file_or_log_error(tmpdir): False) submission_order = [node_a, node_b] test_path = tmpdir.mkdir("test-data").join("DataImportOrderPath.txt") - succeeded = write_submission_order_to_file_or_log_error(test_path, submission_order) + succeeded = try_writing_to_file(test_path, write_submission_order_to_file(submission_order)) assert succeeded and os.path.getsize(test_path) > 0 From 743a89df76e5484f7400a5892026fd0bd8e29eac Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Thu, 28 Mar 2024 16:08:54 -0500 Subject: [PATCH 17/24] fix(refactor): scrapping everything. we don't need the try catch after all I think --- datasimulator/file_handling.py | 27 +++------------------------ datasimulator/main.py | 4 ++-- tests/test_file_handling.py | 4 ++-- 3 files changed, 7 insertions(+), 28 deletions(-) diff --git a/datasimulator/file_handling.py b/datasimulator/file_handling.py index 6b5fe94..4dfce3f 100644 --- a/datasimulator/file_handling.py +++ b/datasimulator/file_handling.py @@ -2,30 +2,9 @@ logger = get_logger("data-simulator", log_level="info") -def try_writing_to_file(file_path, action): - """ - Args: - file_path (str): alleged path to file - action (callable): whatever you want to do with the open file - - Returns: result dict in the form - - "success" (bool): whether it succeeded - - "result": action return if successful - - "error": exception if failed - """ - try: - output_file = open(file_path, "w") - result = action(output_file) - output_file.close() - return {"success": True, "result": result} - except Exception as e: - logger.error(f"Error trying to open a new file '{file_path}'. Error: {e}") - return {"success": False, "result": e} - - -def write_submission_order_to_file(submission_order): - def write_node_to_file(output_file): +def write_submission_order_to_file(submission_order, file_path): + with open(file_path, "w") as output_file: for node in submission_order: output_file.write(f"{node.name}\t{node.category}\n") - return write_node_to_file + return True diff --git a/datasimulator/main.py b/datasimulator/main.py index 121b98d..04c3383 100755 --- a/datasimulator/main.py +++ b/datasimulator/main.py @@ -5,7 +5,7 @@ from .graph import Graph from .submit_data_utils import submit_test_data -from .file_handling import write_submission_order_to_file, try_writing_to_file +from .file_handling import write_submission_order_to_file from cdislogging import get_logger logger = get_logger("data-simulator", log_level="info") @@ -148,7 +148,7 @@ def run_submission_order_generation(graph, data_path, node_name=None): if not path_exists: logger.error(f"Cannot create file because path does not exist. Here is the path we expect: '{data_path}'") else: - try_writing_to_file(file_path, write_submission_order_to_file(submission_order)) + write_submission_order_to_file(submission_order, file_path) diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py index d920c31..17b4d0f 100644 --- a/tests/test_file_handling.py +++ b/tests/test_file_handling.py @@ -1,5 +1,5 @@ import os -from datasimulator.file_handling import write_submission_order_to_file, try_writing_to_file +from datasimulator.file_handling import write_submission_order_to_file from datasimulator.node import Node @@ -17,5 +17,5 @@ def test_write_to_file_or_log_error(tmpdir): False) submission_order = [node_a, node_b] test_path = tmpdir.mkdir("test-data").join("DataImportOrderPath.txt") - succeeded = try_writing_to_file(test_path, write_submission_order_to_file(submission_order)) + succeeded = write_submission_order_to_file(submission_order, test_path) assert succeeded and os.path.getsize(test_path) > 0 From 0a1d42e1e4bd9c1348a79ebe9006b38c9e0889d1 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Fri, 29 Mar 2024 10:46:53 -0500 Subject: [PATCH 18/24] fix(fix): switching error log to exception --- datasimulator/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasimulator/main.py b/datasimulator/main.py index 04c3383..65d1779 100755 --- a/datasimulator/main.py +++ b/datasimulator/main.py @@ -146,7 +146,7 @@ def run_submission_order_generation(graph, data_path, node_name=None): file_path = os.path.join(data_path, "DataImportOrderPath.txt") path_exists = os.path.exists(data_path) if not path_exists: - logger.error(f"Cannot create file because path does not exist. Here is the path we expect: '{data_path}'") + raise Exception(f"Cannot create file because path does not exist. Here is the path we expect: '{data_path}'") else: write_submission_order_to_file(submission_order, file_path) From 24e72770947f577de6b0b397d6221641f35f1df8 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Fri, 29 Mar 2024 10:47:24 -0500 Subject: [PATCH 19/24] fix(test): adding fail test check --- datasimulator/utils.py | 7 +++++++ tests/test_file_handling.py | 32 +++++++++++++++++++------------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/datasimulator/utils.py b/datasimulator/utils.py index 64284d6..9dce081 100644 --- a/datasimulator/utils.py +++ b/datasimulator/utils.py @@ -5,6 +5,13 @@ from .errors import UserError +def attempt(action): + try: + return {"success": True, "result": action()} + except Exception as e: + return {"success": False, "error": e} + + def is_mixed_type(arr): # An enum is said "mixed type" if the enum items don't all have the same type. The only # exception to this is NoneType, which is allowed in enums regardless of the type of other diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py index 17b4d0f..a25c3c8 100644 --- a/tests/test_file_handling.py +++ b/tests/test_file_handling.py @@ -1,21 +1,27 @@ import os +from datasimulator.utils import attempt from datasimulator.file_handling import write_submission_order_to_file from datasimulator.node import Node +node_a = Node( + 'first_node', + {"category": "foo", "properties": "bar", "links": "biz", "oneOf": "baz"}, + 'test_project', + False) +node_b = Node( + 'second_node', + {"category": "aoo", "properties": "aar", "links": "aiz", "oneOf": "aaz"}, + 'test_project', + False) +example_submission_order = [node_a, node_b] -def test_write_to_file_or_log_error(tmpdir): + +def test_write_to_file_or_log_error_succeeds(tmpdir): """ asking for tmpdir in the parameter provides an empty test directory for testing """ - node_a = Node( - 'first_node', - {"category": "foo", "properties": "bar", "links": "biz", "oneOf": "baz"}, - 'test_project', - False) - node_b = Node( - 'second_node', - {"category": "aoo", "properties": "aar", "links": "aiz", "oneOf": "aaz"}, - 'test_project', - False) - submission_order = [node_a, node_b] test_path = tmpdir.mkdir("test-data").join("DataImportOrderPath.txt") - succeeded = write_submission_order_to_file(submission_order, test_path) + succeeded = write_submission_order_to_file(example_submission_order, test_path) assert succeeded and os.path.getsize(test_path) > 0 + +def test_write_to_file_or_log_error_fails(): + write_result = attempt(lambda: write_submission_order_to_file(example_submission_order, "/test-path")) + assert not write_result["success"] \ No newline at end of file From 23e275532021ddb078eee3ca19fe72e1263b2be4 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Mon, 1 Apr 2024 09:33:40 -0500 Subject: [PATCH 20/24] fix(docs): adding documentation --- datasimulator/file_handling.py | 1 + tests/test_file_handling.py | 1 + 2 files changed, 2 insertions(+) diff --git a/datasimulator/file_handling.py b/datasimulator/file_handling.py index 4dfce3f..3e7ca5a 100644 --- a/datasimulator/file_handling.py +++ b/datasimulator/file_handling.py @@ -3,6 +3,7 @@ def write_submission_order_to_file(submission_order, file_path): + """ given a correct file path, writes a name-category pair to the file from a list of nodes (submissions) """ with open(file_path, "w") as output_file: for node in submission_order: output_file.write(f"{node.name}\t{node.category}\n") diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py index a25c3c8..50b10ce 100644 --- a/tests/test_file_handling.py +++ b/tests/test_file_handling.py @@ -23,5 +23,6 @@ def test_write_to_file_or_log_error_succeeds(tmpdir): assert succeeded and os.path.getsize(test_path) > 0 def test_write_to_file_or_log_error_fails(): + """ Tests that write_submission_order_to_file fails when the directory does not exist""" write_result = attempt(lambda: write_submission_order_to_file(example_submission_order, "/test-path")) assert not write_result["success"] \ No newline at end of file From 97f2fa3a940463d14c9f53a99e3436b618e5a47e Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Mon, 1 Apr 2024 11:02:59 -0500 Subject: [PATCH 21/24] fix(refactor): removing attempt, can just use pytest raise --- datasimulator/utils.py | 7 ------- tests/test_file_handling.py | 9 ++++++--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/datasimulator/utils.py b/datasimulator/utils.py index 9dce081..64284d6 100644 --- a/datasimulator/utils.py +++ b/datasimulator/utils.py @@ -5,13 +5,6 @@ from .errors import UserError -def attempt(action): - try: - return {"success": True, "result": action()} - except Exception as e: - return {"success": False, "error": e} - - def is_mixed_type(arr): # An enum is said "mixed type" if the enum items don't all have the same type. The only # exception to this is NoneType, which is allowed in enums regardless of the type of other diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py index 50b10ce..5a06c45 100644 --- a/tests/test_file_handling.py +++ b/tests/test_file_handling.py @@ -1,5 +1,7 @@ import os -from datasimulator.utils import attempt + +import pytest + from datasimulator.file_handling import write_submission_order_to_file from datasimulator.node import Node @@ -22,7 +24,8 @@ def test_write_to_file_or_log_error_succeeds(tmpdir): succeeded = write_submission_order_to_file(example_submission_order, test_path) assert succeeded and os.path.getsize(test_path) > 0 + def test_write_to_file_or_log_error_fails(): """ Tests that write_submission_order_to_file fails when the directory does not exist""" - write_result = attempt(lambda: write_submission_order_to_file(example_submission_order, "/test-path")) - assert not write_result["success"] \ No newline at end of file + with pytest.raises(Exception): + write_submission_order_to_file(example_submission_order, "/test-path") From 8a3975c8c23eed56d7c7979fb8bd351cccf30317 Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Mon, 1 Apr 2024 11:21:42 -0500 Subject: [PATCH 22/24] fix(refactor): specifying error --- tests/test_file_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py index 5a06c45..280ac2e 100644 --- a/tests/test_file_handling.py +++ b/tests/test_file_handling.py @@ -27,5 +27,5 @@ def test_write_to_file_or_log_error_succeeds(tmpdir): def test_write_to_file_or_log_error_fails(): """ Tests that write_submission_order_to_file fails when the directory does not exist""" - with pytest.raises(Exception): + with pytest.raises(IOError): write_submission_order_to_file(example_submission_order, "/test-path") From d0f148f377c1cf04d7b9bb17f1696015623441ff Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Mon, 1 Apr 2024 11:36:53 -0500 Subject: [PATCH 23/24] fix(testing): ensure content is as expected --- tests/test_file_handling.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_file_handling.py b/tests/test_file_handling.py index 280ac2e..b46534f 100644 --- a/tests/test_file_handling.py +++ b/tests/test_file_handling.py @@ -18,11 +18,17 @@ example_submission_order = [node_a, node_b] +def get_contents(test_path): + with open(test_path, "r") as file: + contents = file.read() + return contents + + def test_write_to_file_or_log_error_succeeds(tmpdir): """ asking for tmpdir in the parameter provides an empty test directory for testing """ test_path = tmpdir.mkdir("test-data").join("DataImportOrderPath.txt") succeeded = write_submission_order_to_file(example_submission_order, test_path) - assert succeeded and os.path.getsize(test_path) > 0 + assert succeeded and os.path.getsize(test_path) > 0 and get_contents(test_path) == 'first_node\tfoo\nsecond_node\taoo\n' def test_write_to_file_or_log_error_fails(): From da03845e14a23f0175d2cf5ed3a72e2140fdb7ee Mon Sep 17 00:00:00 2001 From: Albert Snow Date: Mon, 1 Apr 2024 11:38:28 -0500 Subject: [PATCH 24/24] fix(docs): remove submissions --- datasimulator/file_handling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasimulator/file_handling.py b/datasimulator/file_handling.py index 3e7ca5a..22eab12 100644 --- a/datasimulator/file_handling.py +++ b/datasimulator/file_handling.py @@ -3,7 +3,7 @@ def write_submission_order_to_file(submission_order, file_path): - """ given a correct file path, writes a name-category pair to the file from a list of nodes (submissions) """ + """ given a correct file path, writes a name-category pair to the file from a list of nodes """ with open(file_path, "w") as output_file: for node in submission_order: output_file.write(f"{node.name}\t{node.category}\n")