Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PXP-11291 - Fix/data import directory notice #105

Merged
merged 26 commits into from
Apr 1, 2024
Merged
Show file tree
Hide file tree
Changes from 21 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
7fb4eee
fix(functions, test): functions added, need to test since it probably…
AlbertSnows Mar 27, 2024
c9e39b9
fix(test): trying to add tests but it's complaining
AlbertSnows Mar 27, 2024
e9b965d
fix(test, import, .gitignore): working on tests, plus imports, minor …
AlbertSnows Mar 28, 2024
eb2e178
fix(bug): minor fixes in main
AlbertSnows Mar 28, 2024
b251929
fix(function): finalizing file writer
AlbertSnows Mar 28, 2024
53f22ac
fix(test): finalizing test
AlbertSnows Mar 28, 2024
b02b05e
Merge branch 'master' into fix/data_import_directory_notice
AlbertSnows Mar 28, 2024
8283590
rephrasing
AlbertSnows Mar 28, 2024
5bbef2b
Update datasimulator/file_handling.py
AlbertSnows Mar 28, 2024
b58c50f
Update datasimulator/file_handling.py
AlbertSnows Mar 28, 2024
631e46c
fix(refactor): removing function
AlbertSnows Mar 28, 2024
ef16031
fix(file): closing file
AlbertSnows Mar 28, 2024
e6f70e8
fix(refactor): hard coding test
AlbertSnows Mar 28, 2024
3b290b7
fix(cleanup): remove unused import
AlbertSnows Mar 28, 2024
00ba780
Merge remote-tracking branch 'origin/fix/data_import_directory_notice…
AlbertSnows Mar 28, 2024
409adc9
fix(imports): fixing merge imports
AlbertSnows Mar 28, 2024
5c2a1b2
fix(refactor): refactor attempt 1, currying
AlbertSnows Mar 28, 2024
761e2d8
fix(refactor): refactor attempt 1, fix test case
AlbertSnows Mar 28, 2024
743a89d
fix(refactor): scrapping everything. we don't need the try catch afte…
AlbertSnows Mar 28, 2024
0a1d42e
fix(fix): switching error log to exception
AlbertSnows Mar 29, 2024
24e7277
fix(test): adding fail test check
AlbertSnows Mar 29, 2024
23e2755
fix(docs): adding documentation
AlbertSnows Apr 1, 2024
97f2fa3
fix(refactor): removing attempt, can just use pytest raise
AlbertSnows Apr 1, 2024
8a3975c
fix(refactor): specifying error
AlbertSnows Apr 1, 2024
d0f148f
fix(testing): ensure content is as expected
AlbertSnows Apr 1, 2024
da03845
fix(docs): remove submissions
AlbertSnows Apr 1, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ sample_test_data/
*.egg-info/

tests/TestData
/.idea/
10 changes: 10 additions & 0 deletions datasimulator/file_handling.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
from cdislogging import get_logger
logger = get_logger("data-simulator", log_level="info")


def write_submission_order_to_file(submission_order, file_path):
AlbertSnows marked this conversation as resolved.
Show resolved Hide resolved
with open(file_path, "w") as output_file:
for node in submission_order:
output_file.write(f"{node.name}\t{node.category}\n")
return True

13 changes: 8 additions & 5 deletions datasimulator/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,8 @@

from .graph import Graph
from .submit_data_utils import submit_test_data

from .file_handling import write_submission_order_to_file
from cdislogging import get_logger

logger = get_logger("data-simulator", log_level="info")


Expand Down Expand Up @@ -144,9 +143,13 @@ def run_submission_order_generation(graph, data_path, node_name=None):
else:
submission_order = graph.generate_submission_order()

with open(os.path.join(data_path, "DataImportOrderPath.txt"), "w") as outfile:
for node in submission_order:
outfile.write(node.name + "\t" + node.category + "\n")
file_path = os.path.join(data_path, "DataImportOrderPath.txt")
AlbertSnows marked this conversation as resolved.
Show resolved Hide resolved
path_exists = os.path.exists(data_path)
if not path_exists:
raise Exception(f"Cannot create file because path does not exist. Here is the path we expect: '{data_path}'")
else:
write_submission_order_to_file(submission_order, file_path)



# python main.py simulate --url https://s3.amazonaws.com/dictionary-artifacts/bhcdictionary/0.4.3/schema.json --path ./data-simulator/sample_test_data --program DEV --project test
Expand Down
8 changes: 7 additions & 1 deletion datasimulator/utils.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
import os
import random
import json
from functools import reduce

from .errors import UserError


def attempt(action):
try:
AlbertSnows marked this conversation as resolved.
Show resolved Hide resolved
return {"success": True, "result": action()}
except Exception as e:
return {"success": False, "error": e}


def is_mixed_type(arr):
# An enum is said "mixed type" if the enum items don't all have the same type. The only
# exception to this is NoneType, which is allowed in enums regardless of the type of other
Expand Down
27 changes: 27 additions & 0 deletions tests/test_file_handling.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import os
from datasimulator.utils import attempt
from datasimulator.file_handling import write_submission_order_to_file
from datasimulator.node import Node

node_a = Node(
'first_node',
{"category": "foo", "properties": "bar", "links": "biz", "oneOf": "baz"},
'test_project',
False)
node_b = Node(
'second_node',
{"category": "aoo", "properties": "aar", "links": "aiz", "oneOf": "aaz"},
'test_project',
False)
example_submission_order = [node_a, node_b]


def test_write_to_file_or_log_error_succeeds(tmpdir):
""" asking for tmpdir in the parameter provides an empty test directory for testing """
test_path = tmpdir.mkdir("test-data").join("DataImportOrderPath.txt")
succeeded = write_submission_order_to_file(example_submission_order, test_path)
assert succeeded and os.path.getsize(test_path) > 0
AlbertSnows marked this conversation as resolved.
Show resolved Hide resolved

def test_write_to_file_or_log_error_fails():
AlbertSnows marked this conversation as resolved.
Show resolved Hide resolved
write_result = attempt(lambda: write_submission_order_to_file(example_submission_order, "/test-path"))
assert not write_result["success"]
AlbertSnows marked this conversation as resolved.
Show resolved Hide resolved
Loading