From cdb9634edce2f23288aeddb00772f3e2f62537af Mon Sep 17 00:00:00 2001 From: Mihir Trivedi Date: Sun, 22 Oct 2023 21:53:33 -0400 Subject: [PATCH 1/4] load hooks function --- prism/infra/hooks.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/prism/infra/hooks.py b/prism/infra/hooks.py index 10622c5..85b2a67 100644 --- a/prism/infra/hooks.py +++ b/prism/infra/hooks.py @@ -15,6 +15,7 @@ from typing import Any, Optional # Prism-specific imports +from prism.cli.base import get_project_dir from prism.infra import project as prism_project import prism.constants import prism.exceptions @@ -131,3 +132,21 @@ def dbt_ref(self, df = dbt_project.handle_ref(target_1, target_2, target_version) return df + + +# Function to load hooks in a script or environment +def load_hooks(): + """ + Load the PrismHooks associated with the current project + """ + project_dir = get_project_dir() + project = prism_project.PrismProject( + project_dir=project_dir, + user_context={}, + which="run" + ) + project.setup() + + # Hooks object + hooks = PrismHooks(project) + return hooks From f2a018975de25c6bcae303072bc8390d77cebd63 Mon Sep 17 00:00:00 2001 From: Mihir Trivedi Date: Mon, 23 Oct 2023 23:13:10 -0400 Subject: [PATCH 2/4] add test cases and warning print msg --- prism/infra/hooks.py | 4 ++ prism/tests/integration/test_load_hooks.py | 71 ++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 prism/tests/integration/test_load_hooks.py diff --git a/prism/infra/hooks.py b/prism/infra/hooks.py index 85b2a67..a68758f 100644 --- a/prism/infra/hooks.py +++ b/prism/infra/hooks.py @@ -149,4 +149,8 @@ def load_hooks(): # Hooks object hooks = PrismHooks(project) + + # Print a warning if the hooks are empty + if hooks.project.adapters_object_dict == {}: + print("WARNING: Your hooks are empty! Create a profile YAML to populate your hooks") # noqa return hooks diff --git a/prism/tests/integration/test_load_hooks.py b/prism/tests/integration/test_load_hooks.py new file mode 100644 index 0000000..9f628dc --- /dev/null +++ b/prism/tests/integration/test_load_hooks.py @@ -0,0 +1,71 @@ +""" +Load hooks unit test + +Table of Contents: +- Imports +- Test case class definition +""" + +########### +# Imports # +########### + +# Standard library imports +import os +from pathlib import Path + +# Prism imports +from prism.infra.hooks import PrismHooks, load_hooks +from prism.profiles import snowflake, pyspark, postgres +from prism.tests.integration import integration_test_class + +# Paths +TEST_PROJECTS = Path(__file__).parent / 'test_projects' + + +############################## +# Test case class definition # +############################## + +class TestLoadHooks(integration_test_class.IntegrationTestCase): + + def test_load_hooks_in_project(self): + """ + `load_hook` produces a PrismHooks object when the user is in a project directory + """ + PROJECT_PATH = TEST_PROJECTS / '013_hooks_sql_spark' + os.chdir(PROJECT_PATH) + + # Load hooks + hooks = load_hooks() + adapter_dict = hooks.project.adapters_object_dict + + # Test + self.assertTrue(isinstance(hooks, PrismHooks)) + self.assertTrue("snowflake_base" in adapter_dict.keys()) + self.assertTrue("pyspark_base" in adapter_dict.keys()) + self.assertTrue("postgres_base" in adapter_dict.keys()) + + self.assertTrue(isinstance(adapter_dict["snowflake_base"], snowflake.Snowflake)) + self.assertTrue(isinstance(adapter_dict["pyspark_base"], pyspark.Pyspark)) + self.assertTrue(isinstance(adapter_dict["postgres_base"], postgres.Postgres)) + + # Cleanup + hooks.project.cleanup(hooks.project.run_context) + self._set_up_wkdir() + + def test_load_hooks_not_in_project(self): + """ + `load_hook` produces a PrismHooks object when the user is in a project directory + """ + PROJECT_PATH = TEST_PROJECTS / '005_simple_project_no_null_tasks' + os.chdir(PROJECT_PATH) + + # Load hooks + hooks = load_hooks() + self.assertTrue(isinstance(hooks, PrismHooks)) + self.assertEqual({}, hooks.project.adapters_object_dict) + + # Cleanup + hooks.project.cleanup(hooks.project.run_context) + self._set_up_wkdir() From 6afc6aa41fe44f862e3591d0b67b1aa4661c823c Mon Sep 17 00:00:00 2001 From: Mihir Trivedi Date: Wed, 25 Oct 2023 07:58:44 -0400 Subject: [PATCH 3/4] allow project dir as an argument to load_hooks --- prism/infra/hooks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/prism/infra/hooks.py b/prism/infra/hooks.py index a68758f..4a17e24 100644 --- a/prism/infra/hooks.py +++ b/prism/infra/hooks.py @@ -13,6 +13,7 @@ # Standard library imports import pandas as pd from typing import Any, Optional +from pathlib import Path # Prism-specific imports from prism.cli.base import get_project_dir @@ -135,11 +136,12 @@ def dbt_ref(self, # Function to load hooks in a script or environment -def load_hooks(): +def load_hooks(project_dir: Optional[Path] = None): """ Load the PrismHooks associated with the current project """ - project_dir = get_project_dir() + if project_dir is None: + project_dir = get_project_dir() project = prism_project.PrismProject( project_dir=project_dir, user_context={}, From 0b92481cdd085463001addf01a5ab520642f9cb7 Mon Sep 17 00:00:00 2001 From: Mihir Trivedi Date: Wed, 25 Oct 2023 08:36:46 -0400 Subject: [PATCH 4/4] stable API test case --- .../output/andreas_mogensen.txt | 1 - .../019_dec_targets/output/astros.json | 1 - .../019_dec_targets/output/dmitry_petelin.txt | 1 - .../019_dec_targets/output/frank_rubio.txt | 1 - .../019_dec_targets/output/gui_haichow.txt | 1 - .../output/jasmin_moghbeli.txt | 1 - .../019_dec_targets/output/jing_haiping.txt | 1 - .../output/konstantin_borisov.txt | 1 - .../019_dec_targets/output/loral_o'hara.txt | 1 - .../019_dec_targets/output/nikolai_chub.txt | 1 - .../019_dec_targets/output/oleg_kononenko.txt | 1 - .../output/satoshi_furukawa.txt | 1 - .../019_dec_targets/output/second_target.txt | 1 - .../output/sergey_prokopyev.txt | 1 - .../019_dec_targets/output/zhu_yangzhu.txt | 1 - .../019_dec_targets/tasks/extract.py | 4 +- .../019_dec_targets/tasks/load.py | 11 ++-- .../020_dec_retries/output/astros.json | 1 - .../020_dec_retries/tasks/extract.py | 4 +- prism/tests/integration/test_run.py | 61 ++++++------------- 20 files changed, 27 insertions(+), 69 deletions(-) delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/andreas_mogensen.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/astros.json delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/dmitry_petelin.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/frank_rubio.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/gui_haichow.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/jasmin_moghbeli.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/jing_haiping.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/konstantin_borisov.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/loral_o'hara.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/nikolai_chub.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/oleg_kononenko.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/satoshi_furukawa.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/second_target.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/sergey_prokopyev.txt delete mode 100644 prism/tests/integration/test_projects/019_dec_targets/output/zhu_yangzhu.txt delete mode 100644 prism/tests/integration/test_projects/020_dec_retries/output/astros.json diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/andreas_mogensen.txt b/prism/tests/integration/test_projects/019_dec_targets/output/andreas_mogensen.txt deleted file mode 100644 index b9e3407..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/andreas_mogensen.txt +++ /dev/null @@ -1 +0,0 @@ -Andreas Mogensen \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/astros.json b/prism/tests/integration/test_projects/019_dec_targets/output/astros.json deleted file mode 100644 index b1949d1..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/astros.json +++ /dev/null @@ -1 +0,0 @@ -{"people": [{"name": "Sergey Prokopyev", "craft": "ISS"}, {"name": "Dmitry Petelin", "craft": "ISS"}, {"name": "Frank Rubio", "craft": "ISS"}, {"name": "Jing Haiping", "craft": "Tiangong"}, {"name": "Gui Haichow", "craft": "Tiangong"}, {"name": "Zhu Yangzhu", "craft": "Tiangong"}, {"name": "Jasmin Moghbeli", "craft": "ISS"}, {"name": "Andreas Mogensen", "craft": "ISS"}, {"name": "Satoshi Furukawa", "craft": "ISS"}, {"name": "Konstantin Borisov", "craft": "ISS"}, {"name": "Oleg Kononenko", "craft": "ISS"}, {"name": "Nikolai Chub", "craft": "ISS"}, {"name": "Loral O'Hara", "craft": "ISS"}], "number": 13, "message": "success"} \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/dmitry_petelin.txt b/prism/tests/integration/test_projects/019_dec_targets/output/dmitry_petelin.txt deleted file mode 100644 index dd2754f..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/dmitry_petelin.txt +++ /dev/null @@ -1 +0,0 @@ -Dmitry Petelin \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/frank_rubio.txt b/prism/tests/integration/test_projects/019_dec_targets/output/frank_rubio.txt deleted file mode 100644 index f8cd030..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/frank_rubio.txt +++ /dev/null @@ -1 +0,0 @@ -Frank Rubio \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/gui_haichow.txt b/prism/tests/integration/test_projects/019_dec_targets/output/gui_haichow.txt deleted file mode 100644 index 9056cfb..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/gui_haichow.txt +++ /dev/null @@ -1 +0,0 @@ -Gui Haichow \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/jasmin_moghbeli.txt b/prism/tests/integration/test_projects/019_dec_targets/output/jasmin_moghbeli.txt deleted file mode 100644 index 4bea93b..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/jasmin_moghbeli.txt +++ /dev/null @@ -1 +0,0 @@ -Jasmin Moghbeli \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/jing_haiping.txt b/prism/tests/integration/test_projects/019_dec_targets/output/jing_haiping.txt deleted file mode 100644 index e870b50..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/jing_haiping.txt +++ /dev/null @@ -1 +0,0 @@ -Jing Haiping \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/konstantin_borisov.txt b/prism/tests/integration/test_projects/019_dec_targets/output/konstantin_borisov.txt deleted file mode 100644 index aa2540f..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/konstantin_borisov.txt +++ /dev/null @@ -1 +0,0 @@ -Konstantin Borisov \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/loral_o'hara.txt b/prism/tests/integration/test_projects/019_dec_targets/output/loral_o'hara.txt deleted file mode 100644 index 232ecb4..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/loral_o'hara.txt +++ /dev/null @@ -1 +0,0 @@ -Loral O'Hara \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/nikolai_chub.txt b/prism/tests/integration/test_projects/019_dec_targets/output/nikolai_chub.txt deleted file mode 100644 index 495d7c0..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/nikolai_chub.txt +++ /dev/null @@ -1 +0,0 @@ -Nikolai Chub \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/oleg_kononenko.txt b/prism/tests/integration/test_projects/019_dec_targets/output/oleg_kononenko.txt deleted file mode 100644 index 1a7c44f..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/oleg_kononenko.txt +++ /dev/null @@ -1 +0,0 @@ -Oleg Kononenko \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/satoshi_furukawa.txt b/prism/tests/integration/test_projects/019_dec_targets/output/satoshi_furukawa.txt deleted file mode 100644 index 13ba18e..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/satoshi_furukawa.txt +++ /dev/null @@ -1 +0,0 @@ -Satoshi Furukawa \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/second_target.txt b/prism/tests/integration/test_projects/019_dec_targets/output/second_target.txt deleted file mode 100644 index 5226279..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/second_target.txt +++ /dev/null @@ -1 +0,0 @@ -second target \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/sergey_prokopyev.txt b/prism/tests/integration/test_projects/019_dec_targets/output/sergey_prokopyev.txt deleted file mode 100644 index 219e99a..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/sergey_prokopyev.txt +++ /dev/null @@ -1 +0,0 @@ -Sergey Prokopyev \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/output/zhu_yangzhu.txt b/prism/tests/integration/test_projects/019_dec_targets/output/zhu_yangzhu.txt deleted file mode 100644 index bbc88f7..0000000 --- a/prism/tests/integration/test_projects/019_dec_targets/output/zhu_yangzhu.txt +++ /dev/null @@ -1 +0,0 @@ -Zhu Yangzhu \ No newline at end of file diff --git a/prism/tests/integration/test_projects/019_dec_targets/tasks/extract.py b/prism/tests/integration/test_projects/019_dec_targets/tasks/extract.py index 44cbe0c..714eaf5 100644 --- a/prism/tests/integration/test_projects/019_dec_targets/tasks/extract.py +++ b/prism/tests/integration/test_projects/019_dec_targets/tasks/extract.py @@ -12,12 +12,12 @@ # Task @task( targets=[ - target(type=JSON, loc=prism_project.OUTPUT / 'astros.json'), + target(type=JSON, loc=prism_project.OUTPUT / 'todos.json'), target(type=Txt, loc=prism_project.OUTPUT / 'second_target.txt') ], ) def extract(tasks, hooks): - url = "http://api.open-notify.org/astros.json" + url = "https://jsonplaceholder.typicode.com/todos" resp = requests.get(url) json_dict = json.loads(resp.text) second_target_str = "second target" diff --git a/prism/tests/integration/test_projects/019_dec_targets/tasks/load.py b/prism/tests/integration/test_projects/019_dec_targets/tasks/load.py index a966acc..dcaa2d5 100644 --- a/prism/tests/integration/test_projects/019_dec_targets/tasks/load.py +++ b/prism/tests/integration/test_projects/019_dec_targets/tasks/load.py @@ -12,12 +12,9 @@ def load(tasks, hooks): data, _ = tasks.ref("extract.py") # Names - names = {} - for ppl in data["people"]: - - # Formatted - name = ppl["name"].lower().replace(" ", "_") - names[f"{name}.txt"] = ppl["name"] + todos = {} + for todo in data: + todos[f"todo_{todo['id']}.txt"] = todo['title'] # Return - return names + return todos diff --git a/prism/tests/integration/test_projects/020_dec_retries/output/astros.json b/prism/tests/integration/test_projects/020_dec_retries/output/astros.json deleted file mode 100644 index b1949d1..0000000 --- a/prism/tests/integration/test_projects/020_dec_retries/output/astros.json +++ /dev/null @@ -1 +0,0 @@ -{"people": [{"name": "Sergey Prokopyev", "craft": "ISS"}, {"name": "Dmitry Petelin", "craft": "ISS"}, {"name": "Frank Rubio", "craft": "ISS"}, {"name": "Jing Haiping", "craft": "Tiangong"}, {"name": "Gui Haichow", "craft": "Tiangong"}, {"name": "Zhu Yangzhu", "craft": "Tiangong"}, {"name": "Jasmin Moghbeli", "craft": "ISS"}, {"name": "Andreas Mogensen", "craft": "ISS"}, {"name": "Satoshi Furukawa", "craft": "ISS"}, {"name": "Konstantin Borisov", "craft": "ISS"}, {"name": "Oleg Kononenko", "craft": "ISS"}, {"name": "Nikolai Chub", "craft": "ISS"}, {"name": "Loral O'Hara", "craft": "ISS"}], "number": 13, "message": "success"} \ No newline at end of file diff --git a/prism/tests/integration/test_projects/020_dec_retries/tasks/extract.py b/prism/tests/integration/test_projects/020_dec_retries/tasks/extract.py index 15a4f99..7e08a64 100644 --- a/prism/tests/integration/test_projects/020_dec_retries/tasks/extract.py +++ b/prism/tests/integration/test_projects/020_dec_retries/tasks/extract.py @@ -11,9 +11,9 @@ # Task @task( - targets=[target(type=JSON, loc=prism_project.OUTPUT / 'astros.json')], + targets=[target(type=JSON, loc=prism_project.OUTPUT / 'todos.json')], ) def extract(tasks, hooks): - url = "http://api.open-notify.org/astros.json" + url = "https://jsonplaceholder.typicode.com/todos" resp = requests.get(url) return json.loads(resp.text) diff --git a/prism/tests/integration/test_run.py b/prism/tests/integration/test_run.py index f490d1d..b028218 100644 --- a/prism/tests/integration/test_run.py +++ b/prism/tests/integration/test_run.py @@ -1142,14 +1142,14 @@ def test_decorator_tasks_with_targets(self): self.assertEqual(' | '.join(expected_events), runtask_run_results) # Check output of 'extract' task - self.assertTrue(Path(wkdir / 'output' / 'astros.json').is_file()) + self.assertTrue(Path(wkdir / 'output' / 'todos.json').is_file()) self.assertTrue(Path(wkdir / 'output' / 'second_target.txt').is_file()) # Astros JSON file - with open(Path(wkdir / 'output' / 'astros.json'), 'r') as f: - astros_str = f.read() - astros_dict = json.loads(astros_str) - self.assertEqual(astros_dict["message"], "success") + with open(Path(wkdir / 'output' / 'todos.json'), 'r') as f: + todos_str = f.read() + todos_dict = json.loads(todos_str) + self.assertEqual(len(todos_dict), 200) # Dummy second target text file with open(Path(wkdir / 'output' / 'second_target.txt'), 'r') as f: @@ -1157,27 +1157,15 @@ def test_decorator_tasks_with_targets(self): self.assertEqual(second_target, "second target") # Check output of 'load' task - names = [ - "Sergey Prokopyev", - "Dmitry Petelin", - "Frank Rubio", - "Jing Haiping", - "Gui Haichow", - "Zhu Yangzhu", - "Jasmin Moghbeli", - "Andreas Mogensen", - "Satoshi Furukawa", - "Konstantin Borisov", - ] - for n in names: - formatted_name = n.lower().replace(" ", "_") - self.assertTrue(Path(wkdir / 'output' / f'{formatted_name}.txt').is_file()) - with open(Path(wkdir / 'output' / f'{formatted_name}.txt'), 'r') as f: + for i in range(1, 201): + self.assertTrue(Path(wkdir / 'output' / f'todo_{i}.txt').is_file()) + with open(Path(wkdir / 'output' / f'todo_{i}.txt'), 'r') as f: contents = f.read() - self.assertEqual(contents, n) + self.assertEqual(contents, todos_dict[i - 1]['title']) # Remove the .compiled directory, if it exists self._remove_compiled_dir(wkdir) + self._remove_files_in_output(wkdir) # Set up the working directory self._set_up_wkdir() @@ -1230,31 +1218,20 @@ def test_decorator_tasks_with_retries(self): self.assertEqual(' | '.join(expected_events), runtask_run_results) # Check output of 'extract' task - self.assertTrue(Path(wkdir / 'output' / 'astros.json').is_file()) - with open(Path(wkdir / 'output' / 'astros.json'), 'r') as f: - astros_str = f.read() - astros_dict = json.loads(astros_str) - self.assertEqual(astros_dict["message"], "success") + self.assertTrue(Path(wkdir / 'output' / 'todos.json').is_file()) + with open(Path(wkdir / 'output' / 'todos.json'), 'r') as f: + todos_str = f.read() + todos_dict = json.loads(todos_str) + self.assertEqual(len(todos_dict), 200) # Output of 'load' task was not created - names = [ - "Andrey Fedyaev", - "Deng Qingming", - "Dmitry Petelin", - "Fei Junlong", - "Frank Rubio", - "Sergey Prokopyev", - "Stephen Bowen", - "Sultan Alneyadi", - "Warren Hoburg", - "Zhang Lu", - ] - for n in names: - formatted_name = n.lower().replace(" ", "_") - self.assertFalse(Path(wkdir / 'output' / f'{formatted_name}.txt').is_file()) + # Check output of 'load' task + for i in range(1, 201): + self.assertFalse(Path(wkdir / 'output' / f'todo_{i}.txt').is_file()) # Remove the .compiled directory, if it exists self._remove_compiled_dir(wkdir) + self._remove_files_in_output(wkdir) # Set up the working directory self._set_up_wkdir()