Merge pull request #151 from icanbwell/print_execution_plan_if_loglev…

…el_is_debug print execution plan if loglevel is DEBUG
icanbwell · Nov 30, 2022 · e177d4d · e177d4d
2 parents d5372a3 + 2786129
commit e177d4d
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 42 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -14,29 +14,11 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: '3.7'
 
-      - name: Set cache permissions
-        run: |
-          mkdir ~/.local && mkdir ~/.local/share && mkdir ~/.local/share/virtualenvs
-          chown -R $(whoami) ~/.local/share/virtualenvs/
-
-      - name: Cache pip dependencies
-        uses: actions/cache@v2
-        env:
-          cache-name: cache-pip-deps-warehouse-p
-        with:
-          path: ~/.local/share/virtualenvs/
-          key: ${{ runner.os }}-${{ env.CACHE_SEED }}-${{ github.job }}-${{ env.cache-name }}-${{ hashFiles('Pipfile.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ env.CACHE_SEED }}-${{ github.job }}-${{ env.cache-name }}-
-            ${{ runner.os }}-${{ env.CACHE_SEED }}-${{ github.job }}-
-            ${{ runner.os }}-${{ env.CACHE_SEED }}-
-            ${{ runner.os }}-
-
       - name: pre-commit
         run: make run-pre-commit
 

diff --git a/spark_auto_mapper/automappers/automapper.py b/spark_auto_mapper/automappers/automapper.py
@@ -215,19 +215,22 @@ def _transform_with_data_frame_single_select(
             if not self.drop_key_columns:
                 column_specs = [col(f"b.{c}") for c in keys] + column_specs
 
-            self.logger.debug(f"-------- automapper ({self.view}) column specs ------")
-            self.logger.debug(self.to_debug_string(source_df=source_df))
-            self.logger.debug(
-                f"-------- end automapper ({self.view}) column specs ------"
-            )
-            self.logger.debug(
-                f"-------- automapper ({self.source_view}) source_df schema ------"
-            )
-            # noinspection PyProtectedMember
-            self.logger.debug(source_df._jdf.schema().treeString())
-            self.logger.debug(
-                f"-------- end automapper ({self.source_view}) source_df schema ------"
-            )
+            if self.log_level and self.log_level == "DEBUG":
+                self.logger.debug(
+                    f"-------- automapper ({self.view}) column specs ------"
+                )
+                self.logger.debug(self.to_debug_string(source_df=source_df))
+                self.logger.debug(
+                    f"-------- end automapper ({self.view}) column specs ------"
+                )
+                self.logger.debug(
+                    f"-------- automapper ({self.source_view}) source_df schema ------"
+                )
+                # noinspection PyProtectedMember
+                self.logger.debug(source_df._jdf.schema().treeString())
+                self.logger.debug(
+                    f"-------- end automapper ({self.source_view}) source_df schema ------"
+                )
 
             if self.check_schema_for_all_columns:
                 for column_name, mapper in self.mappers.items():
@@ -247,6 +250,13 @@ def _transform_with_data_frame_single_select(
 
             # run all the selects
             df = source_df.alias("b").select(*column_specs)
+
+            if self.log_level and self.log_level == "DEBUG":
+                print(
+                    f"------------  Start Execution Plan for view {self.view} -----------"
+                )
+                df.explain(extended="cost")
+                print("------------  End Execution Plan -----------")
             # write out final checkpoint for this automapper
             if self.checkpoint_path:
                 checkpoint_path = (
@@ -264,14 +274,15 @@ def _transform_with_data_frame_single_select(
             self.logger.warning(
                 f"-------- end automapper ({self.view}) column specs ------"
             )
-            self.logger.debug(
-                f"-------- automapper ({self.source_view}) source_df schema ------"
-            )
-            # noinspection PyProtectedMember
-            self.logger.debug(source_df._jdf.schema().treeString())
-            self.logger.debug(
-                f"-------- end automapper ({self.source_view}) source_df schema ------"
-            )
+            if self.log_level and self.log_level == "DEBUG":
+                self.logger.debug(
+                    f"-------- automapper ({self.source_view}) source_df schema ------"
+                )
+                # noinspection PyProtectedMember
+                self.logger.debug(source_df._jdf.schema().treeString())
+                self.logger.debug(
+                    f"-------- end automapper ({self.source_view}) source_df schema ------"
+                )
             # iterate through each column to find the problem child
             for column_name, mapper in self.mappers.items():
                 try:

diff --git a/tests/amount/test_automapper_amount_typed.py b/tests/amount/test_automapper_amount_typed.py
@@ -29,7 +29,7 @@ def test_auto_mapper_amount_typed(spark_session: SparkSession) -> None:
 
     # Act
     mapper = AutoMapper(
-        view="members", source_view="patients", keys=["member_id"]
+        view="members", source_view="patients", keys=["member_id"], log_level="DEBUG"
     ).columns(age=A.amount(A.column("my_age")))
 
     assert isinstance(mapper, AutoMapper)