feat: Add --ddl-file option and support offload to an existing empty table

.gitignore

@@ -14,7 +14,6 @@ build
 eggs
 .eggs
 parts
-bin
 var
 sdist
 develop-eggs

bin/offload

@@ -17,11 +17,12 @@
 import sys
 
 from goe.config import config_file
+from goe.exceptions import OffloadOptionError
 from goe.goe import (
     get_options,
     OFFLOAD_OP_NAME,
 )
-from goe.offload.offload import OffloadOptionError, get_offload_options
+from goe.offload.offload import get_offload_options
 from goe.orchestration.cli_entry_points import offload_by_cli
 
 

sql/oracle/source/sql/create_offload_repo_104.sql

@@ -0,0 +1,45 @@
+/*
+# Copyright 2016 The GOE Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+define goe_offload_repo_version = '1.0.4'
+define goe_offload_repo_comments = "GOE repo upgrades for &goe_offload_repo_version."
+
+PROMPT Installing GOE repository &goe_offload_repo_version....
+
+-- New seed data
+-- -----------------------------------------------------------------------------------------------
+
+DECLARE
+    PROCEDURE add_command_step ( p_code  IN command_step.code%TYPE,
+                                 p_title IN command_step.title%TYPE ) IS
+    BEGIN
+        INSERT INTO command_step
+            (id, code, title, create_time)
+        VALUES
+            (command_step_seq.NEXTVAL, p_code, p_title, SYSTIMESTAMP);
+    END add_command_step;
+BEGIN
+    add_command_step('DDL_FILE', 'Create DDL file');
+END;
+/
+
+--------------------------------------------------------------------------------------------------
+@@upgrade_offload_repo_version.sql
+
+PROMPT GOE repository &goe_offload_repo_version. installed.
+
+undefine goe_offload_repo_version
+undefine goe_offload_repo_comments

sql/oracle/source/sql/install_offload_repo.sql

@@ -21,5 +21,6 @@ prompt Installing GOE repository...
 alter session set current_schema = &goe_db_repo_user;
 -- Start offload repo version files...
 @@create_offload_repo_100.sql
+@@create_offload_repo_104.sql
 -- End offload repo version files.
 @@install_offload_repo_code.sql

sql/oracle/source/sql/upgrade_offload_repo_deltas.sql

@@ -60,6 +60,7 @@ begin
 
     -- Follow this pattern for each repo version file in sequence...
     check_version(v_current_version, '1.0.0');
+    check_version(v_current_version, '1.0.4');
 
 end;
 /

src/goe/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 2024 The GOE Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.metadata
+
+__version__ = importlib.metadata.version("goe")

src/goe/conductor/hybrid_view_service.py

@@ -193,6 +193,7 @@ def _get_backend_table(self):
             self._connection_options,
             self._messages,
             hybrid_metadata=self._offload_metadata,
+            dry_run=self._dry_run,
         )
 
     def _get_backend_detail(self, attribute_name=None):
@@ -315,6 +316,7 @@ def validate_by_aggregation(self, lower_hv=None, upper_hv=None, as_json=True):
             messages=self._messages,
             backend_db=self._backend_table_owner,
             backend_table=self._backend_table_name,
+            execute=(not self._dry_run),
         )
         status, agg_msg = validator.validate(
             safe=False,

src/goe/config/option_descriptions.py

@@ -0,0 +1,38 @@
+#! /usr/bin/env python3
+
+# Copyright 2016 The GOE Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" option_descriptions: Library of constants defining descriptions for options.
+    In the future we expect to refactor all option processing, including descriptions, and this module will
+    may become redundant at that time.
+"""
+
+DATA_SAMPLE_PARALLELISM = (
+    "Degree of parallelism to use when sampling RDBMS data for columns with no precision/scale properties. "
+    "Values of 0 or 1 will execute the query without parallelism"
+)
+
+RESET_BACKEND_TABLE = "Remove backend data table. Use with caution - this will delete previously offloaded data for this table!"
+
+REUSE_BACKEND_TABLE = (
+    "Allow Offload to re-use an empty backend table when there is already Offload metadata. "
+    "This may be useful if a backend table had data removed by an administrator and a re-offload is required"
+)
+
+VERIFY_PARALLELISM = (
+    "Degree of parallelism to use for the RDBMS query executed when validating an offload. "
+    "Values of 0 or 1 will execute the query without parallelism. Values > 1 will force a parallel query of the given degree. "
+    "If unset, the RDBMS query will fall back to using the behavior specified by RDBMS defaults"
+)

src/goe/config/orchestration_config.py

@@ -79,7 +79,6 @@
     "data_governance_auto_tags_csv",
     "data_governance_auto_properties_csv",
     "dev_log_level",
-    "execute",
     "error_on_token",
     "frontend_odbc_driver_name",
     "google_dataproc_batches_subnet",
@@ -252,7 +251,6 @@ class OrchestrationConfig:
     db_type: str
     dev_log_level: str
     error_on_token: Optional[str]
-    execute: bool
     frontend_odbc_driver_name: Optional[str]
     google_dataproc_batches_subnet: Optional[str]
     google_dataproc_batches_ttl: Optional[str]
@@ -429,9 +427,6 @@ def from_dict(config_dict: dict, do_not_connect=False):
                 "dev_log_level", orchestration_defaults.dev_log_level_default()
             ),
             error_on_token=config_dict.get("error_on_token"),
-            execute=config_dict.get(
-                "execute", orchestration_defaults.execute_default()
-            ),
             frontend_odbc_driver_name=config_dict.get(
                 "frontend_odbc_driver_name",
                 orchestration_defaults.frontend_odbc_driver_name_default(),

src/goe/connect/connect.py

@@ -447,7 +447,7 @@ def check_environment(options, orchestration_config):
 def test_offload_fs_container(orchestration_config, messages):
     test_name = "Offload filesystem container"
     test_header(test_name)
-    dfs_client = get_dfs_from_options(orchestration_config, messages)
+    dfs_client = get_dfs_from_options(orchestration_config, messages, dry_run=False)
     display_uri = dfs_client.gen_uri(
         orchestration_config.offload_fs_scheme,
         orchestration_config.offload_fs_container,
@@ -466,7 +466,6 @@ def test_offload_fs_container(orchestration_config, messages):
 
 def get_config_with_connect_overrides(connect_options):
     override_dict = {
-        "execute": True,
         "verbose": connect_options.verbose,
         "hive_timeout_s": CONNECT_HIVE_TIMEOUT_S,
     }

src/goe/connect/connect_backend.py

@@ -121,10 +121,11 @@ def test_raw_conn(hadoop_host, hadoop_port):
 
 
 def get_cli_hdfs(orchestration_config, host, messages):
+    # dry_run always = False in connect.
     return CliHdfs(
         host,
         orchestration_config.hadoop_ssh_user,
-        dry_run=(not orchestration_config.execute),
+        dry_run=False,
         messages=messages,
         db_path_suffix=orchestration_config.hdfs_db_path_suffix,
         hdfs_data=orchestration_config.hdfs_data,
@@ -241,7 +242,7 @@ def test_webhdfs_config(orchestration_config, messages):
         orchestration_config.hadoop_ssh_user,
         True if orchestration_config.kerberos_service else False,
         orchestration_config.webhdfs_verify_ssl,
-        dry_run=not orchestration_config.execute,
+        dry_run=False,
         messages=messages,
         db_path_suffix=orchestration_config.hdfs_db_path_suffix,
         hdfs_data=orchestration_config.hdfs_data,
@@ -264,7 +265,7 @@ def test_sentry_privs(orchestration_config, backend_api, messages):
         log("Skipping Sentry steps due to backend system", detail=VVERBOSE)
         return
 
-    dfs_client = get_dfs_from_options(orchestration_config, messages)
+    dfs_client = get_dfs_from_options(orchestration_config, messages, dry_run=False)
     uris_left_to_check = get_hdfs_dirs(
         orchestration_config, dfs_client, include_hdfs_home=False
     )

src/goe/data_governance/hadoop_data_governance.py

@@ -208,12 +208,12 @@ def data_governance_auto_property_defaults(
     """
     property_defaults = {}
     now = datetime.datetime.now().replace(microsecond=0)
-    property_defaults[
-        DATA_GOVERNANCE_DYNAMIC_PROPERTY_INITIAL_OPERATION_DATETIME
-    ] = now.isoformat()
-    property_defaults[
-        DATA_GOVERNANCE_DYNAMIC_PROPERTY_LATEST_OPERATION_DATETIME
-    ] = now.isoformat()
+    property_defaults[DATA_GOVERNANCE_DYNAMIC_PROPERTY_INITIAL_OPERATION_DATETIME] = (
+        now.isoformat()
+    )
+    property_defaults[DATA_GOVERNANCE_DYNAMIC_PROPERTY_LATEST_OPERATION_DATETIME] = (
+        now.isoformat()
+    )
     if source_rdbms_object:
         property_defaults[DATA_GOVERNANCE_DYNAMIC_PROPERTY_SOURCE_RDBMS_OBJECT] = (
             "%s.%s.%s" % (rdbms_name, rdbms_schema, source_rdbms_object)
@@ -223,9 +223,9 @@ def data_governance_auto_property_defaults(
             "%s.%s.%s" % (rdbms_name, rdbms_schema, target_rdbms_object)
         ).upper()
     if goe_object_type:
-        property_defaults[
-            DATA_GOVERNANCE_DYNAMIC_PROPERTY_GOE_OBJECT_TYPE
-        ] = goe_object_type
+        property_defaults[DATA_GOVERNANCE_DYNAMIC_PROPERTY_GOE_OBJECT_TYPE] = (
+            goe_object_type
+        )
     property_defaults = filter_properties_by_goe_object_type(
         property_defaults, goe_object_type
     )
@@ -436,8 +436,8 @@ def data_governance_register_new_db(
 def data_governance_register_new_db_step(
     hadoop_db, data_gov_client, messages, goe_object_type, options=None
 ):
-    opts_execute = options.execute if options else True
     if data_gov_client:
+        opts_execute = options.execute if options else True
 
         def step_fn():
             data_governance_register_new_db(
@@ -463,8 +463,8 @@ def data_governance_register_new_object_step(
     renaming_from_object_name=None,
     dg_object_type=None,
 ):
-    opts_execute = options.execute if options else True
     if data_gov_client:
+        opts_execute = options.execute if options else True
 
         def step_fn():
             data_governance_register_new_object(
@@ -526,8 +526,8 @@ def data_governance_register_new_view_step(
 def data_governance_update_metadata_step(
     hadoop_db, hadoop_object_name, data_gov_client, messages, options=None
 ):
-    opts_execute = options.execute if options else True
     if data_gov_client:
+        opts_execute = options.execute if options else True
 
         def step_fn():
             data_governance_update_metadata(
@@ -550,8 +550,8 @@ def data_governance_register_new_multi_db_step(
     """
     assert hadoop_db_list
     assert type(hadoop_db_list) is list
-    opts_execute = options.execute if options else True
     if data_gov_client:
+        opts_execute = options.execute if options else True
 
         def step_fn():
             for hadoop_db, goe_object_type in hadoop_db_list:

src/goe/exceptions.py

@@ -0,0 +1,25 @@
+# Copyright 2024 The GOE Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class OffloadException(Exception):
+    pass
+
+
+class OffloadOptionError(Exception):
+    def __init__(self, detail):
+        self.detail = detail
+
+    def __str__(self):
+        return repr(self.detail)