Merge branch 'develop' of github.com:utPLSQL/utPLSQL into fix/oracle_…

…bug_with_xmlspaces

docs/about/authors.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 ### utPLSQL v3 Major Contributors 
 

docs/about/license.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Version Information
 

docs/about/project-details.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # utPLSQL Project Details
 

docs/about/support.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # How to get support
 

docs/index.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Introduction to utPLSQL
 

docs/userguide/advanced_data_comparison.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Advanced data comparison
 

docs/userguide/annotations.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Annotations
 

docs/userguide/best-practices.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Best Practices
 

docs/userguide/coverage.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Coverage
 utPLSQL comes with a built-in coverage reporting engine. The code coverage reporting is based on the DBMS_PROFILER package provided with Oracle database.

docs/userguide/exception-reporting.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Exception handling and reporting
 

docs/userguide/expectations.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Expectation concepts 
 Validation of the code under test (the tested logic of procedure/function etc.) is performed by comparing the actual data against the expected data.

docs/userguide/getting-started.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Getting started with TDD and utPLSQL
 

docs/userguide/install.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Downloading latest version of utPLSQL
 

docs/userguide/querying_suites.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Qyerying for test suites
 

docs/userguide/reporters.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 utPLSQL provides the following reporting formats.
 

docs/userguide/running-unit-tests.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Running tests
 

docs/userguide/upgrade.md

@@ -1,4 +1,4 @@
-![version](https://img.shields.io/badge/version-v3.1.7.2808--develop-blue.svg)
+![version](https://img.shields.io/badge/version-v3.1.7.2851--develop-blue.svg)
 
 # Upgrading from version 2
 

source/core/ut_expectation_processor.pkb

@@ -88,7 +88,7 @@ create or replace package body ut_expectation_processor as
   begin
     add_expectation_result(ut_expectation_result(ut_utils.gc_failure, null, a_message));
   end;
-
+  
   function get_session_parameters return tt_nls_params is
     l_session_params tt_nls_params;
   begin

source/core/ut_expectation_processor.pks

@@ -35,7 +35,7 @@ create or replace package ut_expectation_processor authid current_user as
   procedure add_expectation_result(a_expectation_result ut_expectation_result);
 
   procedure report_failure(a_message in varchar2);
-
+  
   procedure set_xml_nls_params;
 
   procedure reset_nls_params;

source/core/ut_utils.pkb

@@ -16,6 +16,13 @@ create or replace package body ut_utils is
   limitations under the License.
   */
 
+  /**
+  * Constants regex used to validate XML name
+  */
+  gc_invalid_first_xml_char  constant varchar2(50)  := '[^_a-zA-Z]';
+  gc_invalid_xml_char        constant varchar2(50)  := '[^_a-zA-Z0-9\.-]';
+  gc_full_valid_xml_name     constant varchar2(50)  := '^([_a-zA-Z])([_a-zA-Z0-9\.-])*$';
+
   function surround_with(a_value varchar2, a_quote_char varchar2) return varchar2 is
   begin
     return case when a_quote_char is not null then a_quote_char||a_value||a_quote_char else a_value end;
@@ -737,5 +744,59 @@ create or replace package body ut_utils is
     return l_results;
   end;
 
+  function remove_error_from_stack(a_error_stack varchar2, a_ora_code number) return varchar2 is
+    l_caller_stack_line          varchar2(4000);
+    l_ora_search_pattern         varchar2(500) := '^ORA'||a_ora_code||': (.*)$';
+  begin
+   l_caller_stack_line := regexp_replace(srcstr     => a_error_stack
+                          ,pattern    => l_ora_search_pattern
+                          ,replacestr => null
+                          ,position   => 1
+                          ,occurrence => 1
+                          ,modifier   => 'm');
+   return l_caller_stack_line;
+  end;
+
+  /**
+  * Change string into unicode to match xmlgen format _00<unicode>_
+  * https://docs.oracle.com/en/database/oracle/oracle-database/12.2/adxdb/generation-of-XML-data-from-relational-data.html#GUID-5BE09A7D-80D8-4734-B9AF-4A61F27FA9B2
+  * secion v3.1.7.2851-develop
+  */  
+  function char_to_xmlgen_unicode(a_character varchar2) return varchar2 is
+  begin
+    return '_x00'||rawtohex(utl_raw.cast_to_raw(a_character))||'_';
+  end;
+
+  /**
+  * Build valid XML column name as element names can contain letters, digits, hyphens, underscores, and periods
+  */  
+  function build_valid_xml_name(a_preprocessed_name varchar2) return varchar2 is
+    l_post_processed varchar2(4000);
+  begin
+    for i in (select regexp_substr( a_preprocessed_name ,'(.{1})', 1, level, null, 1 ) AS string_char,level level_no
+              from   dual connect by level <= regexp_count(a_preprocessed_name, '(.{1})'))
+    loop
+      if i.level_no = 1 and regexp_like(i.string_char,gc_invalid_first_xml_char) then
+        l_post_processed := l_post_processed || char_to_xmlgen_unicode(i.string_char);
+      elsif regexp_like(i.string_char,gc_invalid_xml_char) then
+        l_post_processed := l_post_processed || char_to_xmlgen_unicode(i.string_char);
+      else
+        l_post_processed := l_post_processed || i.string_char;
+      end if;
+    end loop;
+    return l_post_processed;  
+  end;
+
+  function get_valid_xml_name(a_name varchar2) return varchar2 is
+    l_valid_name varchar2(4000);
+  begin
+    if regexp_like(a_name,gc_full_valid_xml_name) then
+      l_valid_name := a_name;
+    else
+      l_valid_name := build_valid_xml_name(a_name);
+    end if;
+    return l_valid_name;
+  end;
+
 end ut_utils;
 /

source/core/ut_utils.pks

@@ -21,8 +21,8 @@ create or replace package ut_utils authid definer is
    *
    */
 
-  gc_version                 constant varchar2(50) := 'v3.1.7.2808-develop';
-
+  gc_version                 constant varchar2(50) := 'v3.1.7.2851-develop';
+    
   subtype t_executable_type      is varchar2(30);
   gc_before_all                  constant t_executable_type := 'beforeall';
   gc_before_each                 constant t_executable_type := 'beforeeach';
@@ -109,6 +109,14 @@ create or replace package ut_utils authid definer is
   gc_value_too_large constant pls_integer := -20217;
   pragma exception_init (ex_value_too_large, -20217);
 
+  ex_xml_processing exception;
+  gc_xml_processing constant pls_integer := -19202;
+  pragma exception_init (ex_xml_processing, -19202);
+
+  ex_failed_open_cur exception;
+  gc_failed_open_cur constant pls_integer := -20218;
+  pragma exception_init (ex_failed_open_cur, -20218);  
+
   gc_max_storage_varchar2_len constant integer := 4000;
   gc_max_output_string_length constant integer := 4000;
   gc_more_data_string         constant varchar2(5) := '[...]';
@@ -368,6 +376,16 @@ create or replace package ut_utils authid definer is
    * Returns list of sub-type reporters for given list of super-type reporters
    */
   function get_child_reporters(a_for_reporters ut_reporters_info := null) return ut_reporters_info;
-
+
+  /**
+  * Remove given ORA error from stack
+  */
+  function remove_error_from_stack(a_error_stack varchar2, a_ora_code number) return varchar2;
+
+  /**
+  * Check if xml name is valid if not build a valid name
+  */
+  function get_valid_xml_name(a_name varchar2) return varchar2;
+
 end ut_utils;
 /

source/expectations/data_values/ut_compound_data_helper.pkb

@@ -18,8 +18,10 @@ create or replace package body ut_compound_data_helper is
 
   g_diff_count        integer;
   type t_type_name_map is table of varchar2(128) index by binary_integer;
+  type t_types_no_length is table of varchar2(128) index by varchar2(128);
   g_type_name_map           t_type_name_map;
   g_anytype_name_map        t_type_name_map;
+  g_type_no_length_map      t_types_no_length;
 
   g_compare_sql_template varchar2(4000) :=
   q'[
@@ -78,13 +80,13 @@ create or replace package body ut_compound_data_helper is
   begin
     execute immediate q'[with
           expected_cols as (
-            select access_path exp_column_name,column_position exp_col_pos,
+            select display_path exp_column_name,column_position exp_col_pos,
                    replace(column_type_name,'VARCHAR2','CHAR') exp_col_type_compare, column_type_name exp_col_type
               from table(:a_expected)
               where parent_name is null and hierarchy_level = 1 and column_name is not null
           ),
           actual_cols as (
-            select access_path act_column_name,column_position act_col_pos,
+            select display_path act_column_name,column_position act_col_pos,
                    replace(column_type_name,'VARCHAR2','CHAR') act_col_type_compare, column_type_name act_col_type
               from table(:a_actual)
               where parent_name is null and hierarchy_level = 1 and column_name is not null
@@ -231,7 +233,7 @@ create or replace package body ut_compound_data_helper is
     elsif  a_data_info.is_sql_diffable = 1  and a_data_info.column_type in ('DATE','TIMESTAMP','TIMESTAMP WITH TIME ZONE',
       'TIMESTAMP WITH LOCAL TIME ZONE') then
       l_col_type := 'VARCHAR2(50)';
-    elsif  a_data_info.is_sql_diffable = 1  and a_data_info.column_type in ('INTERVAL DAY TO SECOND','INTERVAL YEAR TO MONTH') then
+    elsif  a_data_info.is_sql_diffable = 1  and type_no_length(a_data_info.column_type) then
       l_col_type := a_data_info.column_type;
     elsif a_data_info.is_sql_diffable = 1  and a_data_info.column_type in ('VARCHAR2','CHAR') then
       l_col_type := 'VARCHAR2('||greatest(a_data_info.column_len,4000)||')';
@@ -571,6 +573,8 @@ create or replace package body ut_compound_data_helper is
     --clob/blob/xmltype/object/nestedcursor/nestedtable
     if a_type_name IN (g_type_name_map(dbms_sql.blob_type),
                        g_type_name_map(dbms_sql.clob_type),
+                       g_type_name_map(dbms_sql.long_type),
+                       g_type_name_map(dbms_sql.long_raw_type),
                        g_type_name_map(dbms_sql.bfile_type),
                        g_anytype_name_map(dbms_types.typecode_namedcollection))
     then    
@@ -597,6 +601,24 @@ create or replace package body ut_compound_data_helper is
     open l_diff_cursor for a_diff_cursor_text using a_self_id, a_other_id;
     return l_diff_cursor;
   end;
+
+  function create_err_cursor_msg(a_error_stack varchar2) return varchar2 is
+  begin
+    return 'SQL exception thrown when fetching data from cursor:'||
+      ut_utils.remove_error_from_stack(sqlerrm,ut_utils.gc_xml_processing)||chr(10)||
+      ut_expectation_processor.who_called_expectation(a_error_stack)||
+      'Check the query and data for errors.';   
+  end; 
+
+  function type_no_length ( a_type_name varchar2) return boolean is
+  begin
+    return case 
+      when g_type_no_length_map.exists(a_type_name) then
+        true
+      else
+        false
+      end;
+  end;
 
 begin
   g_anytype_name_map(dbms_types.typecode_date)             := 'DATE';
@@ -643,6 +665,15 @@ begin
   g_type_name_map( dbms_sql.urowid_type )                  := 'UROWID';  
   g_type_name_map( dbms_sql.user_defined_type )            := 'USER_DEFINED_TYPE';
   g_type_name_map( dbms_sql.ref_type )                     := 'REF_TYPE';
-
+
+
+  /**
+  * List of types that have no length but can produce a max_len from desc_cursor function.
+  */
+  g_type_no_length_map('ROWID')                            := 'ROWID';
+  g_type_no_length_map('INTERVAL DAY TO SECOND')           := 'INTERVAL DAY TO SECOND';
+  g_type_no_length_map('INTERVAL YEAR TO MONTH')           := 'INTERVAL YEAR TO MONTH';
+  g_type_no_length_map('BINARY_DOUBLE')                    := 'BINARY_DOUBLE';
+  g_type_no_length_map('BINARY_FLOAT')                     := 'BINARY_FLOAT';
 end;
 /

source/expectations/data_values/ut_compound_data_helper.pks

@@ -92,5 +92,12 @@ create or replace package ut_compound_data_helper authid definer is
 
   function get_compare_cursor(a_diff_cursor_text in clob,a_self_id raw, a_other_id raw) return sys_refcursor;
 
+  function create_err_cursor_msg(a_error_stack varchar2) return varchar2;
+
+  /*
+  * Function to return true or false if the type dont have an length
+  */
+  function type_no_length ( a_type_name varchar2) return boolean;
+
 end;
 /

source/expectations/data_values/ut_cursor_column.tpb

@@ -13,17 +13,24 @@ create or replace type body ut_cursor_column as
       self.column_len       := a_col_max_len; --length of column
       self.column_name      := TRIM( BOTH '''' FROM a_col_name); --name of the column
       self.column_type_name := coalesce(a_col_type_name,a_col_type); --type name e.g. test_dummy_object or varchar2
-      self.access_path      := case when a_access_path is null then 
+      self.xml_valid_name   := ut_utils.get_valid_xml_name(self.column_name);
+      self.display_path     := case when a_access_path is null then 
                                  self.column_name 
                                else 
                                  a_access_path||'/'||self.column_name 
-                               end; --Access path used for incldue exclude eg/ TEST_DUMMY_OBJECT/VARCHAR2
-      self.xml_valid_name   := '"'||self.column_name||'"'; --User friendly column name
-      self.transformed_name := case when self.parent_name is null then 
-                                self.xml_valid_name
-                             else 
-                                '"'||ut_compound_data_helper.get_fixed_size_hash(self.parent_name||self.column_name)||'"'
-                             end; --when is nestd we need to hash name to make sure we dont exceed 30 char
+                               end; --Access path used for incldue exclude eg/ TEST_DUMMY_OBJECT/VARCHAR2       
+      self.access_path      := case when a_access_path is null then 
+                                 self.xml_valid_name 
+                               else 
+                                 a_access_path||'/'||self.xml_valid_name 
+                               end; --Access path used for incldue exclude eg/ TEST_DUMMY_OBJECT/VARCHAR2     
+      self.transformed_name := case when length(self.xml_valid_name) > 30 then
+                                 '"'||ut_compound_data_helper.get_fixed_size_hash(self.parent_name||self.xml_valid_name)||'"'
+                               when self.parent_name is null then 
+                                 '"'||self.xml_valid_name||'"'
+                               else 
+                                 '"'||ut_compound_data_helper.get_fixed_size_hash(self.parent_name||self.xml_valid_name)||'"'
+                               end; --when is nestd we need to hash name to make sure we dont exceed 30 char
       self.column_type      := a_col_type; --column type e.g. user_defined , varchar2
       self.column_schema    := a_col_schema_name; -- schema name
       self.is_sql_diffable  := case 

source/expectations/data_values/ut_cursor_column.tps

@@ -17,12 +17,13 @@ create or replace type ut_cursor_column force authid current_user as object (
   */
    parent_name      varchar2(4000),
    access_path      varchar2(4000),
+   display_path     varchar2(4000),
    has_nested_col   number(1,0),
-   transformed_name varchar2(32),
+   transformed_name varchar2(2000),
    hierarchy_level  number,
    column_position  number,
-   xml_valid_name   varchar2(128),
-   column_name      varchar2(128),
+   xml_valid_name   varchar2(2000),
+   column_name      varchar2(2000),
    column_type      varchar2(128),
    column_type_name varchar2(128),
    column_schema    varchar2(128),