Merge 62d683a into df67dee

.travis/install.sh

@@ -10,7 +10,7 @@ set verify off
 
 --alter session set plsql_warnings = 'ENABLE:ALL', 'DISABLE:(5004,5018,6000,6001,6003,6009,6010,7206)';
 alter session set plsql_optimize_level=0;
-@install_headless.sql $UT3_OWNER $UT3_OWNER_PASSWORD
+@install_headless_with_trigger.sql $UT3_OWNER $UT3_OWNER_PASSWORD
 SQL
 
 #Run this step only on second child job (12.1 - at it's fastest)
@@ -25,14 +25,18 @@ if [[ "${TRAVIS_JOB_NUMBER}" =~ \.2$ ]]; then
     set verify off
 
     @uninstall_all.sql $UT3_OWNER
+    whenever sqlerror exit failure rollback
     declare
       v_leftover_objects_count integer;
     begin
       select sum(cnt)
         into v_leftover_objects_count
-        from (select count(1) cnt from dba_objects where owner = '$UT3_OWNER'
-        union all
-        select count(1) cnt from dba_synonyms where table_owner = '$UT3_OWNER'
+        from (
+          select count(1) cnt from dba_objects where owner = '$UT3_OWNER'
+           where object_name not like 'PLSQL_PROFILER%' and object_name not like 'DBMSPCC_%'
+          union all
+          select count(1) cnt from dba_synonyms where table_owner = '$UT3_OWNER'
+           where table_name not like 'PLSQL_PROFILER%' and table_name not like 'DBMSPCC_%'
         );
       if v_leftover_objects_count > 0 then
         raise_application_error(-20000, 'Not all objects were successfully uninstalled - leftover objects count='||v_leftover_objects_count);
@@ -47,6 +51,7 @@ SQL
 
     alter session set plsql_optimize_level=0;
     @install.sql $UT3_OWNER
+    @install_ddl_trigger.sql $UT3_OWNER
     @create_synonyms_and_grants_for_public.sql $UT3_OWNER
 SQL
 
@@ -65,6 +70,9 @@ SQL
 time "$SQLCLI" sys/$ORACLE_PWD@//$CONNECTION_STR AS SYSDBA <<-SQL
 set feedback off
 @create_utplsql_owner.sql $UT3_TESTER $UT3_TESTER_PASSWORD $UT3_TABLESPACE
+--needed for disabling DDL trigger and testint parser without trigger enabled/present
+grant alter any trigger to ut3_tester;
+grant administer database trigger to $UT3_TESTER;
 exit
 SQL
 
@@ -88,5 +96,7 @@ set feedback on
 --Needed for testing coverage outside of main UT3 schema.
 grant create any procedure, drop any procedure, execute any procedure, create any type, drop any type, execute any type, under any type, select any table, update any table, insert any table, delete any table, create any table, drop any table, alter any table, select any dictionary, create any synonym, drop any synonym to $UT3_TESTER_HELPER;
 grant create job to $UT3_TESTER_HELPER;
+--Needed to allow for enable/disable of annotation triggers
+grant administer database trigger to $UT3_TESTER_HELPER;
 exit
 SQL

.travis/install_utplsql_release.sh

@@ -36,9 +36,14 @@ end;
 /
 SQL
 
+INSTALL_FILE="install_headless_with_trigger.sql"
+if [[ ! -f "${INSTALL_FILE}" ]]; then
+ INSTALL_FILE="install_headless.sql"
+fi
+
 "$SQLCLI" sys/$ORACLE_PWD@//$CONNECTION_STR AS SYSDBA <<SQL
 alter session set plsql_optimize_level=0;
-@install_headless.sql ${UT3_RELEASE_VERSION_SCHEMA}
+@${INSTALL_FILE} ${UT3_RELEASE_VERSION_SCHEMA}
 exit
 SQL
 

development/utplsql_style_check.sql

@@ -40,8 +40,8 @@ select i.name, i.type, i.object_name, i.object_type, i.usage, i.line, i.col, cou
    and i.usage_context_id = p.usage_id
  where i.type like 'VARIABLE' and i.usage = 'DECLARATION'
    and i.object_type not in ('TYPE')
-   and (i.name not like 'L#_%' escape '#' and p.type in ('PROCEDURE','FUNCTION','ITERATOR')
-       or i.name not like 'G#_%' escape '#' and p.type not in ('PROCEDURE','FUNCTION','ITERATOR'))
+   and (i.name not like 'L#_%' escape '#' and p.type in ('PROCEDURE','FUNCTION','ITERATOR','TRIGGER')
+       or i.name not like 'G#_%' escape '#' and p.type not in ('PROCEDURE','FUNCTION','ITERATOR','TRIGGER'))
    and p.type != 'RECORD'
  order by object_name, object_type, line, col
 ;

docs/userguide/advanced_data_comparison.md

@@ -430,7 +430,7 @@ Unable to join sets:
   Please make sure that your join clause is not refferring to collection element
 ```
 
-***Note***
+**Note**
 >`join_by` option is slower to process as it needs to perform a cursor join.
 
 ## Defining item lists in option

docs/userguide/install.md

@@ -72,16 +72,44 @@ The utPLSQL may be installed on any supported version of Oracle Database [see](h
 * 12c
 * 12c R2
 * 18c
+* 19c
 
 # Headless installation
 
-To install the utPLSQL into a new database schema and grant it to public, execute the script `install_headless.sql` as SYSDBA.
+utPLSQL can be installed with DDL trigger, to enable tracking of DDL changes to your unit test packages.
+This is the recommended installation approach, when you want to compile and run unit test packages in a schema containing huge amount of database packages (for example Oracle EBS installation schema).
+The reason for having DDL trigger is to enable in-time annotation parsing for utPLSQL.
+Without DDL trigger, utPLSQL needs to investigate your schema objects last_ddl_timestamp each time tests are executed to check if any of DB packages were changed in given schema and if they need scanning for annotation changes.
+This process can be time-consuming if DB schema is large.     
 
-The script accepts three optional parameters that define:
+The headless scripts accept three optional parameters that define:
 - username to create as owner of utPLSQL (default `ut3`)  
 - password for owner of utPLSQL (default `XNtxj8eEgA6X6b6f`)
 - tablespace to use for storage of profiler data (default `users`)  
 
+The scripts need to be executed by `SYSDBA`, in order to grant access to `DBMS_LOCK` and `DBMS_CRYPTO` system packages.
+
+**Note:**
+> Grant on `DBMS_LOCK` is required only for installation on Oracle versions below 18c. For versions 18c and above, utPLSQL uses `DBMS_SESSION.SLEEP` so access to `DBMS_LOCK` package is no longer needed. 
+
+**Note:**
+> The user performing the installation must have the `ADMINISTER DATABASE TRIGGER` privilege. This is required for installation of trigger that is responsible for parsing annotations at at compile-time of a package.
+
+**Note:**
+> When installing with DDL trigger, utPLSQL will not be registering unit tests for any of oracle-maintained schemas.
+For Oracle 11g following users are excluded:
+> ANONYMOUS, APPQOSSYS, AUDSYS, DBSFWUSER, DBSNMP, DIP, GGSYS, GSMADMIN_INTERNAL, GSMCATUSER, GSMUSER, ORACLE_OCM, OUTLN, REMOTE_SCHEDULER_AGENT, SYS, SYS$UMF, SYSBACKUP, SYSDG, SYSKM, SYSRAC, SYSTEM, WMSYS, XDB, XS$NULL 
+>
+> For Oracle 12c and above the users returned by below query are excluded by utPLSQL:
+>
+>```sql
+>  select username from all_users where oracle_maintained='Y';
+>```
+ 
+## Installation without DDL trigger
+
+To install the utPLSQL into a new database schema and grant it to public, execute the script `install_headless.sql` as SYSDBA.
+
 Example invocation of the script from command line:
 ```bash
 cd source
@@ -94,16 +122,34 @@ cd source
 sqlplus sys/sys_pass@db as sysdba @install_headless.sql utp3 my_verySecret_password utp3_tablespace   
 ```
 
-The script needs to be executed by `SYSDBA`, in order to grant access to `DBMS_LOCK` and `DBMS_CRYPTO` system packages.
+## Installation with DDL trigger
+
+To install the utPLSQL into a new database schema and grant it to public, execute the script `install_headless_with_trigger.sql` as SYSDBA.
+
+Example invocation of the script from command line:
+```bash
+cd source
+sqlplus sys/sys_pass@db as sysdba @install_headless_with_trigger.sql  
+```
 
-*Note:* Grant on `DBMS_LOCK` is required on Oracle versions below 18c
+Invoking script with parameters:
+```bash
+cd source
+sqlplus sys/sys_pass@db as sysdba @install_headless_with_trigger.sql utp3 my_verySecret_password utp3_tablespace   
+```
 
+**Note:** 
+>When installing utPLSQL into database with existing unit test packages, utPLSQL will not be able to already-existing unit test packages. When utPSLQL was installed with DDL trigger, you have to do one of:
+>- Recompile existing Unit Test packages to make utPLSQL aware of their existence 
+>- Invoke `exec ut_runner.rebuild_annotation_cache(a_schema_name=> ... );` for every schema containing unit tests in your database
+>
+> Steps above are required to assure annotation cache is populated properly from existing objects. Rebuilding annotation cache might be faster than code recompilation.     
 
 # Recommended Schema
 It is highly recommended to install utPLSQL in it's own schema. You are free to choose any name for this schema.
 Installing uPLSQL into shared schema is really not recommended as you loose isolation of framework.
 
-If the installation and utPLSQL owner user is one and the same, the user must have the following Oracle system permissions before you can proceed with the installation.
+If the installing user and utPLSQL owner is one and the same, the user must have the following Oracle system permissions before you can proceed with the installation.
 
   - CREATE SESSION
   - CREATE PROCEDURE
@@ -113,6 +159,7 @@ If the installation and utPLSQL owner user is one and the same, the user must ha
   - CREATE VIEW
   - CREATE SYNONYM
   - ALTER SESSION
+  - CREATE TRIGGER
 
 In addition the user must be granted the execute privilege on `DBMS_LOCK` and `DBMS_CRYPTO` packages.
 
@@ -123,7 +170,7 @@ It is up to DBA to maintain the storage of the profiler tables.
 
 # Manual installation procedure
 
-### Creating schema for utPLSQL
+## Creating schema for utPLSQL
 To create the utPLSQL schema and grant all the required privileges execute script `create_utplsql_owner.sql` from the `source` directory with parameters:
 
   - `user name` - the name of the user that will own of utPLSQL object
@@ -136,8 +183,8 @@ cd source
 sqlplus sys/sys_password@database as sysdba @create_utPLSQL_owner.sql ut3 ut3 users  
 ```
 
-### Installing utPLSQL
-To install the utPLSQL framework into your database run the `/source/install.sql` script and provide `schema_name` where utPLSQL is to be installed.  
+## Installing utPLSQL
+To install the utPLSQL framework into your database, go to `source` directory, run the `install.sql` providing the `schema_name` for utPLSQL as parameter.  
 Schema must be created prior to calling the `install` script.
 You may install utPLSQL from any account that has sufficient privileges to create objects in other users schema.  
 
@@ -147,7 +194,26 @@ cd source
 sqlplus admin/admins_password@database @install.sql ut3  
 ```
 
-### Allowing other users to access the utPLSQL framework
+## Installing DDL trigger
+To minimize startup time of utPLSQL framework (especially on a database with large schema) it is recommended to install utPLSQL DDL trigger to enable utPLSQL annotation to be updated at compile-time.
+
+It's recommended to install DDL trigger when connected as `SYSDBA` user. Trigger is created in utPLSQL schema.
+If using the owner schema of utPLSQL to install trigger, the owner needs to have `ADMINISTER DATABASE TRIGGER` and `CREATE TRIGGER` system privileges. 
+If using different user to install trigger, the user needs to have `ADMINISTER DATABASE TRIGGER` and `CREATE ANY TRIGGER` system privileges. 
+
+To install DDL trigger go to `source` directory, run the `install_ddl_trigger.sql` providing the `schema_name` for utPLSQL as parameter.
+
+Example invocation:
+```bash
+cd source
+sqlplus admin/admins_password@database @install_ddl_trigger.sql ut3  
+```
+
+**Note:**
+>Trigger can be installed ant any point in time.
+
+
+## Allowing other users to access the utPLSQL framework
 In order to allow other users to access utPLSQL, synonyms must be created and privileges granted.
 You have two options:
 

source/check_sys_grants.sql

@@ -1,15 +1,15 @@
+define expected_grants = "&1"
 declare
-  c_expected_grants constant dbmsoutput_linesarray
-  := dbmsoutput_linesarray(
-      'CREATE TYPE','CREATE VIEW','CREATE SYNONYM','CREATE SEQUENCE','CREATE PROCEDURE','CREATE TABLE'
-  );
+  c_expected_grants constant dbmsoutput_linesarray := dbmsoutput_linesarray( &expected_grants );
 
   l_expected_grants dbmsoutput_linesarray := c_expected_grants;
   l_missing_grants varchar2(4000);
 begin
   if user != SYS_CONTEXT('userenv','current_schema') then
     for i in 1 .. l_expected_grants.count loop
-      l_expected_grants(i) := replace(l_expected_grants(i),' ',' ANY ');
+      if l_expected_grants(i) != 'ADMINISTER DATABASE TRIGGER' then
+        l_expected_grants(i) := replace(l_expected_grants(i),' ',' ANY ');
+      end if;
     end loop;
   end if;
   select listagg(' -  '||privilege,CHR(10)) within group(order by privilege)

source/core/annotations/ut_annotation_cache_info.sql

@@ -18,6 +18,6 @@ create table ut_annotation_cache_info (
   object_type     varchar2(250) not null,
   parse_time      timestamp     not null,
   constraint ut_annotation_cache_info_pk primary key(cache_id),
-  constraint ut_annotation_cache_info_uk unique (object_owner, object_name, object_type)
+  constraint ut_annotation_cache_info_uk unique (object_owner, object_type, object_name)
 ) organization index;
 

source/core/annotations/ut_annotation_cache_manager.pkb

@@ -18,22 +18,40 @@ create or replace package body ut_annotation_cache_manager as
 
   procedure update_cache(a_object ut_annotated_object) is
     l_cache_id       integer;
+    l_new_objects_count integer := 0;
     pragma autonomous_transaction;
   begin
-    update ut_annotation_cache_info i
-       set i.parse_time = systimestamp
-     where (i.object_owner, i.object_name, i.object_type)
-        in ((a_object.object_owner, a_object.object_name, a_object.object_type))
-      returning cache_id into l_cache_id;
-    if sql%rowcount = 0 then
-      insert into ut_annotation_cache_info
-             (cache_id, object_owner, object_name, object_type, parse_time)
-      values (ut_annotation_cache_seq.nextval, a_object.object_owner, a_object.object_name, a_object.object_type, systimestamp)
+    -- if not in trigger, or object has annotations
+    if ora_sysevent is null or a_object.annotations is not null and a_object.annotations.count > 0 then
+
+      update ut_annotation_cache_info i
+         set i.parse_time = systimestamp
+       where (i.object_owner, i.object_name, i.object_type)
+          in ((a_object.object_owner, a_object.object_name, a_object.object_type))
         returning cache_id into l_cache_id;
+
+      if sql%rowcount = 0 then
+
+        insert into ut_annotation_cache_info
+               (cache_id, object_owner, object_name, object_type, parse_time)
+        values (ut_annotation_cache_seq.nextval, a_object.object_owner, a_object.object_name, a_object.object_type, systimestamp)
+          returning cache_id into l_cache_id;
+        l_new_objects_count := 1;
+      end if;
+
     end if;
 
-    delete from ut_annotation_cache c
-      where cache_id = l_cache_id;
+    update ut_annotation_cache_schema s
+    set s.object_count = s.object_count + l_new_objects_count, s.max_parse_time = systimestamp
+    where s.object_type = a_object.object_type and s.object_owner = a_object.object_owner;
+
+    if sql%rowcount = 0 then
+      insert into ut_annotation_cache_schema s
+          (object_owner, object_type, object_count, max_parse_time)
+      values (a_object.object_owner, a_object.object_type, l_new_objects_count, systimestamp);
+    end if;
+
+    delete from ut_annotation_cache c where cache_id = l_cache_id;
 
     if a_object.annotations is not null and a_object.annotations.count > 0 then
       insert into ut_annotation_cache
@@ -73,17 +91,32 @@ create or replace package body ut_annotation_cache_manager as
     commit;
   end;
 
+  function get_cache_schema_info(a_object_owner varchar2, a_object_type varchar2) return t_cache_schema_info is
+    l_result t_cache_schema_info;
+  begin
+    begin
+      select *
+             into l_result
+        from ut_annotation_cache_schema s
+       where s.object_type = a_object_type and s.object_owner = a_object_owner;
+    exception
+      when no_data_found then
+        null;
+    end;
+    return l_result;
+  end;
+
   procedure remove_from_cache(a_objects ut_annotation_objs_cache_info) is
     pragma autonomous_transaction;
   begin
 
     delete from ut_annotation_cache_info i
-      where exists (
-        select 1 from table (a_objects) o
-        where o.object_name = i.object_name
-        and o.object_type = i.object_type
-        and o.object_owner = i.object_owner
-      );
+     where exists (
+             select 1 from table (a_objects) o
+              where o.object_name = i.object_name
+                and o.object_type = i.object_type
+                and o.object_owner = i.object_owner
+             );
 
     commit;
   end;
@@ -139,6 +172,12 @@ create or replace package body ut_annotation_cache_manager as
       delete from ut_annotation_cache_info i
        where ' || l_filter
     using a_object_owner, a_object_type;
+
+    execute immediate '
+      delete from ut_annotation_cache_schema s
+       where ' || l_filter
+    using a_object_owner, a_object_type;
+
     commit;
   end;
 

source/core/annotations/ut_annotation_cache_manager.pks

@@ -15,7 +15,7 @@ create or replace package ut_annotation_cache_manager authid definer as
   See the License for the specific language governing permissions and
   limitations under the License.
   */
-
+  subtype t_cache_schema_info is ut_annotation_cache_schema%rowtype;
   /**
    * Populates cache with information about object and it's annotations
    * Cache information for individual object is modified by this code
@@ -34,6 +34,8 @@ create or replace package ut_annotation_cache_manager authid definer as
    */
   function get_annotations_for_objects(a_cached_objects ut_annotation_objs_cache_info, a_parse_time timestamp) return sys_refcursor;
 
+  function get_cache_schema_info(a_object_owner varchar2, a_object_type varchar2) return t_cache_schema_info;
+
   /**
    * Removes cached information about annotations for objects on the list and updates parse_time in cache info table.
    *