Merge remote-tracking branch 'refs/remotes/origin/master'

config/sql/oracle-4.3-all.sql

@@ -1211,6 +1211,8 @@ CREATE INDEX iObjectExtLongItemId ON M_OBJECT_EXT_LONG(ITEM_ID) INITRANS 30;
 CREATE INDEX iObjectExtPolyItemId ON M_OBJECT_EXT_POLY(ITEM_ID) INITRANS 30;
 CREATE INDEX iObjectExtReferenceItemId ON M_OBJECT_EXT_REFERENCE(ITEM_ID) INITRANS 30;
 CREATE INDEX iObjectExtStringItemId ON M_OBJECT_EXT_STRING(ITEM_ID) INITRANS 30;
+CREATE INDEX iObjectExtString_OT_II_LVAL
+    ON M_OBJECT_EXT_STRING(ownerType, ITEM_ID, (lower(stringvalue))) INITRANS 30;
 CREATE INDEX iObjectSubtypeOid ON M_OBJECT_SUBTYPE(OBJECT_OID) INITRANS 30;
 CREATE INDEX iOrgOrgTypeOid ON M_ORG_ORG_TYPE(ORG_OID) INITRANS 30;
 CREATE INDEX iServiceTypeOid ON M_SERVICE_TYPE(SERVICE_OID) INITRANS 30;

config/sql/oracle-upgrade-4.2-4.3.sql

@@ -4,6 +4,10 @@ ALTER TABLE m_operation_execution ADD recordType NUMBER(10, 0);
 -- MID-3669
 ALTER TABLE m_focus ADD lockoutStatus NUMBER(10, 0);
 
+-- MID-6691
+CREATE INDEX iObjectExtString_OT_II_LVAL
+    ON M_OBJECT_EXT_STRING(ownerType, ITEM_ID, (lower(stringvalue))) INITRANS 30;
+
 -- WRITE CHANGES ABOVE ^^
 UPDATE m_global_metadata SET value = '4.3' WHERE name = 'databaseSchemaVersion';
 

config/sql/postgresql-4.3-all.sql

@@ -1206,6 +1206,8 @@ CREATE INDEX iObjectExtLongItemId ON M_OBJECT_EXT_LONG(ITEM_ID);
 CREATE INDEX iObjectExtPolyItemId ON M_OBJECT_EXT_POLY(ITEM_ID);
 CREATE INDEX iObjectExtReferenceItemId ON M_OBJECT_EXT_REFERENCE(ITEM_ID);
 CREATE INDEX iObjectExtStringItemId ON M_OBJECT_EXT_STRING(ITEM_ID);
+CREATE INDEX iObjectExtStringOwnerTypeItemIdLcValue
+    ON M_OBJECT_EXT_STRING(ownerType, ITEM_ID, (lower(stringvalue)));
 CREATE INDEX iObjectSubtypeOid ON M_OBJECT_SUBTYPE(OBJECT_OID);
 CREATE INDEX iOrgOrgTypeOid ON M_ORG_ORG_TYPE(ORG_OID);
 CREATE INDEX iServiceTypeOid ON M_SERVICE_TYPE(SERVICE_OID);

config/sql/postgresql-upgrade-4.2-4.3.sql

@@ -4,6 +4,15 @@ ALTER TABLE m_operation_execution ADD COLUMN recordType INT4;
 -- MID-3669
 ALTER TABLE m_focus ADD COLUMN lockoutStatus INT4;
 
+-- MID-6691
+CREATE INDEX CONCURRENTLY IF NOT EXISTS iObjectExtStringOwnerTypeItemIdLcValue
+    ON M_OBJECT_EXT_STRING(ownerType, ITEM_ID, (lower(stringvalue)));
+/* TODO what if this is better?
+CREATE INDEX CONCURRENTLY IF NOT EXISTS iObjectExtStringItemIdLcValue0
+    ON M_OBJECT_EXT_STRING(ITEM_ID, (lower(stringvalue)))
+         WHERE ownerType = 0;
+*/
+
 -- WRITE CHANGES ABOVE ^^
 UPDATE m_global_metadata SET value = '4.3' WHERE name = 'databaseSchemaVersion';
 

pom.xml

@@ -57,8 +57,6 @@
     Running unit+IT tests of a selected module (-pl) with output to stdout:
     mvn clean install -DredirectTestOutputToFile=false -pl model/model-intest
 
-    Running GUI tests from Schrodinger module only with custom location of ChromeDriver (Chrome must be installed too):
-    mvn clean install -P -dist -DskipTests -DskipSchrodingerTests=false -DwebdriverLocation=/c/work/tools/bin/chromedriver.exe
     -->
 
     <parent>
@@ -1538,46 +1536,6 @@
                 <version>1.0-alpha-2</version>
             </dependency>
 
-            <!-- Schrodinger -->
-            <dependency>
-                <groupId>com.codeborne</groupId>
-                <artifactId>selenide</artifactId>
-                <version>5.3.1</version>
-                <exclusions>
-                    <!-- Excluding to exclude commons-logging lower -->
-                    <exclusion>
-                        <groupId>io.netty</groupId>
-                        <artifactId>netty-all</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>com.google.guava</groupId>
-                        <artifactId>guava</artifactId>
-                    </exclusion>
-                </exclusions>
-            </dependency>
-            <dependency>
-                <groupId>org.seleniumhq.selenium</groupId>
-                <artifactId>selenium-java</artifactId>
-                <version>${selenium.version}</version>
-                <exclusions>
-                    <!-- replaced by org.slf4j:jcl-over-slf4j -->
-                    <exclusion>
-                        <groupId>commons-logging</groupId>
-                        <artifactId>commons-logging</artifactId>
-                    </exclusion>
-                </exclusions>
-            </dependency>
-            <dependency>
-                <groupId>org.seleniumhq.selenium</groupId>
-                <artifactId>selenium-api</artifactId>
-                <version>${selenium.version}</version>
-            </dependency>
-            <dependency>
-                <!-- Resolves dependency conflict in selenium -->
-                <groupId>com.squareup.okio</groupId>
-                <artifactId>okio</artifactId>
-                <version>1.14.1</version>
-            </dependency>
             <dependency>
                 <groupId>io.netty</groupId>
                 <artifactId>netty-all</artifactId>

repo/repo-sqale/sql/pgnew-experiments.sql

@@ -86,7 +86,7 @@ SELECT 'resource-' || LPAD(r::text, 10, '0'),
     1
 from generate_series(1, 10) as r;
 
-INSERT INTO m_user (name_norm, name_orig, fullobject, ext, version)
+INSERT INTO m_user (name_norm, name_orig, fullobject, ext, policySituations, version)
 SELECT 'user-' || LPAD(r::text, 10, '0'),
     'user-' || LPAD(r::text, 10, '0'),
     random_bytea(100, 2000),
@@ -108,10 +108,109 @@ SELECT 'user-' || LPAD(r::text, 10, '0'),
             ('{"hired": "' || current_date - width_bucket(random(), 0, 1, 1000) || '",' ||
                 '"hobbies": ' || array_to_json(random_pick(ARRAY['eating', 'books', 'music', 'dancing', 'walking', 'jokes', 'video', 'photo'], 0.5))::text || '}')::jsonb
         END,
+    CASE
+        WHEN r % 10 > 4 THEN
+            -- let's add some policy situations (IDs of M_URI entries)
+            random_pick(ARRAY(SELECT a.n FROM generate_series(1, 100) AS a(n)), r % 10 / 100::decimal)
+        -- ELSE NULL is default and redundant
+        END,
     1
-from generate_series(2000001, 3000000) as r;
+from generate_series(100001,1000000) as r;
+
+EXPLAIN (ANALYZE, VERBOSE, BUFFERS)
+select oid, policysituations from m_user
+where policysituations @> '{10}'
+-- order by name_norm desc
+;
 
-select * from m_user;
+/* 1k rows
+Seq Scan on public.m_user  (cost=0.00..14947.00 rows=3437 width=65) (actual time=0.029..39.071 rows=3567 loops=1)
+"  Output: oid, policysituations"
+  Filter: (m_user.policysituations @> '{10}'::integer[])
+  Rows Removed by Filter: 96433
+  Buffers: shared hit=11650 read=2047
+Planning:
+  Buffers: shared hit=7
+Planning Time: 0.076 ms
+Execution Time: 39.246 ms
+
+1M rows (without analyze 125ms for first 500 rows):
+Gather  (cost=1000.00..146779.83 rows=34406 width=65) (actual time=40.305..2390.356 rows=34949 loops=1)
+"  Output: oid, policysituations"
+  Workers Planned: 2
+  Workers Launched: 2
+  Buffers: shared hit=10990 read=126405 dirtied=89590 written=78799
+  ->  Parallel Seq Scan on public.m_user  (cost=0.00..142339.23 rows=14336 width=65) (actual time=42.050..2216.133 rows=11650 loops=3)
+"        Output: oid, policysituations"
+        Filter: (m_user.policysituations @> '{10}'::integer[])
+        Rows Removed by Filter: 321684
+        Buffers: shared hit=10990 read=126405 dirtied=89590 written=78799
+        Worker 0:  actual time=37.016..2129.323 rows=10913 loops=1
+          JIT:
+            Functions: 4
+"            Options: Inlining false, Optimization false, Expressions true, Deforming true"
+"            Timing: Generation 4.363 ms, Inlining 0.000 ms, Optimization 9.042 ms, Emission 24.312 ms, Total 37.717 ms"
+          Buffers: shared hit=3011 read=39940 dirtied=28314 written=25406
+        Worker 1:  actual time=50.567..2173.420 rows=11177 loops=1
+          JIT:
+            Functions: 4
+"            Options: Inlining false, Optimization false, Expressions true, Deforming true"
+"            Timing: Generation 4.199 ms, Inlining 0.000 ms, Optimization 3.489 ms, Emission 33.603 ms, Total 41.291 ms"
+          Buffers: shared hit=3637 read=40546 dirtied=28917 written=25375
+Planning Time: 0.088 ms
+JIT:
+  Functions: 12
+"  Options: Inlining false, Optimization false, Expressions true, Deforming true"
+"  Timing: Generation 20.183 ms, Inlining 0.000 ms, Optimization 14.305 ms, Emission 87.116 ms, Total 121.604 ms"
+Execution Time: 2411.721 ms
+ */
+DROP INDEX m_user_policySituation_idx;
+/* 1k rows
+Bitmap Heap Scan on public.m_user  (cost=38.63..7971.34 rows=3437 width=65) (actual time=0.895..6.448 rows=3567 loops=1)
+"  Output: oid, policysituations"
+  Recheck Cond: (m_user.policysituations @> '{10}'::integer[])
+  Heap Blocks: exact=3133
+  Buffers: shared hit=2514 read=622
+  ->  Bitmap Index Scan on m_user_policysituation_idx  (cost=0.00..37.77 rows=3437 width=0) (actual time=0.550..0.550 rows=3567 loops=1)
+        Index Cond: (m_user.policysituations @> '{10}'::integer[])
+        Buffers: shared hit=3
+Planning:
+  Buffers: shared hit=25
+Planning Time: 0.228 ms
+Execution Time: 6.603 ms
+
+1M rows (but not parallel, much lower cost (1000 to 330), fast without analyze, 96ms for first 500 rows):
+Bitmap Heap Scan on public.m_user  (cost=326.49..80473.40 rows=34902 width=64) (actual time=23.726..3046.666 rows=34949 loops=1)
+"  Output: oid, policysituations"
+  Recheck Cond: (m_user.policysituations @> '{10}'::integer[])
+  Heap Blocks: exact=31249
+  Buffers: shared hit=13 read=31249 written=4948
+  ->  Bitmap Index Scan on m_user_policysituation_idx  (cost=0.00..317.77 rows=34902 width=0) (actual time=15.035..15.035 rows=34949 loops=1)
+        Index Cond: (m_user.policysituations @> '{10}'::integer[])
+        Buffers: shared hit=13
+Planning:
+  Buffers: shared hit=73
+Planning Time: 0.434 ms
+Execution Time: 3053.032 ms
+ */
+CREATE INDEX m_user_policySituation_idx
+    ON m_user USING GIN(policysituations);
+
+/* 1M rows, clear winner, ~60ms without analyze/select of first 500 rows (also the index built under 1.5s, so it's faster even with its creation):
+Bitmap Heap Scan on public.m_user  (cost=326.48..80471.49 rows=34900 width=64) (actual time=12.360..120.267 rows=34949 loops=1)
+"  Output: oid, policysituations"
+  Recheck Cond: (m_user.policysituations @> '{10}'::integer[])
+  Heap Blocks: exact=31249
+  Buffers: shared hit=450 read=30812 written=165
+  ->  Bitmap Index Scan on m_user_policysituation_idx2  (cost=0.00..317.75 rows=34900 width=0) (actual time=6.909..6.910 rows=34949 loops=1)
+        Index Cond: (m_user.policysituations @> '{10}'::integer[])
+        Buffers: shared hit=13
+Planning:
+  Buffers: shared hit=1
+Planning Time: 0.099 ms
+Execution Time: 122.310 ms
+ */
+CREATE EXTENSION IF NOT EXISTS intarray;
 
 select * from m_object
 where oid<'812c5e7a-8a94-4bd1-944d-389e7294b831'
@@ -229,14 +328,17 @@ vacuum full analyze;
 SELECT pg_size_pretty(pg_database_size('midpoint'));
 
 -- show tables + their toast tables ordered from the largest toast table
--- ut = user table, tt = toast table
-select ut.oid, ut.relname, ut.relkind, tt.relkind, tt.relname, tt.relpages, tt.reltuples
-from pg_class ut
-    inner join pg_class tt on ut.reltoastrelid = tt.oid
-    inner join pg_namespace ns ON ut.relnamespace = ns.oid
-where ut.relkind = 'r' and tt.relkind = 't'
-    and ns.nspname = 'public'
-order by relpages desc;
+-- t = table, tt = toast table
+select t.oid as table_oid,
+    t.relname as table_name,
+    tt.relname as toast_name,
+    pg_size_pretty(pg_relation_size(t.oid)) AS table_size,
+    pg_size_pretty(pg_relation_size(tt.oid)) AS toast_size,
+    pg_size_pretty(pg_relation_size(t.oid) + coalesce(pg_relation_size(tt.oid), 0)) AS total_size
+from pg_class t
+         left join pg_class tt on t.reltoastrelid = tt.oid and tt.relkind = 't'
+where t.relkind = 'r' and t.relnamespace = (select oid from pg_namespace where nspname = 'public')
+order by total_size desc;
 
 -- find sequence name for serial column (e.g. to alter its value later)
 select pg_get_serial_sequence('m_qname', 'id');
@@ -268,9 +370,10 @@ AS $$
     FROM generate_series(2, $1 + width_bucket(random(), 0, 1, $2 - $1 + 1));
 $$;
 
--- should return 10 and 20, just to check the ranges
+/* Test: should return 10 and 20, just to check the ranges
 select min(length(i)), max(length(i))
 from (select random_bytea(10, 20) as i from generate_series(1, 200)) q;
+*/
 
 -- returns random element from array (NULL for empty arrays)
 CREATE OR REPLACE FUNCTION random_pick(vals ANYARRAY)

repo/repo-sqale/sql/pgnew-repo.sql

@@ -14,6 +14,7 @@
 -- just in case PUBLIC schema was dropped (fastest way to remove all midpoint objects)
 -- drop schema public cascade;
 CREATE SCHEMA IF NOT EXISTS public;
+CREATE EXTENSION IF NOT EXISTS intarray; -- support for indexing INTEGER[] columns
 
 -- region custom enum types
 -- Some enums are from schema, some are only defined in repo-sqale.
@@ -201,7 +202,8 @@ CREATE TABLE m_object (
     cid_seq BIGINT NOT NULL DEFAULT 1, -- sequence for container id, next free cid
     version INTEGER NOT NULL DEFAULT 1,
     -- complex DB columns, add indexes as needed per concrete table, e.g. see m_user
-    policySituations TEXT[], -- TODO experimental, compare with [] in JSONB, check performance, indexing, etc. first
+    -- TODO compare with [] in JSONB, check performance, indexing, etc. first
+    policySituations INTEGER[], -- soft-references m_uri, add index per table as/if needed
     subtypes TEXT[],
     textInfo TEXT[], -- TODO not mapped yet
     ext JSONB,
@@ -428,6 +430,7 @@ CREATE TRIGGER m_user_oid_delete_tr AFTER DELETE ON m_user
 
 CREATE INDEX m_user_name_orig_idx ON m_user (name_orig);
 ALTER TABLE m_user ADD CONSTRAINT m_user_name_norm_key UNIQUE (name_norm);
+CREATE INDEX m_user_policySituation_idx ON m_user USING GIN(policysituations gin__int_ops);
 CREATE INDEX m_user_ext_idx ON m_user USING gin (ext);
 CREATE INDEX m_user_fullName_orig_idx ON m_user (fullName_orig);
 CREATE INDEX m_user_familyName_orig_idx ON m_user (familyName_orig);
@@ -719,6 +722,7 @@ CREATE TRIGGER m_shadow_oid_delete_tr AFTER DELETE ON m_shadow
 
 CREATE INDEX m_shadow_name_orig_idx ON m_shadow (name_orig);
 ALTER TABLE m_shadow ADD CONSTRAINT m_shadow_name_norm_key UNIQUE (name_norm);
+CREATE INDEX m_shadow_policySituation_idx ON m_shadow USING GIN(policysituations gin__int_ops);
 CREATE INDEX m_shadow_ext_idx ON m_shadow USING gin (ext);
 /*
 TODO: reconsider, especially boolean things like dead (perhaps WHERE in other indexes?)
@@ -1111,6 +1115,7 @@ CREATE TABLE m_assignment_type (
     -- TODO what is this? see RAssignment.getExtension (both extId/Oid)
     extId INTEGER,
     extOid TEXT/*VARCHAR(36)*/, -- is this UUID too?
+    policySituations INTEGER[], -- soft-references m_uri, add index per table
     ext JSONB,
     -- construction
     resourceRef_targetOid UUID,
@@ -1155,6 +1160,7 @@ CREATE TABLE m_assignment (
 )
     INHERITS(m_assignment_type);
 
+CREATE INDEX m_assignment_policySituation_idx ON m_assignment USING GIN(policysituations gin__int_ops);
 CREATE INDEX m_assignment_ext_idx ON m_assignment USING gin (ext);
 -- TODO was: CREATE INDEX iAssignmentAdministrative ON m_assignment (administrativeStatus);
 -- administrativeStatus has 3 states (ENABLED/DISABLED/ARCHIVED), not sure it's worth indexing
@@ -1197,17 +1203,6 @@ ALTER TABLE m_assignment_ref_modify_approver ADD CONSTRAINT m_assignment_ref_mod
 
 -- TODO index targetOid, relation_id?
 
-/* TODO - this is also not mapped in Java, obviously
-CREATE TABLE m_assignment_policy_situation (
-  assignment_owner_oid UUID NOT NULL,
-  assignment_cid INTEGER NOT NULL,
-  policySituation TEXT/*VARCHAR(255)*/
-);
-CREATE INDEX iAssignmentPolicySituationId ON M_ASSIGNMENT_POLICY_SITUATION(ASSIGNMENT_OID, ASSIGNMENT_ID);
-ALTER TABLE m_assignment_policy_situation
-  ADD CONSTRAINT fk_assignment_policy_situation FOREIGN KEY (assignment_oid, assignment_id) REFERENCES m_assignment;
-*/
-
 CREATE TABLE m_inducement (
     owner_oid UUID NOT NULL REFERENCES m_object_oid(oid) ON DELETE CASCADE,
     containerType ContainerType GENERATED ALWAYS AS ('INDUCEMENT') STORED,
@@ -1762,8 +1757,6 @@ ALTER TABLE m_object_ext_reference
 ALTER TABLE m_object_ext_string
   ADD CONSTRAINT fk_o_ext_string_item FOREIGN KEY (item_id) REFERENCES m_ext_item;
 
-ALTER TABLE m_object_subtype
-  ADD CONSTRAINT fk_object_subtype FOREIGN KEY (object_oid) REFERENCES m_object;
 ALTER TABLE m_object_text_info
   ADD CONSTRAINT fk_object_text_info_owner FOREIGN KEY (owner_oid) REFERENCES m_object;
 ALTER TABLE m_operation_execution
@@ -1811,7 +1804,6 @@ CREATE INDEX iObjectExtLongItemId ON M_OBJECT_EXT_LONG(ITEM_ID);
 CREATE INDEX iObjectExtPolyItemId ON M_OBJECT_EXT_POLY(ITEM_ID);
 CREATE INDEX iObjectExtReferenceItemId ON M_OBJECT_EXT_REFERENCE(ITEM_ID);
 CREATE INDEX iObjectExtStringItemId ON M_OBJECT_EXT_STRING(ITEM_ID);
-CREATE INDEX iObjectSubtypeOid ON M_OBJECT_SUBTYPE(OBJECT_OID);
 CREATE INDEX iOrgOrgTypeOid ON M_ORG_ORG_TYPE(ORG_OID);
 
 -- Thanks to Patrick Lightbody for submitting this...

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleRepoContext.java

@@ -16,7 +16,7 @@
 import com.evolveum.midpoint.repo.sqale.qmodel.common.MContainerType;
 import com.evolveum.midpoint.repo.sqale.qmodel.object.MObjectType;
 import com.evolveum.midpoint.repo.sqale.qmodel.ref.MReferenceType;
-import com.evolveum.midpoint.repo.sqale.support.QuerydslJsonbType;
+import com.evolveum.midpoint.repo.sqlbase.querydsl.QuerydslJsonbType;
 import com.evolveum.midpoint.repo.sqlbase.JdbcRepositoryConfiguration;
 import com.evolveum.midpoint.repo.sqlbase.JdbcSession;
 import com.evolveum.midpoint.repo.sqlbase.SqlRepoContext;

repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/qmodel/assignment/MAssignment.java

@@ -12,6 +12,7 @@
 import com.evolveum.midpoint.repo.sqale.qmodel.common.MContainer;
 import com.evolveum.midpoint.repo.sqale.qmodel.object.MObjectType;
 import com.evolveum.midpoint.repo.sqale.qmodel.ref.MReferenceOwner;
+import com.evolveum.midpoint.repo.sqlbase.querydsl.Jsonb;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.ActivationStatusType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.TimeIntervalStatusType;
 
@@ -34,7 +35,8 @@ public class MAssignment extends MContainer implements MReferenceOwner<MAssignme
     public Integer tenantRefRelationId;
     public Integer extId;
     public String extOid;
-    public byte[] ext; // TODO JSONB?
+    public Integer[] policySituations;
+    public Jsonb ext;
     // construction
     public UUID resourceRefTargetOid;
     public MObjectType resourceRefTargetType;