Merge remote-tracking branch 'origin/master' into feature/gui-config

Evolveum · Aug 30, 2021 · 1992e39 · 1992e39
2 parents d66f0b6 + 60e1f9a
commit 1992e39
Show file tree

Hide file tree

Showing 35 changed files with 2,222 additions and 498 deletions.
diff --git a/repo/repo-sqale/pom.xml b/repo/repo-sqale/pom.xml
@@ -85,10 +85,6 @@
         </dependency>
 
         <!-- SPRING -->
-        <dependency>
-            <groupId>org.springframework</groupId>
-            <artifactId>spring-core</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-beans</artifactId>

diff --git a/repo/repo-sqale/sql/pgnew-repo-audit.sql b/repo/repo-sqale/sql/pgnew-repo-audit.sql
@@ -115,7 +115,7 @@ CREATE TABLE ma_audit_event (
     changedItemPaths TEXT[],
     resourceOids UUID[],
     properties JSONB,
-    customColumnProperties JSONB,
+    ext JSONB, -- extension container + old custom properties?
 
     PRIMARY KEY (id, timestamp)
 ) PARTITION BY RANGE (timestamp);
@@ -158,7 +158,7 @@ CREATE INDEX ma_audit_delta_recordId_timestamp_idx ON ma_audit_delta (recordId,
 
 -- TODO: any unique combination within single recordId? name+oid+type perhaps?
 CREATE TABLE ma_audit_ref (
-    id BIGSERIAL NOT NULL,
+    id BIGSERIAL NOT NULL, -- unique technical PK
     recordId BIGINT NOT NULL, -- references ma_audit_event.id
     timestamp TIMESTAMPTZ NOT NULL, -- references ma_audit_event.timestamp
     name TEXT, -- multiple refs can have the same name, conceptually it's a Map(name -> refs[])
@@ -167,7 +167,7 @@ CREATE TABLE ma_audit_ref (
     targetNameOrig TEXT,
     targetNameNorm TEXT,
 
-    PRIMARY KEY (id, timestamp)
+    PRIMARY KEY (id, timestamp) -- real PK must contain partition key (timestamp)
 ) PARTITION BY RANGE (timestamp);
 
 /* Similar FK is created PER PARTITION only

diff --git a/repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleRepoContext.java b/repo/repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleRepoContext.java
@@ -14,6 +14,7 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import com.evolveum.midpoint.prism.delta.ChangeType;
 import com.evolveum.midpoint.repo.sqale.jsonb.QuerydslJsonbType;
 import com.evolveum.midpoint.repo.sqale.qmodel.common.MContainerType;
 import com.evolveum.midpoint.repo.sqale.qmodel.common.QUri;
@@ -27,6 +28,8 @@
 import com.evolveum.midpoint.repo.sqlbase.mapping.QueryModelMappingRegistry;
 import com.evolveum.midpoint.schema.SchemaService;
 import com.evolveum.midpoint.util.QNameUtil;
+import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventStageType;
+import com.evolveum.midpoint.xml.ns._public.common.audit_3.AuditEventTypeType;
 import com.evolveum.midpoint.xml.ns._public.common.common_3.*;
 
 /**
@@ -47,7 +50,10 @@ public SqaleRepoContext(
         // each enum type must be registered if we want to map it as objects (to PG enum types)
         querydslConfig.register(new EnumAsObjectType<>(AccessCertificationCampaignStateType.class));
         querydslConfig.register(new EnumAsObjectType<>(ActivationStatusType.class));
+        querydslConfig.register(new EnumAsObjectType<>(AuditEventStageType.class));
+        querydslConfig.register(new EnumAsObjectType<>(AuditEventTypeType.class));
         querydslConfig.register(new EnumAsObjectType<>(AvailabilityStatusType.class));
+        querydslConfig.register(new EnumAsObjectType<>(ChangeType.class));
         querydslConfig.register(new EnumAsObjectType<>(MContainerType.class));
         querydslConfig.register(new EnumAsObjectType<>(MExtItemHolderType.class));
         querydslConfig.register(new EnumAsObjectType<>(MExtItemCardinality.class));

diff --git a/.../repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleRepositoryBeanConfig.java b/.../repo-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleRepositoryBeanConfig.java
@@ -13,12 +13,13 @@
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
-import org.springframework.core.env.Environment;
 
+import com.evolveum.midpoint.audit.api.AuditServiceFactory;
 import com.evolveum.midpoint.common.configuration.api.MidpointConfiguration;
 import com.evolveum.midpoint.repo.api.RepositoryServiceFactoryException;
 import com.evolveum.midpoint.repo.api.SqlPerformanceMonitorsCollection;
 import com.evolveum.midpoint.repo.api.SystemConfigurationChangeDispatcher;
+import com.evolveum.midpoint.repo.sqale.audit.SqaleAuditServiceFactory;
 import com.evolveum.midpoint.repo.sqale.qmodel.accesscert.QAccessCertificationCampaignMapping;
 import com.evolveum.midpoint.repo.sqale.qmodel.accesscert.QAccessCertificationCaseMapping;
 import com.evolveum.midpoint.repo.sqale.qmodel.accesscert.QAccessCertificationDefinitionMapping;
@@ -89,10 +90,8 @@ public class SqaleRepositoryBeanConfig {
 
     @Bean
     public SqaleRepositoryConfiguration sqaleRepositoryConfiguration(
-            Environment env,
             MidpointConfiguration midpointConfiguration) {
-
-        return new SqaleRepositoryConfiguration(env,
+        return new SqaleRepositoryConfiguration(
                 midpointConfiguration.getConfiguration(
                         MidpointConfiguration.REPOSITORY_CONFIGURATION));
     }
@@ -205,6 +204,19 @@ public SqaleRepositoryService repositoryService(
                 sqlPerformanceMonitorsCollection);
     }
 
+    @Bean
+    public AuditServiceFactory sqlAuditServiceFactory(
+            SqaleRepositoryConfiguration sqaleRepositoryConfiguration,
+            SchemaService schemaService,
+            DataSource dataSource,
+            SqlPerformanceMonitorsCollection sqlPerformanceMonitorsCollection) {
+        return new SqaleAuditServiceFactory(
+                sqaleRepositoryConfiguration,
+                schemaService,
+                dataSource,
+                sqlPerformanceMonitorsCollection);
+    }
+
     // TODO @Bean for AuditServiceFactory later
 
     // TODO rethink? using Spring events

diff --git a/...po-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleRepositoryConfiguration.java b/...po-sqale/src/main/java/com/evolveum/midpoint/repo/sqale/SqaleRepositoryConfiguration.java
@@ -14,9 +14,7 @@
 import org.apache.commons.configuration2.Configuration;
 import org.apache.commons.lang3.StringUtils;
 import org.jetbrains.annotations.NotNull;
-import org.springframework.core.env.Environment;
 
-import com.evolveum.midpoint.common.configuration.api.MidpointConfiguration;
 import com.evolveum.midpoint.prism.PrismContext;
 import com.evolveum.midpoint.repo.api.RepositoryServiceFactoryException;
 import com.evolveum.midpoint.repo.sqlbase.JdbcRepositoryConfiguration;
@@ -39,7 +37,6 @@ public class SqaleRepositoryConfiguration implements JdbcRepositoryConfiguration
 
     private static final int DEFAULT_ITERATIVE_SEARCH_PAGE_SIZE = 100;
 
-    @NotNull private final Environment env; // for better Spring properties/override integration
     @NotNull private final Configuration configuration;
 
     /** Database kind - either explicitly configured or derived from other options. */
@@ -64,27 +61,24 @@ public class SqaleRepositoryConfiguration implements JdbcRepositoryConfiguration
     private int performanceStatisticsLevel;
 
     private int iterativeSearchByPagingBatchSize;
+    private boolean createMissingCustomColumns;
 
-    public SqaleRepositoryConfiguration(
-            @NotNull Environment env,
-            @NotNull Configuration configuration) {
-        this.env = env;
+    // Provided with configuration node "midpoint.repository".
+    public SqaleRepositoryConfiguration(@NotNull Configuration configuration) {
         this.configuration = configuration;
     }
 
     @PostConstruct
     public void init() throws RepositoryServiceFactoryException {
-        // TODO the rest from SqlRepoConf#validate except for Hibernate of course
-        dataSource = getString(PROPERTY_DATASOURCE);
+        dataSource = configuration.getString(PROPERTY_DATASOURCE);
 
-        jdbcUrl = configuration.getString(PROPERTY_JDBC_URL, defaultJdbcUrl());
-        jdbcUsername = getString(PROPERTY_JDBC_USERNAME);
+        jdbcUrl = configuration.getString(PROPERTY_JDBC_URL);
+        jdbcUsername = configuration.getString(PROPERTY_JDBC_USERNAME);
 
-        // TODO perhaps add warnings that other values are ignored anyway?
         databaseType = DEFAULT_DATABASE;
         driverClassName = DEFAULT_DRIVER;
 
-        String jdbcPasswordFile = getString(PROPERTY_JDBC_PASSWORD_FILE);
+        String jdbcPasswordFile = configuration.getString(PROPERTY_JDBC_PASSWORD_FILE);
         if (jdbcPasswordFile != null) {
             try {
                 jdbcPassword = Files.readString(Path.of(jdbcPasswordFile));
@@ -93,49 +87,29 @@ public void init() throws RepositoryServiceFactoryException {
                         + jdbcPasswordFile + "': " + e.getMessage(), e);
             }
         } else {
-            jdbcPassword = System.getProperty(PROPERTY_JDBC_PASSWORD,
-                    getString(PROPERTY_JDBC_PASSWORD));
+            jdbcPassword = configuration.getString(PROPERTY_JDBC_PASSWORD);
         }
 
         minPoolSize = configuration.getInt(PROPERTY_MIN_POOL_SIZE, DEFAULT_MIN_POOL_SIZE);
         maxPoolSize = configuration.getInt(PROPERTY_MAX_POOL_SIZE, DEFAULT_MAX_POOL_SIZE);
 
         useZip = configuration.getBoolean(PROPERTY_USE_ZIP, false);
         useZipAudit = configuration.getBoolean(PROPERTY_USE_ZIP_AUDIT, true);
-        fullObjectFormat = getString(PROPERTY_FULL_OBJECT_FORMAT, PrismContext.LANG_XML)
+        fullObjectFormat = configuration.getString(PROPERTY_FULL_OBJECT_FORMAT, PrismContext.LANG_XML)
                 .toLowerCase(); // all language string constants are lower-cases
 
-        performanceStatisticsFile = getString(PROPERTY_PERFORMANCE_STATISTICS_FILE);
+        performanceStatisticsFile = configuration.getString(PROPERTY_PERFORMANCE_STATISTICS_FILE);
         performanceStatisticsLevel = configuration.getInt(PROPERTY_PERFORMANCE_STATISTICS_LEVEL,
                 SqlPerformanceMonitorImpl.LEVEL_LOCAL_STATISTICS);
 
         iterativeSearchByPagingBatchSize = configuration.getInt(
                 PROPERTY_ITERATIVE_SEARCH_BY_PAGING_BATCH_SIZE, DEFAULT_ITERATIVE_SEARCH_PAGE_SIZE);
+        createMissingCustomColumns =
+                configuration.getBoolean(PROPERTY_CREATE_MISSING_CUSTOM_COLUMNS, false);
 
         validateConfiguration();
     }
 
-    private String getString(String property) {
-        return getString(property, null);
-    }
-
-    /**
-     * Returns property value as string in this precedence:
-     *
-     * * using just property name (local, short name) against Spring environment, see
-     * https://docs.spring.io/spring-boot/docs/current/reference/html/spring-boot-features.html#boot-features-external-config[this documentation]
-     * for particular order of value resolution;
-     * * using property name with repository configuration prefix (`midpoint.repository.*`) which is
-     * future proof if we want to migrate to native Spring properties configuration;
-     * * finally, {@link Configuration} object representing content of `midpoint/repository`
-     * elements is used.
-     */
-    private String getString(String property, String defaultValue) {
-        return env.getProperty(property,
-                env.getProperty(MidpointConfiguration.REPOSITORY_CONFIGURATION + '.' + property,
-                        configuration.getString(property, defaultValue)));
-    }
-
     private void validateConfiguration() throws RepositoryServiceFactoryException {
         if (dataSource == null) {
             notEmpty(jdbcUrl, "JDBC Url is empty or not defined.");
@@ -145,10 +119,6 @@ private void validateConfiguration() throws RepositoryServiceFactoryException {
         }
     }
 
-    protected String defaultJdbcUrl() {
-        return null;
-    }
-
     public @NotNull SupportedDatabase getDatabaseType() {
         return databaseType;
     }
@@ -250,13 +220,6 @@ public long getInitializationFailTimeout() {
         return 0;
     }
 
-    @Override
-    public boolean isFatalException(Throwable ex) {
-        // TODO implement
-        // by default, any exception is fatal, unless specified otherwise (not yet implemented)
-        return true;
-    }
-
     @Override
     public String getPerformanceStatisticsFile() {
         return performanceStatisticsFile;
@@ -276,4 +239,9 @@ public int getIterativeSearchByPagingBatchSize() {
     public void setIterativeSearchByPagingBatchSize(int iterativeSearchByPagingBatchSize) {
         this.iterativeSearchByPagingBatchSize = iterativeSearchByPagingBatchSize;
     }
+
+    @Override
+    public boolean isCreateMissingCustomColumns() {
+        return createMissingCustomColumns;
+    }
 }