diff --git a/api/src/org/labkey/api/data/ContainerManager.java b/api/src/org/labkey/api/data/ContainerManager.java index 8c9c7b793f1..55c4daaec01 100644 --- a/api/src/org/labkey/api/data/ContainerManager.java +++ b/api/src/org/labkey/api/data/ContainerManager.java @@ -801,7 +801,7 @@ public static void updateDescription(Container container, String description, Us { ColumnValidators.validate(CORE.getTableInfoContainers().getColumn("Title"), null, 1, description); - //For some reason there is no primary key defined on core.containers + //For some reason, there is no primary key defined on core.containers //so we can't use Table.update here SQLFragment sql = new SQLFragment("UPDATE "); sql.append(CORE.getTableInfoContainers()); @@ -817,7 +817,7 @@ public static void updateDescription(Container container, String description, Us public static void updateSearchable(Container container, boolean searchable, User user) { - //For some reason there is no primary key defined on core.containers + //For some reason, there is no primary key defined on core.containers //so we can't use Table.update here SQLFragment sql = new SQLFragment("UPDATE "); sql.append(CORE.getTableInfoContainers()); @@ -2715,7 +2715,7 @@ public static int updateContainer(TableInfo dataTable, String idField, Collectio } /** - * If a container at the given path does not exist create one and set permissions. If the container does exist, + * If a container at the given path does not exist, create one and set permissions. If the container does exist, * permissions are only set if there is no explicit ACL for the container. This prevents us from resetting * permissions if all users are dropped. Implicitly done as an admin-level service user. */ diff --git a/api/src/org/labkey/api/data/DbSchema.java b/api/src/org/labkey/api/data/DbSchema.java index 94910a0deca..75a786a224b 100644 --- a/api/src/org/labkey/api/data/DbSchema.java +++ b/api/src/org/labkey/api/data/DbSchema.java @@ -148,7 +148,7 @@ public static Pair getDbScopeAndSchemaName(String fullyQualifie } } - Module getModule() + public Module getModule() { return _module; } diff --git a/api/src/org/labkey/api/data/ForeignKey.java b/api/src/org/labkey/api/data/ForeignKey.java index e751e96da80..baeb862751e 100644 --- a/api/src/org/labkey/api/data/ForeignKey.java +++ b/api/src/org/labkey/api/data/ForeignKey.java @@ -94,7 +94,7 @@ default String getLookupSchemaName() return Objects.toString(getLookupSchemaKey(),null); } - /* Schema path relative to the DefaultSchema (e.g. container) */ + /* Schema path relative to the DefaultSchema (e.g., container) */ SchemaKey getLookupSchemaKey(); /** diff --git a/api/src/org/labkey/api/data/SqlScriptExecutor.java b/api/src/org/labkey/api/data/SqlScriptExecutor.java index c63e46594f6..9cca1dbd625 100644 --- a/api/src/org/labkey/api/data/SqlScriptExecutor.java +++ b/api/src/org/labkey/api/data/SqlScriptExecutor.java @@ -15,6 +15,8 @@ */ package org.labkey.api.data; +import org.apache.commons.lang3.mutable.MutableBoolean; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -31,6 +33,7 @@ import java.util.Collections; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Executes a single module upgrade SQL script, including finding calls into Java code that are embedded using @@ -51,13 +54,14 @@ public class SqlScriptExecutor /** * Splits a SQL string into blocks and executes each block, one at a time. Blocks are determined in a dialect-specific * way, using splitPattern and procPattern. - * @param scriptName Name of the SQL script for logging purposes - * @param sql The SQL string to split and execute - * @param splitPattern Dialect-specific regex pattern for splitting normal SQL statements into blocks. Null means no need to split. - * @param procPattern Dialect-specific regex pattern for finding executeJavaCode procedure calls in the SQL. See SqlDialect.getSqlScriptProcPattern() for details. - * @param schema Current schema. Null is allowed for testing purposes. + * + * @param scriptName Name of the SQL script for logging purposes + * @param sql The SQL string to split and execute + * @param splitPattern Dialect-specific regex pattern for splitting normal SQL statements into blocks. Null means no need to split. + * @param procPattern Dialect-specific regex pattern for finding executeJavaCode procedure calls in the SQL. See SqlDialect.getSqlScriptProcPattern() for details. + * @param schema Current schema. Null is allowed for testing purposes. * @param moduleContext Current ModuleContext - * @param conn Connection to use, if non-null + * @param conn Connection to use, if non-null */ public SqlScriptExecutor(String scriptName, String sql, @Nullable Pattern splitPattern, @NotNull Pattern procPattern, @Nullable DbSchema schema, ModuleContext moduleContext, @Nullable Connection conn) { @@ -79,10 +83,51 @@ public void execute() } } + private static final String START_ANNOTATION = "@SkipOnEmptySchemasBegin"; + private static final String END_ANNOTATION = "@SkipOnEmptySchemasEnd"; + private Collection getBlocks() { + String sql; + + if (ModuleLoader.getInstance().shouldInsertData()) + { + sql = _sql; + } + else + { + // Strip all statements (typically inserts) between the empty-schema START and END annotations, inclusive + MutableBoolean skipping = new MutableBoolean(false); + MutableInt lineCount = new MutableInt(0); + sql = _sql.lines() + .filter(line -> { + lineCount.increment(); + if (line.contains(START_ANNOTATION)) + { + if (skipping.booleanValue()) + throw new IllegalStateException("Unexpected " + START_ANNOTATION + " at line " + lineCount.intValue()); + + skipping.setValue(true); + } + + boolean ret = !skipping.booleanValue(); + + if (line.contains(END_ANNOTATION)) + { + if (!skipping.booleanValue()) + throw new IllegalStateException("Unexpected " + END_ANNOTATION + " at line " + lineCount.intValue()); + + skipping.setValue(false); + } + + return ret; + } + ) + .collect(Collectors.joining("\n")); + } + // Strip all comments from the script -- PostgreSQL JDBC driver goes berserk if it sees ; or ? inside a comment - StringBuilder stripped = new SqlScanner(_sql).stripComments(); + StringBuilder stripped = new SqlScanner(sql).stripComments(); Collection sqlBlocks; diff --git a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java index f341051b1e8..245fd01920b 100644 --- a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java @@ -931,13 +931,6 @@ public DatabaseIdentifier makeDatabaseIdentifier(String alias) private static final Pattern PROC_PATTERN = Pattern.compile("^\\s*SELECT\\s+core\\.(executeJava(?:Upgrade|Initialization)Code\\s*\\(\\s*'(.+)'\\s*\\))\\s*;\\s*$", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE); - @Override - // No need to split up PostgreSQL scripts; execute all statements in a single block (unless we have a special stored proc call). - protected Pattern getSQLScriptSplitPattern() - { - return null; - } - @NotNull @Override protected Pattern getSQLScriptProcPattern() diff --git a/api/src/org/labkey/api/exp/api/ProvisionedDbSchema.java b/api/src/org/labkey/api/exp/api/ProvisionedDbSchema.java index ecd47f2b4f3..090180242d7 100644 --- a/api/src/org/labkey/api/exp/api/ProvisionedDbSchema.java +++ b/api/src/org/labkey/api/exp/api/ProvisionedDbSchema.java @@ -31,7 +31,6 @@ /** * A schema in the underlying database that is populated by tables created (provisioned) dynamically * based on administrator or other input into what columns/fields should be tracked. - * Created by klum on 2/23/14. */ public class ProvisionedDbSchema extends DbSchema { diff --git a/api/src/org/labkey/api/exp/api/StorageProvisioner.java b/api/src/org/labkey/api/exp/api/StorageProvisioner.java index 8412f240440..52a05ca79a0 100644 --- a/api/src/org/labkey/api/exp/api/StorageProvisioner.java +++ b/api/src/org/labkey/api/exp/api/StorageProvisioner.java @@ -77,6 +77,12 @@ static TableInfo createTableInfo(@NotNull Domain domain) */ String ensureStorageTable(Domain domain, DomainKind kind, DbScope scope); + /** + * Used by DatabaseMigration only. Creates the storage table associated with this domain, using the storage table + * name provided by the domain. + */ + void createStorageTable(Domain domain, DomainKind kind, DbScope scope); + void dropNotRequiredIndices(Domain domain); void addMissingRequiredIndices(Domain domain); void addTableIndices(Domain domain, Set indices, TableChange.IndexSizeMode sizeMode); diff --git a/api/src/org/labkey/api/module/DatabaseMigration.java b/api/src/org/labkey/api/module/DatabaseMigration.java new file mode 100644 index 00000000000..f3132d5bc92 --- /dev/null +++ b/api/src/org/labkey/api/module/DatabaseMigration.java @@ -0,0 +1,376 @@ +package org.labkey.api.module; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.CopyOnWriteCaseInsensitiveHashMap; +import org.labkey.api.collections.LabKeyCollectors; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DatabaseTableType; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SchemaTableInfo; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.query.TableSorter; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.BiFunction; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +// Handles SQL Server to PostgreSQL data migration +public class DatabaseMigration +{ + private static final Logger LOG = LogHelper.getLogger(DatabaseMigration.class, "Progress of SQL Server to PostgreSQL database migration"); + + // If associated properties are set: clear schemas, verify empty schemas, and migrate data from the external SQL + // Server data source into the just-created empty PostgreSQL schemas. + public static void migrate(boolean shouldInsertData, @Nullable String migrationDataSource) + { + if (!shouldInsertData) + { + clearSchemas(); + verifyEmptySchemas(); + + if (migrationDataSource != null) + migrateDatabase(migrationDataSource); + + System.exit(0); + } + } + + // Clear containers needed for bootstrap + private static void clearSchemas() + { + TableInfo containers = CoreSchema.getInstance().getTableInfoContainers(); + Table.delete(containers); // Now that we've bootstrapped, delete root and shared containers + DbScope targetScope = DbScope.getLabKeyScope(); + new SqlExecutor(targetScope).execute("ALTER SEQUENCE core.containers_rowid_seq RESTART"); // Reset Containers sequence + } + + // Verify that no data rows were inserted and no sequences were incremented + private static void verifyEmptySchemas() + { + DbScope scope = DbScope.getLabKeyScope(); + + Map> schemaMap = scope.getSchemaNames().stream() + .map(name -> scope.getSchema(name, DbSchemaType.Unknown)) + .collect(Collectors.partitioningBy(schema -> schema.getModule() != null && schema.getModule().getSupportedDatabasesSet().contains(SupportedDatabase.mssql))); + + List targetSchemas = schemaMap.get(true); + List tableWarnings = targetSchemas.stream() + .flatMap(schema -> schema.getTableNames().stream() + .map(schema::getTable) + .filter(table -> table.getTableType() != DatabaseTableType.NOT_IN_DB) + ) + .map(table -> { + long rowCount = new TableSelector(table).getRowCount(); + if (rowCount > 0) + return table.getSelectName() + " has " + StringUtilsLabKey.pluralize(rowCount, "row"); + else + return null; + }) + .filter(Objects::nonNull) + .toList(); + + if (!tableWarnings.isEmpty()) + { + LOG.warn("{} rows", StringUtilsLabKey.pluralize(tableWarnings.size(), "table has", "tables have")); + tableWarnings.forEach(LOG::warn); + } + + List schemasToIgnore = schemaMap.get(false).stream() + .map(DbSchema::getName) + .toList(); + String qs = StringUtils.join(Collections.nCopies(schemasToIgnore.size(), "?"), ", "); + List sequenceWarnings = new SqlSelector(scope, new SQLFragment( + "SELECT schemaname || '.' || sequencename FROM pg_sequences WHERE last_value IS NOT NULL AND schemaname NOT IN (" + qs + ")", + schemasToIgnore + )) + .stream(String.class) + .toList(); + + if (!sequenceWarnings.isEmpty()) + { + LOG.warn("{} a value:", StringUtilsLabKey.pluralize(sequenceWarnings.size(), "sequence has", "sequences have")); + sequenceWarnings.forEach(LOG::warn); + } + } + + private record Sequence(String schemaName, String tableName, String columnName, int lastValue) {} + + private static void migrateDatabase(String migrationDataSource) + { + LOG.info("Starting database migration"); + + DbScope targetScope = DbScope.getLabKeyScope(); + DbScope sourceScope = DbScope.getDbScope(migrationDataSource); + if (null == sourceScope) + throw new ConfigurationException("Migration data source not found: " + migrationDataSource); + if (!sourceScope.getSqlDialect().isSqlServer()) + throw new ConfigurationException("Migration data source is not SQL Server: " + migrationDataSource); + + // Verify that all sequences in the target schema have an increment of 1, since that's an assumption below + Collection sequencesNonOneIncrement = new SqlSelector(targetScope, new SQLFragment("SELECT schemaname || '.' || sequencename || ': ' || increment_by FROM pg_sequences WHERE increment_by != 1")).getCollection(String.class); + if (!sequencesNonOneIncrement.isEmpty()) + { + throw new IllegalStateException(StringUtilsLabKey.pluralize(sequencesNonOneIncrement.size(), "sequence has", "sequences have") + " an increment other than 1: " + sequencesNonOneIncrement); + } + + // Select the SQL Server sequences with non-null last value. We'll use the results to set PostgreSQL sequences after copying data. + String sequenceQuery = """ + SELECT + OBJECT_SCHEMA_NAME(tables.object_id, db_id()) AS SchemaName, + tables.name AS TableName, + identity_columns.name AS ColumnName, + identity_columns.seed_value, + identity_columns.increment_value, + identity_columns.last_value + FROM + sys.tables tables + JOIN + sys.identity_columns identity_columns ON tables.object_id = identity_columns.object_id + WHERE last_value IS NOT NULL"""; + Map> sequenceMap = new HashMap<>(); + new SqlSelector(sourceScope, sequenceQuery).forEach(rs -> { + Sequence sequence = new Sequence(rs.getString("SchemaName"), rs.getString("TableName"), rs.getString("ColumnName"), rs.getInt("last_value")); + Map schemaMap = sequenceMap.computeIfAbsent(sequence.schemaName(), s -> new HashMap<>()); + schemaMap.put(sequence.tableName(), sequence); + }); + + // Get the target module schemas in module order, which helps with foreign key relationships + List targetModuleSchemas = ModuleLoader.getInstance().getModules().stream() + .flatMap(module -> module.getSchemaNames().stream().filter(name -> !module.getProvisionedSchemaNames().contains(name))) + .map(name -> targetScope.getSchema(name, DbSchemaType.Module)) + .toList(); + + // Migrate all data in the module schemas + migrateSchemas(migrationDataSource, sourceScope, targetScope, targetModuleSchemas, (schema, handler) -> handler.getTablesToCopy(schema), sequenceMap); + + // Create all provisioned tables listed in exp.DomainDescriptor + PropertyService svc = PropertyService.get(); + StorageProvisioner provisioner = StorageProvisioner.get(); + new SqlSelector(targetScope, "SELECT Container, DomainURI, Name FROM exp.DomainDescriptor WHERE StorageSchemaName IS NOT NULL").forEach(rs -> { + Container c = ContainerManager.getForId(rs.getString("Container")); + if (c != null) + { + String domainURI = rs.getString("DomainURI"); + String name = rs.getString("Name"); + Domain d = svc.ensureDomain(c, null, domainURI, name); + provisioner.createStorageTable(d, d.getDomainKind(), targetScope); + } + }); + + // Get the target provisioned schemas + List targetProvisionedSchemas = ModuleLoader.getInstance().getModules().stream() + .flatMap(module -> module.getSchemaNames().stream().filter(name -> module.getProvisionedSchemaNames().contains(name))) + .map(name -> targetScope.getSchema(name, DbSchemaType.Bare)) + .toList(); + + // Migrate all data in the provisioned schemas + migrateSchemas(migrationDataSource, sourceScope, targetScope, targetProvisionedSchemas, (schema, handler) -> getTables(schema), sequenceMap); + + LOG.info("Database migration is complete"); + } + + private static void migrateSchemas(String migrationDataSource, DbScope sourceScope, DbScope targetScope, List targetSchemas, BiFunction> tableProducer, Map> sequenceMap) + { + for (DbSchema targetSchema : targetSchemas) + { + DbSchema sourceSchema = sourceScope.getSchema(targetSchema.getName(), DbSchemaType.Bare); + if (!sourceSchema.existsInDatabase()) + { + LOG.warn("{} has no schema named '{}'", migrationDataSource, targetSchema.getName()); + } + else + { + MigrationHandler handler = getHandler(targetSchema); + handler.beforeSchema(targetSchema); + + Set sourceTableNames = getTables(sourceSchema).stream().map(TableInfo::getName).collect(LabKeyCollectors.toCaseInsensitiveHashSet()); + Set targetTableNames = getTables(targetSchema).stream().map(TableInfo::getName).collect(LabKeyCollectors.toCaseInsensitiveHashSet()); + Set sourceTableNamesCopy = new CaseInsensitiveHashSet(sourceTableNames); + sourceTableNames.removeAll(targetTableNames); + targetTableNames.removeAll(sourceTableNamesCopy); + if (!sourceTableNames.isEmpty() || !targetTableNames.isEmpty()) + LOG.warn("Table differences in {} schema: {} and {}", sourceSchema.getName(), sourceTableNames, targetTableNames); + + Map schemaSequenceMap = sequenceMap.getOrDefault(sourceSchema.getName(), Map.of()); + + for (TableInfo targetTable : tableProducer.apply(targetSchema, handler)) + { + String targetTableName = targetTable.getName(); + SchemaTableInfo sourceTable = sourceSchema.getTable(targetTableName); + + if (sourceTable == null) + { + LOG.warn("Source schema has no table named '{}'", targetTableName); + } + else + { + LOG.info("Migrating '{}'", targetSchema.getName() + "." + targetTableName); + // Inspect target table to determine column names to select from source table + Set selectColumnNames = targetTable.getColumns().stream() + .filter(column -> column.getWrappedColumnName() == null) // Ignore wrapped columns + .map(ColumnInfo::getName) + .filter(name -> !name.equals("_ts")) + .collect(Collectors.toSet()); + + TableSelector sourceSelector = new TableSelector(sourceTable, selectColumnNames).setJdbcCaching(false); + + try (Stream> mapStream = sourceSelector.uncachedMapStream(); Connection conn = targetScope.getConnection()) + { + Collection sourceColumns = sourceSelector.getSelectedColumns(); + // Map the selected source columns to the target columns so we get the right order and casing for INSERT, etc. + Collection targetColumns = sourceColumns.stream() + .map(sourceCol -> targetTable.getColumn(sourceCol.getName())) + .toList(); + String q = StringUtils.join(Collections.nCopies(sourceColumns.size(), "?"), ", "); + SQLFragment sql = new SQLFragment("INSERT INTO ") + .append(targetTable) + .append("("); + + String sep = ""; + for (ColumnInfo targetColumn : targetColumns) + { + sql.append(sep) + .appendIdentifier(targetColumn.getSelectIdentifier()); + sep = ", "; + } + + sql.append(") VALUES (") + .append(q) + .append(")"); + + PreparedStatement statement = conn.prepareStatement(sql.getRawSQL()); + + mapStream.forEach(map -> { + try + { + int i = 1; + + for (ColumnInfo col : sourceColumns) + statement.setObject(i++, col.getValue(map)); + + statement.execute(); + } + catch (SQLException e) + { + throw new RuntimeException("Exception while migrating data from " + sourceTable, e); + } + }); + + Sequence sequence = schemaSequenceMap.get(targetTable.getName()); + if (sequence != null) + { + ColumnInfo targetColumn = targetTable.getColumn(sequence.columnName()); + String sequenceName = new SqlSelector(targetSchema, "SELECT pg_get_serial_sequence(?, ?)", targetSchema.getName() + "." + targetTable.getName(), targetColumn.getSelectIdentifier().getId()) + .getObject(String.class); + new SqlExecutor(targetScope).execute("SELECT setval(?, ?)", sequenceName, sequence.lastValue()); + } + } + catch (Exception e) + { + LOG.error("Exception: ", e); + } + } + } + + handler.afterSchema(targetSchema); + } + } + } + + private static List getTables(DbSchema schema) + { + return new ArrayList<>(schema.getTableNames().stream() + .map(schema::getTable) + .filter(table -> table.getTableType() == DatabaseTableType.TABLE) + .toList()); + } + + public interface MigrationHandler + { + void beforeSchema(DbSchema targetSchema); + + List getTablesToCopy(DbSchema targetSchema); + + void afterSchema(DbSchema targetSchema); + } + + public static class DefaultMigrationHandler implements MigrationHandler + { + @Override + public void beforeSchema(DbSchema targetSchema) + { + } + + @Override + public List getTablesToCopy(DbSchema targetSchema) + { + Set sortedTables = new LinkedHashSet<>(TableSorter.sort(targetSchema, true)); + + Set allTables = targetSchema.getTableNames().stream() + .map(targetSchema::getTable) + .collect(Collectors.toCollection(HashSet::new)); + allTables.removeAll(sortedTables); + + if (!allTables.isEmpty()) + { + LOG.info("These tables were removed by TableSorter: {}", allTables); + } + + return sortedTables.stream() + // Skip all views and virtual tables (e.g., test.Containers2, which is a table on SS but a view on PG) + .filter(table -> table.getTableType() == DatabaseTableType.TABLE) + .collect(Collectors.toCollection(ArrayList::new)); // Ensure mutable + } + + @Override + public void afterSchema(DbSchema targetSchema) + { + } + } + + private static final Map MIGRATION_HANDLERS = new CopyOnWriteCaseInsensitiveHashMap<>(); + private static final MigrationHandler DEFAULT_MIGRATION_HANDLER = new DefaultMigrationHandler(); + + public static void registerHandler(DbSchema schema, MigrationHandler handler) + { + MIGRATION_HANDLERS.put(schema.getName(), handler); + } + + private static MigrationHandler getHandler(DbSchema schema) + { + MigrationHandler handler = MIGRATION_HANDLERS.get(schema.getName()); + return handler != null ? handler : DEFAULT_MIGRATION_HANDLER; + } +} diff --git a/api/src/org/labkey/api/module/ModuleLoader.java b/api/src/org/labkey/api/module/ModuleLoader.java index 84cd327c601..35d7f87e908 100644 --- a/api/src/org/labkey/api/module/ModuleLoader.java +++ b/api/src/org/labkey/api/module/ModuleLoader.java @@ -185,6 +185,12 @@ public class ModuleLoader implements MemTrackerListener, ShutdownListener private final SqlScriptRunner _upgradeScriptRunner = new SqlScriptRunner(); + // This argument is used to specify a Microsoft SQL Server database that is the source of migration to PostgreSQL + private final String _migrationDataSource = System.getProperty("migrationDataSource"); + // This argument is used to bootstrap all the database schemas without populating them with any data rows. It can + // be used by itself to test the empty schema handling without attempting a full migration. + private final boolean _emptySchemas = Boolean.valueOf(System.getProperty("emptySchemas")) || _migrationDataSource != null; + // NOTE: the following startup fields are synchronized under STARTUP_LOCK private StartupState _startupState = StartupState.StartupIncomplete; private String _startingUpMessage = null; @@ -630,7 +636,7 @@ private void doInit(Execution execution) throws ServletException { _log.info("Check complete: all LabKey-managed modules are recent enough to upgrade"); } - } + } boolean coreRequiredUpgrade = upgradeCoreModule(lockFile); @@ -758,6 +764,10 @@ public void addStaticWarnings(@NotNull Warnings warnings, boolean showAllWarning if (!modulesRequiringUpgrade.isEmpty() || !additionalSchemasRequiringUpgrade.isEmpty()) setUpgradeState(UpgradeState.UpgradeRequired); + // Don't accept any requests if we're bootstrapping empty schemas or migrating from SQL Server + if (!shouldInsertData()) + execution = Execution.Synchronous; + startNonCoreUpgradeAndStartup(execution, lockFile); _log.info("LabKey Server startup is complete; {}", execution.getLogMessage()); @@ -1874,7 +1884,7 @@ private void startNonCoreUpgradeAndStartup(Execution execution, File lockFile) } } - // Final step in upgrade process: set the upgrade state to complete, perform post-upgrade tasks, and start up the modules. + // Final step in the upgrade process: set the upgrade state to complete, perform post-upgrade tasks, and start up the modules. private void afterUpgrade(File lockFile) { setUpgradeState(UpgradeState.UpgradeComplete); @@ -1887,6 +1897,7 @@ private void afterUpgrade(File lockFile) lockFile.delete(); verifyRequiredModules(); + DatabaseMigration.migrate(shouldInsertData(), getMigrationDataSource()); } // If the "requiredModules" parameter is present in application.properties then fail startup if any specified module is missing. @@ -1978,10 +1989,21 @@ public boolean isNewInstall() return _newInstall; } + // Are we bootstrapping a PostgreSQL database with empty schemas? + public boolean shouldInsertData() + { + return !(_emptySchemas && isNewInstall() && DbScope.getLabKeyScope().getSqlDialect().isPostgreSQL()); + } + + public @Nullable String getMigrationDataSource() + { + return _migrationDataSource != null && isNewInstall() && DbScope.getLabKeyScope().getSqlDialect().isPostgreSQL() ? _migrationDataSource : null; + } + public void destroy() { // In the case of a startup failure, _modules may be null. We want to allow a context reload to succeed in this case, - // since the reload may contain the code change to fix the problem + // since the reload may contain a code change to fix the problem. var modules = getModules(); if (modules != null) { diff --git a/api/src/org/labkey/api/query/TableSorter.java b/api/src/org/labkey/api/query/TableSorter.java index 3dc927cad60..5af06601395 100644 --- a/api/src/org/labkey/api/query/TableSorter.java +++ b/api/src/org/labkey/api/query/TableSorter.java @@ -15,7 +15,6 @@ */ package org.labkey.api.query; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; @@ -25,6 +24,7 @@ import org.labkey.api.data.MultiValuedForeignKey; import org.labkey.api.data.TableInfo; import org.labkey.api.util.Tuple3; +import org.labkey.api.util.logging.LogHelper; import java.util.ArrayList; import java.util.Collections; @@ -36,13 +36,9 @@ import java.util.Set; import java.util.stream.Stream; -/** - * User: kevink - * Date: 4/30/13 - */ public final class TableSorter { - private static final Logger LOG = LogManager.getLogger(TableSorter.class); + private static final Logger LOG = LogHelper.getLogger(TableSorter.class, "Warnings about foreign key traversal"); /** * Get a topologically sorted list of TableInfos within this schema. @@ -57,7 +53,7 @@ public static List sort(UserSchema schema) for (String tableName : tableNames) tables.put(tableName, schema.getTable(tableName)); - return sort(schemaName, tables); + return sort(schemaName, tables, false); } /** @@ -66,6 +62,16 @@ public static List sort(UserSchema schema) * @throws IllegalStateException if a loop is detected. */ public static List sort(DbSchema schema) + { + return sort(schema, false); + } + + /** + * Get a topologically sorted list of TableInfos within this schema. + * + * @throws IllegalStateException if a loop is detected and tolerateLoops is false. + */ + public static List sort(DbSchema schema, boolean tolerateLoops) { String schemaName = schema.getName(); Set tableNames = new HashSet<>(schema.getTableNames()); @@ -73,10 +79,10 @@ public static List sort(DbSchema schema) for (String tableName : tableNames) tables.put(tableName, schema.getTable(tableName)); - return sort(schemaName, tables); + return sort(schemaName, tables, tolerateLoops); } - private static List sort(String schemaName, Map tables) + private static List sort(String schemaName, Map tables, boolean tolerateLoops) { if (tables.isEmpty()) return Collections.emptyList(); @@ -99,6 +105,7 @@ private static List sort(String schemaName, Map ta if (fk == null || fk instanceof RowIdForeignKey || fk instanceof MultiValuedForeignKey) continue; + String lookupSchemaName = fk.getLookupSchemaName(); // Unfortunately, we need to get the lookup table since some FKs don't expose .getLookupSchemaName() or .getLookupTableName() TableInfo t = null; try @@ -109,12 +116,12 @@ private static List sort(String schemaName, Map ta { // ignore and try to continue String msg = String.format("Failed to traverse fk (%s, %s, %s) from (%s, %s)", - fk.getLookupSchemaName(), fk.getLookupTableName(), fk.getLookupColumnName(), tableName, column.getName()); + lookupSchemaName, fk.getLookupTableName(), fk.getLookupColumnName(), tableName, column.getName()); LOG.warn(msg, qpe); } // Skip lookups to other schemas - if (!(schemaName.equalsIgnoreCase(fk.getLookupSchemaName()) || (t != null && schemaName.equalsIgnoreCase(t.getPublicSchemaName())))) + if (!(schemaName.equalsIgnoreCase(lookupSchemaName) || (t != null && schemaName.equalsIgnoreCase(t.getPublicSchemaName())))) continue; // Get the lookupTableName: Attempt to use FK name first, then use the actual table name if it exists and is in the set of known tables. @@ -123,7 +130,7 @@ private static List sort(String schemaName, Map ta lookupTableName = t.getName(); // Skip self-referencing FKs - if (schemaName.equalsIgnoreCase(fk.getLookupSchemaName()) && lookupTableName.equals(table.getName())) + if (schemaName.equalsIgnoreCase(lookupSchemaName) && lookupTableName.equalsIgnoreCase(table.getName())) continue; // Remove the lookup table from the set of tables with no incoming FK @@ -145,21 +152,23 @@ private static List sort(String schemaName, Map ta Set visited = new HashSet<>(tables.size()); List sorted = new ArrayList<>(tables.size()); for (String tableName : startTables) - depthFirstWalk(schemaName, tables, tables.get(tableName), visited, new LinkedList<>(), sorted); + depthFirstWalk(schemaName, tables, tables.get(tableName), visited, new LinkedList<>(), sorted, tolerateLoops); return sorted; } - private static void depthFirstWalk(String schemaName, Map tables, TableInfo table, Set visited, LinkedList> visitingPath, List sorted) + private static void depthFirstWalk(String schemaName, Map tables, TableInfo table, Set visited, LinkedList> visitingPath, List sorted, boolean tolerateLoops) { if (hasLoop(visitingPath, table)) { String msg = "Loop detected in schema '" + schemaName + "':\n" + formatPath(visitingPath); - if (anyHaveContainerColumn(visitingPath)) + if (!tolerateLoops && anyHaveContainerColumn(visitingPath)) throw new IllegalStateException(msg); LOG.warn(msg); - return; + + if (!tolerateLoops) + return; } if (visited.contains(table)) @@ -179,6 +188,7 @@ private static void depthFirstWalk(String schemaName, Map tab if (fk == null || fk instanceof RowIdForeignKey || fk instanceof MultiValuedForeignKey) continue; + String lookupSchemaName = fk.getLookupSchemaName(); // Unfortunately, we need to get the lookup table since some FKs don't expose .getLookupSchemaName() or .getLookupTableName() TableInfo t = null; try @@ -191,7 +201,7 @@ private static void depthFirstWalk(String schemaName, Map tab } // Skip lookups to other schemas - if (!(schemaName.equalsIgnoreCase(fk.getLookupSchemaName()) || (t != null && schemaName.equalsIgnoreCase(t.getPublicSchemaName())))) + if (!(schemaName.equalsIgnoreCase(lookupSchemaName) || (t != null && schemaName.equalsIgnoreCase(t.getPublicSchemaName())))) continue; // Get the lookupTableName: Attempt to use FK name first, then use the actual table name if it exists and is in the set of known tables. @@ -200,7 +210,7 @@ private static void depthFirstWalk(String schemaName, Map tab lookupTableName = t.getName(); // Skip self-referencing FKs - if (schemaName.equalsIgnoreCase(fk.getLookupSchemaName()) && lookupTableName.equals(table.getName())) + if (schemaName.equalsIgnoreCase(lookupSchemaName) && lookupTableName.equalsIgnoreCase(table.getName())) continue; // Continue depthFirstWalk if the lookup table is found in the schema (e.g. it exists in this schema and isn't a query) @@ -208,7 +218,7 @@ private static void depthFirstWalk(String schemaName, Map tab if (lookupTable != null) { visitingPath.addLast(Tuple3.of(table, column, lookupTable)); - depthFirstWalk(schemaName, tables, lookupTable, visited, visitingPath, sorted); + depthFirstWalk(schemaName, tables, lookupTable, visited, visitingPath, sorted, tolerateLoops); visitingPath.removeLast(); } } diff --git a/api/src/org/labkey/api/settings/AppProps.java b/api/src/org/labkey/api/settings/AppProps.java index c7c7cc781fb..a4befe5bce7 100644 --- a/api/src/org/labkey/api/settings/AppProps.java +++ b/api/src/org/labkey/api/settings/AppProps.java @@ -48,7 +48,7 @@ public interface AppProps String EXPERIMENTAL_BLOCKER = "blockMaliciousClients"; String EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS = "resolve-property-uri-columns"; String DEPRECATED_OBJECT_LEVEL_DISCUSSIONS = "deprecatedObjectLevelDiscussions"; - String ALLOWED_EXTERNAL_RESOURCES = "allowedExternalResources"; + String ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES = "allowedExternalResources"; String UNKNOWN_VERSION = "Unknown Release Version"; diff --git a/api/src/org/labkey/api/settings/AppPropsImpl.java b/api/src/org/labkey/api/settings/AppPropsImpl.java index 80d9b622895..cf1bb5c4c9d 100644 --- a/api/src/org/labkey/api/settings/AppPropsImpl.java +++ b/api/src/org/labkey/api/settings/AppPropsImpl.java @@ -376,7 +376,7 @@ public String getServerGUID() } } String serverGUID = lookupStringValue(SERVER_GUID, SERVER_SESSION_GUID); - if (serverGUID.equals(SERVER_SESSION_GUID)) + if (serverGUID.equals(SERVER_SESSION_GUID) && ModuleLoader.getInstance().shouldInsertData()) { try (var ignore = SpringActionController.ignoreSqlUpdates()) { @@ -745,6 +745,6 @@ public List getExternalSourceHosts() @Override public @NotNull String getAllowedExternalResourceHosts() { - return lookupStringValue(ALLOWED_EXTERNAL_RESOURCES, "[]"); + return lookupStringValue(ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES, "[]"); } } diff --git a/api/src/org/labkey/api/settings/WriteableAppProps.java b/api/src/org/labkey/api/settings/WriteableAppProps.java index b806b0249f4..2012dfe6922 100644 --- a/api/src/org/labkey/api/settings/WriteableAppProps.java +++ b/api/src/org/labkey/api/settings/WriteableAppProps.java @@ -245,7 +245,7 @@ public void setAllowedFileExtensions(Collection allowedFileExtensions) public void setAllowedExternalResourceHosts(String jsonArray) { - storeStringValue(ALLOWED_EXTERNAL_RESOURCES, jsonArray); + storeStringValue(ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES, jsonArray); } public void setFeatureEnabled(String feature, boolean enabled) diff --git a/api/src/org/labkey/api/util/SystemMaintenanceStartupListener.java b/api/src/org/labkey/api/util/SystemMaintenanceStartupListener.java index 5893d22b79c..995a7eb066f 100644 --- a/api/src/org/labkey/api/util/SystemMaintenanceStartupListener.java +++ b/api/src/org/labkey/api/util/SystemMaintenanceStartupListener.java @@ -55,7 +55,9 @@ public void handle(Map properties) .collect( Collectors.partitioningBy(prop -> Boolean.valueOf(prop.getValue()), Collectors.mapping(StartupPropertyEntry::getName, Collectors.toSet())) ); - SystemMaintenance.ensureTaskProperties(map.get(true), map.get(false)); + + if (!properties.isEmpty()) + SystemMaintenance.ensureTaskProperties(map.get(true), map.get(false)); } } } \ No newline at end of file diff --git a/assay/resources/schemas/dbscripts/postgresql/assay-24.001-24.002.sql b/assay/resources/schemas/dbscripts/postgresql/assay-24.001-24.002.sql index 371e9cbbca9..d86ed7e0ad6 100644 --- a/assay/resources/schemas/dbscripts/postgresql/assay-24.001-24.002.sql +++ b/assay/resources/schemas/dbscripts/postgresql/assay-24.001-24.002.sql @@ -10,6 +10,7 @@ CREATE TABLE assay.PlateType CONSTRAINT UQ_PlateType_Rows_Cols UNIQUE (Rows, Columns) ); +-- @SkipOnEmptySchemasBegin INSERT INTO assay.PlateType (Rows, Columns, Description) VALUES (3, 4, '12 well (3x4)'); INSERT INTO assay.PlateType (Rows, Columns, Description) VALUES (4, 6, '24 well (4x6)'); INSERT INTO assay.PlateType (Rows, Columns, Description) VALUES (6, 8, '48 well (6x8)'); @@ -17,6 +18,7 @@ INSERT INTO assay.PlateType (Rows, Columns, Description) VALUES (8, 12, '96 well INSERT INTO assay.PlateType (Rows, Columns, Description) VALUES (16, 24, '384 well (16x24)'); INSERT INTO assay.PlateType (Rows, Columns, Description, Archived) VALUES (32, 48, '1536 well (32x48)', TRUE); INSERT INTO assay.PlateType (Rows, Columns, Description, Archived) VALUES (0, 0, 'Invalid Plate Type (Plates which were created with non-valid row & column combinations)', TRUE); +-- @SkipOnEmptySchemasEnd -- Rename type column to assayType ALTER TABLE assay.Plate RENAME COLUMN Type TO AssayType; diff --git a/audit/src/org/labkey/audit/AuditLogImpl.java b/audit/src/org/labkey/audit/AuditLogImpl.java index 369e7e7fdb2..48b89f72463 100644 --- a/audit/src/org/labkey/audit/AuditLogImpl.java +++ b/audit/src/org/labkey/audit/AuditLogImpl.java @@ -38,6 +38,7 @@ import org.labkey.api.data.Sort; import org.labkey.api.data.TableSelector; import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.module.ModuleLoader; import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; @@ -92,7 +93,9 @@ public static AuditLogImpl get() private AuditLogImpl() { - ContextListener.addStartupListener(this); + // If we're migrating, avoid creating all the audit log tables and inserting the queued events + if (ModuleLoader.getInstance().shouldInsertData()) + ContextListener.addStartupListener(this); } @Override diff --git a/core/resources/schemas/dbscripts/postgresql/core-0.000-24.000.sql b/core/resources/schemas/dbscripts/postgresql/core-0.000-24.000.sql index c3b1dfccb35..095d3c08490 100644 --- a/core/resources/schemas/dbscripts/postgresql/core-0.000-24.000.sql +++ b/core/resources/schemas/dbscripts/postgresql/core-0.000-24.000.sql @@ -50,7 +50,7 @@ CREATE TABLE core.Principals Container ENTITYID, -- NULL for all users, NOT NULL for _ALL_ groups OwnerId ENTITYID NULL, Name VARCHAR(64), -- email (must contain @ and .), group name (no punctuation), or hidden (no @) - Type CHAR(1), -- 'u'=user 'g'=group (NYI 'r'=role, 'm'=managed(module specific) + Type CHAR(1), -- 'u'=user 'g'=group 'm'=module-specific Active BOOLEAN NOT NULL DEFAULT TRUE, CONSTRAINT PK_Principals PRIMARY KEY (UserId), @@ -474,6 +474,7 @@ CREATE TABLE core.EmailOptions CONSTRAINT PK_EmailOptions PRIMARY KEY (EmailOptionId) ); +-- @SkipOnEmptySchemasBegin INSERT INTO core.EmailOptions (EmailOptionId, EmailOption) VALUES (0, 'No Email'); INSERT INTO core.EmailOptions (EmailOptionId, EmailOption) VALUES (1, 'All conversations'); INSERT INTO core.EmailOptions (EmailOptionId, EmailOption) VALUES (2, 'My conversations'); @@ -493,6 +494,7 @@ INSERT INTO core.emailOptions (EmailOptionId, EmailOption, Type) VALUES (702, 'A -- labbook email notification options INSERT INTO core.emailOptions (EmailOptionId, EmailOption, Type) VALUES (801, 'No Email', 'labbook'); INSERT INTO core.emailOptions (EmailOptionId, EmailOption, Type) VALUES (802, 'All emails', 'labbook'); +-- @SkipOnEmptySchemasEnd CREATE TABLE core.EmailPrefs ( diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index d3c559450b5..165dc6b85a0 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -89,6 +89,8 @@ import org.labkey.api.files.FileContentService; import org.labkey.api.markdown.MarkdownService; import org.labkey.api.message.settings.MessageConfigService; +import org.labkey.api.module.DatabaseMigration; +import org.labkey.api.module.DatabaseMigration.DefaultMigrationHandler; import org.labkey.api.module.FolderType; import org.labkey.api.module.FolderTypeManager; import org.labkey.api.module.Module; @@ -372,8 +374,6 @@ public class CoreModule extends SpringModule implements SearchService.DocumentPr private static final Logger LOG = LogHelper.getLogger(CoreModule.class, "Errors during server startup and shut down"); public static final String PROJECTS_WEB_PART_NAME = "Projects"; - static Runnable _afterUpdateRunnable = null; - static { // Accept most of the standard Quartz properties, but set the misfire threshold to five minutes. This prevents @@ -842,7 +842,8 @@ public void afterUpdate(ModuleContext moduleContext) } // Increment on every core module upgrade to defeat browser caching of static resources. - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + if (ModuleLoader.getInstance().shouldInsertData()) + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); // Allow dialect to make adjustments to the just upgraded core database (e.g., install aggregate functions, etc.) CoreSchema.getInstance().getSqlDialect().afterCoreUpgrade(moduleContext); @@ -858,66 +859,76 @@ public void afterUpdate(ModuleContext moduleContext) // core.Containers metadata and a few common containers. This may prevent some deadlocks during upgrade, #33550. CacheManager.addListener(() -> { ContainerManager.getRoot(); - ContainerManager.getHomeContainer(); ContainerManager.getSharedContainer(); + if (ModuleLoader.getInstance().shouldInsertData()) + { + ContainerManager.getHomeContainer(); + } }); - - if (_afterUpdateRunnable != null) - _afterUpdateRunnable.run(); } private void bootstrap() { - // Create the initial groups - GroupManager.bootstrapGroup(Group.groupUsers, "Users"); - GroupManager.bootstrapGroup(Group.groupGuests, "Guests"); - - // Other containers inherit permissions from root; admins get all permissions, users & guests none - Role noPermsRole = RoleManager.getRole(NoPermissionsRole.class); - Role readerRole = RoleManager.getRole(ReaderRole.class); + if (ModuleLoader.getInstance().shouldInsertData()) + { + // Create the initial groups + GroupManager.bootstrapGroup(Group.groupUsers, "Users"); + GroupManager.bootstrapGroup(Group.groupGuests, "Guests"); - ContainerManager.bootstrapContainer("/", noPermsRole, noPermsRole); - Container rootContainer = ContainerManager.getRoot(); + // Other containers inherit permissions from root; admins get all permissions, users & guests none + Role noPermsRole = RoleManager.getRole(NoPermissionsRole.class); + Role readerRole = RoleManager.getRole(ReaderRole.class); - // Create all the standard containers (Home, Home/support, Shared) using an empty Collaboration folder type - FolderType collaborationType = new CollaborationFolderType(Collections.emptyList()); + ContainerManager.bootstrapContainer("/", noPermsRole, noPermsRole); + Container rootContainer = ContainerManager.getRoot(); - // Users & guests can read from /home - Container home = ContainerManager.bootstrapContainer(ContainerManager.HOME_PROJECT_PATH, readerRole, readerRole); - home.setFolderType(collaborationType, null); + // Create all the standard containers (Home, Home/support, Shared) using an empty Collaboration folder type + FolderType collaborationType = new CollaborationFolderType(Collections.emptyList()); - ContainerManager.createDefaultSupportContainer().setFolderType(collaborationType, null); + // Users & guests can read from /home + Container home = ContainerManager.bootstrapContainer(ContainerManager.HOME_PROJECT_PATH, readerRole, readerRole); + home.setFolderType(collaborationType, null); - // Only users can read from /Shared - ContainerManager.bootstrapContainer(ContainerManager.SHARED_CONTAINER_PATH, readerRole, null).setFolderType(collaborationType, null); + ContainerManager.createDefaultSupportContainer().setFolderType(collaborationType, null); - try - { - // Need to insert standard MV indicators for the root -- okay to call getRoot() since we just created it. - String rootContainerId = rootContainer.getId(); - TableInfo mvTable = CoreSchema.getInstance().getTableInfoMvIndicators(); + // Only users can read from /Shared + ContainerManager.bootstrapContainer(ContainerManager.SHARED_CONTAINER_PATH, readerRole, null).setFolderType(collaborationType, null); - for (Map.Entry qcEntry : MvUtil.getDefaultMvIndicators().entrySet()) + try { - Map params = new HashMap<>(); - params.put("Container", rootContainerId); - params.put("MvIndicator", qcEntry.getKey()); - params.put("Label", qcEntry.getValue()); + // Need to insert standard MV indicators for the root -- okay to call getRoot() since we just created it. + String rootContainerId = rootContainer.getId(); + TableInfo mvTable = CoreSchema.getInstance().getTableInfoMvIndicators(); + + for (Map.Entry qcEntry : MvUtil.getDefaultMvIndicators().entrySet()) + { + Map params = new HashMap<>(); + params.put("Container", rootContainerId); + params.put("MvIndicator", qcEntry.getKey()); + params.put("Label", qcEntry.getValue()); - Table.insert(null, mvTable, params); + Table.insert(null, mvTable, params); + } } + catch (Throwable t) + { + ExceptionUtil.logExceptionToMothership(null, t); + } + + List guids = Stream.of(ContainerManager.HOME_PROJECT_PATH, ContainerManager.SHARED_CONTAINER_PATH) + .map(ContainerManager::getForPath) + .filter(Objects::nonNull) + .map(Container::getEntityId) + .collect(Collectors.toList()); + ContainerManager.setExcludedProjects(guids, () -> {}); } - catch (Throwable t) + else { - ExceptionUtil.logExceptionToMothership(null, t); + // It's very difficult to bootstrap without the root or shared containers in place; create them now and + // we'll delete them later + Container root = ContainerManager.ensureContainer("/", User.getAdminServiceUser()); + Table.insert(null, CoreSchema.getInstance().getTableInfoContainers(), Map.of("Parent", root.getId(), "Name", "Shared")); } - - List guids = Stream.of(ContainerManager.HOME_PROJECT_PATH, ContainerManager.SHARED_CONTAINER_PATH) - .map(ContainerManager::getForPath) - .filter(Objects::nonNull) - .map(Container::getEntityId) - .collect(Collectors.toList()); - ContainerManager.setExcludedProjects(guids, () -> {}); } @@ -954,11 +965,10 @@ public void startupAfterSpringConfig(ModuleContext moduleContext) AnalyticsServiceImpl.get().resetCSP(); - if (moduleContext.isNewInstall()) + if (moduleContext.isNewInstall() && ModuleLoader.getInstance().shouldInsertData()) { - // In order to initialize the portal layout correctly, we need to add the web parts after the folder - // types have been registered. Thus, needs to be here in startupAfterSpringConfig() instead of grouped - // in bootstrap(). + // To initialize the portal layout correctly, we need to add the web parts after the folder types have been + // registered. Thus, it needs to be here in startupAfterSpringConfig() instead of grouped in bootstrap(). Container homeContainer = ContainerManager.getHomeContainer(); int count = Portal.getParts(homeContainer, homeContainer.getFolderType().getDefaultPageId(homeContainer)).size(); addWebPart(PROJECTS_WEB_PART_NAME, homeContainer, HttpView.BODY, count); @@ -1148,7 +1158,8 @@ public void moduleStartupComplete(ServletContext servletContext) if (null != PropertyService.get()) { PropertyService.get().registerDomainKind(new UsersDomainKind()); - UsersDomainKind.ensureDomain(moduleContext); + if (ModuleLoader.getInstance().shouldInsertData()) + UsersDomainKind.ensureDomain(moduleContext); } // Register the standard, wiki-based terms-of-use provider @@ -1256,9 +1267,23 @@ public void moduleStartupComplete(ServletContext servletContext) MessageConfigService.setInstance(new EmailPreferenceConfigServiceImpl()); ContainerManager.addContainerListener(new EmailPreferenceContainerListener()); UserManager.addUserListener(new EmailPreferenceUserListener()); + + DatabaseMigration.registerHandler(CoreSchema.getInstance().getSchema(), new DefaultMigrationHandler() + { + @Override + public List getTablesToCopy(DbSchema targetSchema) + { + List tablesToCopy = super.getTablesToCopy(targetSchema); + tablesToCopy.remove(targetSchema.getTable("Modules")); + tablesToCopy.remove(targetSchema.getTable("SqlScripts")); + tablesToCopy.remove(targetSchema.getTable("UpgradeSteps")); + + return tablesToCopy; + } + }); } - // Issue 7527: Auto-detect missing sql views and attempt to recreate + // Issue 7527: Auto-detect missing SQL views and attempt to recreate private void checkForMissingDbViews() { ModuleLoader.getInstance().getModules().stream() diff --git a/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java b/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java index c653dbb4cc4..5595fda49ce 100644 --- a/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java +++ b/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java @@ -20,9 +20,9 @@ import org.labkey.api.data.DbScope; import org.labkey.api.data.ParameterMarkerInClauseGenerator; import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.dialect.BasePostgreSqlDialect; import org.labkey.api.data.dialect.DialectStringHandler; import org.labkey.api.data.dialect.JdbcHelper; -import org.labkey.api.data.dialect.BasePostgreSqlDialect; import org.labkey.api.data.dialect.StandardJdbcHelper; import org.labkey.api.util.HtmlString; import org.labkey.api.util.StringUtilsLabKey; @@ -204,4 +204,11 @@ private static String truncateBytes(String str, int maxBytes) assert !StringUtilsLabKey.hasBrokenSurrogate(str); return str; } + + @Override + // No need to split up PostgreSQL scripts; execute all statements in a single block (unless we have a special stored proc call). + protected Pattern getSQLScriptSplitPattern() + { + return null; + } } diff --git a/core/src/org/labkey/core/login/LoginController.java b/core/src/org/labkey/core/login/LoginController.java index 4114d563504..ecf726d5144 100644 --- a/core/src/org/labkey/core/login/LoginController.java +++ b/core/src/org/labkey/core/login/LoginController.java @@ -1550,7 +1550,7 @@ public ModelAndView getView(SsoRedirectForm form, BindException errors) url = configuration.getUrl(csrf, getViewContext()); - return HttpView.redirect(url.getURIString()); + return HttpView.redirect(url, true); } @Override diff --git a/core/src/org/labkey/core/security/AllowedExternalResourceHosts.java b/core/src/org/labkey/core/security/AllowedExternalResourceHosts.java index 892e6932e82..d7c09e2041a 100644 --- a/core/src/org/labkey/core/security/AllowedExternalResourceHosts.java +++ b/core/src/org/labkey/core/security/AllowedExternalResourceHosts.java @@ -25,7 +25,7 @@ import java.util.Map; import java.util.stream.Collectors; -import static org.labkey.api.settings.AppProps.ALLOWED_EXTERNAL_RESOURCES; +import static org.labkey.api.settings.AppProps.ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES; public class AllowedExternalResourceHosts { @@ -74,12 +74,12 @@ public static void saveAllowedHosts(@Nullable Collection allowedHos private static void register(Directive dir, String... hosts) { - ContentSecurityPolicyFilter.registerAllowedSources(ALLOWED_EXTERNAL_RESOURCES, dir, hosts); + ContentSecurityPolicyFilter.registerAllowedSources(ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES, dir, hosts); } private static void unregister(Directive dir) { - ContentSecurityPolicyFilter.unregisterAllowedSources(ALLOWED_EXTERNAL_RESOURCES, dir); + ContentSecurityPolicyFilter.unregisterAllowedSources(ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES, dir); } // Returns a mutable list (mutating it won't affect any cached values) diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index c6bbbf15b04..cc805c6acff 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -32,6 +32,7 @@ import org.labkey.api.data.JdbcType; import org.labkey.api.data.NameGenerator; import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.SqlSelector; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TableSelector; @@ -70,7 +71,10 @@ import org.labkey.api.exp.xar.LsidUtils; import org.labkey.api.files.FileContentService; import org.labkey.api.files.TableUpdaterFileListener; +import org.labkey.api.module.DatabaseMigration; +import org.labkey.api.module.DatabaseMigration.DefaultMigrationHandler; import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; import org.labkey.api.module.SpringModule; import org.labkey.api.module.Summary; import org.labkey.api.ontology.OntologyService; @@ -86,6 +90,7 @@ import org.labkey.api.usageMetrics.UsageMetricsService; import org.labkey.api.util.JspTestCase; import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.SystemMaintenance; import org.labkey.api.view.AlwaysAvailableWebPartFactory; import org.labkey.api.view.BaseWebPartFactory; @@ -557,7 +562,8 @@ public void containerDeleted(Container c, User user) // but it should be before the CoreContainerListener ContainerManager.ContainerListener.Order.Last); - SystemProperty.registerProperties(); + if (ModuleLoader.getInstance().shouldInsertData()) + SystemProperty.registerProperties(); FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); if (null != folderRegistry) @@ -848,6 +854,23 @@ HAVING COUNT(*) > 1 return results; }); } + + // Work around foreign key cycle between ExperimentRun <-> ProtocolApplication by temporarily dropping FK_Run_WorfklowTask + DatabaseMigration.registerHandler(OntologyManager.getExpSchema(), new DefaultMigrationHandler() + { + @Override + public void beforeSchema(DbSchema targetSchema) + { + // Yes, the FK name is misspelled + new SqlExecutor(targetSchema).execute("ALTER TABLE exp.ExperimentRun DROP CONSTRAINT FK_Run_WorfklowTask"); + } + + @Override + public void afterSchema(DbSchema targetSchema) + { + new SqlExecutor(targetSchema).execute("ALTER TABLE exp.ExperimentRun ADD CONSTRAINT FK_Run_WorfklowTask FOREIGN KEY (WorkflowTask) REFERENCES exp.ProtocolApplication (RowId) MATCH SIMPLE ON DELETE SET NULL"); + } + }); } @Override @@ -857,7 +880,7 @@ public Collection getSummary(Container c) Collection list = new LinkedList<>(); int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); if (runGroupCount > 0) - list.add("" + runGroupCount + " Run Group" + (runGroupCount > 1 ? "s" : "")); + list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); User user = HttpView.currentContext().getUser(); diff --git a/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java b/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java index 04ddb0813c5..5e451c144da 100644 --- a/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java +++ b/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java @@ -136,7 +136,7 @@ private StorageProvisionerImpl() .expireAfterWrite(1, TimeUnit.DAYS) .build(); - private String _create(DbScope scope, DomainKind kind, Domain domain) + private String _create(DbScope scope, DomainKind kind, Domain domain, boolean useProvidedStorageName) { //noinspection AssertWithSideEffects assert create.start(); @@ -148,18 +148,29 @@ private String _create(DbScope scope, DomainKind kind, Domain domain) DomainDescriptor dd = OntologyManager.getDomainDescriptor(domain.getTypeId()); if (null == dd) { - log.warn("Can't find domain descriptor: " + domain.getTypeId() + " " + domain.getTypeURI()); + log.warn("Can't find domain descriptor: {} {}", domain.getTypeId(), domain.getTypeURI()); transaction.commit(); return null; } String tableName = dd.getStorageTableName(); - if (null != tableName) + + if (useProvidedStorageName) { - transaction.commit(); - return tableName; + if (null == tableName) + { + throw new RuntimeException("Storage table name was null: " + domain.getTypeId() + " " + domain.getTypeURI()); + } } + else + { + if (null != tableName) + { + transaction.commit(); + return tableName; + } - tableName = makeTableName(kind, domain); + tableName = makeTableName(kind, domain); + } TableChange change = new TableChange(domain, ChangeType.CreateTable, tableName); Set base = Sets.newCaseInsensitiveHashSet(); @@ -215,12 +226,15 @@ private String _create(DbScope scope, DomainKind kind, Domain domain) throw re; } - DomainDescriptor editDD = dd.edit() + if (!useProvidedStorageName) + { + DomainDescriptor editDD = dd.edit() .setStorageTableName(tableName) .setStorageSchemaName(kind.getStorageSchemaName()) .build(); - OntologyManager.ensureDomainDescriptor(editDD); + OntologyManager.ensureDomainDescriptor(editDD); + } kind.invalidate(domain); @@ -346,7 +360,7 @@ public void addProperties(Domain domain, Collection properties, if (null == domain.getStorageTableName()) { - _create(scope, kind, domain); + _create(scope, kind, domain, false); return; } @@ -826,13 +840,19 @@ public String ensureStorageTable(Domain domain, DomainKind kind, DbScope scop { try (var ignored = SpringActionController.ignoreSqlUpdates()) { - tableName = _create(scope, kind, domain); + tableName = _create(scope, kind, domain, false); } } return tableName; } + @Override + public void createStorageTable(Domain domain, DomainKind kind, DbScope scope) + { + _create(scope, kind, domain, true); + } + enum RequiredIndicesAction { Drop diff --git a/mothership/src/org/labkey/mothership/MothershipModule.java b/mothership/src/org/labkey/mothership/MothershipModule.java index 471eecdde46..a20aa2d06ad 100644 --- a/mothership/src/org/labkey/mothership/MothershipModule.java +++ b/mothership/src/org/labkey/mothership/MothershipModule.java @@ -22,6 +22,7 @@ import org.labkey.api.module.DefaultModule; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; import org.labkey.api.security.User; import org.labkey.api.security.UserManager; import org.labkey.api.util.MothershipReport; @@ -89,7 +90,7 @@ public boolean hasScripts() @Override public void afterUpdate(ModuleContext moduleContext) { - if (moduleContext.isNewInstall()) + if (moduleContext.isNewInstall() && ModuleLoader.getInstance().shouldInsertData()) bootstrap(moduleContext); } diff --git a/wiki/src/org/labkey/wiki/WikiModule.java b/wiki/src/org/labkey/wiki/WikiModule.java index 096fd62a081..322e160d3a5 100644 --- a/wiki/src/org/labkey/wiki/WikiModule.java +++ b/wiki/src/org/labkey/wiki/WikiModule.java @@ -25,9 +25,14 @@ import org.labkey.api.attachments.AttachmentService; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.TableInfo; import org.labkey.api.module.CodeOnlyModule; +import org.labkey.api.module.DatabaseMigration; +import org.labkey.api.module.DatabaseMigration.DefaultMigrationHandler; import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; import org.labkey.api.search.SearchService; import org.labkey.api.security.User; import org.labkey.api.util.PageFlowUtil; @@ -119,24 +124,51 @@ public void doStartup(ModuleContext moduleContext) WikiSchema.register(this); WikiController.registerAdminConsoleLinks(); + DatabaseMigration.registerHandler(CommSchema.getInstance().getSchema(), new DefaultMigrationHandler() + { + @Override + public void beforeSchema(DbSchema targetSchema) + { + new SqlExecutor(targetSchema).execute("ALTER TABLE comm.Pages DROP CONSTRAINT FK_Pages_PageVersions"); + } + + @Override + public List getTablesToCopy(DbSchema targetSchema) + { + List tablesToCopy = super.getTablesToCopy(targetSchema); + tablesToCopy.add(targetSchema.getTable("Pages")); + tablesToCopy.add(targetSchema.getTable("PageVersions")); + + return tablesToCopy; + } + + @Override + public void afterSchema(DbSchema targetSchema) + { + new SqlExecutor(targetSchema).execute("ALTER TABLE comm.Pages ADD CONSTRAINT FK_Pages_PageVersions FOREIGN KEY (PageVersionId) REFERENCES comm.PageVersions (RowId)"); + } + }); } private void bootstrap(ModuleContext moduleContext) { - Container supportContainer = ContainerManager.getDefaultSupportContainer(); - Container homeContainer = ContainerManager.getHomeContainer(); - Container sharedContainer = ContainerManager.getSharedContainer(); - String defaultPageName = "default"; + if (ModuleLoader.getInstance().shouldInsertData()) + { + Container supportContainer = ContainerManager.getDefaultSupportContainer(); + Container homeContainer = ContainerManager.getHomeContainer(); + Container sharedContainer = ContainerManager.getSharedContainer(); + String defaultPageName = "default"; - loadWikiContent(homeContainer, moduleContext.getUpgradeUser(), defaultPageName, "Welcome to LabKey Server", "/org/labkey/wiki/welcomeWiki.txt"); - loadWikiContent(supportContainer, moduleContext.getUpgradeUser(), defaultPageName, "Welcome to LabKey Support", "/org/labkey/wiki/supportWiki.txt"); - loadWikiContent(sharedContainer, moduleContext.getUpgradeUser(), defaultPageName, "Shared Resources", "/org/labkey/wiki/sharedWiki.txt"); + loadWikiContent(homeContainer, moduleContext.getUpgradeUser(), defaultPageName, "Welcome to LabKey Server", "/org/labkey/wiki/welcomeWiki.txt"); + loadWikiContent(supportContainer, moduleContext.getUpgradeUser(), defaultPageName, "Welcome to LabKey Support", "/org/labkey/wiki/supportWiki.txt"); + loadWikiContent(sharedContainer, moduleContext.getUpgradeUser(), defaultPageName, "Shared Resources", "/org/labkey/wiki/sharedWiki.txt"); - addWebPart(supportContainer, defaultPageName); - addWebPart(sharedContainer, defaultPageName); + addWebPart(supportContainer, defaultPageName); + addWebPart(sharedContainer, defaultPageName); - // Add a wiki webpart with the default content. - addWebPart(homeContainer, defaultPageName); + // Add a wiki webpart with the default content. + addWebPart(homeContainer, defaultPageName); + } } private void addWebPart(@Nullable Container c, String wikiName) @@ -159,7 +191,7 @@ public Collection getSummary(Container c) { int count = WikiSelectManager.getPageCount(c); if (count > 0) - list.add("" + count + " Wiki Page" + (count > 1 ? "s" : "")); + list.add(count + " Wiki Page" + (count > 1 ? "s" : "")); } catch (Exception x) {