diff --git a/api/src/org/labkey/api/data/AES.java b/api/src/org/labkey/api/data/AES.java index 4ed47491b08..37ba7cce81f 100644 --- a/api/src/org/labkey/api/data/AES.java +++ b/api/src/org/labkey/api/data/AES.java @@ -21,7 +21,7 @@ * First reference to {@link org.labkey.api.data.PropertyManager} constructs the stores which causes initialization of the PropertyEncryption enum. * The AES128 enum requires the property manager (we store the standard salt in properties), so we use a holder pattern * (instead of normal static initialization) to implement thread-safe lazy initialization, breaking the loop. - * + *

* This class is only used by the PropertyManager; other encryption users should call Encryption.getAES128() directly. */ class AES diff --git a/api/src/org/labkey/api/data/EncryptedPropertyStore.java b/api/src/org/labkey/api/data/EncryptedPropertyStore.java index bb63774054c..8ca23b6323f 100644 --- a/api/src/org/labkey/api/data/EncryptedPropertyStore.java +++ b/api/src/org/labkey/api/data/EncryptedPropertyStore.java @@ -23,6 +23,7 @@ import org.labkey.api.security.Encryption; import org.labkey.api.security.Encryption.DecryptionException; import org.labkey.api.security.Encryption.EncryptionMigrationHandler; +import org.labkey.api.security.Encryption.AESConfig; import org.labkey.api.settings.AppProps; import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.logging.LogHelper; @@ -94,8 +95,10 @@ protected void appendWhereFilter(SQLFragment sql) } @Override - public void migrateEncryptedContent(String oldPassPhrase, String keySource) + public void migrateEncryptedContent(String oldPassPhrase, String keySource, AESConfig config) { + Encryption.Algorithm oldAes = Encryption.getAES128(oldPassPhrase, keySource, config); + LOG.info(" Attempting to migrate encrypted property store values"); TableInfo sets = PropertySchema.getInstance().getTableInfoPropertySets(); TableInfo props = PropertySchema.getInstance().getTableInfoProperties(); @@ -104,7 +107,7 @@ public void migrateEncryptedContent(String oldPassPhrase, String keySource) int set = (int)map.get("Set"); String encryption = (String)map.get("Encryption"); String propertySetName = "\"" + map.get("Category") + "\" (Set = " + set + ")"; - LOG.info(" Attempting to migrate encrypted property set " + propertySetName); + LOG.info(" Attempting to migrate encrypted property set {}", propertySetName); PropertyEncryption pe = PropertyEncryption.getBySerializedName(encryption); if (null != pe) @@ -117,11 +120,24 @@ public void migrateEncryptedContent(String oldPassPhrase, String keySource) { String name = (String) m.get("Name"); String encryptedValue = (String) m.get("Value"); - LOG.info(" Attempting to decrypt property \"" + name + "\""); - String decryptedValue = pe.decrypt(Base64.decodeBase64(encryptedValue), oldPassPhrase, keySource); - String newEncryptedValue = Base64.encodeBase64String(pe.encrypt(decryptedValue)); - assert decryptedValue.equals(pe.decrypt(Base64.decodeBase64(newEncryptedValue))); - newProps.put(name, newEncryptedValue); + + String newEncryptedValue; + try + { + LOG.info(" Attempting to decrypt property \"{}\"", name); + String decryptedValue = oldAes.decrypt(Base64.decodeBase64(encryptedValue)); + newEncryptedValue = Base64.encodeBase64String(pe.encrypt(decryptedValue)); + assert decryptedValue.equals(pe.decrypt(Base64.decodeBase64(newEncryptedValue))); + if (newEncryptedValue != null) + { + newProps.put(name, newEncryptedValue); + } + + } + catch (DecryptionException e) + { + LOG.warn(" Failed to decrypt property \"{}\". Skipping.", name); + } } for (Map.Entry entry : newProps.entrySet()) @@ -132,15 +148,15 @@ public void migrateEncryptedContent(String oldPassPhrase, String keySource) } catch (RuntimeSQLException e) { - LOG.error("Failed to save re-encrypted property \"" + entry.getKey() + "\"", e); + LOG.error("Failed to save re-encrypted property \"{}\"", entry.getKey(), e); } } - LOG.info(" Successfully migrated encrypted property set " + propertySetName); + LOG.info(" Successfully migrated encrypted property set {}", propertySetName); } catch (DecryptionException e) { - LOG.warn(" Failed to decrypt the previous property. Skipping encrypted property set " + propertySetName); + LOG.warn(" Failed to decrypt the previous property. Skipping encrypted property set {}", propertySetName); } } }); diff --git a/api/src/org/labkey/api/data/PropertyEncryption.java b/api/src/org/labkey/api/data/PropertyEncryption.java index 5b484f64658..0bae97ce6ec 100644 --- a/api/src/org/labkey/api/data/PropertyEncryption.java +++ b/api/src/org/labkey/api/data/PropertyEncryption.java @@ -17,7 +17,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.labkey.api.security.Encryption; import org.labkey.api.settings.AppProps; import org.labkey.api.util.Compress; import org.labkey.api.util.ConfigurationException; @@ -39,19 +38,13 @@ public enum PropertyEncryption None { @Override - public @NotNull byte[] encrypt(@NotNull String plainText) + public byte @NotNull[] encrypt(@NotNull String plainText) { throw new IllegalStateException("Incorrect PropertyStore for this PropertyMap"); } @Override - public @NotNull String decrypt(@NotNull byte[] cipherText) - { - throw new IllegalStateException("Incorrect PropertyStore for this PropertyMap"); - } - - @Override - public @NotNull String decrypt(@NotNull byte[] cipherText, String encryptionPassPhrase, String keySource) + public @NotNull String decrypt(byte @NotNull[] cipherText) { throw new IllegalStateException("Incorrect PropertyStore for this PropertyMap"); } @@ -66,13 +59,13 @@ public enum PropertyEncryption Test { @Override - public @NotNull byte[] encrypt(@NotNull String plainText) + public byte @NotNull[] encrypt(@NotNull String plainText) { return Compress.deflate(plainText); } @Override - public @NotNull String decrypt(@NotNull byte[] cipherText) + public @NotNull String decrypt(byte @NotNull[] cipherText) { try { @@ -84,12 +77,6 @@ public enum PropertyEncryption } } - @Override - public @NotNull String decrypt(@NotNull byte[] cipherText, String encryptionPassPhrase, String keySource) - { - return decrypt(cipherText); - } - @Override public @NotNull String getSerializedName() { @@ -100,19 +87,13 @@ public enum PropertyEncryption NoKey { @Override - public @NotNull byte[] encrypt(@NotNull String plainText) + public byte @NotNull[] encrypt(@NotNull String plainText) { throw getConfigurationException(); } @Override - public @NotNull String decrypt(@NotNull byte[] cipherText) - { - throw getConfigurationException(); - } - - @Override - public @NotNull String decrypt(@NotNull byte[] cipherText, String encryptionPassPhrase, String keySource) + public @NotNull String decrypt(byte @NotNull[] cipherText) { throw getConfigurationException(); } @@ -132,23 +113,17 @@ private ConfigurationException getConfigurationException() AES128 { @Override - public @NotNull byte[] encrypt(@NotNull String plainText) + public byte @NotNull[] encrypt(@NotNull String plainText) { return AES.get().encrypt(plainText); } @Override - public @NotNull String decrypt(@NotNull byte[] cipherText) + public @NotNull String decrypt(byte @NotNull[] cipherText) { return AES.get().decrypt(cipherText); } - @Override - public @NotNull String decrypt(@NotNull byte[] cipherText, String encryptionPassPhrase, String keySource) - { - return Encryption.getAES128(encryptionPassPhrase, keySource).decrypt(cipherText); - } - @Override public @NotNull String getSerializedName() { @@ -156,9 +131,8 @@ private ConfigurationException getConfigurationException() } }; - public abstract @NotNull byte[] encrypt(@NotNull String plainText); - public abstract @NotNull String decrypt(@NotNull byte[] cipherText); - public abstract @NotNull String decrypt(@NotNull byte[] cipherText, String encryptionPassPhrase, String keySource); + public abstract byte @NotNull[] encrypt(@NotNull String plainText); + public abstract @NotNull String decrypt(byte @NotNull[] cipherText); // Canonical name to store in the property set. Do not change these return values, once they are in use! // Consider: if we need to, could change to a collection of names, the first being canonical, for backward diff --git a/api/src/org/labkey/api/data/PropertyManager.java b/api/src/org/labkey/api/data/PropertyManager.java index 23d3bea74df..ef728b1bf27 100644 --- a/api/src/org/labkey/api/data/PropertyManager.java +++ b/api/src/org/labkey/api/data/PropertyManager.java @@ -24,7 +24,6 @@ import org.apache.commons.collections4.map.AbstractMapDecorator; import org.apache.commons.collections4.map.UnmodifiableEntrySet; import org.apache.commons.collections4.set.UnmodifiableSet; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Strings; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; diff --git a/api/src/org/labkey/api/security/AuthenticationManager.java b/api/src/org/labkey/api/security/AuthenticationManager.java index 91f22514a84..4128ca435f5 100644 --- a/api/src/org/labkey/api/security/AuthenticationManager.java +++ b/api/src/org/labkey/api/security/AuthenticationManager.java @@ -248,7 +248,7 @@ public static boolean isLdapOrSsoEmail(ValidEmail validEmail) public static boolean isLdapOrSsoEmail(String emailAddress) { return AuthenticationConfigurationCache.getActiveDomains().stream() - .anyMatch(domain->StringUtils.endsWithIgnoreCase(emailAddress, "@" + domain)); + .anyMatch(domain->Strings.CI.endsWith(emailAddress, "@" + domain)); } public static boolean isRegistrationEnabled() @@ -327,9 +327,9 @@ public static void reorderConfigurations(User user, String name, int[] rowIds) } } - static final EncryptionMigrationHandler ENCRYPTION_MIGRATION_HANDLER = (oldPassPhrase, keySource) -> { + static final EncryptionMigrationHandler ENCRYPTION_MIGRATION_HANDLER = (oldPassPhrase, keySource, oldConfig) -> { + Algorithm decryptAes = Encryption.getAES128(oldPassPhrase, keySource, oldConfig); _log.info(" Attempting to migrate encrypted properties in authentication configurations"); - Algorithm decryptAes = Encryption.getAES128(oldPassPhrase, keySource); TableInfo tinfo = CoreSchema.getInstance().getTableInfoAuthenticationConfigurations(); Map map = new TableSelector(tinfo, PageFlowUtil.set("RowId", "EncryptedProperties"), new SimpleFilter(FieldKey.fromParts("EncryptedProperties"), null, CompareType.NONBLANK), null).getValueMap(Integer.class); @@ -339,15 +339,22 @@ public static void reorderConfigurations(User user, String name, int[] rowIds) try { _log.info(" Migrating encrypted properties for configuration " + key); - String decryptedValue = decryptAes.decrypt(Base64.decodeBase64(value)); - String newEncryptedValue = Base64.encodeBase64String(AES.get().encrypt(decryptedValue)); - saveMap.put("EncryptedProperties", newEncryptedValue); - assert decryptedValue.equals(AES.get().decrypt(Base64.decodeBase64(newEncryptedValue))); - Table.update(null, tinfo, saveMap, key); - } - catch (DecryptionException e) - { - _log.info(" Failed to decrypt encrypted properties for configuration " + key + ". It will be skipped."); + try + { + String decryptedValue = decryptAes.decrypt(Base64.decodeBase64(value)); + String newEncryptedValue = Base64.encodeBase64String(AES.get().encrypt(decryptedValue)); + assert decryptedValue.equals(AES.get().decrypt(Base64.decodeBase64(newEncryptedValue))); + + if (newEncryptedValue != null) + { + saveMap.put("EncryptedProperties", newEncryptedValue); + Table.update(null, tinfo, saveMap, key); + } + } + catch (DecryptionException e) + { + _log.info(" Failed to decrypt encrypted properties for configuration " + key + ". It will be skipped."); + } } catch (Exception e) { diff --git a/api/src/org/labkey/api/security/Encryption.java b/api/src/org/labkey/api/security/Encryption.java index cb9aba53e93..2603cdddcbc 100644 --- a/api/src/org/labkey/api/security/Encryption.java +++ b/api/src/org/labkey/api/security/Encryption.java @@ -1,544 +1,673 @@ -/* - * Copyright (c) 2013-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.security; - -import com.google.common.primitives.Bytes; -import jakarta.servlet.ServletContext; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.ConcurrentHashSet; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.EncryptedPropertyStore; -import org.labkey.api.data.PropertyManager; -import org.labkey.api.data.PropertyManager.WritablePropertyMap; -import org.labkey.api.data.PropertyStore; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.security.permissions.TroubleshooterPermission; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.ConfigurationException; -import org.labkey.api.util.HelpTopic; -import org.labkey.api.util.HtmlStringBuilder; -import org.labkey.api.util.JobRunner; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.template.WarningProvider; -import org.labkey.api.view.template.WarningService; -import org.labkey.api.view.template.Warnings; - -import javax.crypto.BadPaddingException; -import javax.crypto.Cipher; -import javax.crypto.SecretKey; -import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.IvParameterSpec; -import javax.crypto.spec.PBEKeySpec; -import javax.crypto.spec.SecretKeySpec; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.KeySpec; -import java.util.Arrays; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Easy to use wrappers for common encryption algorithms. Also includes related helper methods for shared operations - * such as generating salts & keys, and for retrieving & saving the application.properties encryption key and standard salt. - * WARNING: Do not change the core algorithms or parameters of existing implementations; changes will likely - * render existing data irrecoverable. -*/ - -public class Encryption -{ - private static final Logger LOG = LogHelper.getLogger(Encryption.class, "Encryption operations"); - private static final String CATEGORY = "Encryption"; - private static final String SALT_KEY = "Salt"; - public static final SecureRandom SR; - private static final String ENCRYPTION_PASS_PHRASE; - private static final String KEY_CHANGE_GUIDANCE = "An administrator should change the encryption key back to the previous value, follow the official encryption key change process, or be prepared to re-enter and re-save all saved credentials."; - - static - { - ENCRYPTION_PASS_PHRASE = loadEncryptionPassPhrase(); - - try - { - SR = SecureRandom.getInstanceStrong(); - } - catch (NoSuchAlgorithmException e) - { - throw new ConfigurationException("Could not initialize SecureRandom", e); - } - - WarningService.get().register(new WarningProvider() { - @Override - public void addDynamicWarnings(@NotNull Warnings warnings, @Nullable ViewContext context, boolean showAllWarnings) - { - if (context == null || context.getUser().hasRootPermission(TroubleshooterPermission.class)) - { - if (!isEncryptionPassPhraseSpecified() || showAllWarnings) - warnings.add(HtmlStringBuilder.of("The encryption key property is not set in " + AppProps.getInstance().getWebappConfigurationFilename() + - ". An encryption key is required to save credentials used in various integrations.").append(getEncryptionKeyHelpLink())); - - int count = DECRYPTION_EXCEPTIONS.get(); - - if (count > 0 || showAllWarnings) - warnings.add(HtmlStringBuilder.of("On " + StringUtilsLabKey.pluralize(count, "attempt") + - " the server failed to decrypt encrypted content using the " + - ENCRYPTION_KEY_CHANGED + " " + KEY_CHANGE_GUIDANCE).append(getEncryptionKeyHelpLink())); - } - } - - private HtmlStringBuilder getEncryptionKeyHelpLink() - { - return HtmlStringBuilder.of(" For more information, see ").append(new HelpTopic("labkeyxml", "encrypt").getSimpleLinkHtml("the Encryption Key documentation")).append("."); - } - }); - } - - private static final String TEST_ENCRYPTION_CATEGORY = "encryption-test"; - private static final String TEST_BYTES_NAME = "bytes"; - private static final int TEST_BYTES_LENGTH = 64; // Don't change unless you address backward compatibility - private static final int SHA1_LENGTH = 20; // Don't change hashing algorithm unless you address backward compatibility - - public static void initEncryptionKeyTest() - { - // Run test on a background thread... no need to block startup on a test that might take a few seconds - JobRunner.getDefault().execute(Encryption::testEncryptionKey); - } - - // Proactive test of the encryption key. On first run, encrypt, encode, and store a randomly generated byte string - // plus a SHA1 hash. On subsequent server startups, retrieve, decode, and verify contents are as expected. Any - // decryption failure causes the warning provider (above) to display an admin warning. - private static void testEncryptionKey() - { - if (isEncryptionPassPhraseSpecified()) - { - LOG.info("Attempting to test the integrity of the configured encryption key"); - - try - { - // On trial deployments, encryption key can change between initial bootstrap and the "new install" - // startup, so always delete if "newinstall" file is present. See Issue 48346. - // Use low-level deleteSetDirectly() method to skip decryption attempt and resulting admin warnings. - if (ModuleLoader.getInstance().isNewInstall()) - PropertyManager.deleteSetDirectly(PropertyManager.SHARED_USER, ContainerManager.getRoot().getId(), TEST_ENCRYPTION_CATEGORY, (EncryptedPropertyStore)PropertyManager.getEncryptedStore()); - - // This will likely throw if the encryption key has changed - WritablePropertyMap map = PropertyManager.getEncryptedStore().getWritableProperties(TEST_ENCRYPTION_CATEGORY, true); - - if (map.isEmpty()) - { - byte[] randomBytes = generateRandomBytes(TEST_BYTES_LENGTH); - MessageDigest sha1 = MessageDigest.getInstance("SHA1"); - byte[] hash = sha1.digest(randomBytes); - assert hash.length == SHA1_LENGTH; - byte[] combined = Bytes.concat(randomBytes, hash); - String base64 = Base64.encodeBase64String(combined); - map.put(TEST_BYTES_NAME, base64); - map.save(); - LOG.info("Encryption key test property was generated, encrypted, and stored. It will be tested on subsequent server startups."); - } - else - { - String base64 = map.get(TEST_BYTES_NAME); - byte[] combined = Base64.decodeBase64(base64); // This doesn't seem to throw, no matter what garbage you throw at it - if (combined.length != TEST_BYTES_LENGTH + SHA1_LENGTH) - { - // Base64 decoding problem -- log it and treat as a decryption failure - LOG.error("Encryption key test failed: Base64 decoding failed"); - logFailureGuidance(); - DECRYPTION_EXCEPTIONS.incrementAndGet(); - } - else - { - byte[] storedHash = Arrays.copyOfRange(combined, TEST_BYTES_LENGTH, TEST_BYTES_LENGTH + SHA1_LENGTH); - byte[] randomBytes = Arrays.copyOf(combined, TEST_BYTES_LENGTH); - MessageDigest sha1 = MessageDigest.getInstance("SHA1"); - byte[] hash = sha1.digest(randomBytes); - if (Arrays.equals(storedHash, hash)) - { - LOG.info("Encryption key test succeeded: encryption key has not changed"); - } - else - { - // Hashes didn't match -- log it and treat as a decryption failure - LOG.error("Encryption key test failed: SHA1 hashes did not match"); - logFailureGuidance(); - DECRYPTION_EXCEPTIONS.incrementAndGet(); - } - } - } - } - catch (DecryptionException de) - { - // getWritableProperties() has already incremented the exception count, so just log - LOG.error("Encryption key test failed: decryption of test property failed", de); - logFailureGuidance(); - } - catch (NoSuchAlgorithmException ae) - { - // All unexpected exceptions are handled by the JobRunner - throw new RuntimeException(ae); - } - } - } - - private static void logFailureGuidance() - { - LOG.error(KEY_CHANGE_GUIDANCE + " For more information, see " + new HelpTopic("labkeyxml", "encrypt").getHelpTopicHref() + "."); - } - - private Encryption() - { - } - - // Generate an array of random bytes of the specified length using SecureRandom - private static byte[] generateRandomBytes(int byteCount) - { - byte[] bytes = new byte[byteCount]; - SR.nextBytes(bytes); - - return bytes; - } - - - // Generates an encryption key having the specified bit length from a pass phrase, using PKCS #5 v2.0. This algorithm - // uses the lower 8-bits of each character to generate the key, which is appropriate for ASCII pass phrases. - private static byte[] generateSecretKeyFromPassPhrase(String passPhrase, byte[] salt, int keyLength, int iterationCount) throws NoSuchAlgorithmException, InvalidKeySpecException - { - SecretKeyFactory factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1"); - KeySpec spec = new PBEKeySpec(passPhrase.toCharArray(), salt, iterationCount, keyLength); - SecretKey skey = factory.generateSecret(spec); - - return skey.getEncoded(); - } - - - // Generates an encryption key having the specified bit length from a pass phrase, using PKCS #5 v2.0. Uses standard salt and iteration count of 65,536. - private static byte[] generateSecretKeyFromPassPhrase(String passPhrase, int keyLength) throws NoSuchAlgorithmException, InvalidKeySpecException - { - return generateSecretKeyFromPassPhrase(passPhrase, getStandardSalt(), keyLength, 65536); - } - - - // Returns a standard 16-byte random salt, generated once and unique to this database. - private static byte[] getStandardSalt() - { - PropertyStore store = PropertyManager.getNormalStore(); - Map props = store.getProperties(CATEGORY); - String salt = props.get(SALT_KEY); - - if (null != salt) - return Base64.decodeBase64(salt); - - WritablePropertyMap map = store.getWritableProperties(CATEGORY, true); - byte[] bytes = generateRandomBytes(16); - map.put(SALT_KEY, Base64.encodeBase64String(bytes)); - - // Seems very unlikely that we'd attempt to read encrypted data before we'd write any encrypted data on a production - // server, but it could happen during development. Allow this mutating ensure operation during a GET. - try (var ignored = SpringActionController.ignoreSqlUpdates()) - { - map.save(); - } - - return bytes; - } - - - private static final String ENCRYPTION_KEY_PARAMETER_NAME = "EncryptionKey"; - private static final String DEPRECATED_ENCRYPTION_KEY_PARAMETER_NAME = "MasterEncryptionKey"; - private static final String OLD_ENCRYPTION_KEY_PARAMETER_NAME = "OldEncryptionKey"; - - private static @Nullable String loadEncryptionProperty(String... propertyNames) - { - ServletContext context = ModuleLoader.getServletContext(); - - if (null == context) - throw new IllegalStateException("ServletContext is null"); - - String propertyValue = null; - - for (String name : propertyNames) - { - propertyValue = context.getInitParameter(name); - if (null != propertyValue) - break; - } - - return propertyValue; - } - - private static @Nullable String loadEncryptionPassPhrase() - { - String encryptionKey = loadEncryptionProperty(ENCRYPTION_KEY_PARAMETER_NAME, DEPRECATED_ENCRYPTION_KEY_PARAMETER_NAME); - - // Return the encryption key if it's there (not null, not blank, not whitespace, not default value), otherwise return null - if (!StringUtils.isBlank(encryptionKey) && !encryptionKey.trim().equals("@@masterEncryptionKey@@") && !encryptionKey.trim().equals("@@encryptionKey@@")) - return encryptionKey; - else - return null; - } - - public static @Nullable String getEncryptionPassPhrase() - { - return ENCRYPTION_PASS_PHRASE; - } - - public static @Nullable String getOldEncryptionPassPhrase() - { - return loadEncryptionProperty(OLD_ENCRYPTION_KEY_PARAMETER_NAME); - } - - public static boolean isEncryptionPassPhraseSpecified() - { - return null != getEncryptionPassPhrase(); - } - - public interface Algorithm - { - @NotNull byte[] encrypt(@NotNull String plainText); - @NotNull String decrypt(@NotNull byte[] cipherText); - } - - /* - Wrapper class that makes it easier to encrypt/decrypt using AES and a pass phrase. - - Encryption: AES - Mode of operation: CBC - Padding: PKCS #5 - Initialization vector: random 16-byte IV, generated for each encryption - - Key generation: PKCS #5 v2.0 - Salt: Standard server salt - Iteration count: 65,536 - Key length: specified in constructor parameter - */ - public static class AES implements Algorithm - { - private final @Nullable SecretKeySpec _keySpec; - private final String _keySource; - - public AES(String passPhrase, int keyLength, String keySource) - { - _keySource = keySource; - if (null == passPhrase) - throw new IllegalStateException("Pass phrase cannot be null"); - - // Turn pass phrase into a keyLength-bit key using PKCS #5 v2.0, a standard salt and 65,536 iterations - try - { - byte[] key = generateSecretKeyFromPassPhrase(passPhrase, keyLength); - _keySpec = new SecretKeySpec(key, "AES"); - } - catch (NoSuchAlgorithmException | InvalidKeySpecException e) - { - throw new RuntimeException(e); - } - } - - @NotNull - @Override - public byte[] encrypt(@NotNull String plainText) - { - try - { - // Generate a random, 16-byte initialization vector (IV) for use with this one encryption - byte[] iv = generateRandomBytes(16); - - Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); - cipher.init(Cipher.ENCRYPT_MODE, _keySpec, new IvParameterSpec(iv)); - - // First 16 bytes is the iv, remainder is the encrypted bytes - return ArrayUtils.addAll(iv, cipher.doFinal(plainText.getBytes(StringUtilsLabKey.DEFAULT_CHARSET))); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - - @NotNull - @Override - public String decrypt(@NotNull byte[] cipherText) - { - try - { - // Initialization vector (IV) is the first 16 bytes - byte[] iv = ArrayUtils.subarray(cipherText, 0, 16); - byte[] encrypted = ArrayUtils.subarray(cipherText, 16, cipherText.length); - Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); - cipher.init(Cipher.DECRYPT_MODE, _keySpec, new IvParameterSpec(iv)); - return new String(cipher.doFinal(encrypted), StringUtilsLabKey.DEFAULT_CHARSET); - } - catch (BadPaddingException e) - { - // For now, assume that BadPaddingException means the key has been changed and all other - // exceptions are coding issues. That might change in the future... - - // Track all decryption exceptions that aren't caused by TestCase (below) - if (ENCRYPTION_KEY_CHANGED.equals(_keySource)) - DECRYPTION_EXCEPTIONS.incrementAndGet(); - - throw new DecryptionException("Could not decrypt this content using the " + _keySource, e); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - } - - private static final String ENCRYPTION_KEY_CHANGED = "currently configured EncryptionKey; has the key changed in " + AppProps.getInstance().getWebappConfigurationFilename() + "?"; - private static final AtomicInteger DECRYPTION_EXCEPTIONS = new AtomicInteger(0); - - public static class DecryptionException extends ConfigurationException - { - public DecryptionException(String message, Throwable cause) - { - super(message, cause); - } - } - - /** - * Return standard AES encryption algorithm. Generates a 128-bit key from the application.properties encryption key. - * All other encryption parameters are documented in AES(). Pass in a registered EncryptionMigrationHandler to prove - * that you can migrate your encrypted content. - */ - public static Algorithm getAES128(EncryptionMigrationHandler handler) - { - // Ensure that every user of AES128 has registered an EncryptionMigrationHandler - assert null != handler && (EncryptionMigrationHandler.HANDLERS.contains(handler) || handler == TEST_HANDLER); - - if (isEncryptionPassPhraseSpecified()) - return new AES(getEncryptionPassPhrase(), 128, ENCRYPTION_KEY_CHANGED); - else - throw new IllegalStateException("EncryptionKey has not been specified in " + AppProps.getInstance().getWebappConfigurationFilename() + "; this method should not be called"); - } - - /** - * Same as above, but caller specifies the pass phrase. Used for one special case: migrating encrypted properties - * and settings after changing an encryption key. See {@link EncryptionMigrationHandler}. - */ - public static Algorithm getAES128(String encryptionPassPhrase, String keySource) - { - return new AES(encryptionPassPhrase, 128, keySource); - } - - public interface EncryptionMigrationHandler - { - Set HANDLERS = new ConcurrentHashSet<>(); - - static void registerHandler(EncryptionMigrationHandler handler) - { - HANDLERS.add(handler); - } - - void migrateEncryptedContent(String oldPassPhrase, String keySource); - } - - public static void checkMigration() - { - String oldPassPhrase = getOldEncryptionPassPhrase(); - - if (null != oldPassPhrase && isEncryptionPassPhraseSpecified()) - { - String keySource = "OldEncryptionKey specified in " + AppProps.getInstance().getWebappConfigurationFilename(); - LOG.info("OldEncryptionKey was found in " + AppProps.getInstance().getWebappConfigurationFilename() + - ". Attempting to migrate existing encrypted content from OldEncryptionKey to EncryptionKey."); - - EncryptionMigrationHandler.HANDLERS - .forEach(handler -> handler.migrateEncryptedContent(oldPassPhrase, keySource)); - - CacheManager.clearAllKnownCaches(); - LOG.info("Migration of all existing encrypted content from OldEncryptionKey to EncryptionKey is complete"); - LOG.info("IMPORTANT: Since migration is complete you should now remove the " + keySource); - } - } - - private static final EncryptionMigrationHandler TEST_HANDLER = (oldPassPhrase, keySource) -> {}; - - public static class TestCase extends Assert - { - @Test - public void testEncryptionAlgorithms() throws NoSuchAlgorithmException - { - String passPhrase = "Here's my super secret pass phrase"; - - Algorithm aesPassPhrase = new AES(passPhrase, 128, "test pass phrase"); - - test(aesPassPhrase); - - if (isEncryptionPassPhraseSpecified()) - { - Algorithm aes = getAES128(TEST_HANDLER); - test(aes); - - // Test that static factory method matches this configuration - Algorithm aes2 = new AES(getEncryptionPassPhrase(), 128, "test pass phrase"); - - test(aes, aes2); - test(aes2, aes); - } - - if (Cipher.getMaxAllowedKeyLength("AES") >= 256) - { - test(new AES(passPhrase, 256, "test pass phrase")); - } - } - - @Test(expected = DecryptionException.class) - public void testBadKeyException() - { - String textToEncrypt = "this is some text I want to encrypt"; - String passPhrase = "Here's my super secret pass phrase"; - String wrongPassPhrase = passPhrase + " not"; - - // Our AES implementation can usually detect a bad pass phrase (based on padding anomalies), but this is not 100% guaranteed. - // Give the test three tries... by my calculations, this will fail once in every 2.6 million runs, which we can live with. - for (int i = 0; i < 3; i++) - { - Algorithm aesPassPhrase = new AES(passPhrase, 128, "test pass phrase"); - byte[] encrypted = aesPassPhrase.encrypt(textToEncrypt); - - Algorithm aesWrongPassPhrase = new AES(wrongPassPhrase, 128, "test pass phrase"); - aesWrongPassPhrase.decrypt(encrypted); - } - } - - private void test(Algorithm algorithm) - { - test(algorithm, algorithm); - } - - private void test(Algorithm encryptAlgorithm, Algorithm decryptAlgorithm) - { - for (String test : new String[]{"foo", "bar", "this is some text I want to encrypt"}) - assertEquals(test, decryptAlgorithm.decrypt(encryptAlgorithm.encrypt(test))); - } - } -} +/* + * Copyright (c) 2013-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.security; + +import com.google.common.primitives.Bytes; +import jakarta.servlet.ServletContext; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.ConcurrentHashSet; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.EncryptedPropertyStore; +import org.labkey.api.data.NormalPropertyStore; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.PropertyManager.WritablePropertyMap; +import org.labkey.api.data.PropertyStore; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.security.permissions.TroubleshooterPermission; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.HelpTopic; +import org.labkey.api.util.HtmlStringBuilder; +import org.labkey.api.util.JobRunner; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.template.WarningProvider; +import org.labkey.api.view.template.WarningService; +import org.labkey.api.view.template.Warnings; + +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.spec.AlgorithmParameterSpec; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.KeySpec; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Easy to use wrappers for common encryption algorithms. Also includes related helper methods for shared operations + * such as generating salts & keys, and for retrieving & saving the application.properties encryption key and standard salt. + * WARNING: Do not change the core algorithms or parameters of existing implementations; changes will likely + * render existing data irrecoverable. +*/ + +public class Encryption +{ + private static final Logger LOG = LogHelper.getLogger(Encryption.class, "Encryption operations"); + private static final String CATEGORY = "Encryption"; + private static final String SALT_KEY = "Salt"; + public static final SecureRandom SR; + private static final String ENCRYPTION_PASS_PHRASE; + private static String KEY_CHANGE_GUIDANCE = "An administrator should change the encryption key back to the previous value, follow the official encryption key change process, or be prepared to re-enter and re-save all saved credentials."; + + static + { + ENCRYPTION_PASS_PHRASE = loadEncryptionPassPhrase(); + + try + { + SR = SecureRandom.getInstanceStrong(); + } + catch (NoSuchAlgorithmException e) + { + throw new ConfigurationException("Could not initialize SecureRandom", e); + } + + WarningService.get().register(new WarningProvider() { + @Override + public void addDynamicWarnings(@NotNull Warnings warnings, @Nullable ViewContext context, boolean showAllWarnings) + { + if (context == null || context.getUser().hasRootPermission(TroubleshooterPermission.class)) + { + if (!isEncryptionPassPhraseSpecified() || showAllWarnings) + warnings.add(HtmlStringBuilder.of("The encryption key property is not set in " + AppProps.getInstance().getWebappConfigurationFilename() + + ". An encryption key is required to save credentials used in various integrations.").append(getEncryptionKeyHelpLink())); + + int count = DECRYPTION_EXCEPTIONS.get(); + + if (count > 0 || showAllWarnings) + warnings.add(HtmlStringBuilder.of("On " + StringUtilsLabKey.pluralize(count, "attempt") + + " the server failed to decrypt encrypted content using the " + + ENCRYPTION_KEY_CHANGED + " " + KEY_CHANGE_GUIDANCE).append(getEncryptionKeyHelpLink())); + } + } + + private HtmlStringBuilder getEncryptionKeyHelpLink() + { + return HtmlStringBuilder.of(" For more information, see ").append(new HelpTopic("labkeyxml", "encrypt").getSimpleLinkHtml("the Encryption Key documentation")).append("."); + } + }); + } + + + // Used to keep track of the cipher configuration we've used, allowing us to reliably upgrade from one to another + private static final String ENCRYPTION_CIPHER_CATEGORY = "encryption-cipher"; + private static final String CIPHER_PROPERTY = "cipher"; + + // Used to check the key against a testable value stored in the database + private static final String TEST_ENCRYPTION_CATEGORY = "encryption-test"; + private static final String TEST_BYTES_NAME = "bytes"; + private static final int TEST_BYTES_LENGTH = 64; // Don't change unless you address backward compatibility + private static final int SHA1_LENGTH = 20; // Don't change hashing algorithm unless you address backward compatibility + + public static void initEncryptionKeyTest() + { + // Run test on a background thread... no need to block startup on a test that might take a few seconds + JobRunner.getDefault().execute(Encryption::testEncryptionKey); + } + + // Proactive test of the encryption key. On first run, encrypt, encode, and store a randomly generated byte string + // plus a SHA1 hash. On subsequent server startups, retrieve, decode, and verify contents are as expected. Any + // decryption failure causes the warning provider (above) to display an admin warning. + private static void testEncryptionKey() + { + if (isEncryptionPassPhraseSpecified()) + { + testEncryptionKey(getEncryptionPassPhrase(), AESConfig.current, "configured encryption key", true); + } + } + + // Test encryption key with specific passphrase and AES config. Used for testing old keys before migration. + private static void testEncryptionKey(String passPhrase, AESConfig config, String keyDescription, boolean createIfNotSet) + { + if (passPhrase != null) + { + LOG.info("Attempting to test the integrity of the " + keyDescription); + + try + { + // Create a temporary encrypted property store using the specified passphrase and config + Algorithm testAlgorithm = new AES(passPhrase, 128, keyDescription, config); + + // On trial deployments, encryption key can change between initial bootstrap and the "new install" + // startup, so always delete if "newinstall" file is present. See Issue 48346. + // Use low-level deleteSetDirectly() method to skip decryption attempt and resulting admin warnings. + if (ModuleLoader.getInstance().isNewInstall()) + PropertyManager.deleteSetDirectly(PropertyManager.SHARED_USER, ContainerManager.getRoot().getId(), TEST_ENCRYPTION_CATEGORY, (EncryptedPropertyStore)PropertyManager.getEncryptedStore()); + + // Get the raw values from the store so we can attempt decrypt with the old setup + PropertyManager.PropertyMap rawMap = new NormalPropertyStore() + { + @Override + protected boolean isValidPropertyMap(PropertyManager.PropertyMap props) + { + return true; + } + }.getProperties(TEST_ENCRYPTION_CATEGORY); + + + if (rawMap.isEmpty()) + { + if (createIfNotSet) + { + WritablePropertyMap map = PropertyManager.getEncryptedStore().getWritableProperties(TEST_ENCRYPTION_CATEGORY, true); + byte[] randomBytes = generateRandomBytes(TEST_BYTES_LENGTH); + MessageDigest sha1 = MessageDigest.getInstance("SHA1"); + byte[] hash = sha1.digest(randomBytes); + assert hash.length == SHA1_LENGTH; + byte[] combined = Bytes.concat(randomBytes, hash); + String base64 = Base64.encodeBase64String(combined); + map.put(TEST_BYTES_NAME, base64); + map.save(); + LOG.info("Encryption key test property was generated, encrypted, and stored. It will be tested on subsequent server startups."); + } + } + else + { + String rawEncryptedValue = rawMap.get(TEST_BYTES_NAME); + String decrypted = testAlgorithm.decrypt(Base64.decodeBase64(rawEncryptedValue)); + + byte[] combined = Base64.decodeBase64(decrypted); // This doesn't seem to throw, no matter what garbage you throw at it + if (combined.length != TEST_BYTES_LENGTH + SHA1_LENGTH) + { + // Base64 decoding problem -- log it and treat as a decryption failure + LOG.error("Encryption key test failed: Base64 decoding failed"); + logFailureGuidance(); + DECRYPTION_EXCEPTIONS.incrementAndGet(); + } + else + { + byte[] storedHash = Arrays.copyOfRange(combined, TEST_BYTES_LENGTH, TEST_BYTES_LENGTH + SHA1_LENGTH); + byte[] randomBytes = Arrays.copyOf(combined, TEST_BYTES_LENGTH); + MessageDigest sha1 = MessageDigest.getInstance("SHA1"); + byte[] hash = sha1.digest(randomBytes); + if (Arrays.equals(storedHash, hash)) + { + LOG.info("Encryption key test succeeded"); + } + else + { + // Hashes didn't match -- log it and treat as a decryption failure + LOG.error("Encryption key test failed: SHA1 hashes did not match"); + logFailureGuidance(); + DECRYPTION_EXCEPTIONS.incrementAndGet(); + } + } + } + } + catch (DecryptionException de) + { + LOG.error("Encryption key test failed: decryption of test property failed", de); + DECRYPTION_EXCEPTIONS.incrementAndGet(); + logFailureGuidance(); + } + catch (NoSuchAlgorithmException ae) + { + // All unexpected exceptions are handled by the JobRunner + throw new RuntimeException(ae); + } + } + } + + private static void logFailureGuidance() + { + LOG.error(KEY_CHANGE_GUIDANCE + " For more information, see " + new HelpTopic("labkeyxml", "encrypt").getHelpTopicHref() + "."); + } + + private Encryption() + { + } + + // Generate an array of random bytes of the specified length using SecureRandom + private static byte[] generateRandomBytes(int byteCount) + { + byte[] bytes = new byte[byteCount]; + SR.nextBytes(bytes); + + return bytes; + } + + + // Generates an encryption key having the specified bit length from a pass phrase, using PKCS #5 v2.0. This algorithm + // uses the lower 8-bits of each character to generate the key, which is appropriate for ASCII pass phrases. + private static byte[] generateSecretKeyFromPassPhrase(String passPhrase, byte[] salt, int keyLength, int iterationCount) throws NoSuchAlgorithmException, InvalidKeySpecException + { + SecretKeyFactory factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1"); + KeySpec spec = new PBEKeySpec(passPhrase.toCharArray(), salt, iterationCount, keyLength); + SecretKey skey = factory.generateSecret(spec); + + return skey.getEncoded(); + } + + + // Generates an encryption key having the specified bit length from a pass phrase, using PKCS #5 v2.0. Uses standard salt and iteration count of 65,536. + private static byte[] generateSecretKeyFromPassPhrase(String passPhrase, int keyLength) throws NoSuchAlgorithmException, InvalidKeySpecException + { + return generateSecretKeyFromPassPhrase(passPhrase, getStandardSalt(), keyLength, 65536); + } + + + // Returns a standard 16-byte random salt, generated once and unique to this database. + private static byte[] getStandardSalt() + { + PropertyStore store = PropertyManager.getNormalStore(); + Map props = store.getProperties(CATEGORY); + String salt = props.get(SALT_KEY); + + if (null != salt) + return Base64.decodeBase64(salt); + + WritablePropertyMap map = store.getWritableProperties(CATEGORY, true); + byte[] bytes = generateRandomBytes(16); + map.put(SALT_KEY, Base64.encodeBase64String(bytes)); + + // Seems very unlikely that we'd attempt to read encrypted data before we'd write any encrypted data on a production + // server, but it could happen during development. Allow this mutating ensure operation during a GET. + try (var ignored = SpringActionController.ignoreSqlUpdates()) + { + map.save(); + } + + return bytes; + } + + + private static final String ENCRYPTION_KEY_PARAMETER_NAME = "EncryptionKey"; + private static final String DEPRECATED_ENCRYPTION_KEY_PARAMETER_NAME = "MasterEncryptionKey"; + private static final String OLD_ENCRYPTION_KEY_PARAMETER_NAME = "OldEncryptionKey"; + + private static @Nullable String loadEncryptionProperty(String... propertyNames) + { + ServletContext context = ModuleLoader.getServletContext(); + + if (null == context) + throw new IllegalStateException("ServletContext is null"); + + String propertyValue = null; + + for (String name : propertyNames) + { + propertyValue = context.getInitParameter(name); + if (null != propertyValue) + break; + } + + return propertyValue; + } + + private static @Nullable String loadEncryptionPassPhrase() + { + String encryptionKey = loadEncryptionProperty(ENCRYPTION_KEY_PARAMETER_NAME, DEPRECATED_ENCRYPTION_KEY_PARAMETER_NAME); + + // Return the encryption key if it's there (not null, not blank, not whitespace, not default value), otherwise return null + if (!StringUtils.isBlank(encryptionKey) && !encryptionKey.trim().equals("@@masterEncryptionKey@@") && !encryptionKey.trim().equals("@@encryptionKey@@")) + return encryptionKey; + else + return null; + } + + public static @Nullable String getEncryptionPassPhrase() + { + return ENCRYPTION_PASS_PHRASE; + } + + public static @Nullable String getOldEncryptionPassPhrase() + { + return loadEncryptionProperty(OLD_ENCRYPTION_KEY_PARAMETER_NAME); + } + + public static boolean isEncryptionPassPhraseSpecified() + { + return null != getEncryptionPassPhrase(); + } + + public interface Algorithm + { + byte @NotNull[] encrypt(@NotNull String plainText); + @NotNull String decrypt(byte @NotNull[] cipherText); + } + + public enum AESConfig + { + /** Our preferred config */ + current(12, "AES/GCM/NoPadding") + { + @Override + protected AlgorithmParameterSpec createIvSpec(byte[] iv) + { + return new GCMParameterSpec(128, iv); + } + }, + /** Old config needed for upgrading existing values */ + legacy(16, "AES/CBC/PKCS5Padding") + { + @Override + protected AlgorithmParameterSpec createIvSpec(byte[] iv) + { + return new IvParameterSpec(iv); + } + }; + + private final int _ivLength; + private final String _cipherName; + + AESConfig(int ivLength, String cipherName) + { + _ivLength = ivLength; + _cipherName = cipherName; + } + + public int getIvLength() + { + return _ivLength; + } + + public String getCipherName() + { + return _cipherName; + } + + protected abstract AlgorithmParameterSpec createIvSpec(byte[] iv); + } + + /* + Wrapper class that makes it easier to encrypt/decrypt using AES and a pass phrase. + Salt: Standard server salt + Iteration count: 65,536 + Key length: specified in constructor parameter + */ + public static class AES implements Algorithm + { + private final @Nullable SecretKeySpec _keySpec; + private final String _keySource; + private final AESConfig _config; + + public AES(String passPhrase, int keyLength, String keySource) + { + this(passPhrase, keyLength, keySource, AESConfig.current); + } + + public AES(String passPhrase, int keyLength, String keySource, AESConfig config) + { + _keySource = keySource; + _config = config; + if (null == passPhrase) + throw new IllegalStateException("Pass phrase cannot be null"); + + // Turn pass phrase into a keyLength-bit key using PKCS #5 v2.0, a standard salt and 65,536 iterations + try + { + byte[] key = generateSecretKeyFromPassPhrase(passPhrase, keyLength); + _keySpec = new SecretKeySpec(key, "AES"); + } + catch (NoSuchAlgorithmException | InvalidKeySpecException e) + { + throw new RuntimeException(e); + } + } + + @Override + public byte @NotNull[] encrypt(@NotNull String plainText) + { + try + { + // Generate a random initialization vector (IV) for use with this one encryption + byte[] iv = generateRandomBytes(_config.getIvLength()); + + Cipher cipher = Cipher.getInstance(_config.getCipherName()); + cipher.init(Cipher.ENCRYPT_MODE, _keySpec, _config.createIvSpec(iv)); + + // First bytes is the iv, remainder is the encrypted bytes + return ArrayUtils.addAll(iv, cipher.doFinal(plainText.getBytes(StringUtilsLabKey.DEFAULT_CHARSET))); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + @NotNull + @Override + public String decrypt(byte @NotNull[] cipherText) + { + try + { + // Initialization vector (IV) is the first bytes according to config + int ivLength = _config.getIvLength(); + byte[] iv = ArrayUtils.subarray(cipherText, 0, ivLength); + byte[] encrypted = ArrayUtils.subarray(cipherText, ivLength, cipherText.length); + Cipher cipher = Cipher.getInstance(_config.getCipherName()); + cipher.init(Cipher.DECRYPT_MODE, _keySpec, _config.createIvSpec(iv)); + return new String(cipher.doFinal(encrypted), StringUtilsLabKey.DEFAULT_CHARSET); + } + catch (BadPaddingException e) + { + // For now, assume that BadPaddingException means the key has been changed and all other + // exceptions are coding issues. That might change in the future... + + // Track all decryption exceptions that aren't caused by TestCase (below) + if (ENCRYPTION_KEY_CHANGED.equals(_keySource)) + DECRYPTION_EXCEPTIONS.incrementAndGet(); + + throw new DecryptionException("Could not decrypt this content using the " + _keySource, e); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + } + + private static final String ENCRYPTION_KEY_CHANGED = "currently configured EncryptionKey; has the key changed in " + AppProps.getInstance().getWebappConfigurationFilename() + "?"; + private static final AtomicInteger DECRYPTION_EXCEPTIONS = new AtomicInteger(0); + + public static class DecryptionException extends ConfigurationException + { + public DecryptionException(String message, Throwable cause) + { + super(message, cause); + } + } + + /** + * Return standard AES encryption algorithm. Generates a 128-bit key from the application.properties encryption key. + * All other encryption parameters are documented in AES(). Pass in a registered EncryptionMigrationHandler to prove + * that you can migrate your encrypted content. + */ + public static Algorithm getAES128(EncryptionMigrationHandler handler) + { + // Ensure that every user of AES128 has registered an EncryptionMigrationHandler + assert null != handler && (EncryptionMigrationHandler.HANDLERS.contains(handler) || handler == TEST_HANDLER); + + if (isEncryptionPassPhraseSpecified()) + return new AES(getEncryptionPassPhrase(), 128, ENCRYPTION_KEY_CHANGED); + else + throw new IllegalStateException("EncryptionKey has not been specified in " + AppProps.getInstance().getWebappConfigurationFilename() + "; this method should not be called"); + } + + /** + * Same as above, but caller specifies the pass phrase and AES configuration. Used for migrating encrypted content + * from one AES configuration to another. + */ + public static Algorithm getAES128(String encryptionPassPhrase, String keySource, AESConfig config) + { + return new AES(encryptionPassPhrase, 128, keySource, config); + } + + public interface EncryptionMigrationHandler + { + Set HANDLERS = new ConcurrentHashSet<>(); + + static void registerHandler(EncryptionMigrationHandler handler) + { + HANDLERS.add(handler); + } + + void migrateEncryptedContent(String oldPassPhrase, String keySource, AESConfig oldConfig); + } + + public static void checkMigration() + { + String oldPassPhrase = getOldEncryptionPassPhrase(); + AESConfig oldConfig = AESConfig.current; + + if (isEncryptionPassPhraseSpecified()) + { + boolean migrationNeeded = false; + String keySource = null; + + if (null != oldPassPhrase) + { + keySource = "OldEncryptionKey specified in " + AppProps.getInstance().getWebappConfigurationFilename(); + LOG.info("{}. Attempting to migrate existing encrypted content from OldEncryptionKey to EncryptionKey.", keySource); + migrationNeeded = true; + KEY_CHANGE_GUIDANCE = KEY_CHANGE_GUIDANCE + " Note that if a migration has already been performed, the OldEncryptionKey value should be removed."; + } + + PropertyManager.WritablePropertyMap cipherProps = PropertyManager.getNormalStore().getWritableProperties(ENCRYPTION_CIPHER_CATEGORY, true); + String cipher = cipherProps.get(CIPHER_PROPERTY); + if (cipher == null) + { + migrationNeeded = true; + oldConfig = AESConfig.legacy; + LOG.info("Migrating existing encrypted content from legacy AES configuration to current AES configuration."); + } + else if (!cipher.equals(AESConfig.current.getCipherName())) + { + LOG.error("Unexpected cipher configuration: " + cipher); + } + + if (migrationNeeded) + { + final AESConfig migrationConfig = oldConfig; + final String message = keySource; + final String passPhrase = oldPassPhrase != null ? oldPassPhrase : getEncryptionPassPhrase(); + + // Test the old encryption key/algorithm before attempting migration + String testDescription = (keySource != null) ? "old encryption key" : "legacy AES algorithm"; + // Test but don't create a validation value if it doesn't already exist + testEncryptionKey(passPhrase, migrationConfig, testDescription, false); + if (DECRYPTION_EXCEPTIONS.get() == 0) + { + Encryption.EncryptionMigrationHandler.HANDLERS + .forEach(handler -> handler.migrateEncryptedContent(passPhrase, message, migrationConfig)); + + CacheManager.clearAllKnownCaches(); + } + // Test to validate conversion and create a validation value if needed + testEncryptionKey(); + } + + if (DECRYPTION_EXCEPTIONS.get() == 0) + { + if (oldPassPhrase != null) + { + LOG.info("Migration of all existing encrypted content from OldEncryptionKey to EncryptionKey is complete"); + LOG.info("IMPORTANT: Since migration is complete you should now remove the " + keySource); + } + if (cipher == null) + { + cipherProps.put(CIPHER_PROPERTY, AESConfig.current.getCipherName()); + cipherProps.save(); + LOG.info("Migration from existing encrypted content from legacy AES configuration to current AES configuration is complete."); + } + } + } + } + + + private static final EncryptionMigrationHandler TEST_HANDLER = (oldPassPhrase, keySource, oldConfig) -> {}; + + public static class TestCase extends Assert + { + @Test + public void testEncryptionAlgorithms() throws NoSuchAlgorithmException + { + String passPhrase = "Here's my super secret pass phrase"; + + Algorithm aesPassPhrase = new AES(passPhrase, 128, "test pass phrase"); + + test(aesPassPhrase); + + if (isEncryptionPassPhraseSpecified()) + { + Algorithm aes = getAES128(TEST_HANDLER); + test(aes); + + // Test that static factory method matches this configuration + Algorithm aes2 = new AES(getEncryptionPassPhrase(), 128, "test pass phrase"); + + test(aes, aes2); + test(aes2, aes); + } + + if (Cipher.getMaxAllowedKeyLength("AES") >= 256) + { + test(new AES(passPhrase, 256, "test pass phrase")); + } + } + + @Test(expected = DecryptionException.class) + public void testBadKeyException() + { + String textToEncrypt = "this is some text I want to encrypt"; + String passPhrase = "Here's my super secret pass phrase"; + String wrongPassPhrase = passPhrase + " not"; + + // Our AES implementation can usually detect a bad pass phrase (based on padding anomalies), but this is not 100% guaranteed. + // Give the test three tries... by my calculations, this will fail once in every 2.6 million runs, which we can live with. + for (int i = 0; i < 3; i++) + { + Algorithm aesPassPhrase = new AES(passPhrase, 128, "test pass phrase"); + byte[] encrypted = aesPassPhrase.encrypt(textToEncrypt); + + Algorithm aesWrongPassPhrase = new AES(wrongPassPhrase, 128, "test pass phrase"); + aesWrongPassPhrase.decrypt(encrypted); + } + } + + private void test(Algorithm algorithm) + { + test(algorithm, algorithm); + } + + private void test(Algorithm encryptAlgorithm, Algorithm decryptAlgorithm) + { + for (String test : new String[]{"foo", "bar", "this is some text I want to encrypt"}) + assertEquals(test, decryptAlgorithm.decrypt(encryptAlgorithm.encrypt(test))); + } + } +} diff --git a/api/src/org/labkey/api/view/FileServlet.java b/api/src/org/labkey/api/view/FileServlet.java index aa4638315a2..2e832937595 100644 --- a/api/src/org/labkey/api/view/FileServlet.java +++ b/api/src/org/labkey/api/view/FileServlet.java @@ -17,8 +17,10 @@ package org.labkey.api.view; import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.URLHelper; @@ -28,6 +30,8 @@ import jakarta.servlet.http.HttpServlet; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import org.labkey.api.util.logging.LogHelper; + import java.io.IOException; /** @@ -35,26 +39,32 @@ */ public class FileServlet extends HttpServlet { - private static final Logger _log = LogManager.getLogger(FileServlet.class); + private static final Logger _log = LogHelper.getLogger(FileServlet.class, "Forwards requests from /files to the FileContent module"); @Override protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { String pathInfo = StringUtils.trimToEmpty(request.getPathInfo()); - int index = pathInfo.lastIndexOf("/@"); // new style URL's: /files//@files// + int index = pathInfo.lastIndexOf("/@"); // new style URL's: /files//@files// or /files//@files//?fileName= if (index < 0) - index = pathInfo.lastIndexOf('/'); // legacy style: /files// + index = pathInfo.lastIndexOf('/'); // legacy style: /files// or /files//?fileName= if (index < 0) { response.setStatus(HttpServletResponse.SC_NOT_FOUND); return; } - //pathInfo is /${extraPath}/${fileName} + //pathInfo is />/ String fileNameParam = StringUtils.trimToNull(request.getParameter("fileName")); String fileName = pathInfo.substring(index + 1); - String extraPath = pathInfo.substring(0, index); + String containerPath = pathInfo.substring(0, index); + Container c = ContainerManager.getForPath(containerPath); + if (c == null) + { + response.setStatus(HttpServletResponse.SC_NOT_FOUND); + return; + } // Store the original URL in case we need to redirect for authentication if (request.getAttribute(ViewServlet.ORIGINAL_URL_STRING) == null) @@ -64,7 +74,9 @@ protected void service(HttpServletRequest request, HttpServletResponse response) request.setAttribute(ViewServlet.ORIGINAL_URL_URLHELPER, helper); } - String dispatchUrl = extraPath + "/filecontent-sendFile.view?" + (null == fileNameParam ? "fileName=" + PageFlowUtil.encodeURIComponent(fileName) : ""); + SimpleMetricsService.get().increment("API", "FileServlet", "urlsDispatched"); + String dispatchUrl = containerPath + "/filecontent-sendFile.view?" + (null == fileNameParam ? "fileName=" + PageFlowUtil.encodeURIComponent(fileName) : ""); + _log.info("FileServlet dispatching " + request.getRequestURL() + " to " + dispatchUrl); // NOTE other parameters seem to get magically propagated... RequestDispatcher r = request.getRequestDispatcher(dispatchUrl); r.forward(request, response); diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index 3dd84bbad36..ca983b4dba7 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -1,1841 +1,1844 @@ -/* - * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.core; - -import com.fasterxml.jackson.core.io.CharTypes; -import com.google.common.collect.Sets; -import jakarta.servlet.ServletContext; -import jakarta.servlet.ServletRegistration; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.pdfbox.pdmodel.font.FontMapper; -import org.apache.pdfbox.pdmodel.font.FontMappers; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.action.ApiJsonWriter; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AdminConsoleService; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.admin.HealthCheck; -import org.labkey.api.admin.HealthCheckRegistry; -import org.labkey.api.admin.TableXmlUtils; -import org.labkey.api.admin.notification.NotificationService; -import org.labkey.api.admin.sitevalidation.SiteValidationService; -import org.labkey.api.analytics.AnalyticsService; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.attachments.DocumentConversionService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.ClientApiAuditProvider; -import org.labkey.api.audit.DefaultAuditProvider; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.ContainerAuditProvider; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.audit.provider.GroupAuditProvider; -import org.labkey.api.audit.provider.ModulePropertiesAuditProvider; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.CompareType.CompareClause; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerService; -import org.labkey.api.data.ContainerServiceImpl; -import org.labkey.api.data.ContainerTypeRegistry; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DataColumn; -import org.labkey.api.data.DataRegion; -import org.labkey.api.data.DatabaseMigrationService; -import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationHandler; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.FileSqlScriptProvider; -import org.labkey.api.data.MvUtil; -import org.labkey.api.data.NormalContainerType; -import org.labkey.api.data.OutOfRangeDisplayColumn; -import org.labkey.api.data.PropertySchema; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SchemaTableInfoFactory; -import org.labkey.api.data.SimpleFilter.FilterClause; -import org.labkey.api.data.SimpleFilter.OrClause; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TSVWriter; -import org.labkey.api.data.TabContainerType; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TempTableTracker; -import org.labkey.api.data.TestSchema; -import org.labkey.api.data.WorkbookContainerType; -import org.labkey.api.data.dialect.BasePostgreSqlDialect; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.data.dialect.SqlDialectManager; -import org.labkey.api.data.dialect.SqlDialectRegistry; -import org.labkey.api.data.statistics.StatsService; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.TestDomainKind; -import org.labkey.api.external.tools.ExternalToolsViewService; -import org.labkey.api.files.FileBrowserConfigImporter; -import org.labkey.api.files.FileBrowserConfigWriter; -import org.labkey.api.files.FileContentService; -import org.labkey.api.markdown.MarkdownService; -import org.labkey.api.message.settings.MessageConfigService; -import org.labkey.api.module.FolderType; -import org.labkey.api.module.FolderTypeManager; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.module.SchemaUpdateType; -import org.labkey.api.module.SpringModule; -import org.labkey.api.module.Summary; -import org.labkey.api.notification.EmailMessage; -import org.labkey.api.notification.EmailService; -import org.labkey.api.notification.NotificationMenuView; -import org.labkey.api.portal.ProjectUrls; -import org.labkey.api.premium.AntiVirusProviderRegistry; -import org.labkey.api.products.ProductRegistry; -import org.labkey.api.qc.DataStateManager; -import org.labkey.api.query.DefaultSchema; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QuerySchema; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.reader.DataLoaderFactory; -import org.labkey.api.reader.DataLoaderService; -import org.labkey.api.reader.ExcelLoader; -import org.labkey.api.reader.FastaDataLoader; -import org.labkey.api.reader.HTMLDataLoader; -import org.labkey.api.reader.JSONDataLoader; -import org.labkey.api.reader.TabLoader; -import org.labkey.api.reports.LabKeyScriptEngineManager; -import org.labkey.api.resource.Resource; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.AuthenticationManager; -import org.labkey.api.security.AuthenticationManager.Priority; -import org.labkey.api.security.AuthenticationSettingsAuditTypeProvider; -import org.labkey.api.security.DbLoginService; -import org.labkey.api.security.DummyAntiVirusService; -import org.labkey.api.security.Group; -import org.labkey.api.security.GroupManager; -import org.labkey.api.security.LimitActiveUsersService; -import org.labkey.api.security.MutableSecurityPolicy; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.SecurityPointcutService; -import org.labkey.api.security.SecurityPolicyManager; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.security.WikiTermsOfUseProvider; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.QCAnalystPermission; -import org.labkey.api.security.permissions.TroubleshooterPermission; -import org.labkey.api.security.roles.NoPermissionsRole; -import org.labkey.api.security.roles.ReaderRole; -import org.labkey.api.security.roles.Role; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.CustomLabelService; -import org.labkey.api.settings.CustomLabelService.CustomLabelServiceImpl; -import org.labkey.api.settings.FolderSettingsCache; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.settings.LookAndFeelPropertiesManager; -import org.labkey.api.settings.LookAndFeelPropertiesManager.ResourceType; -import org.labkey.api.settings.LookAndFeelPropertiesManager.SiteResourceHandler; -import org.labkey.api.settings.OptionalFeatureFlag; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.settings.OptionalFeatureService.FeatureType; -import org.labkey.api.settings.ProductConfiguration; -import org.labkey.api.settings.StandardStartupPropertyHandler; -import org.labkey.api.settings.StartupPropertyEntry; -import org.labkey.api.settings.StashedStartupProperties; -import org.labkey.api.settings.WriteableAppProps; -import org.labkey.api.settings.WriteableLookAndFeelProperties; -import org.labkey.api.stats.AnalyticsProviderRegistry; -import org.labkey.api.stats.SummaryStatisticRegistry; -import org.labkey.api.study.Study; -import org.labkey.api.study.StudyService; -import org.labkey.api.thumbnail.ThumbnailService; -import org.labkey.api.usageMetrics.SimpleMetricsService; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.ContextListener; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JobRunner; -import org.labkey.api.util.MimeMap; -import org.labkey.api.util.MothershipReport; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.ShutdownListener; -import org.labkey.api.util.StartupListener; -import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.util.UsageReportingLevel; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.vcs.VcsService; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.AlwaysAvailableWebPartFactory; -import org.labkey.api.view.BaseWebPartFactory; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.Portal; -import org.labkey.api.view.Portal.WebPart; -import org.labkey.api.view.ShortURLService; -import org.labkey.api.view.VBox; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.ViewService; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.menu.FolderMenu; -import org.labkey.api.view.template.WarningService; -import org.labkey.api.webdav.SimpleDocumentResource; -import org.labkey.api.webdav.WebdavResolverImpl; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.api.webdav.WebdavService; -import org.labkey.api.wiki.WikiRenderingService; -import org.labkey.api.writer.ContainerUser; -import org.labkey.core.admin.ActionsTsvWriter; -import org.labkey.core.admin.AdminConsoleServiceImpl; -import org.labkey.core.admin.AdminController; -import org.labkey.core.admin.AllowListType; -import org.labkey.core.admin.CopyFileRootPipelineJob; -import org.labkey.core.admin.CustomizeMenuForm; -import org.labkey.core.admin.DisplayFormatAnalyzer; -import org.labkey.core.admin.DisplayFormatValidationProviderFactory; -import org.labkey.core.admin.FilesSiteSettingsAction; -import org.labkey.core.admin.MenuViewFactory; -import org.labkey.core.admin.OptionalFeatureServiceImpl; -import org.labkey.core.admin.OptionalFeatureStartupListener; -import org.labkey.core.admin.importer.FolderTypeImporterFactory; -import org.labkey.core.admin.importer.MissingValueImporterFactory; -import org.labkey.core.admin.importer.ModulePropertiesImporterFactory; -import org.labkey.core.admin.importer.PageImporterFactory; -import org.labkey.core.admin.importer.RoleAssignmentsImporterFactory; -import org.labkey.core.admin.importer.SearchSettingsImporterFactory; -import org.labkey.core.admin.importer.SecurityGroupImporterFactory; -import org.labkey.core.admin.importer.SubfolderImporterFactory; -import org.labkey.core.admin.logger.LoggerController; -import org.labkey.core.admin.logger.LoggingTestCase; -import org.labkey.core.admin.miniprofiler.MiniProfilerController; -import org.labkey.core.admin.sitevalidation.SiteValidationServiceImpl; -import org.labkey.core.admin.sql.SqlScriptController; -import org.labkey.core.admin.test.SchemaXMLTestCase; -import org.labkey.core.admin.test.UnknownSchemasTest; -import org.labkey.core.admin.usageMetrics.UsageMetricsServiceImpl; -import org.labkey.core.admin.writer.FolderSerializationRegistryImpl; -import org.labkey.core.admin.writer.FolderTypeWriterFactory; -import org.labkey.core.admin.writer.MissingValueWriterFactory; -import org.labkey.core.admin.writer.ModulePropertiesWriterFactory; -import org.labkey.core.admin.writer.PageWriterFactory; -import org.labkey.core.admin.writer.RoleAssignmentsWriterFactory; -import org.labkey.core.admin.writer.SearchSettingsWriterFactory; -import org.labkey.core.admin.writer.SecurityGroupWriterFactory; -import org.labkey.core.analytics.AnalyticsController; -import org.labkey.core.analytics.AnalyticsServiceImpl; -import org.labkey.core.attachment.AttachmentServiceImpl; -import org.labkey.core.dialect.PostgreSqlDialectFactory; -import org.labkey.core.dialect.PostgreSqlInClauseTest; -import org.labkey.core.dialect.PostgreSqlVersion; -import org.labkey.core.junit.JunitController; -import org.labkey.core.login.DbLoginAuthenticationProvider; -import org.labkey.core.login.DbLoginManager; -import org.labkey.core.login.LoginController; -import org.labkey.core.metrics.SimpleMetricsServiceImpl; -import org.labkey.core.metrics.WebSocketConnectionManager; -import org.labkey.core.notification.EmailPreferenceConfigServiceImpl; -import org.labkey.core.notification.EmailPreferenceContainerListener; -import org.labkey.core.notification.EmailPreferenceUserListener; -import org.labkey.core.notification.EmailServiceImpl; -import org.labkey.core.notification.NotificationController; -import org.labkey.core.notification.NotificationServiceImpl; -import org.labkey.core.portal.CollaborationFolderType; -import org.labkey.core.portal.PortalJUnitTest; -import org.labkey.core.portal.ProjectController; -import org.labkey.core.portal.UtilController; -import org.labkey.core.products.ProductController; -import org.labkey.core.project.FolderNavigationForm; -import org.labkey.core.qc.CoreQCStateHandler; -import org.labkey.core.qc.DataStateImporter; -import org.labkey.core.qc.DataStateWriter; -import org.labkey.core.query.AttachmentAuditProvider; -import org.labkey.core.query.CoreQuerySchema; -import org.labkey.core.query.PostgresUserSchema; -import org.labkey.core.query.UserAuditProvider; -import org.labkey.core.query.UsersDomainKind; -import org.labkey.core.reader.DataLoaderServiceImpl; -import org.labkey.core.reports.DocumentConversionServiceImpl; -import org.labkey.core.reports.ScriptEngineManagerImpl; -import org.labkey.core.script.RhinoService; -import org.labkey.core.security.AllowedExternalResourceHosts; -import org.labkey.core.security.ApiKeyViewProvider; -import org.labkey.core.security.SecurityApiActions; -import org.labkey.core.security.SecurityController; -import org.labkey.core.security.SecurityPointcutServiceImpl; -import org.labkey.core.security.validators.PermissionsValidatorFactory; -import org.labkey.core.statistics.AnalyticsProviderRegistryImpl; -import org.labkey.core.statistics.StatsServiceImpl; -import org.labkey.core.statistics.SummaryStatisticRegistryImpl; -import org.labkey.core.thumbnail.ThumbnailServiceImpl; -import org.labkey.core.user.LimitActiveUsersSettings; -import org.labkey.core.user.UserController; -import org.labkey.core.vcs.VcsServiceImpl; -import org.labkey.core.view.ShortURLServiceImpl; -import org.labkey.core.view.TableViewFormTestCase; -import org.labkey.core.view.external.tools.ExternalToolsViewServiceImpl; -import org.labkey.core.view.template.bootstrap.CoreWarningProvider; -import org.labkey.core.view.template.bootstrap.PageTemplate; -import org.labkey.core.view.template.bootstrap.ViewServiceImpl; -import org.labkey.core.view.template.bootstrap.WarningServiceImpl; -import org.labkey.core.webdav.DavController; -import org.labkey.core.webdav.ModuleStaticResolverImpl; -import org.labkey.core.webdav.WebFilesResolverImpl; -import org.labkey.core.webdav.WebdavServlet; -import org.labkey.core.wiki.MarkdownServiceImpl; -import org.labkey.core.wiki.MarkdownTestCase; -import org.labkey.core.wiki.RadeoxRenderer; -import org.labkey.core.wiki.WikiRenderingServiceImpl; -import org.labkey.core.workbook.WorkbookFolderType; -import org.labkey.core.workbook.WorkbookQueryView; -import org.labkey.core.workbook.WorkbookSearchView; -import org.labkey.filters.ContentSecurityPolicyFilter; -import org.quartz.Scheduler; -import org.quartz.SchedulerException; -import org.quartz.impl.StdSchedulerFactory; -import org.radeox.filter.regex.RegexFilter; -import org.radeox.test.BaseRenderEngineTest; -import org.radeox.test.filter.BasicRegexTest; -import org.radeox.test.filter.BoldFilterTest; -import org.radeox.test.filter.EscapeFilterTest; -import org.radeox.test.filter.FilterPipeTest; -import org.radeox.test.filter.HeadingFilterTest; -import org.radeox.test.filter.HtmlRemoveFilterTest; -import org.radeox.test.filter.ItalicFilterTest; -import org.radeox.test.filter.KeyFilterTest; -import org.radeox.test.filter.LineFilterTest; -import org.radeox.test.filter.LinkTestFilterTest; -import org.radeox.test.filter.ListFilterTest; -import org.radeox.test.filter.NewlineFilterTest; -import org.radeox.test.filter.ParamFilterTest; -import org.radeox.test.filter.SmileyFilterTest; -import org.radeox.test.filter.StrikeThroughFilterTest; -import org.radeox.test.filter.TypographyFilterTest; -import org.radeox.test.filter.UrlFilterTest; -import org.radeox.test.filter.WikiLinkFilterTest; -import org.radeox.test.macro.list.AtoZListFormatterTest; -import org.radeox.test.macro.list.ExampleListFormatterTest; -import org.radeox.test.macro.list.SimpleListTest; - -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.sql.Connection; -import java.sql.SQLException; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.labkey.api.settings.StashedStartupProperties.homeProjectFolderType; -import static org.labkey.api.settings.StashedStartupProperties.homeProjectResetPermissions; -import static org.labkey.api.settings.StashedStartupProperties.homeProjectWebparts; -import static org.labkey.api.settings.StashedStartupProperties.siteAvailableEmailFrom; -import static org.labkey.api.settings.StashedStartupProperties.siteAvailableEmailMessage; -import static org.labkey.api.settings.StashedStartupProperties.siteAvailableEmailSubject; -import static org.labkey.api.util.MothershipReport.EXPERIMENTAL_LOCAL_MARKETING_UPDATE; -import static org.labkey.api.util.MothershipReport.FEATURE_FLAG_EXTENDED_METRICS; -import static org.labkey.filters.ContentSecurityPolicyFilter.FEATURE_FLAG_DISABLE_ENFORCE_CSP; - -public class CoreModule extends SpringModule implements SearchService.DocumentProvider -{ - private static final Logger LOG = LogHelper.getLogger(CoreModule.class, "Errors during server startup and shut down"); - public static final String PROJECTS_WEB_PART_NAME = "Projects"; - - static - { - // Accept most of the standard Quartz properties, but set the misfire threshold to five minutes. This prevents - // Quartz from dropping scheduled work if a lot of items fire at the same time, like a lot of ETLs triggering at 2AM. - // This can overwhelm the thread pool running them so they don't complete in the default 1 minute window. Set it early so - // if any other module touches Quartz in its setup, it initializes with this setting. - Properties props = System.getProperties(); - props.setProperty("org.quartz.jobStore.misfireThreshold", "300000"); - - // Register dialect extra early, since we need to initialize the data sources before calling DefaultModule.initialize() - SqlDialectRegistry.register(new PostgreSqlDialectFactory()); - - try - { - var field = CharTypes.class.getDeclaredField("sOutputEscapes128"); - field.setAccessible(true); - ((int[])field.get(null))['/'] = '/'; - field.setAccessible(false); - } - catch (NoSuchFieldException|IllegalArgumentException|IllegalAccessException x) - { - // pass - } - } - - private CoreWarningProvider _warningProvider; - private ServletRegistration.Dynamic _webdavServletDynamic; - - @Override - public boolean hasScripts() - { - return true; - } - - @Override - protected void init() - { - ContainerService.setInstance(new ContainerServiceImpl()); - FolderSerializationRegistry.setInstance(new FolderSerializationRegistryImpl()); - ExternalToolsViewService.setInstance(new ExternalToolsViewServiceImpl()); - ExternalToolsViewService.get().registerExternalAccessViewProvider(new ApiKeyViewProvider()); - LimitActiveUsersService.setInstance(() -> new LimitActiveUsersSettings().isUserLimitReached()); - - // Register the default DataLoaders during init so they are available to sql upgrade scripts - DataLoaderServiceImpl dls = new DataLoaderServiceImpl(); - dls.registerFactory(new ExcelLoader.Factory()); - dls.registerFactory(new TabLoader.TsvFactory()); - dls.registerFactory(new TabLoader.CsvFactory()); - dls.registerFactory(new HTMLDataLoader.Factory()); - dls.registerFactory(new JSONDataLoader.Factory()); - dls.registerFactory(new FastaDataLoader.Factory()); - DataLoaderService.setInstance(dls); - - addController("admin", AdminController.class); - addController("admin-sql", SqlScriptController.class); - addController("security", SecurityController.class); - addController("user", UserController.class); - addController("login", LoginController.class); - addController("junit", JunitController.class); - addController("core", CoreController.class); - addController("analytics", AnalyticsController.class); - addController("project", ProjectController.class); - addController("util", UtilController.class); - addController("logger", LoggerController.class); - addController("mini-profiler", MiniProfilerController.class); - addController("notification", NotificationController.class); - addController("product", ProductController.class); - - AuthenticationManager.registerProvider(new DbLoginAuthenticationProvider(), Priority.Low); - AttachmentService.setInstance(new AttachmentServiceImpl()); - AnalyticsService.setInstance(new AnalyticsServiceImpl()); - RhinoService.register(); - CacheManager.addListener(RhinoService::clearCaches); - NotificationService.setInstance(NotificationServiceImpl.getInstance()); - - WarningService.setInstance(new WarningServiceImpl()); - - ViewService.setInstance(ViewServiceImpl.getInstance()); - OptionalFeatureService.setInstance(new OptionalFeatureServiceImpl()); - ThumbnailService.setInstance(new ThumbnailServiceImpl()); - ShortURLService.setInstance(new ShortURLServiceImpl()); - StatsService.setInstance(new StatsServiceImpl()); - SiteValidationService.setInstance(new SiteValidationServiceImpl()); - AnalyticsProviderRegistry.setInstance(new AnalyticsProviderRegistryImpl()); - SummaryStatisticRegistry.setInstance(new SummaryStatisticRegistryImpl()); - UsageMetricsService.setInstance(new UsageMetricsServiceImpl()); - CustomLabelService.setInstance(new CustomLabelServiceImpl()); - SecurityPointcutService.setInstance(new SecurityPointcutServiceImpl()); - AdminConsoleService.setInstance(new AdminConsoleServiceImpl()); - WikiRenderingService.setInstance(new WikiRenderingServiceImpl()); - VcsService.setInstance(new VcsServiceImpl()); - LabKeyScriptEngineManager.setInstance(new ScriptEngineManagerImpl()); - DocumentConversionService.setInstance(new DocumentConversionServiceImpl()); - DbLoginService.setInstance(new DbLoginManager()); - - try - { - ContainerTypeRegistry.get().register("normal", new NormalContainerType()); - ContainerTypeRegistry.get().register("tab", new TabContainerType()); - ContainerTypeRegistry.get().register("workbook", new WorkbookContainerType()); - } - catch (Exception e) - { - throw UnexpectedException.wrap(e); - } - - _warningProvider = new CoreWarningProvider(); - WarningService.get().register(_warningProvider); - - WebdavService.get().setResolver(ModuleStaticResolverImpl.get()); - // need to register webdav resolvers in init() instead of startupAfterSpringConfig since static module files are loaded during module startup - WebdavService.get().registerRootResolver(WebdavResolverImpl.get()); - WebdavService.get().registerRootResolver(WebFilesResolverImpl.get()); - - DefaultSchema.registerProvider(CoreQuerySchema.NAME, new DefaultSchema.SchemaProvider(this) - { - @Override - public boolean isAvailable(DefaultSchema schema, Module module) - { - return true; - } - - @Override - public QuerySchema createSchema(DefaultSchema schema, Module module) - { - return new CoreQuerySchema(schema.getUser(), schema.getContainer()); - } - }); - - if (CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) - { - DefaultSchema.registerProvider(BasePostgreSqlDialect.POSTGRES_SCHEMA_NAME, new DefaultSchema.SchemaProvider(this) - { - @Override - public boolean isAvailable(DefaultSchema schema, Module module) - { - return schema.getContainer().isRoot() && schema.getContainer().hasPermission(schema.getUser(), TroubleshooterPermission.class); - } - - @Override - public QuerySchema createSchema(DefaultSchema schema, Module module) - { - return new PostgresUserSchema(schema.getUser(), schema.getContainer()); - } - }); - } - - OptionalFeatureService.get().addExperimentalFeatureFlag(NotificationMenuView.EXPERIMENTAL_NOTIFICATION_MENU, "Notifications Menu", - "Notifications 'inbox' count display in the header bar with click to show the notifications panel of unread notifications.", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(DataColumn.EXPERIMENTAL_USE_QUERYSELECT_COMPONENT, "Use QuerySelect for row insert/update form", - "This feature will switch the query based select inputs on the row insert/update form to use the React QuerySelect" + - "component. This will allow for a user to view the first 100 options in the select but then use type ahead" + - "search to find the other select values.", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(SQLFragment.FEATUREFLAG_DISABLE_STRICT_CHECKS, "Disable SQLFragment strict checks", - "SQLFragment now has very strict usage validation, these checks may cause errors in code that has not been updated. Turn on this feature to disable checks.", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(LoginController.FEATUREFLAG_DISABLE_LOGIN_XFRAME, "Disable Login X-FRAME-OPTIONS=DENY", - "By default LabKey disables all framing of login related actions. Disabling this feature will revert to using the standard site settings.", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(PageTemplate.EXPERIMENTAL_SHORT_CIRCUIT_ROBOTS, - "Short-circuit robots", - "Save resources by not rendering pages marked as 'noindex' for robots. This is experimental as not all robots are search engines.", - false); - OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(TabLoader.FEATUREFLAG_UNESCAPE_BACKSLASH, - "Unescape backslash character on import", - "Treat backslash '\\' character as an escape character when loading data from file.", - false, false, OptionalFeatureService.FeatureType.Deprecated - )); - OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(AppProps.GENERATE_CONTROLLER_FIRST_URLS, - "Restore controller-first URLS", - "Generate URLs in a legacy format that puts the controller name before the folder path. This option will be removed in LabKey Server 26.3.", - false, false, OptionalFeatureService.FeatureType.Deprecated - )); - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.REJECT_CONTROLLER_FIRST_URLS, - "Reject controller-first URLs", - "Require standard path-first URLs. Note: This option will be ignored if the deprecated feature for generating controller-first URLs is enabled.", - false - ); - - SiteValidationService svc = SiteValidationService.get(); - if (null != svc) - { - svc.registerProviderFactory(getName(), new PermissionsValidatorFactory()); - svc.registerProviderFactory(getName(), new DisplayFormatValidationProviderFactory()); - } - - registerHealthChecks(); - - ContextListener.addNewInstallCompleteListener(() -> sendSystemReadyEmail(UserManager.getAppAdmins())); - - ScriptEngineManagerImpl.registerEncryptionMigrationHandler(); - - deleteTempFiles(); - } - - private void deleteTempFiles() - { - try - { - // Issue 46598 - clean up previously created temp files from file uploads - FileUtil.deleteDirectoryContents(SpringActionController.getTempUploadDir()); - } - catch (IOException e) - { - LOG.warn("Failed to clean up previously uploaded files from {}", SpringActionController.getTempUploadDir(), e); - } - } - - private void registerHealthChecks() - { - HealthCheckRegistry.get().registerHealthCheck("database", HealthCheckRegistry.DEFAULT_CATEGORY, () -> - { - Map healthValues = new HashMap<>(); - boolean allConnected = true; - for (DbScope dbScope : DbScope.getDbScopes()) - { - boolean dbConnected; - try (Connection conn = dbScope.getConnection()) - { - dbConnected = conn != null; - } - catch (SQLException e) - { - dbConnected = false; - } - - healthValues.put(dbScope.getDatabaseName(), dbConnected); - allConnected &= dbConnected; - } - - return new HealthCheck.Result(allConnected, healthValues); - } - ); - - HealthCheckRegistry.get().registerHealthCheck("modules", HealthCheckRegistry.TRIAL_INSTANCES_CATEGORY, () -> { - Map failures = ModuleLoader.getInstance().getModuleFailures(); - Map failureDetails = new HashMap<>(); - for (Map.Entry failure : failures.entrySet()) - { - failureDetails.put(failure.getKey(), failure.getValue().getMessage()); - } - return new HealthCheck.Result(failures.isEmpty(), failureDetails); - }); - - HealthCheckRegistry.get().registerHealthCheck("users", HealthCheckRegistry.TRIAL_INSTANCES_CATEGORY, () -> { - Map userHealth = new HashMap<>(); - ZonedDateTime now = ZonedDateTime.now(); - int userCount = UserManager.getUserCount(Date.from(now.toInstant())); - userHealth.put("registeredUsers", userCount); - return new HealthCheck.Result(userCount > 0, userHealth); - }); - } - - private void sendSystemReadyEmail(List users) - { - if (users.isEmpty()) - return; - - Map map = AppProps.getInstance().getStashedStartupProperties(); - String fromEmail = getValue(map, siteAvailableEmailFrom); - String subject = getValue(map, siteAvailableEmailSubject); - String body = getValue(map, siteAvailableEmailMessage); - - if (fromEmail == null || subject == null || body == null) - return; - - EmailService svc = EmailService.get(); - List messages = new ArrayList<>(); - for (User user: users) - { - EmailMessage message = svc.createMessage(fromEmail, Collections.singletonList(user.getEmail()), subject); - message.addContent(MimeMap.MimeType.HTML, body); - messages.add(message); - } - // For audit purposes, we use the first user as the originator of the message. - // Would be better to have this be a site admin, but we aren't guaranteed to have such a user - // for hosted sites. Another option is to use the guest user here, but that's strange. - svc.sendMessages(messages, users.get(0), ContainerManager.getRoot()); - } - - private @Nullable String getValue(Map map, StashedStartupProperties prop) - { - StartupPropertyEntry entry = map.get(prop); - return null != entry ? StringUtils.trimToNull(entry.getValue()) : null; - } - - @NotNull - @Override - protected Collection createWebPartFactories() - { - return List.of( - new AlwaysAvailableWebPartFactory("Contacts") - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext ctx, @NotNull WebPart webPart) - { - UserSchema schema = QueryService.get().getUserSchema(ctx.getUser(), ctx.getContainer(), CoreQuerySchema.NAME); - QuerySettings settings = new QuerySettings(ctx, QueryView.DATAREGIONNAME_DEFAULT); - - settings.setQueryName(CoreQuerySchema.USERS_TABLE_NAME); - - QueryView view = schema.createView(ctx, settings, null); - view.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); - view.setFrame(WebPartView.FrameType.PORTAL); - view.setTitle("Project Contacts"); - - return view; - } - }, - new BaseWebPartFactory("FolderNav") - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) - { - FolderNavigationForm form = getForm(portalCtx); - - form.setFolderMenu(new FolderMenu(portalCtx)); - - JspView view = new JspView<>("/org/labkey/core/project/folderNav.jsp", form); - view.setTitle("Folder Navigation"); - view.setFrame(WebPartView.FrameType.NONE); - return view; - } - - @Override - public boolean isAvailable(Container c, String scope, String location) - { - return false; - } - - private FolderNavigationForm getForm(ViewContext context) - { - FolderNavigationForm form = new FolderNavigationForm(); - form.setPortalContext(context); - return form; - } - }, - new BaseWebPartFactory("Workbooks") - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) - { - UserSchema schema = QueryService.get().getUserSchema(portalCtx.getUser(), portalCtx.getContainer(), SchemaKey.fromParts(CoreQuerySchema.NAME)); - WorkbookQueryView wbqview = new WorkbookQueryView(portalCtx, schema); - VBox box = new VBox(new WorkbookSearchView(wbqview), wbqview); - box.setFrame(WebPartView.FrameType.PORTAL); - box.setTitle("Workbooks"); - return box; - } - - @Override - public boolean isAvailable(Container c, String scope, String location) - { - return !c.isWorkbook() && "folder".equals(scope) && location.equalsIgnoreCase(HttpView.BODY); - } - }, - new BaseWebPartFactory("Workbook Description") - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) - { - JspView view = new JspView<>("/org/labkey/core/workbook/workbookDescription.jsp"); - view.setTitle("Workbook Description"); - view.setFrame(WebPartView.FrameType.NONE); - return view; - } - - @Override - public boolean isAvailable(Container c, String scope, String location) - { - return false; - } - }, - new AlwaysAvailableWebPartFactory(PROJECTS_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) - { - JspView view = new JspView<>("/org/labkey/core/project/projects.jsp", webPart); - - String title = webPart.getPropertyMap().getOrDefault("title", "Projects"); - view.setTitle(title); - - if (portalCtx.hasPermission(getClass().getName(), AdminPermission.class)) - { - NavTree customize = new NavTree(""); - customize.setScript("customizeProjectWebpart" + webPart.getRowId() + "();"); - view.setCustomize(customize); - } - return view; - } - }, - new AlwaysAvailableWebPartFactory("Subfolders", WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPart createWebPart() - { - // Issue 44913: Set the default properties for all new instances of the Subfolders webpart - WebPart webPart = super.createWebPart(); - return setDefaultProperties(webPart); - } - - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) - { - if (webPart.getPropertyMap().isEmpty()) - { - // Configure to show subfolders if not previously configured - webPart = setDefaultProperties(new WebPart(webPart)); - } - - JspView view = new JspView<>("/org/labkey/core/project/projects.jsp", webPart); - view.setTitle(webPart.getPropertyMap().get("title")); - - if (portalCtx.hasPermission(getClass().getName(), AdminPermission.class)) - { - NavTree customize = new NavTree(""); - customize.setScript("customizeProjectWebpart" + webPart.getRowId() + "(" + webPart.getRowId() + ", " + PageFlowUtil.jsString(webPart.getPageId()) + ", " + webPart.getIndex() + ");"); - view.setCustomize(customize); - } - - return view; - } - - private WebPart setDefaultProperties(WebPart webPart) - { - webPart.setProperty("title", "Subfolders"); - webPart.setProperty("containerFilter", ContainerFilter.Type.CurrentAndFirstChildren.name()); - webPart.setProperty("containerTypes", "folder"); - return webPart; - } - }, - new AlwaysAvailableWebPartFactory("Custom Menu", true, true, WebPartFactory.LOCATION_MENUBAR) - { - @Override - public WebPartView getWebPartView(@NotNull final ViewContext portalCtx, @NotNull WebPart webPart) - { - final CustomizeMenuForm form = AdminController.getCustomizeMenuForm(webPart); - String title = "My Menu"; - if (form.getTitle() != null && !form.getTitle().isEmpty()) - title = form.getTitle(); - - WebPartView view; - if (form.isChoiceListQuery()) - { - view = MenuViewFactory.createMenuQueryView(portalCtx, title, form); - } - else - { - view = MenuViewFactory.createMenuFolderView(portalCtx, title, form); - } - view.setFrame(WebPartView.FrameType.PORTAL); - return view; - } - - @Override - public HttpView getEditView(WebPart webPart, ViewContext context) - { - CustomizeMenuForm form = AdminController.getCustomizeMenuForm(webPart); - JspView view = new JspView<>("/org/labkey/core/admin/customizeMenu.jsp", form); - view.setTitle(form.getTitle()); - view.setFrame(WebPartView.FrameType.PORTAL); - return view; - } - } - ); - } - - @Override - public void afterUpdate(ModuleContext moduleContext) - { - if (moduleContext.isNewInstall()) - { - bootstrap(); - } - - // Increment on every core module upgrade to defeat browser caching of static resources. - if (ModuleLoader.getInstance().shouldInsertData()) - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - - // Allow dialect to make adjustments to the just upgraded core database (e.g., install aggregate functions, etc.) - CoreSchema.getInstance().getSqlDialect().afterCoreUpgrade(moduleContext); - - // The core SQL scripts install aggregate functions and other objects that dialects need to know about. Prepare - // the previously initialized dialects again to make sure they're aware of all the changes. Prepare all the - // initialized scopes because we could have more than one scope pointed at the core database (e.g., external - // schemas). See #17077 (pg example) and #19177 (ss example). - for (DbScope scope : DbScope.getInitializedDbScopes()) - scope.getSqlDialect().prepare(scope); - - // Now that we know the standard containers have been created, add a listener that warms the just-cleared caches with - // core.Containers metadata and a few common containers. This may prevent some deadlocks during upgrade, #33550. - CacheManager.addListener(() -> { - ContainerManager.getRoot(); - ContainerManager.getSharedContainer(); - if (ModuleLoader.getInstance().shouldInsertData()) - { - ContainerManager.getHomeContainer(); - } - }); - } - - private void bootstrap() - { - if (ModuleLoader.getInstance().shouldInsertData()) - { - // Create the initial groups - GroupManager.bootstrapGroup(Group.groupUsers, "Users"); - GroupManager.bootstrapGroup(Group.groupGuests, "Guests"); - - // Other containers inherit permissions from root; admins get all permissions, users & guests none - Role noPermsRole = RoleManager.getRole(NoPermissionsRole.class); - Role readerRole = RoleManager.getRole(ReaderRole.class); - - ContainerManager.bootstrapContainer("/", noPermsRole, noPermsRole); - Container rootContainer = ContainerManager.getRoot(); - - // Create all the standard containers (Home, Home/support, Shared) using an empty Collaboration folder type - FolderType collaborationType = new CollaborationFolderType(Collections.emptyList()); - - // Users & guests can read from /home - Container home = ContainerManager.bootstrapContainer(ContainerManager.HOME_PROJECT_PATH, readerRole, readerRole); - home.setFolderType(collaborationType, null); - - ContainerManager.createDefaultSupportContainer().setFolderType(collaborationType, null); - - // Only users can read from /Shared - ContainerManager.bootstrapContainer(ContainerManager.SHARED_CONTAINER_PATH, readerRole, null).setFolderType(collaborationType, null); - - try - { - // Need to insert standard MV indicators for the root -- okay to call getRoot() since we just created it. - String rootContainerId = rootContainer.getId(); - TableInfo mvTable = CoreSchema.getInstance().getTableInfoMvIndicators(); - - for (Map.Entry qcEntry : MvUtil.getDefaultMvIndicators().entrySet()) - { - Map params = new HashMap<>(); - params.put("Container", rootContainerId); - params.put("MvIndicator", qcEntry.getKey()); - params.put("Label", qcEntry.getValue()); - - Table.insert(null, mvTable, params); - } - } - catch (Throwable t) - { - ExceptionUtil.logExceptionToMothership(null, t); - } - - List guids = Stream.of(ContainerManager.HOME_PROJECT_PATH, ContainerManager.SHARED_CONTAINER_PATH) - .map(ContainerManager::getForPath) - .filter(Objects::nonNull) - .map(Container::getEntityId) - .collect(Collectors.toList()); - ContainerManager.setExcludedProjects(guids, () -> {}); - } - else - { - // It's very difficult to bootstrap without the root or shared containers in place; create them now and - // we'll delete them later - Container root = ContainerManager.ensureContainer("/", User.getAdminServiceUser()); - Table.insert(null, CoreSchema.getInstance().getTableInfoContainers(), Map.of("Parent", root.getId(), "Name", "Shared")); - } - } - - - @Override - public CoreUpgradeCode getUpgradeCode() - { - return new CoreUpgradeCode(); - } - - - @Override - public void destroy() - { - super.destroy(); - UsageReportingLevel.shutdown(); - } - - - @Override - public void startupAfterSpringConfig(ModuleContext moduleContext) - { - // Any containers in the cache have bogus folder types since they aren't registered until startup(). See #10310 - ContainerManager.clearCache(); - - checkForMissingDbViews(); - - ProductConfiguration.handleStartupProperties(); - // This listener deletes all properties; make sure it executes after most of the other listeners - ContainerManager.addContainerListener(new CoreContainerListener(), ContainerManager.ContainerListener.Order.Last); - ContainerManager.addContainerListener(new FolderSettingsCache.FolderSettingsCacheListener()); - SecurityManager.init(); - FolderTypeManager.get().registerFolderType(this, FolderType.NONE); - FolderTypeManager.get().registerFolderType(this, new CollaborationFolderType()); - - AnalyticsServiceImpl.get().resetCSP(); - - if (moduleContext.isNewInstall() && ModuleLoader.getInstance().shouldInsertData()) - { - // To initialize the portal layout correctly, we need to add the web parts after the folder types have been - // registered. Thus, it needs to be here in startupAfterSpringConfig() instead of grouped in bootstrap(). - Container homeContainer = ContainerManager.getHomeContainer(); - int count = Portal.getParts(homeContainer, homeContainer.getFolderType().getDefaultPageId(homeContainer)).size(); - addWebPart(PROJECTS_WEB_PART_NAME, homeContainer, HttpView.BODY, count); - } - - EmailService.setInstance(new EmailServiceImpl()); - - if (null != AuditLogService.get() && AuditLogService.get().getClass() != DefaultAuditProvider.class) - { - AuditLogService.get().registerAuditType(new UserAuditProvider()); - AuditLogService.get().registerAuditType(new GroupAuditProvider()); - AuditLogService.get().registerAuditType(new AttachmentAuditProvider()); - AuditLogService.get().registerAuditType(new ContainerAuditProvider()); - AuditLogService.get().registerAuditType(new FileSystemAuditProvider()); - AuditLogService.get().registerAuditType(new ClientApiAuditProvider()); - AuditLogService.get().registerAuditType(new AuthenticationSettingsAuditTypeProvider()); - AuditLogService.get().registerAuditType(new TransactionAuditProvider()); - AuditLogService.get().registerAuditType(new ModulePropertiesAuditProvider()); - - DataStateManager.getInstance().registerDataStateHandler(new CoreQCStateHandler()); - } - ContextListener.addShutdownListener(TempTableTracker.getShutdownListener()); - ContextListener.addShutdownListener(DavController.getShutdownListener()); - ContextListener.addShutdownListener(ShutdownListener.of("Temp file cleanup", null, this::deleteTempFiles)); - - SimpleMetricsService.setInstance(new SimpleMetricsServiceImpl()); - - // Export action stats on graceful shutdown - ContextListener.addShutdownListener(new ShutdownListener() { - @Override - public String getName() - { - return "Action stats export"; - } - - @Override - public void shutdownPre() - { - try - { - // Halt firing of Quartz triggers - Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler(); - scheduler.standby(); - } - catch (SchedulerException ignored) - { - } - - Logger logger = LogManager.getLogger(ActionsTsvWriter.class); - - if (null != logger) - { - StringBuilder buf = new StringBuilder(); - - try (TSVWriter writer = new ActionsTsvWriter()) - { - writer.write(buf); - } - catch (IOException e) - { - LOG.error("Exception exporting action stats", e); - } - - logger.info(buf.toString()); - LOG.info("Completed logging statistics for actions prior to web application shut down"); - } - } - - @Override - public void shutdownStarted() - { - try - { - // Clean up Quartz resources and wait for jobs to complete - Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler(); - scheduler.shutdown(true); - } - catch (SchedulerException ignored) - { - } - } - }); - - // populate look and feel settings and site settings with values read from startup properties as appropriate for not bootstrap - populateLookAndFeelResourcesWithStartupProps(); - AllowedExternalResourceHosts.registerStartupProperties(); - AllowedExternalResourceHosts.registerHosts(); - WriteableLookAndFeelProperties.populateLookAndFeelWithStartupProps(); - WriteableAppProps.populateSiteSettingsWithStartupProps(); - // create users and groups and assign roles with values read from startup properties as appropriate for not bootstrap - SecurityManager.populateStartupProperties(); - // This method depends on resources (FolderType) from other modules, so invoke after startup - ContextListener.addStartupListener(new StartupListener() - { - @Override - public String getName() - { - return "CoreModule.populateSiteSettingsWithStartupProps"; - } - - @Override - public void moduleStartupComplete(ServletContext servletContext) - { - populateSiteSettingsWithStartupProps(); - } - }); - - // Handle optional feature startup properties as late as possible; we want all optional features to be registered first - ContextListener.addStartupListener(new OptionalFeatureStartupListener()); - - LabKeyScriptEngineManager svc = LabKeyScriptEngineManager.get(); - // populate script engine definitions values read from startup properties - if (svc instanceof ScriptEngineManagerImpl) - ((ScriptEngineManagerImpl)svc).populateScriptEngineDefinitionsWithStartupProps(); - - // populate folder types from startup properties as appropriate for not bootstrap - FolderTypeManager.get().populateWithStartupProps(); - LimitActiveUsersSettings.populateStartupProperties(); - - AdminController.registerAdminConsoleLinks(); - AdminController.registerManagementTabs(); - AnalyticsController.registerAdminConsoleLinks(); - UserController.registerAdminConsoleLinks(); - LoggerController.registerAdminConsoleLinks(); - CoreController.registerAdminConsoleLinks(); - - FolderTypeManager.get().registerFolderType(this, new WorkbookFolderType()); - - SecurityManager.addViewFactory(new SecurityController.GroupDiagramViewFactory()); - - FolderSerializationRegistry fsr = FolderSerializationRegistry.get(); - if (null != fsr) - { - fsr.addFactories(new FolderTypeWriterFactory(), new FolderTypeImporterFactory()); - fsr.addFactories(new MissingValueWriterFactory(), new MissingValueImporterFactory()); - fsr.addFactories(new SearchSettingsWriterFactory(), new SearchSettingsImporterFactory()); - fsr.addFactories(new PageWriterFactory(), new PageImporterFactory()); - fsr.addFactories(new ModulePropertiesWriterFactory(), new ModulePropertiesImporterFactory()); - fsr.addFactories(new SecurityGroupWriterFactory(), new SecurityGroupImporterFactory()); - fsr.addFactories(new RoleAssignmentsWriterFactory(), new RoleAssignmentsImporterFactory()); - fsr.addFactories(new DataStateWriter.Factory(), new DataStateImporter.Factory()); - fsr.addFactories(new FileBrowserConfigWriter.Factory(), new FileBrowserConfigImporter.Factory()); - fsr.addImportFactory(new SubfolderImporterFactory()); - } - - SearchService ss = SearchService.get(); - ss.addDocumentParser(new TabLoader.CsvFactoryNoConversions()); - ss.addDocumentProvider(this); - - // Register indexable DataLoaders with the search service - DataLoaderServiceImpl.get().getFactories() - .stream() - .filter(DataLoaderFactory::indexable) - .forEach(ss::addDocumentParser); - - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_NO_GUESTS, - "No Guest Account", - "Disable the guest account", - false); - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_BLOCKER, - "Block malicious clients", - "Reject requests from clients that appear malicious. Turn this feature off if you want to run a security scanner.", - false); - OptionalFeatureService.get().addExperimentalFeatureFlag(FEATURE_FLAG_DISABLE_ENFORCE_CSP, - "Disable enforce Content Security Policy", - "Stop sending the " + ContentSecurityPolicyFilter.ContentSecurityPolicyType.Enforce.getHeaderName() + " header to browsers, " + - "but continue sending the " + ContentSecurityPolicyFilter.ContentSecurityPolicyType.Report.getHeaderName() + " header. " + - "This turns off an important layer of security for the entire site, so use it as a last resort only on a temporary basis " + - "(e.g., if an enforce CSP breaks critical functionality).", - false); - OptionalFeatureService.get().addExperimentalFeatureFlag(DataRegion.EXPERIMENTAL_DATA_REGION_ASYNC_TOTAL_ROWS, - "Data Region Async Total Rows", - "Enable asynchronous calculation of total rows for data regions. This can improve performance for large datasets.", - false); - - OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(EXPERIMENTAL_LOCAL_MARKETING_UPDATE, - "Self test marketing updates", "Test marketing updates from this local server (requires the mothership module).", false, true, FeatureType.Experimental)); - OptionalFeatureService.get().addFeatureListener(EXPERIMENTAL_LOCAL_MARKETING_UPDATE, (feature, enabled) -> { - // update the timer task when this setting changes - MothershipReport.setSelfTestMarketingUpdates(enabled); - UsageReportingLevel.reportNow(); - }); - - if (null != PropertyService.get()) - { - PropertyService.get().registerDomainKind(new UsersDomainKind()); - if (ModuleLoader.getInstance().shouldInsertData()) - UsersDomainKind.ensureDomain(moduleContext); - } - - // Register the standard, wiki-based terms-of-use provider - SecurityManager.addTermsOfUseProvider(new WikiTermsOfUseProvider()); - - if (null != PropertyService.get()) - PropertyService.get().registerDomainKind(new TestDomainKind()); - - AuthenticationManager.populateSettingsWithStartupProps(); - AnalyticsServiceImpl.populateSettingsWithStartupProps(); - - UsageMetricsService.get().registerUsageMetrics(getName(), () -> { - Map results = new HashMap<>(); - Map javaInfo = new HashMap<>(); - javaInfo.put("java.vendor", System.getProperty("java.vendor")); - javaInfo.put("java.vm.name", System.getProperty("java.vm.name")); - results.put("javaRuntime", javaInfo); - results.put("distributionFilename", AppProps.getInstance().getDistributionFilename()); - results.put("applicationMenuDisplayMode", LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getApplicationMenuDisplayMode()); - results.put("optionalFeatures", OptionalFeatureService.get().getOptionalFeatureFlags().stream() - .collect(Collectors.groupingBy(optionalFeatureFlag -> optionalFeatureFlag.getType().name().toLowerCase(), - Collectors.mapping(flag -> flag, Collectors.toMap(OptionalFeatureFlag::getFlag, OptionalFeatureFlag::isEnabled)) - )) - ); - results.put("productFeaturesEnabled", ProductRegistry.getProductFeatureSet()); - results.put("analyticsTrackingStatus", AnalyticsServiceImpl.get().getTrackingStatus().toString()); - String labkeyContextPath = AppProps.getInstance().getContextPath(); - results.put("webappContextPath", labkeyContextPath); - results.put("embeddedTomcat", true); - boolean customLog4JConfig = false; - if (ModuleLoader.getServletContext() != null) - { - customLog4JConfig = Boolean.parseBoolean(ModuleLoader.getServletContext().getInitParameter("org.labkey.customLog4JConfig")); - } - results.put("customLog4JConfig", customLog4JConfig); - results.put("containerRelativeURL", AppProps.getInstance().getUseContainerRelativeURL()); - results.put("runtimeMode", AppProps.getInstance().isDevMode() ? "development" : "production"); - Set deployedApps = new HashSet<>(CoreWarningProvider.collectAllDeployedApps()); - deployedApps.remove(labkeyContextPath); - if (labkeyContextPath.startsWith("/")) - { - deployedApps.remove(labkeyContextPath.substring(1)); - } - results.put("otherDeployedWebapps", StringUtils.join(deployedApps, ",")); - - // Report the total number of login entries in the audit log - results.put("totalLogins", UserManager.getAuthCount(null, false, false, false)); - results.put("apiKeyLogins", UserManager.getAuthCount(null, false, true, false)); - results.put("sessionTimeout", ModuleLoader.getServletContext().getSessionTimeout()); - results.put("userLimits", new LimitActiveUsersSettings().getMetricsMap()); - results.put("systemUserCount", UserManager.getSystemUserCount()); - Calendar cal = new GregorianCalendar(); - cal.add(Calendar.DATE, -30); - results.put("uniqueRecentUserCount", UserManager.getAuthCount(cal.getTime(), false, false, true)); - results.put("uniqueRecentNonSystemUserCount", UserManager.getAuthCount(cal.getTime(), true, false, true)); - if (OptionalFeatureService.get().isFeatureEnabled(FEATURE_FLAG_EXTENDED_METRICS)) - { - // Optionally include a list of active users, Issue #53050 - results.put("activeUsers", UserManager.getActiveUsers().stream() - .filter(u -> !u.isSystem()) - .map(User::getEmail) - .toList() - ); - } - - results.put("workbookCount", ContainerManager.getWorkbookCount()); - results.put("archivedFolderCount", ContainerManager.getArchivedContainerCount()); - results.put("databaseSize", CoreSchema.getInstance().getSchema().getScope().getDatabaseSize()); - results.put("scriptEngines", LabKeyScriptEngineManager.get().getScriptEngineMetrics()); - results.put("customLabels", CustomLabelService.get().getCustomLabelMetrics()); - Map roleAssignments = new HashMap<>(); - final String roleCountSql = "SELECT COUNT(*) FROM core.RoleAssignments WHERE userid > 0 AND role = ?"; - roleAssignments.put("assayDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.assay.security.AssayDesignerRole").getObject(Long.class)); - roleAssignments.put("dataClassDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.DataClassDesignerRole").getObject(Long.class)); - roleAssignments.put("sampleTypeDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.SampleTypeDesignerRole").getObject(Long.class)); - results.put("roleAssignments", roleAssignments); - - Map allowListCounts = new HashMap<>(); - for (AllowListType type : AllowListType.values()) - { - allowListCounts.put(type.name(), type.getValues().size()); - } - results.put("allowListCounts", allowListCounts); - - return results; - }); - - UsageMetricsService.get().registerUsageMetrics(getName(), WebSocketConnectionManager.getInstance()); - UsageMetricsService.get().registerUsageMetrics(getName(), DbLoginManager.getMetricsProvider()); - UsageMetricsService.get().registerUsageMetrics(getName(), SecurityManager.getMetricsProvider()); - UsageMetricsService.get().registerUsageMetrics(getName(), DisplayFormatAnalyzer.getMetricsProvider()); - UsageMetricsService.get().registerUsageMetrics(getName(), Portal.getMetricsProvider()); - - if (AppProps.getInstance().isDevMode()) - AntiVirusProviderRegistry.get().registerAntiVirusProvider(new DummyAntiVirusService.Provider()); - - FileContentService fileContentService = FileContentService.get(); - if (fileContentService != null) - fileContentService.addFileListener(WebFilesResolverImpl.get()); - - RoleManager.registerPermission(new QCAnalystPermission()); - MarkdownService.setInstance(new MarkdownServiceImpl()); - - // initialize email preference service and listeners - MessageConfigService.setInstance(new EmailPreferenceConfigServiceImpl()); - ContainerManager.addContainerListener(new EmailPreferenceContainerListener()); - UserManager.addUserListener(new EmailPreferenceUserListener()); - - DatabaseMigrationService.get().registerHandler(new DefaultMigrationHandler(CoreSchema.getInstance().getSchema()) - { - @Override - public void beforeVerification() - { - super.beforeVerification(); - - // Delete root and shared containers that were needed for bootstrapping - TableInfo containers = CoreSchema.getInstance().getTableInfoContainers(); - Table.delete(containers); - DbScope targetScope = DbScope.getLabKeyScope(); - new SqlExecutor(targetScope).execute("ALTER SEQUENCE core.containers_rowid_seq RESTART"); // Reset Containers sequence - } - - @Override - public void beforeSchema() - { - new SqlExecutor(getSchema()).execute("ALTER TABLE core.Containers DROP CONSTRAINT FK_Containers_Containers"); - new SqlExecutor(getSchema()).execute("ALTER TABLE core.ViewCategory DROP CONSTRAINT FK_ViewCategory_Parent"); - } - - @Override - public List getTablesToCopy() - { - List tablesToCopy = super.getTablesToCopy(); - tablesToCopy.remove(CoreSchema.getInstance().getTableInfoModules()); - tablesToCopy.remove(CoreSchema.getInstance().getTableInfoSqlScripts()); - tablesToCopy.remove(CoreSchema.getInstance().getTableInfoUpgradeSteps()); - - return tablesToCopy; - } - - @Override - public @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable) - { - return switch (sourceTable.getName()) - { - case "ContainerAliases" -> FieldKey.fromParts("ContainerRowId", "EntityId"); - case "Containers" -> FieldKey.fromParts("EntityId"); - case "Report" -> FieldKey.fromParts("ContainerId"); - case "APIKeys", "AuthenticationConfigurations", "EmailOptions", "Logins", "ReportEngines", "ShortURL", "UsersData" -> SITE_WIDE_TABLE; - default -> super.getContainerFieldKey(sourceTable); - }; - } - - @Override - public FilterClause getContainerClause(TableInfo sourceTable, FieldKey containerFieldKey, Set containers) - { - FilterClause containerClause = super.getContainerClause(sourceTable, containerFieldKey, containers); - - // Users and root groups have container == null, so add that as an OR clause - if (sourceTable.getName().equals("Principals") || sourceTable.getName().equals("Members")) - { - OrClause orClause = new OrClause(); - orClause.addClause(containerClause); - orClause.addClause(new CompareClause(containerFieldKey, CompareType.ISBLANK, null)); - containerClause = orClause; - } - - return containerClause; - } - - @Override - public void afterSchema() - { - new SqlExecutor(getSchema()).execute("ALTER TABLE core.Containers ADD CONSTRAINT FK_Containers_Containers FOREIGN KEY (Parent) REFERENCES core.Containers(EntityId)"); - new SqlExecutor(getSchema()).execute("ALTER TABLE core.ViewCategory ADD CONSTRAINT FK_ViewCategory_Parent FOREIGN KEY (Parent) REFERENCES core.ViewCategory(RowId)"); - } - }); - - DatabaseMigrationService.get().registerHandler(new DefaultMigrationHandler(PropertySchema.getInstance().getSchema()){ - @Override - public @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable) - { - return sourceTable.getName().equals("PropertySets") ? FieldKey.fromParts("ObjectId") : super.getContainerFieldKey(sourceTable); - } - }); - - DatabaseMigrationService.get().registerHandler(new DefaultMigrationHandler(TestSchema.getInstance().getSchema()){ - @Override - public List getTablesToCopy() - { - return List.of(); // Skip all test tables - } - }); - - // TODO: Temporary, until "clone" migration type copies schemas with a registered handler only - if (ModuleLoader.getInstance().getModule(DbScope.getLabKeyScope(), "vehicle") != null) - { - DatabaseMigrationService.get().registerHandler(new DefaultMigrationHandler(DbSchema.get("vehicle", DbSchemaType.Module)) - { - @Override - public List getTablesToCopy() - { - return List.of(); // Skip all vehicle tables - } - }); - } - } - - // Issue 7527: Auto-detect missing SQL views and attempt to recreate - private void checkForMissingDbViews() - { - ModuleLoader.getInstance().getModules().stream() - .map(FileSqlScriptProvider::new) - .flatMap(p -> p.getSchemas().stream() - .filter(schema-> SchemaUpdateType.Before.getScript(p, schema) != null || SchemaUpdateType.After.getScript(p, schema) != null) - ) - .filter(schema -> TableXmlUtils.compareXmlToMetaData(schema, false, false, true).hasViewProblem()) - .findAny() - .ifPresent(schema -> - { - LOG.warn("At least one database view was not as expected in the {} schema. Attempting to recreate views automatically", schema.getName()); - ModuleLoader.getInstance().recreateViews(); - }); - } - - @Override - public void registerServlets(ServletContext servletCtx) - { -// even though there is one webdav tree rooted at "/" we still use two servlet bindings. -// This is because we want /_webdav/* to be resolved BEFORE all other servlet-mappings -// and /* to resolve AFTER all other servlet-mappings - _webdavServletDynamic = servletCtx.addServlet("static", new WebdavServlet(true)); - _webdavServletDynamic.setMultipartConfig(SpringActionController.getMultiPartConfigElement()); - _webdavServletDynamic.addMapping("/_webdav/*"); - } - - @Override - public void registerFinalServlets(ServletContext servletCtx) - { - _webdavServletDynamic.addMapping("/"); - } - - @Override - public void startBackgroundThreads() - { - SystemMaintenance.setTimer(); - ThumbnailServiceImpl.startThread(); - // Launch in the background, but delay by 10 seconds to reduce impact on other startup tasks - _warningProvider.startSchemaCheck(10); - - // Start up the default Quartz scheduler, used in many places - try - { - StdSchedulerFactory.getDefaultScheduler().start(); - } - catch (SchedulerException e) - { - throw UnexpectedException.wrap(e); - } - - if (MothershipReport.shouldReceiveMarketingUpdates()) - { - if (AppProps.getInstance().getUsageReportingLevel() == UsageReportingLevel.NONE) - { - // force the usage reporting level to on for community edition distributions - WriteableAppProps appProps = AppProps.getWriteableInstance(); - appProps.setUsageReportingLevel(UsageReportingLevel.ON); - appProps.save(User.getAdminServiceUser()); - } - } - // On bootstrap in production mode, this will send an initial ping with very little information, as the admin will - // not have set up their account yet. On later startups, depending on the reporting level, this will send an immediate - // ping, and then once every 24 hours. - UsageReportingLevel.reportNow(); - TempTableTracker.init(); - - // Loading the PDFBox font cache can be very slow on some agents; fill it proactively. Issue 50601 - JobRunner.getDefault().execute(() -> { - try - { - long start = System.currentTimeMillis(); - FontMapper mapper = FontMappers.instance(); - Method method = mapper.getClass().getMethod("getProvider"); - method.setAccessible(true); - method.invoke(mapper); - long duration = System.currentTimeMillis() - start; - LOG.info("Ensuring PDFBox on-disk font cache took {} seconds", Math.round(duration / 100.0) / 10.0); - } - catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) - { - LOG.warn("Unable to initialize PDFBox font cache", e); - } - }); - } - - @Override - public @NotNull List

getDetailedSummary(Container c, User user) - { - long childContainerCount = ContainerManager.getChildren(c).stream().filter(Container::isInFolderNav).count(); - if (childContainerCount == 0) - return Collections.emptyList(); - return List.of(new Summary(childContainerCount, "Subfolder")); - } - - @Override - public JSONObject getPageContextJson(ContainerUser context) - { - JSONObject json = new JSONObject(getDefaultPageContextJson(context.getContainer())); - json.put("productFeatures", ProductRegistry.getProductFeatureSet()); - json.put("primaryApplicationId", ProductRegistry.get().getPrimaryApplicationId(context.getContainer())); - return json; - } - - @Override - public String getTabName(ViewContext context) - { - return "Portal"; - } - - - @Override - public ActionURL getTabURL(Container c, User user) - { - if (user == null) - return AppProps.getInstance().getHomePageActionURL(); - - return PageFlowUtil.urlProvider(ProjectUrls.class).getBeginURL(c); - } - - @Override - public TabDisplayMode getTabDisplayMode() - { - return TabDisplayMode.DISPLAY_USER_PREFERENCE_DEFAULT; - } - - @Override - public @NotNull Set> getIntegrationTests() - { - // Must be mutable since we add the dialect tests below - Set> testClasses = Sets.newHashSet - ( - AdminController.SchemaVersionTestCase.class, - AdminController.SerializationTest.class, - AdminController.TestCase.class, - AdminController.WorkbookDeleteTestCase.class, - AllowListType.TestCase.class, - AttachmentServiceImpl.TestCase.class, - CoreController.TestCase.class, - DataRegion.TestCase.class, - DavController.TestCase.class, - EmailServiceImpl.TestCase.class, - FilesSiteSettingsAction.TestCase.class, - LoggerController.TestCase.class, - LoggingTestCase.class, - LoginController.TestCase.class, - MarkdownTestCase.class, - ModuleInfoTestCase.class, - ModulePropertiesTestCase.class, - ModuleStaticResolverImpl.TestCase.class, - NotificationServiceImpl.TestCase.class, - PortalJUnitTest.class, - PostgreSqlInClauseTest.class, - ProductRegistry.TestCase.class, - RadeoxRenderer.RadeoxRenderTest.class, - RhinoService.TestCase.class, - SchemaXMLTestCase.class, - SecurityApiActions.TestCase.class, - SecurityController.TestCase.class, - SqlDialect.DialectTestCase.class, - SqlScriptController.TestCase.class, - TableViewFormTestCase.class, - UnknownSchemasTest.class, - UserController.TestCase.class - ); - - testClasses.addAll(SqlDialectManager.getAllJUnitTests()); - - return testClasses; - } - - @Override - public @NotNull Set> getUnitTests() - { - return Set.of( - ApiJsonWriter.TestCase.class, - ClassLoaderTestCase.class, - CopyFileRootPipelineJob.TestCase.class, - OutOfRangeDisplayColumn.TestCase.class, - PostgreSqlVersion.TestCase.class, - ScriptEngineManagerImpl.TestCase.class, - StatsServiceImpl.TestCase.class, - - - // Radeox tests - SimpleListTest.class, - ExampleListFormatterTest.class, - AtoZListFormatterTest.class, - BaseRenderEngineTest.class, - BasicRegexTest.class, - ItalicFilterTest.class, - BoldFilterTest.class, - KeyFilterTest.class, - NewlineFilterTest.class, - LineFilterTest.class, - TypographyFilterTest.class, - HtmlRemoveFilterTest.class, - StrikeThroughFilterTest.class, - UrlFilterTest.class, - ParamFilterTest.class, - FilterPipeTest.class, - EscapeFilterTest.class, - LinkTestFilterTest.class, - WikiLinkFilterTest.class, - SmileyFilterTest.class, - ListFilterTest.class, - HeadingFilterTest.class, - RegexFilter.TestCase.class - ); - } - - @Override - public DbSchema createModuleDbSchema(DbScope scope, String metaDataName, Map tableInfoFactoryMap) - { - // Special case for the "labkey" schema we create in every module data source - if ("labkey".equals(metaDataName)) - return new LabKeyDbSchema(scope, tableInfoFactoryMap); - - return super.createModuleDbSchema(scope, metaDataName, tableInfoFactoryMap); - } - - @Override - @NotNull - public Collection getSchemaNames() - { - return List.of - ( - CoreSchema.getInstance().getSchemaName(), // core - PropertySchema.getInstance().getSchemaName(), // prop - TestSchema.getInstance().getSchemaName(), // test - DbSchema.TEMP_SCHEMA_NAME // temp - ); - } - - @NotNull - @Override - public Collection getProvisionedSchemaNames() - { - return Collections.singleton(DbSchema.TEMP_SCHEMA_NAME); - } - - @NotNull - @Override - public Set getSchemasToTest() - { - Set result = new LinkedHashSet<>(super.getSchemasToTest()); - - // Add the "labkey" schema in all module data sources as well... should match application.properties - for (String dataSourceName : ModuleLoader.getInstance().getAllModuleDataSourceNames()) - { - DbScope scope = DbScope.getDbScope(dataSourceName); - if (scope != null) - { - result.add(scope.getLabKeySchema()); - } - } - - return result; - } - - @Override - public void enumerateDocuments(SearchService.TaskIndexingQueue queue, Date since) - { - Container c = queue.getContainer(); - if (c.isRoot()) - return; - - Runnable r = () -> { - Container p = c.getProject(); - if (null == p) - return; - String title; - String keywords; - String body; - - // UNDONE: generalize to other folder types - StudyService svc = StudyService.get(); - Study study = svc != null ? svc.getStudy(c) : null; - - if (null != study) - { - title = study.getSearchDisplayTitle(); - keywords = study.getSearchKeywords(); - body = study.getSearchBody(); - } - else - { - String type = c.getContainerNoun(true); - - String containerTitle = c.getTitle(); - - String description = StringUtils.trimToEmpty(c.getDescription()); - title = type + " -- " + containerTitle; - User u_user = UserManager.getUser(c.getCreatedBy()); - String user = (u_user == null) ? "" : u_user.getDisplayName(User.getSearchUser()); - keywords = description + " " + type + " " + user; - body = type + " " + containerTitle + (c.isProject() ? "" : " in Project " + p.getName()); - body += "\n" + description; - } - - String identifiers = c.getName(); - - Map properties = new HashMap<>(); - - assert (null != keywords); - properties.put(SearchService.PROPERTY.identifiersMed.toString(), identifiers); - properties.put(SearchService.PROPERTY.keywordsMed.toString(), keywords); - properties.put(SearchService.PROPERTY.title.toString(), title); - properties.put(SearchService.PROPERTY.categories.toString(), SearchService.navigationCategory.getName()); - ActionURL startURL = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(c); - startURL.setExtraPath(c.getId()); - WebdavResource doc = new SimpleDocumentResource(c.getParsedPath(), - "link:" + c.getId(), - c.getEntityId(), - "text/plain", - body, - startURL, - UserManager.getUser(c.getCreatedBy()), c.getCreated(), - null, null, - properties); - queue.addResource(doc); - }; - r.run(); - } - - @Override - public void indexDeleted() - { - new SqlExecutor(CoreSchema.getInstance().getSchema()).execute("UPDATE core.Documents SET LastIndexed = NULL"); - } - - /** - * Handles startup props for LookAndFeelSettings resources - */ - private void populateLookAndFeelResourcesWithStartupProps() - { - ModuleLoader.getInstance().handleStartupProperties(new StandardStartupPropertyHandler<>(WriteableLookAndFeelProperties.SCOPE_LOOK_AND_FEEL_SETTINGS, ResourceType.class) - { - @Override - public void handle(Map map) - { - boolean incrementRevision = false; - - for (Map.Entry entry : map.entrySet()) - { - SiteResourceHandler handler = getResourceHandler(entry.getKey()); - if (handler != null) - incrementRevision |= setSiteResource(handler, entry.getValue(), User.guest); - } - - // Bump the look & feel revision so browsers retrieve the new logo, custom stylesheet, etc. - if (incrementRevision) - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - } - }); - } - - /** - * This method handles the home project settings - */ - private void populateSiteSettingsWithStartupProps() - { - Map props = AppProps.getInstance().getStashedStartupProperties(); - - StartupPropertyEntry folderTypeEntry = props.get(homeProjectFolderType); - if (null != folderTypeEntry) - { - FolderType folderType = FolderTypeManager.get().getFolderType(folderTypeEntry.getValue()); - if (folderType != null) - // using guest user since the server startup doesn't have a true user (this will be used for audit events) - ContainerManager.getHomeContainer().setFolderType(folderType, User.guest); - else - LOG.error("Unable to find folder type for home project during server startup: " + folderTypeEntry.getValue()); - } - - StartupPropertyEntry resetPermissionsEntry = props.get(homeProjectResetPermissions); - if (null != resetPermissionsEntry && Boolean.valueOf(resetPermissionsEntry.getValue())) - { - // reset the home project permissions to remove the default assignments given at server install - MutableSecurityPolicy homePolicy = new MutableSecurityPolicy(ContainerManager.getHomeContainer()); - SecurityPolicyManager.savePolicy(homePolicy, User.getAdminServiceUser()); - // remove the guest role assignment from the support subfolder - Group guests = SecurityManager.getGroup(Group.groupGuests); - if (null != guests) - { - Container supportFolder = ContainerManager.getDefaultSupportContainer(); - if (supportFolder != null) - { - MutableSecurityPolicy supportPolicy = new MutableSecurityPolicy(supportFolder.getPolicy()); - for (Role assignedRole : supportPolicy.getAssignedRoles(guests)) - supportPolicy.removeRoleAssignment(guests, assignedRole); - SecurityPolicyManager.savePolicy(supportPolicy, User.getAdminServiceUser()); - } - } - } - - StartupPropertyEntry webparts = props.get(homeProjectWebparts); - if (null != webparts) - { - // Clear existing webparts added by core and wiki modules - Container homeContainer = ContainerManager.getHomeContainer(); - Portal.saveParts(homeContainer, Collections.emptyList()); - - for (String webpartName : StringUtils.split(webparts.getValue(), ';')) - { - WebPartFactory webPartFactory = Portal.getPortalPart(webpartName); - if (webPartFactory != null) - addWebPart(webPartFactory.getName(), homeContainer, HttpView.BODY); - } - } - } - - private @Nullable SiteResourceHandler getResourceHandler(@NotNull ResourceType type) - { - return LookAndFeelPropertiesManager.get().getResourceHandler(type); - } - - private boolean setSiteResource(SiteResourceHandler resourceHandler, StartupPropertyEntry prop, User user) - { - Resource resource = getModuleResourceFromPropValue(prop.getValue()); - if (resource != null) - { - try - { - resourceHandler.accept(resource, ContainerManager.getRoot(), user); - return true; - } - catch(Exception e) - { - LOG.error("Exception setting {} during server startup.", prop.getName(), e); - } - } - - LOG.error("Unable to find {} resource during server startup: {}", prop.getName(), prop.getValue()); - return false; - } - - private Resource getModuleResourceFromPropValue(String propValue) - { - if (propValue != null) - { - // split the prop value on the separator char to get the module name and resource path in that module - String moduleName = propValue.substring(0, propValue.indexOf(":")); - String resourcePath = propValue.substring(propValue.indexOf(":") + 1); - - Module module = ModuleLoader.getInstance().getModule(moduleName); - if (module != null) - return module.getModuleResource(resourcePath); - } - - return null; - } - - public void rerunSchemaCheck() - { - // Queue a job without delay. This avoids executing multiple overlapping schema checks. Not bothering with a - // more surgical approach since this variant is likely being called during development. - _warningProvider.startSchemaCheck(0); - } -} +/* + * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.core; + +import com.fasterxml.jackson.core.io.CharTypes; +import com.google.common.collect.Sets; +import jakarta.servlet.ServletContext; +import jakarta.servlet.ServletRegistration; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.pdfbox.pdmodel.font.FontMapper; +import org.apache.pdfbox.pdmodel.font.FontMappers; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.action.ApiJsonWriter; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AdminConsoleService; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.admin.HealthCheck; +import org.labkey.api.admin.HealthCheckRegistry; +import org.labkey.api.admin.TableXmlUtils; +import org.labkey.api.admin.notification.NotificationService; +import org.labkey.api.admin.sitevalidation.SiteValidationService; +import org.labkey.api.analytics.AnalyticsService; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.attachments.DocumentConversionService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.ClientApiAuditProvider; +import org.labkey.api.audit.DefaultAuditProvider; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.ContainerAuditProvider; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.audit.provider.GroupAuditProvider; +import org.labkey.api.audit.provider.ModulePropertiesAuditProvider; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.CompareType.CompareClause; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ContainerService; +import org.labkey.api.data.ContainerServiceImpl; +import org.labkey.api.data.ContainerTypeRegistry; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DataColumn; +import org.labkey.api.data.DataRegion; +import org.labkey.api.data.DatabaseMigrationService; +import org.labkey.api.data.DatabaseMigrationService.DefaultMigrationHandler; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.FileSqlScriptProvider; +import org.labkey.api.data.MvUtil; +import org.labkey.api.data.NormalContainerType; +import org.labkey.api.data.OutOfRangeDisplayColumn; +import org.labkey.api.data.PropertySchema; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SchemaTableInfoFactory; +import org.labkey.api.data.SimpleFilter.FilterClause; +import org.labkey.api.data.SimpleFilter.OrClause; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TSVWriter; +import org.labkey.api.data.TabContainerType; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TempTableTracker; +import org.labkey.api.data.TestSchema; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.data.dialect.BasePostgreSqlDialect; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.data.dialect.SqlDialectManager; +import org.labkey.api.data.dialect.SqlDialectRegistry; +import org.labkey.api.data.statistics.StatsService; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.TestDomainKind; +import org.labkey.api.external.tools.ExternalToolsViewService; +import org.labkey.api.files.FileBrowserConfigImporter; +import org.labkey.api.files.FileBrowserConfigWriter; +import org.labkey.api.files.FileContentService; +import org.labkey.api.markdown.MarkdownService; +import org.labkey.api.message.settings.MessageConfigService; +import org.labkey.api.module.FolderType; +import org.labkey.api.module.FolderTypeManager; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.SchemaUpdateType; +import org.labkey.api.module.SpringModule; +import org.labkey.api.module.Summary; +import org.labkey.api.notification.EmailMessage; +import org.labkey.api.notification.EmailService; +import org.labkey.api.notification.NotificationMenuView; +import org.labkey.api.portal.ProjectUrls; +import org.labkey.api.premium.AntiVirusProviderRegistry; +import org.labkey.api.products.ProductRegistry; +import org.labkey.api.qc.DataStateManager; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QuerySchema; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.reader.DataLoaderFactory; +import org.labkey.api.reader.DataLoaderService; +import org.labkey.api.reader.ExcelLoader; +import org.labkey.api.reader.FastaDataLoader; +import org.labkey.api.reader.HTMLDataLoader; +import org.labkey.api.reader.JSONDataLoader; +import org.labkey.api.reader.TabLoader; +import org.labkey.api.reports.LabKeyScriptEngineManager; +import org.labkey.api.resource.Resource; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.AuthenticationManager; +import org.labkey.api.security.AuthenticationManager.Priority; +import org.labkey.api.security.AuthenticationSettingsAuditTypeProvider; +import org.labkey.api.security.DbLoginService; +import org.labkey.api.security.DummyAntiVirusService; +import org.labkey.api.security.Encryption; +import org.labkey.api.security.Group; +import org.labkey.api.security.GroupManager; +import org.labkey.api.security.LimitActiveUsersService; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPointcutService; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.WikiTermsOfUseProvider; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.QCAnalystPermission; +import org.labkey.api.security.permissions.TroubleshooterPermission; +import org.labkey.api.security.roles.NoPermissionsRole; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.security.roles.Role; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.CustomLabelService; +import org.labkey.api.settings.CustomLabelService.CustomLabelServiceImpl; +import org.labkey.api.settings.FolderSettingsCache; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.settings.LookAndFeelPropertiesManager; +import org.labkey.api.settings.LookAndFeelPropertiesManager.ResourceType; +import org.labkey.api.settings.LookAndFeelPropertiesManager.SiteResourceHandler; +import org.labkey.api.settings.OptionalFeatureFlag; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.settings.OptionalFeatureService.FeatureType; +import org.labkey.api.settings.ProductConfiguration; +import org.labkey.api.settings.StandardStartupPropertyHandler; +import org.labkey.api.settings.StartupPropertyEntry; +import org.labkey.api.settings.StashedStartupProperties; +import org.labkey.api.settings.WriteableAppProps; +import org.labkey.api.settings.WriteableLookAndFeelProperties; +import org.labkey.api.stats.AnalyticsProviderRegistry; +import org.labkey.api.stats.SummaryStatisticRegistry; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; +import org.labkey.api.thumbnail.ThumbnailService; +import org.labkey.api.usageMetrics.SimpleMetricsService; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.ContextListener; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JobRunner; +import org.labkey.api.util.MimeMap; +import org.labkey.api.util.MothershipReport; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.ShutdownListener; +import org.labkey.api.util.StartupListener; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.util.UsageReportingLevel; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.vcs.VcsService; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.AlwaysAvailableWebPartFactory; +import org.labkey.api.view.BaseWebPartFactory; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.Portal; +import org.labkey.api.view.Portal.WebPart; +import org.labkey.api.view.ShortURLService; +import org.labkey.api.view.VBox; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.ViewService; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.menu.FolderMenu; +import org.labkey.api.view.template.WarningService; +import org.labkey.api.webdav.SimpleDocumentResource; +import org.labkey.api.webdav.WebdavResolverImpl; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; +import org.labkey.api.wiki.WikiRenderingService; +import org.labkey.api.writer.ContainerUser; +import org.labkey.core.admin.ActionsTsvWriter; +import org.labkey.core.admin.AdminConsoleServiceImpl; +import org.labkey.core.admin.AdminController; +import org.labkey.core.admin.AllowListType; +import org.labkey.core.admin.CopyFileRootPipelineJob; +import org.labkey.core.admin.CustomizeMenuForm; +import org.labkey.core.admin.DisplayFormatAnalyzer; +import org.labkey.core.admin.DisplayFormatValidationProviderFactory; +import org.labkey.core.admin.FilesSiteSettingsAction; +import org.labkey.core.admin.MenuViewFactory; +import org.labkey.core.admin.OptionalFeatureServiceImpl; +import org.labkey.core.admin.OptionalFeatureStartupListener; +import org.labkey.core.admin.importer.FolderTypeImporterFactory; +import org.labkey.core.admin.importer.MissingValueImporterFactory; +import org.labkey.core.admin.importer.ModulePropertiesImporterFactory; +import org.labkey.core.admin.importer.PageImporterFactory; +import org.labkey.core.admin.importer.RoleAssignmentsImporterFactory; +import org.labkey.core.admin.importer.SearchSettingsImporterFactory; +import org.labkey.core.admin.importer.SecurityGroupImporterFactory; +import org.labkey.core.admin.importer.SubfolderImporterFactory; +import org.labkey.core.admin.logger.LoggerController; +import org.labkey.core.admin.logger.LoggingTestCase; +import org.labkey.core.admin.miniprofiler.MiniProfilerController; +import org.labkey.core.admin.sitevalidation.SiteValidationServiceImpl; +import org.labkey.core.admin.sql.SqlScriptController; +import org.labkey.core.admin.test.SchemaXMLTestCase; +import org.labkey.core.admin.test.UnknownSchemasTest; +import org.labkey.core.admin.usageMetrics.UsageMetricsServiceImpl; +import org.labkey.core.admin.writer.FolderSerializationRegistryImpl; +import org.labkey.core.admin.writer.FolderTypeWriterFactory; +import org.labkey.core.admin.writer.MissingValueWriterFactory; +import org.labkey.core.admin.writer.ModulePropertiesWriterFactory; +import org.labkey.core.admin.writer.PageWriterFactory; +import org.labkey.core.admin.writer.RoleAssignmentsWriterFactory; +import org.labkey.core.admin.writer.SearchSettingsWriterFactory; +import org.labkey.core.admin.writer.SecurityGroupWriterFactory; +import org.labkey.core.analytics.AnalyticsController; +import org.labkey.core.analytics.AnalyticsServiceImpl; +import org.labkey.core.attachment.AttachmentServiceImpl; +import org.labkey.core.dialect.PostgreSqlDialectFactory; +import org.labkey.core.dialect.PostgreSqlInClauseTest; +import org.labkey.core.dialect.PostgreSqlVersion; +import org.labkey.core.junit.JunitController; +import org.labkey.core.login.DbLoginAuthenticationProvider; +import org.labkey.core.login.DbLoginManager; +import org.labkey.core.login.LoginController; +import org.labkey.core.metrics.SimpleMetricsServiceImpl; +import org.labkey.core.metrics.WebSocketConnectionManager; +import org.labkey.core.notification.EmailPreferenceConfigServiceImpl; +import org.labkey.core.notification.EmailPreferenceContainerListener; +import org.labkey.core.notification.EmailPreferenceUserListener; +import org.labkey.core.notification.EmailServiceImpl; +import org.labkey.core.notification.NotificationController; +import org.labkey.core.notification.NotificationServiceImpl; +import org.labkey.core.portal.CollaborationFolderType; +import org.labkey.core.portal.PortalJUnitTest; +import org.labkey.core.portal.ProjectController; +import org.labkey.core.portal.UtilController; +import org.labkey.core.products.ProductController; +import org.labkey.core.project.FolderNavigationForm; +import org.labkey.core.qc.CoreQCStateHandler; +import org.labkey.core.qc.DataStateImporter; +import org.labkey.core.qc.DataStateWriter; +import org.labkey.core.query.AttachmentAuditProvider; +import org.labkey.core.query.CoreQuerySchema; +import org.labkey.core.query.PostgresUserSchema; +import org.labkey.core.query.UserAuditProvider; +import org.labkey.core.query.UsersDomainKind; +import org.labkey.core.reader.DataLoaderServiceImpl; +import org.labkey.core.reports.DocumentConversionServiceImpl; +import org.labkey.core.reports.ScriptEngineManagerImpl; +import org.labkey.core.script.RhinoService; +import org.labkey.core.security.AllowedExternalResourceHosts; +import org.labkey.core.security.ApiKeyViewProvider; +import org.labkey.core.security.SecurityApiActions; +import org.labkey.core.security.SecurityController; +import org.labkey.core.security.SecurityPointcutServiceImpl; +import org.labkey.core.security.validators.PermissionsValidatorFactory; +import org.labkey.core.statistics.AnalyticsProviderRegistryImpl; +import org.labkey.core.statistics.StatsServiceImpl; +import org.labkey.core.statistics.SummaryStatisticRegistryImpl; +import org.labkey.core.thumbnail.ThumbnailServiceImpl; +import org.labkey.core.user.LimitActiveUsersSettings; +import org.labkey.core.user.UserController; +import org.labkey.core.vcs.VcsServiceImpl; +import org.labkey.core.view.ShortURLServiceImpl; +import org.labkey.core.view.TableViewFormTestCase; +import org.labkey.core.view.external.tools.ExternalToolsViewServiceImpl; +import org.labkey.core.view.template.bootstrap.CoreWarningProvider; +import org.labkey.core.view.template.bootstrap.PageTemplate; +import org.labkey.core.view.template.bootstrap.ViewServiceImpl; +import org.labkey.core.view.template.bootstrap.WarningServiceImpl; +import org.labkey.core.webdav.DavController; +import org.labkey.core.webdav.ModuleStaticResolverImpl; +import org.labkey.core.webdav.WebFilesResolverImpl; +import org.labkey.core.webdav.WebdavServlet; +import org.labkey.core.wiki.MarkdownServiceImpl; +import org.labkey.core.wiki.MarkdownTestCase; +import org.labkey.core.wiki.RadeoxRenderer; +import org.labkey.core.wiki.WikiRenderingServiceImpl; +import org.labkey.core.workbook.WorkbookFolderType; +import org.labkey.core.workbook.WorkbookQueryView; +import org.labkey.core.workbook.WorkbookSearchView; +import org.labkey.filters.ContentSecurityPolicyFilter; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.impl.StdSchedulerFactory; +import org.radeox.filter.regex.RegexFilter; +import org.radeox.test.BaseRenderEngineTest; +import org.radeox.test.filter.BasicRegexTest; +import org.radeox.test.filter.BoldFilterTest; +import org.radeox.test.filter.EscapeFilterTest; +import org.radeox.test.filter.FilterPipeTest; +import org.radeox.test.filter.HeadingFilterTest; +import org.radeox.test.filter.HtmlRemoveFilterTest; +import org.radeox.test.filter.ItalicFilterTest; +import org.radeox.test.filter.KeyFilterTest; +import org.radeox.test.filter.LineFilterTest; +import org.radeox.test.filter.LinkTestFilterTest; +import org.radeox.test.filter.ListFilterTest; +import org.radeox.test.filter.NewlineFilterTest; +import org.radeox.test.filter.ParamFilterTest; +import org.radeox.test.filter.SmileyFilterTest; +import org.radeox.test.filter.StrikeThroughFilterTest; +import org.radeox.test.filter.TypographyFilterTest; +import org.radeox.test.filter.UrlFilterTest; +import org.radeox.test.filter.WikiLinkFilterTest; +import org.radeox.test.macro.list.AtoZListFormatterTest; +import org.radeox.test.macro.list.ExampleListFormatterTest; +import org.radeox.test.macro.list.SimpleListTest; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.sql.Connection; +import java.sql.SQLException; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.labkey.api.settings.StashedStartupProperties.homeProjectFolderType; +import static org.labkey.api.settings.StashedStartupProperties.homeProjectResetPermissions; +import static org.labkey.api.settings.StashedStartupProperties.homeProjectWebparts; +import static org.labkey.api.settings.StashedStartupProperties.siteAvailableEmailFrom; +import static org.labkey.api.settings.StashedStartupProperties.siteAvailableEmailMessage; +import static org.labkey.api.settings.StashedStartupProperties.siteAvailableEmailSubject; +import static org.labkey.api.util.MothershipReport.EXPERIMENTAL_LOCAL_MARKETING_UPDATE; +import static org.labkey.api.util.MothershipReport.FEATURE_FLAG_EXTENDED_METRICS; +import static org.labkey.filters.ContentSecurityPolicyFilter.FEATURE_FLAG_DISABLE_ENFORCE_CSP; + +public class CoreModule extends SpringModule implements SearchService.DocumentProvider +{ + private static final Logger LOG = LogHelper.getLogger(CoreModule.class, "Errors during server startup and shut down"); + public static final String PROJECTS_WEB_PART_NAME = "Projects"; + + static + { + // Accept most of the standard Quartz properties, but set the misfire threshold to five minutes. This prevents + // Quartz from dropping scheduled work if a lot of items fire at the same time, like a lot of ETLs triggering at 2AM. + // This can overwhelm the thread pool running them so they don't complete in the default 1 minute window. Set it early so + // if any other module touches Quartz in its setup, it initializes with this setting. + Properties props = System.getProperties(); + props.setProperty("org.quartz.jobStore.misfireThreshold", "300000"); + + // Register dialect extra early, since we need to initialize the data sources before calling DefaultModule.initialize() + SqlDialectRegistry.register(new PostgreSqlDialectFactory()); + + try + { + var field = CharTypes.class.getDeclaredField("sOutputEscapes128"); + field.setAccessible(true); + ((int[])field.get(null))['/'] = '/'; + field.setAccessible(false); + } + catch (NoSuchFieldException|IllegalArgumentException|IllegalAccessException x) + { + // pass + } + } + + private CoreWarningProvider _warningProvider; + private ServletRegistration.Dynamic _webdavServletDynamic; + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + protected void init() + { + ContainerService.setInstance(new ContainerServiceImpl()); + FolderSerializationRegistry.setInstance(new FolderSerializationRegistryImpl()); + ExternalToolsViewService.setInstance(new ExternalToolsViewServiceImpl()); + ExternalToolsViewService.get().registerExternalAccessViewProvider(new ApiKeyViewProvider()); + LimitActiveUsersService.setInstance(() -> new LimitActiveUsersSettings().isUserLimitReached()); + + // Register the default DataLoaders during init so they are available to sql upgrade scripts + DataLoaderServiceImpl dls = new DataLoaderServiceImpl(); + dls.registerFactory(new ExcelLoader.Factory()); + dls.registerFactory(new TabLoader.TsvFactory()); + dls.registerFactory(new TabLoader.CsvFactory()); + dls.registerFactory(new HTMLDataLoader.Factory()); + dls.registerFactory(new JSONDataLoader.Factory()); + dls.registerFactory(new FastaDataLoader.Factory()); + DataLoaderService.setInstance(dls); + + addController("admin", AdminController.class); + addController("admin-sql", SqlScriptController.class); + addController("security", SecurityController.class); + addController("user", UserController.class); + addController("login", LoginController.class); + addController("junit", JunitController.class); + addController("core", CoreController.class); + addController("analytics", AnalyticsController.class); + addController("project", ProjectController.class); + addController("util", UtilController.class); + addController("logger", LoggerController.class); + addController("mini-profiler", MiniProfilerController.class); + addController("notification", NotificationController.class); + addController("product", ProductController.class); + + AuthenticationManager.registerProvider(new DbLoginAuthenticationProvider(), Priority.Low); + AttachmentService.setInstance(new AttachmentServiceImpl()); + AnalyticsService.setInstance(new AnalyticsServiceImpl()); + RhinoService.register(); + CacheManager.addListener(RhinoService::clearCaches); + NotificationService.setInstance(NotificationServiceImpl.getInstance()); + + WarningService.setInstance(new WarningServiceImpl()); + + ViewService.setInstance(ViewServiceImpl.getInstance()); + OptionalFeatureService.setInstance(new OptionalFeatureServiceImpl()); + ThumbnailService.setInstance(new ThumbnailServiceImpl()); + ShortURLService.setInstance(new ShortURLServiceImpl()); + StatsService.setInstance(new StatsServiceImpl()); + SiteValidationService.setInstance(new SiteValidationServiceImpl()); + AnalyticsProviderRegistry.setInstance(new AnalyticsProviderRegistryImpl()); + SummaryStatisticRegistry.setInstance(new SummaryStatisticRegistryImpl()); + UsageMetricsService.setInstance(new UsageMetricsServiceImpl()); + CustomLabelService.setInstance(new CustomLabelServiceImpl()); + SecurityPointcutService.setInstance(new SecurityPointcutServiceImpl()); + AdminConsoleService.setInstance(new AdminConsoleServiceImpl()); + WikiRenderingService.setInstance(new WikiRenderingServiceImpl()); + VcsService.setInstance(new VcsServiceImpl()); + LabKeyScriptEngineManager.setInstance(new ScriptEngineManagerImpl()); + DocumentConversionService.setInstance(new DocumentConversionServiceImpl()); + DbLoginService.setInstance(new DbLoginManager()); + + try + { + ContainerTypeRegistry.get().register("normal", new NormalContainerType()); + ContainerTypeRegistry.get().register("tab", new TabContainerType()); + ContainerTypeRegistry.get().register("workbook", new WorkbookContainerType()); + } + catch (Exception e) + { + throw UnexpectedException.wrap(e); + } + + _warningProvider = new CoreWarningProvider(); + WarningService.get().register(_warningProvider); + + WebdavService.get().setResolver(ModuleStaticResolverImpl.get()); + // need to register webdav resolvers in init() instead of startupAfterSpringConfig since static module files are loaded during module startup + WebdavService.get().registerRootResolver(WebdavResolverImpl.get()); + WebdavService.get().registerRootResolver(WebFilesResolverImpl.get()); + + DefaultSchema.registerProvider(CoreQuerySchema.NAME, new DefaultSchema.SchemaProvider(this) + { + @Override + public boolean isAvailable(DefaultSchema schema, Module module) + { + return true; + } + + @Override + public QuerySchema createSchema(DefaultSchema schema, Module module) + { + return new CoreQuerySchema(schema.getUser(), schema.getContainer()); + } + }); + + if (CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) + { + DefaultSchema.registerProvider(BasePostgreSqlDialect.POSTGRES_SCHEMA_NAME, new DefaultSchema.SchemaProvider(this) + { + @Override + public boolean isAvailable(DefaultSchema schema, Module module) + { + return schema.getContainer().isRoot() && schema.getContainer().hasPermission(schema.getUser(), TroubleshooterPermission.class); + } + + @Override + public QuerySchema createSchema(DefaultSchema schema, Module module) + { + return new PostgresUserSchema(schema.getUser(), schema.getContainer()); + } + }); + } + + OptionalFeatureService.get().addExperimentalFeatureFlag(NotificationMenuView.EXPERIMENTAL_NOTIFICATION_MENU, "Notifications Menu", + "Notifications 'inbox' count display in the header bar with click to show the notifications panel of unread notifications.", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(DataColumn.EXPERIMENTAL_USE_QUERYSELECT_COMPONENT, "Use QuerySelect for row insert/update form", + "This feature will switch the query based select inputs on the row insert/update form to use the React QuerySelect" + + "component. This will allow for a user to view the first 100 options in the select but then use type ahead" + + "search to find the other select values.", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(SQLFragment.FEATUREFLAG_DISABLE_STRICT_CHECKS, "Disable SQLFragment strict checks", + "SQLFragment now has very strict usage validation, these checks may cause errors in code that has not been updated. Turn on this feature to disable checks.", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(LoginController.FEATUREFLAG_DISABLE_LOGIN_XFRAME, "Disable Login X-FRAME-OPTIONS=DENY", + "By default LabKey disables all framing of login related actions. Disabling this feature will revert to using the standard site settings.", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(PageTemplate.EXPERIMENTAL_SHORT_CIRCUIT_ROBOTS, + "Short-circuit robots", + "Save resources by not rendering pages marked as 'noindex' for robots. This is experimental as not all robots are search engines.", + false); + OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(TabLoader.FEATUREFLAG_UNESCAPE_BACKSLASH, + "Unescape backslash character on import", + "Treat backslash '\\' character as an escape character when loading data from file.", + false, false, OptionalFeatureService.FeatureType.Deprecated + )); + OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(AppProps.GENERATE_CONTROLLER_FIRST_URLS, + "Restore controller-first URLS", + "Generate URLs in a legacy format that puts the controller name before the folder path. This option will be removed in LabKey Server 26.3.", + false, false, OptionalFeatureService.FeatureType.Deprecated + )); + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.REJECT_CONTROLLER_FIRST_URLS, + "Reject controller-first URLs", + "Require standard path-first URLs. Note: This option will be ignored if the deprecated feature for generating controller-first URLs is enabled.", + false + ); + + SiteValidationService svc = SiteValidationService.get(); + if (null != svc) + { + svc.registerProviderFactory(getName(), new PermissionsValidatorFactory()); + svc.registerProviderFactory(getName(), new DisplayFormatValidationProviderFactory()); + } + + registerHealthChecks(); + + ContextListener.addNewInstallCompleteListener(() -> sendSystemReadyEmail(UserManager.getAppAdmins())); + + ScriptEngineManagerImpl.registerEncryptionMigrationHandler(); + + deleteTempFiles(); + } + + private void deleteTempFiles() + { + try + { + // Issue 46598 - clean up previously created temp files from file uploads + FileUtil.deleteDirectoryContents(SpringActionController.getTempUploadDir()); + } + catch (IOException e) + { + LOG.warn("Failed to clean up previously uploaded files from {}", SpringActionController.getTempUploadDir(), e); + } + } + + private void registerHealthChecks() + { + HealthCheckRegistry.get().registerHealthCheck("database", HealthCheckRegistry.DEFAULT_CATEGORY, () -> + { + Map healthValues = new HashMap<>(); + boolean allConnected = true; + for (DbScope dbScope : DbScope.getDbScopes()) + { + boolean dbConnected; + try (Connection conn = dbScope.getConnection()) + { + dbConnected = conn != null; + } + catch (SQLException e) + { + dbConnected = false; + } + + healthValues.put(dbScope.getDatabaseName(), dbConnected); + allConnected &= dbConnected; + } + + return new HealthCheck.Result(allConnected, healthValues); + } + ); + + HealthCheckRegistry.get().registerHealthCheck("modules", HealthCheckRegistry.TRIAL_INSTANCES_CATEGORY, () -> { + Map failures = ModuleLoader.getInstance().getModuleFailures(); + Map failureDetails = new HashMap<>(); + for (Map.Entry failure : failures.entrySet()) + { + failureDetails.put(failure.getKey(), failure.getValue().getMessage()); + } + return new HealthCheck.Result(failures.isEmpty(), failureDetails); + }); + + HealthCheckRegistry.get().registerHealthCheck("users", HealthCheckRegistry.TRIAL_INSTANCES_CATEGORY, () -> { + Map userHealth = new HashMap<>(); + ZonedDateTime now = ZonedDateTime.now(); + int userCount = UserManager.getUserCount(Date.from(now.toInstant())); + userHealth.put("registeredUsers", userCount); + return new HealthCheck.Result(userCount > 0, userHealth); + }); + } + + private void sendSystemReadyEmail(List users) + { + if (users.isEmpty()) + return; + + Map map = AppProps.getInstance().getStashedStartupProperties(); + String fromEmail = getValue(map, siteAvailableEmailFrom); + String subject = getValue(map, siteAvailableEmailSubject); + String body = getValue(map, siteAvailableEmailMessage); + + if (fromEmail == null || subject == null || body == null) + return; + + EmailService svc = EmailService.get(); + List messages = new ArrayList<>(); + for (User user: users) + { + EmailMessage message = svc.createMessage(fromEmail, Collections.singletonList(user.getEmail()), subject); + message.addContent(MimeMap.MimeType.HTML, body); + messages.add(message); + } + // For audit purposes, we use the first user as the originator of the message. + // Would be better to have this be a site admin, but we aren't guaranteed to have such a user + // for hosted sites. Another option is to use the guest user here, but that's strange. + svc.sendMessages(messages, users.get(0), ContainerManager.getRoot()); + } + + private @Nullable String getValue(Map map, StashedStartupProperties prop) + { + StartupPropertyEntry entry = map.get(prop); + return null != entry ? StringUtils.trimToNull(entry.getValue()) : null; + } + + @NotNull + @Override + protected Collection createWebPartFactories() + { + return List.of( + new AlwaysAvailableWebPartFactory("Contacts") + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext ctx, @NotNull WebPart webPart) + { + UserSchema schema = QueryService.get().getUserSchema(ctx.getUser(), ctx.getContainer(), CoreQuerySchema.NAME); + QuerySettings settings = new QuerySettings(ctx, QueryView.DATAREGIONNAME_DEFAULT); + + settings.setQueryName(CoreQuerySchema.USERS_TABLE_NAME); + + QueryView view = schema.createView(ctx, settings, null); + view.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); + view.setFrame(WebPartView.FrameType.PORTAL); + view.setTitle("Project Contacts"); + + return view; + } + }, + new BaseWebPartFactory("FolderNav") + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) + { + FolderNavigationForm form = getForm(portalCtx); + + form.setFolderMenu(new FolderMenu(portalCtx)); + + JspView view = new JspView<>("/org/labkey/core/project/folderNav.jsp", form); + view.setTitle("Folder Navigation"); + view.setFrame(WebPartView.FrameType.NONE); + return view; + } + + @Override + public boolean isAvailable(Container c, String scope, String location) + { + return false; + } + + private FolderNavigationForm getForm(ViewContext context) + { + FolderNavigationForm form = new FolderNavigationForm(); + form.setPortalContext(context); + return form; + } + }, + new BaseWebPartFactory("Workbooks") + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) + { + UserSchema schema = QueryService.get().getUserSchema(portalCtx.getUser(), portalCtx.getContainer(), SchemaKey.fromParts(CoreQuerySchema.NAME)); + WorkbookQueryView wbqview = new WorkbookQueryView(portalCtx, schema); + VBox box = new VBox(new WorkbookSearchView(wbqview), wbqview); + box.setFrame(WebPartView.FrameType.PORTAL); + box.setTitle("Workbooks"); + return box; + } + + @Override + public boolean isAvailable(Container c, String scope, String location) + { + return !c.isWorkbook() && "folder".equals(scope) && location.equalsIgnoreCase(HttpView.BODY); + } + }, + new BaseWebPartFactory("Workbook Description") + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) + { + JspView view = new JspView<>("/org/labkey/core/workbook/workbookDescription.jsp"); + view.setTitle("Workbook Description"); + view.setFrame(WebPartView.FrameType.NONE); + return view; + } + + @Override + public boolean isAvailable(Container c, String scope, String location) + { + return false; + } + }, + new AlwaysAvailableWebPartFactory(PROJECTS_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) + { + JspView view = new JspView<>("/org/labkey/core/project/projects.jsp", webPart); + + String title = webPart.getPropertyMap().getOrDefault("title", "Projects"); + view.setTitle(title); + + if (portalCtx.hasPermission(getClass().getName(), AdminPermission.class)) + { + NavTree customize = new NavTree(""); + customize.setScript("customizeProjectWebpart" + webPart.getRowId() + "();"); + view.setCustomize(customize); + } + return view; + } + }, + new AlwaysAvailableWebPartFactory("Subfolders", WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPart createWebPart() + { + // Issue 44913: Set the default properties for all new instances of the Subfolders webpart + WebPart webPart = super.createWebPart(); + return setDefaultProperties(webPart); + } + + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull WebPart webPart) + { + if (webPart.getPropertyMap().isEmpty()) + { + // Configure to show subfolders if not previously configured + webPart = setDefaultProperties(new WebPart(webPart)); + } + + JspView view = new JspView<>("/org/labkey/core/project/projects.jsp", webPart); + view.setTitle(webPart.getPropertyMap().get("title")); + + if (portalCtx.hasPermission(getClass().getName(), AdminPermission.class)) + { + NavTree customize = new NavTree(""); + customize.setScript("customizeProjectWebpart" + webPart.getRowId() + "(" + webPart.getRowId() + ", " + PageFlowUtil.jsString(webPart.getPageId()) + ", " + webPart.getIndex() + ");"); + view.setCustomize(customize); + } + + return view; + } + + private WebPart setDefaultProperties(WebPart webPart) + { + webPart.setProperty("title", "Subfolders"); + webPart.setProperty("containerFilter", ContainerFilter.Type.CurrentAndFirstChildren.name()); + webPart.setProperty("containerTypes", "folder"); + return webPart; + } + }, + new AlwaysAvailableWebPartFactory("Custom Menu", true, true, WebPartFactory.LOCATION_MENUBAR) + { + @Override + public WebPartView getWebPartView(@NotNull final ViewContext portalCtx, @NotNull WebPart webPart) + { + final CustomizeMenuForm form = AdminController.getCustomizeMenuForm(webPart); + String title = "My Menu"; + if (form.getTitle() != null && !form.getTitle().isEmpty()) + title = form.getTitle(); + + WebPartView view; + if (form.isChoiceListQuery()) + { + view = MenuViewFactory.createMenuQueryView(portalCtx, title, form); + } + else + { + view = MenuViewFactory.createMenuFolderView(portalCtx, title, form); + } + view.setFrame(WebPartView.FrameType.PORTAL); + return view; + } + + @Override + public HttpView getEditView(WebPart webPart, ViewContext context) + { + CustomizeMenuForm form = AdminController.getCustomizeMenuForm(webPart); + JspView view = new JspView<>("/org/labkey/core/admin/customizeMenu.jsp", form); + view.setTitle(form.getTitle()); + view.setFrame(WebPartView.FrameType.PORTAL); + return view; + } + } + ); + } + + @Override + public void afterUpdate(ModuleContext moduleContext) + { + if (moduleContext.isNewInstall()) + { + bootstrap(); + } + + // Increment on every core module upgrade to defeat browser caching of static resources. + if (ModuleLoader.getInstance().shouldInsertData()) + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + + // Allow dialect to make adjustments to the just upgraded core database (e.g., install aggregate functions, etc.) + CoreSchema.getInstance().getSqlDialect().afterCoreUpgrade(moduleContext); + + // The core SQL scripts install aggregate functions and other objects that dialects need to know about. Prepare + // the previously initialized dialects again to make sure they're aware of all the changes. Prepare all the + // initialized scopes because we could have more than one scope pointed at the core database (e.g., external + // schemas). See #17077 (pg example) and #19177 (ss example). + for (DbScope scope : DbScope.getInitializedDbScopes()) + scope.getSqlDialect().prepare(scope); + + // Now that we know the standard containers have been created, add a listener that warms the just-cleared caches with + // core.Containers metadata and a few common containers. This may prevent some deadlocks during upgrade, #33550. + CacheManager.addListener(() -> { + ContainerManager.getRoot(); + ContainerManager.getSharedContainer(); + if (ModuleLoader.getInstance().shouldInsertData()) + { + ContainerManager.getHomeContainer(); + } + }); + } + + private void bootstrap() + { + if (ModuleLoader.getInstance().shouldInsertData()) + { + // Create the initial groups + GroupManager.bootstrapGroup(Group.groupUsers, "Users"); + GroupManager.bootstrapGroup(Group.groupGuests, "Guests"); + + // Other containers inherit permissions from root; admins get all permissions, users & guests none + Role noPermsRole = RoleManager.getRole(NoPermissionsRole.class); + Role readerRole = RoleManager.getRole(ReaderRole.class); + + ContainerManager.bootstrapContainer("/", noPermsRole, noPermsRole); + Container rootContainer = ContainerManager.getRoot(); + + // Create all the standard containers (Home, Home/support, Shared) using an empty Collaboration folder type + FolderType collaborationType = new CollaborationFolderType(Collections.emptyList()); + + // Users & guests can read from /home + Container home = ContainerManager.bootstrapContainer(ContainerManager.HOME_PROJECT_PATH, readerRole, readerRole); + home.setFolderType(collaborationType, null); + + ContainerManager.createDefaultSupportContainer().setFolderType(collaborationType, null); + + // Only users can read from /Shared + ContainerManager.bootstrapContainer(ContainerManager.SHARED_CONTAINER_PATH, readerRole, null).setFolderType(collaborationType, null); + + try + { + // Need to insert standard MV indicators for the root -- okay to call getRoot() since we just created it. + String rootContainerId = rootContainer.getId(); + TableInfo mvTable = CoreSchema.getInstance().getTableInfoMvIndicators(); + + for (Map.Entry qcEntry : MvUtil.getDefaultMvIndicators().entrySet()) + { + Map params = new HashMap<>(); + params.put("Container", rootContainerId); + params.put("MvIndicator", qcEntry.getKey()); + params.put("Label", qcEntry.getValue()); + + Table.insert(null, mvTable, params); + } + } + catch (Throwable t) + { + ExceptionUtil.logExceptionToMothership(null, t); + } + + List guids = Stream.of(ContainerManager.HOME_PROJECT_PATH, ContainerManager.SHARED_CONTAINER_PATH) + .map(ContainerManager::getForPath) + .filter(Objects::nonNull) + .map(Container::getEntityId) + .collect(Collectors.toList()); + ContainerManager.setExcludedProjects(guids, () -> {}); + } + else + { + // It's very difficult to bootstrap without the root or shared containers in place; create them now and + // we'll delete them later + Container root = ContainerManager.ensureContainer("/", User.getAdminServiceUser()); + Table.insert(null, CoreSchema.getInstance().getTableInfoContainers(), Map.of("Parent", root.getId(), "Name", "Shared")); + } + } + + + @Override + public CoreUpgradeCode getUpgradeCode() + { + return new CoreUpgradeCode(); + } + + + @Override + public void destroy() + { + super.destroy(); + UsageReportingLevel.shutdown(); + } + + + @Override + public void startupAfterSpringConfig(ModuleContext moduleContext) + { + // Any containers in the cache have bogus folder types since they aren't registered until startup(). See #10310 + ContainerManager.clearCache(); + + checkForMissingDbViews(); + + ProductConfiguration.handleStartupProperties(); + // This listener deletes all properties; make sure it executes after most of the other listeners + ContainerManager.addContainerListener(new CoreContainerListener(), ContainerManager.ContainerListener.Order.Last); + ContainerManager.addContainerListener(new FolderSettingsCache.FolderSettingsCacheListener()); + SecurityManager.init(); + FolderTypeManager.get().registerFolderType(this, FolderType.NONE); + FolderTypeManager.get().registerFolderType(this, new CollaborationFolderType()); + + AnalyticsServiceImpl.get().resetCSP(); + + if (moduleContext.isNewInstall() && ModuleLoader.getInstance().shouldInsertData()) + { + // To initialize the portal layout correctly, we need to add the web parts after the folder types have been + // registered. Thus, it needs to be here in startupAfterSpringConfig() instead of grouped in bootstrap(). + Container homeContainer = ContainerManager.getHomeContainer(); + int count = Portal.getParts(homeContainer, homeContainer.getFolderType().getDefaultPageId(homeContainer)).size(); + addWebPart(PROJECTS_WEB_PART_NAME, homeContainer, HttpView.BODY, count); + } + + EmailService.setInstance(new EmailServiceImpl()); + + if (null != AuditLogService.get() && AuditLogService.get().getClass() != DefaultAuditProvider.class) + { + AuditLogService.get().registerAuditType(new UserAuditProvider()); + AuditLogService.get().registerAuditType(new GroupAuditProvider()); + AuditLogService.get().registerAuditType(new AttachmentAuditProvider()); + AuditLogService.get().registerAuditType(new ContainerAuditProvider()); + AuditLogService.get().registerAuditType(new FileSystemAuditProvider()); + AuditLogService.get().registerAuditType(new ClientApiAuditProvider()); + AuditLogService.get().registerAuditType(new AuthenticationSettingsAuditTypeProvider()); + AuditLogService.get().registerAuditType(new TransactionAuditProvider()); + AuditLogService.get().registerAuditType(new ModulePropertiesAuditProvider()); + + DataStateManager.getInstance().registerDataStateHandler(new CoreQCStateHandler()); + } + ContextListener.addShutdownListener(TempTableTracker.getShutdownListener()); + ContextListener.addShutdownListener(DavController.getShutdownListener()); + ContextListener.addShutdownListener(ShutdownListener.of("Temp file cleanup", null, this::deleteTempFiles)); + + SimpleMetricsService.setInstance(new SimpleMetricsServiceImpl()); + + // Export action stats on graceful shutdown + ContextListener.addShutdownListener(new ShutdownListener() { + @Override + public String getName() + { + return "Action stats export"; + } + + @Override + public void shutdownPre() + { + try + { + // Halt firing of Quartz triggers + Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler(); + scheduler.standby(); + } + catch (SchedulerException ignored) + { + } + + Logger logger = LogManager.getLogger(ActionsTsvWriter.class); + + if (null != logger) + { + StringBuilder buf = new StringBuilder(); + + try (TSVWriter writer = new ActionsTsvWriter()) + { + writer.write(buf); + } + catch (IOException e) + { + LOG.error("Exception exporting action stats", e); + } + + logger.info(buf.toString()); + LOG.info("Completed logging statistics for actions prior to web application shut down"); + } + } + + @Override + public void shutdownStarted() + { + try + { + // Clean up Quartz resources and wait for jobs to complete + Scheduler scheduler = StdSchedulerFactory.getDefaultScheduler(); + scheduler.shutdown(true); + } + catch (SchedulerException ignored) + { + } + } + }); + + // populate look and feel settings and site settings with values read from startup properties as appropriate for not bootstrap + populateLookAndFeelResourcesWithStartupProps(); + AllowedExternalResourceHosts.registerStartupProperties(); + AllowedExternalResourceHosts.registerHosts(); + WriteableLookAndFeelProperties.populateLookAndFeelWithStartupProps(); + WriteableAppProps.populateSiteSettingsWithStartupProps(); + // create users and groups and assign roles with values read from startup properties as appropriate for not bootstrap + SecurityManager.populateStartupProperties(); + // This method depends on resources (FolderType) from other modules, so invoke after startup + ContextListener.addStartupListener(new StartupListener() + { + @Override + public String getName() + { + return "CoreModule.populateSiteSettingsWithStartupProps"; + } + + @Override + public void moduleStartupComplete(ServletContext servletContext) + { + populateSiteSettingsWithStartupProps(); + } + }); + + // Handle optional feature startup properties as late as possible; we want all optional features to be registered first + ContextListener.addStartupListener(new OptionalFeatureStartupListener()); + + LabKeyScriptEngineManager svc = LabKeyScriptEngineManager.get(); + // populate script engine definitions values read from startup properties + if (svc instanceof ScriptEngineManagerImpl) + ((ScriptEngineManagerImpl)svc).populateScriptEngineDefinitionsWithStartupProps(); + + // populate folder types from startup properties as appropriate for not bootstrap + FolderTypeManager.get().populateWithStartupProps(); + LimitActiveUsersSettings.populateStartupProperties(); + + AdminController.registerAdminConsoleLinks(); + AdminController.registerManagementTabs(); + AnalyticsController.registerAdminConsoleLinks(); + UserController.registerAdminConsoleLinks(); + LoggerController.registerAdminConsoleLinks(); + CoreController.registerAdminConsoleLinks(); + + FolderTypeManager.get().registerFolderType(this, new WorkbookFolderType()); + + SecurityManager.addViewFactory(new SecurityController.GroupDiagramViewFactory()); + + FolderSerializationRegistry fsr = FolderSerializationRegistry.get(); + if (null != fsr) + { + fsr.addFactories(new FolderTypeWriterFactory(), new FolderTypeImporterFactory()); + fsr.addFactories(new MissingValueWriterFactory(), new MissingValueImporterFactory()); + fsr.addFactories(new SearchSettingsWriterFactory(), new SearchSettingsImporterFactory()); + fsr.addFactories(new PageWriterFactory(), new PageImporterFactory()); + fsr.addFactories(new ModulePropertiesWriterFactory(), new ModulePropertiesImporterFactory()); + fsr.addFactories(new SecurityGroupWriterFactory(), new SecurityGroupImporterFactory()); + fsr.addFactories(new RoleAssignmentsWriterFactory(), new RoleAssignmentsImporterFactory()); + fsr.addFactories(new DataStateWriter.Factory(), new DataStateImporter.Factory()); + fsr.addFactories(new FileBrowserConfigWriter.Factory(), new FileBrowserConfigImporter.Factory()); + fsr.addImportFactory(new SubfolderImporterFactory()); + } + + SearchService ss = SearchService.get(); + ss.addDocumentParser(new TabLoader.CsvFactoryNoConversions()); + ss.addDocumentProvider(this); + + // Register indexable DataLoaders with the search service + DataLoaderServiceImpl.get().getFactories() + .stream() + .filter(DataLoaderFactory::indexable) + .forEach(ss::addDocumentParser); + + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_NO_GUESTS, + "No Guest Account", + "Disable the guest account", + false); + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_BLOCKER, + "Block malicious clients", + "Reject requests from clients that appear malicious. Turn this feature off if you want to run a security scanner.", + false); + OptionalFeatureService.get().addExperimentalFeatureFlag(FEATURE_FLAG_DISABLE_ENFORCE_CSP, + "Disable enforce Content Security Policy", + "Stop sending the " + ContentSecurityPolicyFilter.ContentSecurityPolicyType.Enforce.getHeaderName() + " header to browsers, " + + "but continue sending the " + ContentSecurityPolicyFilter.ContentSecurityPolicyType.Report.getHeaderName() + " header. " + + "This turns off an important layer of security for the entire site, so use it as a last resort only on a temporary basis " + + "(e.g., if an enforce CSP breaks critical functionality).", + false); + OptionalFeatureService.get().addExperimentalFeatureFlag(DataRegion.EXPERIMENTAL_DATA_REGION_ASYNC_TOTAL_ROWS, + "Data Region Async Total Rows", + "Enable asynchronous calculation of total rows for data regions. This can improve performance for large datasets.", + false); + + OptionalFeatureService.get().addFeatureFlag(new OptionalFeatureFlag(EXPERIMENTAL_LOCAL_MARKETING_UPDATE, + "Self test marketing updates", "Test marketing updates from this local server (requires the mothership module).", false, true, FeatureType.Experimental)); + OptionalFeatureService.get().addFeatureListener(EXPERIMENTAL_LOCAL_MARKETING_UPDATE, (feature, enabled) -> { + // update the timer task when this setting changes + MothershipReport.setSelfTestMarketingUpdates(enabled); + UsageReportingLevel.reportNow(); + }); + + if (null != PropertyService.get()) + { + PropertyService.get().registerDomainKind(new UsersDomainKind()); + if (ModuleLoader.getInstance().shouldInsertData()) + UsersDomainKind.ensureDomain(moduleContext); + } + + // Register the standard, wiki-based terms-of-use provider + SecurityManager.addTermsOfUseProvider(new WikiTermsOfUseProvider()); + + if (null != PropertyService.get()) + PropertyService.get().registerDomainKind(new TestDomainKind()); + + AuthenticationManager.populateSettingsWithStartupProps(); + AnalyticsServiceImpl.populateSettingsWithStartupProps(); + + UsageMetricsService.get().registerUsageMetrics(getName(), () -> { + Map results = new HashMap<>(); + Map javaInfo = new HashMap<>(); + javaInfo.put("java.vendor", System.getProperty("java.vendor")); + javaInfo.put("java.vm.name", System.getProperty("java.vm.name")); + results.put("javaRuntime", javaInfo); + results.put("distributionFilename", AppProps.getInstance().getDistributionFilename()); + results.put("applicationMenuDisplayMode", LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getApplicationMenuDisplayMode()); + results.put("optionalFeatures", OptionalFeatureService.get().getOptionalFeatureFlags().stream() + .collect(Collectors.groupingBy(optionalFeatureFlag -> optionalFeatureFlag.getType().name().toLowerCase(), + Collectors.mapping(flag -> flag, Collectors.toMap(OptionalFeatureFlag::getFlag, OptionalFeatureFlag::isEnabled)) + )) + ); + results.put("productFeaturesEnabled", ProductRegistry.getProductFeatureSet()); + results.put("analyticsTrackingStatus", AnalyticsServiceImpl.get().getTrackingStatus().toString()); + String labkeyContextPath = AppProps.getInstance().getContextPath(); + results.put("webappContextPath", labkeyContextPath); + results.put("embeddedTomcat", true); + boolean customLog4JConfig = false; + if (ModuleLoader.getServletContext() != null) + { + customLog4JConfig = Boolean.parseBoolean(ModuleLoader.getServletContext().getInitParameter("org.labkey.customLog4JConfig")); + } + results.put("customLog4JConfig", customLog4JConfig); + results.put("containerRelativeURL", AppProps.getInstance().getUseContainerRelativeURL()); + results.put("runtimeMode", AppProps.getInstance().isDevMode() ? "development" : "production"); + Set deployedApps = new HashSet<>(CoreWarningProvider.collectAllDeployedApps()); + deployedApps.remove(labkeyContextPath); + if (labkeyContextPath.startsWith("/")) + { + deployedApps.remove(labkeyContextPath.substring(1)); + } + results.put("otherDeployedWebapps", StringUtils.join(deployedApps, ",")); + + // Report the total number of login entries in the audit log + results.put("totalLogins", UserManager.getAuthCount(null, false, false, false)); + results.put("apiKeyLogins", UserManager.getAuthCount(null, false, true, false)); + results.put("sessionTimeout", ModuleLoader.getServletContext().getSessionTimeout()); + results.put("userLimits", new LimitActiveUsersSettings().getMetricsMap()); + results.put("systemUserCount", UserManager.getSystemUserCount()); + Calendar cal = new GregorianCalendar(); + cal.add(Calendar.DATE, -30); + results.put("uniqueRecentUserCount", UserManager.getAuthCount(cal.getTime(), false, false, true)); + results.put("uniqueRecentNonSystemUserCount", UserManager.getAuthCount(cal.getTime(), true, false, true)); + if (OptionalFeatureService.get().isFeatureEnabled(FEATURE_FLAG_EXTENDED_METRICS)) + { + // Optionally include a list of active users, Issue #53050 + results.put("activeUsers", UserManager.getActiveUsers().stream() + .filter(u -> !u.isSystem()) + .map(User::getEmail) + .toList() + ); + } + + results.put("workbookCount", ContainerManager.getWorkbookCount()); + results.put("archivedFolderCount", ContainerManager.getArchivedContainerCount()); + results.put("databaseSize", CoreSchema.getInstance().getSchema().getScope().getDatabaseSize()); + results.put("scriptEngines", LabKeyScriptEngineManager.get().getScriptEngineMetrics()); + results.put("customLabels", CustomLabelService.get().getCustomLabelMetrics()); + Map roleAssignments = new HashMap<>(); + final String roleCountSql = "SELECT COUNT(*) FROM core.RoleAssignments WHERE userid > 0 AND role = ?"; + roleAssignments.put("assayDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.assay.security.AssayDesignerRole").getObject(Long.class)); + roleAssignments.put("dataClassDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.DataClassDesignerRole").getObject(Long.class)); + roleAssignments.put("sampleTypeDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.SampleTypeDesignerRole").getObject(Long.class)); + results.put("roleAssignments", roleAssignments); + + Map allowListCounts = new HashMap<>(); + for (AllowListType type : AllowListType.values()) + { + allowListCounts.put(type.name(), type.getValues().size()); + } + results.put("allowListCounts", allowListCounts); + + return results; + }); + + UsageMetricsService.get().registerUsageMetrics(getName(), WebSocketConnectionManager.getInstance()); + UsageMetricsService.get().registerUsageMetrics(getName(), DbLoginManager.getMetricsProvider()); + UsageMetricsService.get().registerUsageMetrics(getName(), SecurityManager.getMetricsProvider()); + UsageMetricsService.get().registerUsageMetrics(getName(), DisplayFormatAnalyzer.getMetricsProvider()); + UsageMetricsService.get().registerUsageMetrics(getName(), Portal.getMetricsProvider()); + + if (AppProps.getInstance().isDevMode()) + AntiVirusProviderRegistry.get().registerAntiVirusProvider(new DummyAntiVirusService.Provider()); + + FileContentService fileContentService = FileContentService.get(); + if (fileContentService != null) + fileContentService.addFileListener(WebFilesResolverImpl.get()); + + RoleManager.registerPermission(new QCAnalystPermission()); + MarkdownService.setInstance(new MarkdownServiceImpl()); + + // initialize email preference service and listeners + MessageConfigService.setInstance(new EmailPreferenceConfigServiceImpl()); + ContainerManager.addContainerListener(new EmailPreferenceContainerListener()); + UserManager.addUserListener(new EmailPreferenceUserListener()); + + DatabaseMigrationService.get().registerHandler(new DefaultMigrationHandler(CoreSchema.getInstance().getSchema()) + { + @Override + public void beforeVerification() + { + super.beforeVerification(); + + // Delete root and shared containers that were needed for bootstrapping + TableInfo containers = CoreSchema.getInstance().getTableInfoContainers(); + Table.delete(containers); + DbScope targetScope = DbScope.getLabKeyScope(); + new SqlExecutor(targetScope).execute("ALTER SEQUENCE core.containers_rowid_seq RESTART"); // Reset Containers sequence + } + + @Override + public void beforeSchema() + { + new SqlExecutor(getSchema()).execute("ALTER TABLE core.Containers DROP CONSTRAINT FK_Containers_Containers"); + new SqlExecutor(getSchema()).execute("ALTER TABLE core.ViewCategory DROP CONSTRAINT FK_ViewCategory_Parent"); + } + + @Override + public List getTablesToCopy() + { + List tablesToCopy = super.getTablesToCopy(); + tablesToCopy.remove(CoreSchema.getInstance().getTableInfoModules()); + tablesToCopy.remove(CoreSchema.getInstance().getTableInfoSqlScripts()); + tablesToCopy.remove(CoreSchema.getInstance().getTableInfoUpgradeSteps()); + + return tablesToCopy; + } + + @Override + public @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable) + { + return switch (sourceTable.getName()) + { + case "ContainerAliases" -> FieldKey.fromParts("ContainerRowId", "EntityId"); + case "Containers" -> FieldKey.fromParts("EntityId"); + case "Report" -> FieldKey.fromParts("ContainerId"); + case "APIKeys", "AuthenticationConfigurations", "EmailOptions", "Logins", "ReportEngines", "ShortURL", "UsersData" -> SITE_WIDE_TABLE; + default -> super.getContainerFieldKey(sourceTable); + }; + } + + @Override + public FilterClause getContainerClause(TableInfo sourceTable, FieldKey containerFieldKey, Set containers) + { + FilterClause containerClause = super.getContainerClause(sourceTable, containerFieldKey, containers); + + // Users and root groups have container == null, so add that as an OR clause + if (sourceTable.getName().equals("Principals") || sourceTable.getName().equals("Members")) + { + OrClause orClause = new OrClause(); + orClause.addClause(containerClause); + orClause.addClause(new CompareClause(containerFieldKey, CompareType.ISBLANK, null)); + containerClause = orClause; + } + + return containerClause; + } + + @Override + public void afterSchema() + { + new SqlExecutor(getSchema()).execute("ALTER TABLE core.Containers ADD CONSTRAINT FK_Containers_Containers FOREIGN KEY (Parent) REFERENCES core.Containers(EntityId)"); + new SqlExecutor(getSchema()).execute("ALTER TABLE core.ViewCategory ADD CONSTRAINT FK_ViewCategory_Parent FOREIGN KEY (Parent) REFERENCES core.ViewCategory(RowId)"); + } + }); + + DatabaseMigrationService.get().registerHandler(new DefaultMigrationHandler(PropertySchema.getInstance().getSchema()){ + @Override + public @Nullable FieldKey getContainerFieldKey(TableInfo sourceTable) + { + return sourceTable.getName().equals("PropertySets") ? FieldKey.fromParts("ObjectId") : super.getContainerFieldKey(sourceTable); + } + }); + + DatabaseMigrationService.get().registerHandler(new DefaultMigrationHandler(TestSchema.getInstance().getSchema()){ + @Override + public List getTablesToCopy() + { + return List.of(); // Skip all test tables + } + }); + + // TODO: Temporary, until "clone" migration type copies schemas with a registered handler only + if (ModuleLoader.getInstance().getModule(DbScope.getLabKeyScope(), "vehicle") != null) + { + DatabaseMigrationService.get().registerHandler(new DefaultMigrationHandler(DbSchema.get("vehicle", DbSchemaType.Module)) + { + @Override + public List getTablesToCopy() + { + return List.of(); // Skip all vehicle tables + } + }); + } + + Encryption.checkMigration(); + } + + // Issue 7527: Auto-detect missing SQL views and attempt to recreate + private void checkForMissingDbViews() + { + ModuleLoader.getInstance().getModules().stream() + .map(FileSqlScriptProvider::new) + .flatMap(p -> p.getSchemas().stream() + .filter(schema-> SchemaUpdateType.Before.getScript(p, schema) != null || SchemaUpdateType.After.getScript(p, schema) != null) + ) + .filter(schema -> TableXmlUtils.compareXmlToMetaData(schema, false, false, true).hasViewProblem()) + .findAny() + .ifPresent(schema -> + { + LOG.warn("At least one database view was not as expected in the {} schema. Attempting to recreate views automatically", schema.getName()); + ModuleLoader.getInstance().recreateViews(); + }); + } + + @Override + public void registerServlets(ServletContext servletCtx) + { +// even though there is one webdav tree rooted at "/" we still use two servlet bindings. +// This is because we want /_webdav/* to be resolved BEFORE all other servlet-mappings +// and /* to resolve AFTER all other servlet-mappings + _webdavServletDynamic = servletCtx.addServlet("static", new WebdavServlet(true)); + _webdavServletDynamic.setMultipartConfig(SpringActionController.getMultiPartConfigElement()); + _webdavServletDynamic.addMapping("/_webdav/*"); + } + + @Override + public void registerFinalServlets(ServletContext servletCtx) + { + _webdavServletDynamic.addMapping("/"); + } + + @Override + public void startBackgroundThreads() + { + SystemMaintenance.setTimer(); + ThumbnailServiceImpl.startThread(); + // Launch in the background, but delay by 10 seconds to reduce impact on other startup tasks + _warningProvider.startSchemaCheck(10); + + // Start up the default Quartz scheduler, used in many places + try + { + StdSchedulerFactory.getDefaultScheduler().start(); + } + catch (SchedulerException e) + { + throw UnexpectedException.wrap(e); + } + + if (MothershipReport.shouldReceiveMarketingUpdates()) + { + if (AppProps.getInstance().getUsageReportingLevel() == UsageReportingLevel.NONE) + { + // force the usage reporting level to on for community edition distributions + WriteableAppProps appProps = AppProps.getWriteableInstance(); + appProps.setUsageReportingLevel(UsageReportingLevel.ON); + appProps.save(User.getAdminServiceUser()); + } + } + // On bootstrap in production mode, this will send an initial ping with very little information, as the admin will + // not have set up their account yet. On later startups, depending on the reporting level, this will send an immediate + // ping, and then once every 24 hours. + UsageReportingLevel.reportNow(); + TempTableTracker.init(); + + // Loading the PDFBox font cache can be very slow on some agents; fill it proactively. Issue 50601 + JobRunner.getDefault().execute(() -> { + try + { + long start = System.currentTimeMillis(); + FontMapper mapper = FontMappers.instance(); + Method method = mapper.getClass().getMethod("getProvider"); + method.setAccessible(true); + method.invoke(mapper); + long duration = System.currentTimeMillis() - start; + LOG.info("Ensuring PDFBox on-disk font cache took {} seconds", Math.round(duration / 100.0) / 10.0); + } + catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) + { + LOG.warn("Unable to initialize PDFBox font cache", e); + } + }); + } + + @Override + public @NotNull List getDetailedSummary(Container c, User user) + { + long childContainerCount = ContainerManager.getChildren(c).stream().filter(Container::isInFolderNav).count(); + if (childContainerCount == 0) + return Collections.emptyList(); + return List.of(new Summary(childContainerCount, "Subfolder")); + } + + @Override + public JSONObject getPageContextJson(ContainerUser context) + { + JSONObject json = new JSONObject(getDefaultPageContextJson(context.getContainer())); + json.put("productFeatures", ProductRegistry.getProductFeatureSet()); + json.put("primaryApplicationId", ProductRegistry.get().getPrimaryApplicationId(context.getContainer())); + return json; + } + + @Override + public String getTabName(ViewContext context) + { + return "Portal"; + } + + + @Override + public ActionURL getTabURL(Container c, User user) + { + if (user == null) + return AppProps.getInstance().getHomePageActionURL(); + + return PageFlowUtil.urlProvider(ProjectUrls.class).getBeginURL(c); + } + + @Override + public TabDisplayMode getTabDisplayMode() + { + return TabDisplayMode.DISPLAY_USER_PREFERENCE_DEFAULT; + } + + @Override + public @NotNull Set> getIntegrationTests() + { + // Must be mutable since we add the dialect tests below + Set> testClasses = Sets.newHashSet + ( + AdminController.SchemaVersionTestCase.class, + AdminController.SerializationTest.class, + AdminController.TestCase.class, + AdminController.WorkbookDeleteTestCase.class, + AllowListType.TestCase.class, + AttachmentServiceImpl.TestCase.class, + CoreController.TestCase.class, + DataRegion.TestCase.class, + DavController.TestCase.class, + EmailServiceImpl.TestCase.class, + FilesSiteSettingsAction.TestCase.class, + LoggerController.TestCase.class, + LoggingTestCase.class, + LoginController.TestCase.class, + MarkdownTestCase.class, + ModuleInfoTestCase.class, + ModulePropertiesTestCase.class, + ModuleStaticResolverImpl.TestCase.class, + NotificationServiceImpl.TestCase.class, + PortalJUnitTest.class, + PostgreSqlInClauseTest.class, + ProductRegistry.TestCase.class, + RadeoxRenderer.RadeoxRenderTest.class, + RhinoService.TestCase.class, + SchemaXMLTestCase.class, + SecurityApiActions.TestCase.class, + SecurityController.TestCase.class, + SqlDialect.DialectTestCase.class, + SqlScriptController.TestCase.class, + TableViewFormTestCase.class, + UnknownSchemasTest.class, + UserController.TestCase.class + ); + + testClasses.addAll(SqlDialectManager.getAllJUnitTests()); + + return testClasses; + } + + @Override + public @NotNull Set> getUnitTests() + { + return Set.of( + ApiJsonWriter.TestCase.class, + ClassLoaderTestCase.class, + CopyFileRootPipelineJob.TestCase.class, + OutOfRangeDisplayColumn.TestCase.class, + PostgreSqlVersion.TestCase.class, + ScriptEngineManagerImpl.TestCase.class, + StatsServiceImpl.TestCase.class, + + + // Radeox tests + SimpleListTest.class, + ExampleListFormatterTest.class, + AtoZListFormatterTest.class, + BaseRenderEngineTest.class, + BasicRegexTest.class, + ItalicFilterTest.class, + BoldFilterTest.class, + KeyFilterTest.class, + NewlineFilterTest.class, + LineFilterTest.class, + TypographyFilterTest.class, + HtmlRemoveFilterTest.class, + StrikeThroughFilterTest.class, + UrlFilterTest.class, + ParamFilterTest.class, + FilterPipeTest.class, + EscapeFilterTest.class, + LinkTestFilterTest.class, + WikiLinkFilterTest.class, + SmileyFilterTest.class, + ListFilterTest.class, + HeadingFilterTest.class, + RegexFilter.TestCase.class + ); + } + + @Override + public DbSchema createModuleDbSchema(DbScope scope, String metaDataName, Map tableInfoFactoryMap) + { + // Special case for the "labkey" schema we create in every module data source + if ("labkey".equals(metaDataName)) + return new LabKeyDbSchema(scope, tableInfoFactoryMap); + + return super.createModuleDbSchema(scope, metaDataName, tableInfoFactoryMap); + } + + @Override + @NotNull + public Collection getSchemaNames() + { + return List.of + ( + CoreSchema.getInstance().getSchemaName(), // core + PropertySchema.getInstance().getSchemaName(), // prop + TestSchema.getInstance().getSchemaName(), // test + DbSchema.TEMP_SCHEMA_NAME // temp + ); + } + + @NotNull + @Override + public Collection getProvisionedSchemaNames() + { + return Collections.singleton(DbSchema.TEMP_SCHEMA_NAME); + } + + @NotNull + @Override + public Set getSchemasToTest() + { + Set result = new LinkedHashSet<>(super.getSchemasToTest()); + + // Add the "labkey" schema in all module data sources as well... should match application.properties + for (String dataSourceName : ModuleLoader.getInstance().getAllModuleDataSourceNames()) + { + DbScope scope = DbScope.getDbScope(dataSourceName); + if (scope != null) + { + result.add(scope.getLabKeySchema()); + } + } + + return result; + } + + @Override + public void enumerateDocuments(SearchService.TaskIndexingQueue queue, Date since) + { + Container c = queue.getContainer(); + if (c.isRoot()) + return; + + Runnable r = () -> { + Container p = c.getProject(); + if (null == p) + return; + String title; + String keywords; + String body; + + // UNDONE: generalize to other folder types + StudyService svc = StudyService.get(); + Study study = svc != null ? svc.getStudy(c) : null; + + if (null != study) + { + title = study.getSearchDisplayTitle(); + keywords = study.getSearchKeywords(); + body = study.getSearchBody(); + } + else + { + String type = c.getContainerNoun(true); + + String containerTitle = c.getTitle(); + + String description = StringUtils.trimToEmpty(c.getDescription()); + title = type + " -- " + containerTitle; + User u_user = UserManager.getUser(c.getCreatedBy()); + String user = (u_user == null) ? "" : u_user.getDisplayName(User.getSearchUser()); + keywords = description + " " + type + " " + user; + body = type + " " + containerTitle + (c.isProject() ? "" : " in Project " + p.getName()); + body += "\n" + description; + } + + String identifiers = c.getName(); + + Map properties = new HashMap<>(); + + assert (null != keywords); + properties.put(SearchService.PROPERTY.identifiersMed.toString(), identifiers); + properties.put(SearchService.PROPERTY.keywordsMed.toString(), keywords); + properties.put(SearchService.PROPERTY.title.toString(), title); + properties.put(SearchService.PROPERTY.categories.toString(), SearchService.navigationCategory.getName()); + ActionURL startURL = PageFlowUtil.urlProvider(ProjectUrls.class).getStartURL(c); + startURL.setExtraPath(c.getId()); + WebdavResource doc = new SimpleDocumentResource(c.getParsedPath(), + "link:" + c.getId(), + c.getEntityId(), + "text/plain", + body, + startURL, + UserManager.getUser(c.getCreatedBy()), c.getCreated(), + null, null, + properties); + queue.addResource(doc); + }; + r.run(); + } + + @Override + public void indexDeleted() + { + new SqlExecutor(CoreSchema.getInstance().getSchema()).execute("UPDATE core.Documents SET LastIndexed = NULL"); + } + + /** + * Handles startup props for LookAndFeelSettings resources + */ + private void populateLookAndFeelResourcesWithStartupProps() + { + ModuleLoader.getInstance().handleStartupProperties(new StandardStartupPropertyHandler<>(WriteableLookAndFeelProperties.SCOPE_LOOK_AND_FEEL_SETTINGS, ResourceType.class) + { + @Override + public void handle(Map map) + { + boolean incrementRevision = false; + + for (Map.Entry entry : map.entrySet()) + { + SiteResourceHandler handler = getResourceHandler(entry.getKey()); + if (handler != null) + incrementRevision |= setSiteResource(handler, entry.getValue(), User.guest); + } + + // Bump the look & feel revision so browsers retrieve the new logo, custom stylesheet, etc. + if (incrementRevision) + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + } + }); + } + + /** + * This method handles the home project settings + */ + private void populateSiteSettingsWithStartupProps() + { + Map props = AppProps.getInstance().getStashedStartupProperties(); + + StartupPropertyEntry folderTypeEntry = props.get(homeProjectFolderType); + if (null != folderTypeEntry) + { + FolderType folderType = FolderTypeManager.get().getFolderType(folderTypeEntry.getValue()); + if (folderType != null) + // using guest user since the server startup doesn't have a true user (this will be used for audit events) + ContainerManager.getHomeContainer().setFolderType(folderType, User.guest); + else + LOG.error("Unable to find folder type for home project during server startup: " + folderTypeEntry.getValue()); + } + + StartupPropertyEntry resetPermissionsEntry = props.get(homeProjectResetPermissions); + if (null != resetPermissionsEntry && Boolean.valueOf(resetPermissionsEntry.getValue())) + { + // reset the home project permissions to remove the default assignments given at server install + MutableSecurityPolicy homePolicy = new MutableSecurityPolicy(ContainerManager.getHomeContainer()); + SecurityPolicyManager.savePolicy(homePolicy, User.getAdminServiceUser()); + // remove the guest role assignment from the support subfolder + Group guests = SecurityManager.getGroup(Group.groupGuests); + if (null != guests) + { + Container supportFolder = ContainerManager.getDefaultSupportContainer(); + if (supportFolder != null) + { + MutableSecurityPolicy supportPolicy = new MutableSecurityPolicy(supportFolder.getPolicy()); + for (Role assignedRole : supportPolicy.getAssignedRoles(guests)) + supportPolicy.removeRoleAssignment(guests, assignedRole); + SecurityPolicyManager.savePolicy(supportPolicy, User.getAdminServiceUser()); + } + } + } + + StartupPropertyEntry webparts = props.get(homeProjectWebparts); + if (null != webparts) + { + // Clear existing webparts added by core and wiki modules + Container homeContainer = ContainerManager.getHomeContainer(); + Portal.saveParts(homeContainer, Collections.emptyList()); + + for (String webpartName : StringUtils.split(webparts.getValue(), ';')) + { + WebPartFactory webPartFactory = Portal.getPortalPart(webpartName); + if (webPartFactory != null) + addWebPart(webPartFactory.getName(), homeContainer, HttpView.BODY); + } + } + } + + private @Nullable SiteResourceHandler getResourceHandler(@NotNull ResourceType type) + { + return LookAndFeelPropertiesManager.get().getResourceHandler(type); + } + + private boolean setSiteResource(SiteResourceHandler resourceHandler, StartupPropertyEntry prop, User user) + { + Resource resource = getModuleResourceFromPropValue(prop.getValue()); + if (resource != null) + { + try + { + resourceHandler.accept(resource, ContainerManager.getRoot(), user); + return true; + } + catch(Exception e) + { + LOG.error("Exception setting {} during server startup.", prop.getName(), e); + } + } + + LOG.error("Unable to find {} resource during server startup: {}", prop.getName(), prop.getValue()); + return false; + } + + private Resource getModuleResourceFromPropValue(String propValue) + { + if (propValue != null) + { + // split the prop value on the separator char to get the module name and resource path in that module + String moduleName = propValue.substring(0, propValue.indexOf(":")); + String resourcePath = propValue.substring(propValue.indexOf(":") + 1); + + Module module = ModuleLoader.getInstance().getModule(moduleName); + if (module != null) + return module.getModuleResource(resourcePath); + } + + return null; + } + + public void rerunSchemaCheck() + { + // Queue a job without delay. This avoids executing multiple overlapping schema checks. Not bothering with a + // more surgical approach since this variant is likely being called during development. + _warningProvider.startSchemaCheck(0); + } +} diff --git a/core/src/org/labkey/core/CoreUpgradeCode.java b/core/src/org/labkey/core/CoreUpgradeCode.java index c732bc1bb99..eec54cf0ae2 100644 --- a/core/src/org/labkey/core/CoreUpgradeCode.java +++ b/core/src/org/labkey/core/CoreUpgradeCode.java @@ -30,6 +30,7 @@ import org.labkey.api.module.ModuleContext; import org.labkey.api.module.ModuleLoader; import org.labkey.api.security.Directive; +import org.labkey.api.security.Encryption; import org.labkey.api.settings.AppProps; import org.labkey.api.util.logging.LogHelper; import org.labkey.core.security.AllowedExternalResourceHosts; diff --git a/core/src/org/labkey/core/reports/ScriptEngineManagerImpl.java b/core/src/org/labkey/core/reports/ScriptEngineManagerImpl.java index ac7722f065c..0f6309ef8a6 100644 --- a/core/src/org/labkey/core/reports/ScriptEngineManagerImpl.java +++ b/core/src/org/labkey/core/reports/ScriptEngineManagerImpl.java @@ -56,6 +56,7 @@ import org.labkey.api.security.Encryption.Algorithm; import org.labkey.api.security.Encryption.DecryptionException; import org.labkey.api.security.Encryption.EncryptionMigrationHandler; +import org.labkey.api.security.Encryption.AESConfig; import org.labkey.api.security.User; import org.labkey.api.settings.LenientStartupPropertyHandler; import org.labkey.api.settings.StartupProperty; @@ -121,11 +122,18 @@ public class ScriptEngineManagerImpl extends ScriptEngineManager implements LabK private static final String PASSWORD_FIELD = "password"; - static final EncryptionMigrationHandler ENCRYPTION_MIGRATION_HANDLER = (oldPassPhrase, keySource) -> { - LOG.info(" Attempting to migrate encrypted content in scripting engine configurations"); - Algorithm decryptAes = Encryption.getAES128(oldPassPhrase, keySource); + static final EncryptionMigrationHandler ENCRYPTION_MIGRATION_HANDLER = (oldPassPhrase, keySource, oldConfig) -> { + String currentPassPhrase = Encryption.getEncryptionPassPhrase(); + if (currentPassPhrase == null) + { + LOG.warn(" Cannot migrate encrypted content: EncryptionKey not specified"); + return; + } + + Algorithm oldAes = Encryption.getAES128(oldPassPhrase, keySource, oldConfig); + Algorithm newAes = Encryption.getAES128(currentPassPhrase, keySource, AESConfig.current); + TableInfo tinfo = CoreSchema.getInstance().getTableInfoReportEngines(); - Algorithm encryptAes = ExternalScriptEngineDefinitionImpl.AES.get(); new TableSelector(tinfo, PageFlowUtil.set("RowId", "Configuration")).getValueMap(Integer.class).forEach((rowId, configuration) -> { JSONObject json = new JSONObject(configuration); String oldEncryptedPassword = json.optString(PASSWORD_FIELD, null); @@ -134,15 +142,24 @@ public class ScriptEngineManagerImpl extends ScriptEngineManager implements LabK LOG.info(" Migrating script engine configuration " + rowId); try { - String decryptedPassword = decryptAes.decrypt(Base64.decodeBase64(oldEncryptedPassword)); - String newEncryptedPassword = Base64.encodeBase64String(encryptAes.encrypt(decryptedPassword)); - json.put(PASSWORD_FIELD, newEncryptedPassword); - assert decryptedPassword.equals(encryptAes.decrypt(Base64.decodeBase64(json.getString(PASSWORD_FIELD)))); - Table.update(null, tinfo, PageFlowUtil.map("Configuration", json.toString()), rowId); - } - catch (DecryptionException e) - { - LOG.info(" Failed to decrypt password for configuration " + rowId + ". This configuration will be skipped."); + String decryptedPassword; + try + { + decryptedPassword = oldAes.decrypt(Base64.decodeBase64(oldEncryptedPassword)); + + String newEncryptedPassword = Base64.encodeBase64String(newAes.encrypt(decryptedPassword)); + assert decryptedPassword.equals(newAes.decrypt(Base64.decodeBase64(newEncryptedPassword))); + + if (newEncryptedPassword != null) + { + json.put(PASSWORD_FIELD, newEncryptedPassword); + Table.update(null, tinfo, PageFlowUtil.map("Configuration", json.toString()), rowId); + } + } + catch (DecryptionException e) + { + LOG.info(" Failed to decrypt password for configuration " + rowId + ". This configuration will be skipped."); + } } catch (Exception e) {