Skip to content

Commit

Permalink
#25184 adding the support for the system table config (#25936)
Browse files Browse the repository at this point in the history
* #25184 adding the support for the system table config

* #25184 adding the support for the system table config

* #25184 avoid recursive issues on system table/config

* #25184 adding fixes and unit test

* #25184 adding fixes and unit test

* #25184 adding feedback

* #25184 adding more coverage test

* #25184 trying some fixes for the init fail

* #25184 reverting undesired changes

* #25184 reverting undesired changes

* #25184 reverting undesired changes

* #25184 decoupling the task from config

* #25184 trying to avoid issues when the db is no ready

* #25184 trying to avoid issues when the db is no ready

* #25184 adding a fix to avoid to run the system table on test

* #25184 adding an event when update the system table

* #25184 making lazy the system table on the config source

* #25184 lazyness of the config dependency

* Attempt to fix Integration Test run

* Attempt to fix Integration Test run

* Attempting to fix Unit Test run

* Attempting to fix Unit Test run

* Reverting unnecessary file changes.

* Implementing SonarQube feedback

* Merging changes from master.

---------

Co-authored-by: Jose Castro <jose.castro@dotcms.com>
  • Loading branch information
jdotcms and jcastro-dotcms committed Oct 17, 2023
1 parent 0a4a8f8 commit 26f0c52
Show file tree
Hide file tree
Showing 17 changed files with 528 additions and 72 deletions.
5 changes: 4 additions & 1 deletion dotCMS/src/integration-test/java/com/dotcms/MainSuite.java
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@
import com.dotmarketing.startup.runonce.Task230701AddHashIndicesToWorkflowTablesTest;
import com.dotmarketing.startup.runonce.Task230713IncreaseDisabledWysiwygColumnSizeTest;
import com.dotmarketing.util.HashBuilderTest;
import com.dotmarketing.util.ITConfigTest;
import com.dotmarketing.util.MaintenanceUtilTest;
import com.dotmarketing.util.ResourceCollectorUtilTest;
import com.dotmarketing.util.TestConfig;
Expand Down Expand Up @@ -652,7 +653,9 @@
Task230701AddHashIndicesToWorkflowTablesTest.class,
Task230713IncreaseDisabledWysiwygColumnSizeTest.class,
BundleFactoryImplTest.class,
DropOldContentVersionsJobTest.class
DropOldContentVersionsJobTest.class,
ITConfigTest.class

})

public class MainSuite {
Expand Down
48 changes: 47 additions & 1 deletion dotCMS/src/integration-test/java/com/dotcms/MainSuite2b.java
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,53 @@
import com.dotmarketing.startup.StartupTasksExecutorDataTest;
import com.dotmarketing.startup.StartupTasksExecutorTest;
import com.dotmarketing.startup.runalways.Task00050LoadAppsSecretsTest;
import com.dotmarketing.startup.runonce.*;
import com.dotmarketing.startup.runonce.Task05370AddAppsPortletToLayoutTest;
import com.dotmarketing.startup.runonce.Task05380ChangeContainerPathToAbsoluteTest;
import com.dotmarketing.startup.runonce.Task05390MakeRoomForLongerJobDetailTest;
import com.dotmarketing.startup.runonce.Task05395RemoveEndpointIdForeignKeyInIntegrityResolverTablesIntegrationTest;
import com.dotmarketing.startup.runonce.Task201013AddNewColumnsToIdentifierTableTest;
import com.dotmarketing.startup.runonce.Task201014UpdateColumnsValuesInIdentifierTableTest;
import com.dotmarketing.startup.runonce.Task201102UpdateColumnSitelicTableTest;
import com.dotmarketing.startup.runonce.Task210218MigrateUserProxyTableTest;
import com.dotmarketing.startup.runonce.Task210319CreateStorageTableTest;
import com.dotmarketing.startup.runonce.Task210321RemoveOldMetadataFilesTest;
import com.dotmarketing.startup.runonce.Task210506UpdateStorageTableTest;
import com.dotmarketing.startup.runonce.Task210510UpdateStorageTableDropMetadataColumnTest;
import com.dotmarketing.startup.runonce.Task210520UpdateAnonymousEmailTest;
import com.dotmarketing.startup.runonce.Task210527DropReviewFieldsFromContentletTableTest;
import com.dotmarketing.startup.runonce.Task210719CleanUpTitleFieldTest;
import com.dotmarketing.startup.runonce.Task210802UpdateStructureTableTest;
import com.dotmarketing.startup.runonce.Task210805DropUserProxyTableTest;
import com.dotmarketing.startup.runonce.Task210816DeInodeRelationshipTest;
import com.dotmarketing.startup.runonce.Task210901UpdateDateTimezonesTest;
import com.dotmarketing.startup.runonce.Task211007RemoveNotNullConstraintFromCompanyMXColumnTest;
import com.dotmarketing.startup.runonce.Task211012AddCompanyDefaultLanguageTest;
import com.dotmarketing.startup.runonce.Task211101AddContentletAsJsonColumnTest;
import com.dotmarketing.startup.runonce.Task211103RenameHostNameLabelTest;
import com.dotmarketing.startup.runonce.Task220202RemoveFKStructureFolderConstraintTest;
import com.dotmarketing.startup.runonce.Task220203RemoveFolderInodeConstraintTest;
import com.dotmarketing.startup.runonce.Task220214AddOwnerAndIDateToFolderTableTest;
import com.dotmarketing.startup.runonce.Task220215MigrateDataFromInodeToFolderTest;
import com.dotmarketing.startup.runonce.Task220330ChangeVanityURLSiteFieldTypeTest;
import com.dotmarketing.startup.runonce.Task220401CreateClusterLockTableTest;
import com.dotmarketing.startup.runonce.Task220402UpdateDateTimezonesTest;
import com.dotmarketing.startup.runonce.Task220413IncreasePublishedPushedAssetIdColTest;
import com.dotmarketing.startup.runonce.Task220512UpdateNoHTMLRegexValueTest;
import com.dotmarketing.startup.runonce.Task220606UpdatePushNowActionletNameTest;
import com.dotmarketing.startup.runonce.Task220822CreateVariantTableTest;
import com.dotmarketing.startup.runonce.Task220824CreateDefaultVariantTest;
import com.dotmarketing.startup.runonce.Task220825CreateVariantFieldTest;
import com.dotmarketing.startup.runonce.Task220829CreateExperimentsTableTest;
import com.dotmarketing.startup.runonce.Task220912UpdateCorrectShowOnMenuPropertyTest;
import com.dotmarketing.startup.runonce.Task220928AddLookbackWindowColumnToExperimentTest;
import com.dotmarketing.startup.runonce.Task221007AddVariantIntoPrimaryKeyTest;
import com.dotmarketing.startup.runonce.Task230110MakeSomeSystemFieldsRemovableByBaseTypeTest;
import com.dotmarketing.startup.runonce.Task230328AddMarkedForDeletionColumnTest;
import com.dotmarketing.startup.runonce.Task230426AlterVarcharLengthOfLockedByColTest;
import com.dotmarketing.startup.runonce.Task230523CreateVariantFieldInContentletIntegrationTest;
import com.dotmarketing.startup.runonce.Task230701AddHashIndicesToWorkflowTablesTest;
import com.dotmarketing.startup.runonce.Task230707CreateSystemTableTest;
import com.dotmarketing.startup.runonce.Task230713IncreaseDisabledWysiwygColumnSizeTest;
import com.dotmarketing.util.MaintenanceUtilTest;
import com.dotmarketing.util.ResourceCollectorUtilTest;
import com.dotmarketing.util.UtilMethodsITest;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@ public void init() throws Exception {
if (initCompleted.compareAndSet(false, true)) {

System.setProperty(TestUtil.DOTCMS_INTEGRATION_TEST, TestUtil.DOTCMS_INTEGRATION_TEST);



Awaitility.setDefaultPollInterval(10, TimeUnit.MILLISECONDS);
Awaitility.setDefaultPollDelay(Duration.ZERO);
Awaitility.setDefaultTimeout(Duration.ONE_MINUTE);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package com.dotmarketing.util;

import com.dotcms.IntegrationTestBase;
import com.dotmarketing.business.APILocator;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;

/**
* Test for the ConfigTest.
*/
public class ITConfigTest extends IntegrationTestBase {

@BeforeClass
public static void beforeInit() throws Exception {
IntegrationTestBase.beforeInit();
Config.enableSystemTableConfigSource = true;
}

@AfterClass
public static void afterInit() throws Exception {
Config.enableSystemTableConfigSource = false;
}

/**
* Method to test: {@link Config#getStringProperty(String, String)}
* Given Scenario: Will request a property that does not exist on config nor system table
* ExpectedResult: Should return default value
*/
@Test
public void test_404_on_system_table () {

final String value = Config.getStringProperty("NON_EXISTING_KEY", "DEFAULT_VALUE");
Assert.assertEquals("Should return default value", "DEFAULT_VALUE", value);
}

/**
* Method to test: {@link Config#getStringProperty(String, String)}
* Given Scenario: Will request a property that exist on the system table
* ExpectedResult: Should return value from the system table
*/
@Test
public void test_resolving_config_on_system_table () {

APILocator.getSystemAPI().getSystemTable().set("NEW_PROPERTY", "NEW_VALUE");
final String value = Config.getStringProperty("NEW_PROPERTY", "DEFAULT_VALUE");
Assert.assertEquals("Should return the value from the system table", "NEW_VALUE", value);
final Object valueFromCache = APILocator.getSystemAPI().getSystemCache().get("NEW_PROPERTY");
Assert.assertNotNull("Should be on the cache", valueFromCache);
Assert.assertEquals("Should return the value from the system table", "NEW_VALUE", valueFromCache);
}

/**
* Method to test: {@link Config#getStringProperty(String, String)}
* Given Scenario: Will request a property that exist on the system table and it is updated
* ExpectedResult: Should return latest value twice
*/
@Test
public void test_resolving_config_on_system_table_and_update () {

APILocator.getSystemAPI().getSystemTable().set("SECOND_PROPERTY", "NEW_VALUE_1");
String value = Config.getStringProperty("SECOND_PROPERTY", "DEFAULT_VALUE");
Assert.assertEquals("Should return the value from the system table", "NEW_VALUE_1", value);
APILocator.getSystemAPI().getSystemTable().set("SECOND_PROPERTY", "NEW_VALUE_2");
value = Config.getStringProperty("SECOND_PROPERTY", "DEFAULT_VALUE");
Assert.assertEquals("Should return the value from the system table", "NEW_VALUE_2", value);
final Object valueFromCache = APILocator.getSystemAPI().getSystemCache().get("SECOND_PROPERTY");
Assert.assertNotNull("Should be on the cache", valueFromCache);
Assert.assertEquals("Should return the value from the system table", "NEW_VALUE_2", valueFromCache);
}

/**
* Method to test: {@link Config#getStringProperty(String, String)}
* Given Scenario: Will request a property that exist on the system table, then updated, finally removed
* ExpectedResult: Should return latest value twice and finally removed
*/
@Test
public void test_resolving_config_on_system_table_update_and_update () {

APILocator.getSystemAPI().getSystemTable().set("THIRD_PROPERTY", "NEW_VALUE_1");
String value = Config.getStringProperty("THIRD_PROPERTY", "DEFAULT_VALUE");
Assert.assertEquals("Should return the value from the system table", "NEW_VALUE_1", value);
APILocator.getSystemAPI().getSystemTable().set("THIRD_PROPERTY", "NEW_VALUE_2");
value = Config.getStringProperty("THIRD_PROPERTY", "DEFAULT_VALUE");
Assert.assertEquals("Should return the value from the system table", "NEW_VALUE_2", value);
APILocator.getSystemAPI().getSystemTable().delete("THIRD_PROPERTY");
value = Config.getStringProperty("THIRD_PROPERTY", "DEFAULT_VALUE");
Assert.assertEquals("Should return default value since the property has been removed", "DEFAULT_VALUE", value);
}
}
12 changes: 8 additions & 4 deletions dotCMS/src/main/java/com/dotcms/business/SystemCache.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.dotcms.business;

import com.dotmarketing.business.Cachable;
import com.dotmarketing.business.CachableSupport;
import com.dotmarketing.business.CacheLocator;
import com.dotmarketing.business.DotCacheAdministrator;
import com.dotmarketing.business.DotCacheException;
Expand All @@ -9,7 +10,7 @@
/**
* Created by jasontesser on 3/17/17.
*/
public class SystemCache implements Cachable {
public class SystemCache implements CachableSupport<String, Object> {

protected static String PRIMARY_GROUP = "SYSTEM_GROUP";
private String[] groupNames = {PRIMARY_GROUP};
Expand All @@ -32,7 +33,8 @@ public void clearCache() {
cache.flushGroup(PRIMARY_GROUP);
}

public Object get(String key) {
@Override
public Object get(final String key) {
Object o = null;
try{
o = (Object) cache.get(key,PRIMARY_GROUP);
Expand All @@ -43,7 +45,8 @@ public Object get(String key) {
return o;
}

public Object put(String key, Object object) {
@Override
public Object put(final String key, final Object object) {
if(object == null){
return null;
}
Expand All @@ -56,7 +59,8 @@ public Object put(String key, Object object) {
/* (non-Javadoc)
* @see org.apache.velocity.runtime.resource.ResourceCache#remove(java.lang.Object)
*/
public void remove(String key) {
@Override
public void remove(final String key) {

try{
cache.remove(key,getPrimaryGroup());
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
package com.dotcms.business;

import com.dotcms.exception.ExceptionUtil;
import com.dotmarketing.business.CacheLocator;
import com.dotmarketing.common.db.DotConnect;
import com.dotmarketing.exception.DoesNotExistException;
import com.dotmarketing.exception.DotDataException;
import com.dotmarketing.exception.DotDuplicateDataException;
import com.dotmarketing.exception.DotRuntimeException;
import com.dotmarketing.util.UtilMethods;
import io.vavr.control.Try;

import java.util.LinkedHashMap;
import java.util.List;
Expand All @@ -32,25 +30,25 @@ public SystemTableFactoryImpl () {
protected Optional<String> find(final String key) throws DotDataException {

Object value = null;
if(UtilMethods.isSet(key)) {

value = this.systemCache.get(key);
if(Objects.isNull(value)) {

final List<Map<String, Object>> result = new DotConnect()
.setSQL(" SELECT * FROM system_table WHERE key = ? ")
.addParam(key)
.loadObjectResults();
if (UtilMethods.isSet(key)) {

value = this.systemCache.getOrUpdate(key, ()-> {

List<Map<String, Object>> result;
try {
result = new DotConnect()
.setSQL(" SELECT * FROM system_table WHERE key = ? ")
.addParam(key)
.loadObjectResults();
} catch (final DotDataException e) {
throw new DotRuntimeException(String.format("Failed to retrieve key '%s': %s"
, key, ExceptionUtil.getErrorMessage(e)), e);
}

value = result.isEmpty() ? null : result.get(0).get("value");
final Object v = null == result || result.isEmpty()? null : result.get(0).get("value");
return Objects.nonNull(v)?v.toString():VALUE_404;

if(Objects.nonNull(value)) {
this.systemCache.put(key, value);
} else {
this.systemCache.put(key, VALUE_404);
value = VALUE_404;
}
}
});
}

return Objects.isNull(value) || VALUE_404.equals(value)?
Expand Down
8 changes: 8 additions & 0 deletions dotCMS/src/main/java/com/dotcms/business/SystemTableImpl.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package com.dotcms.business;

import com.dotmarketing.business.APILocator;
import com.dotmarketing.business.FactoryLocator;
import com.dotmarketing.db.HibernateUtil;
import com.dotmarketing.exception.DotRuntimeException;
import com.dotmarketing.util.Logger;
import io.vavr.control.Try;
Expand Down Expand Up @@ -47,6 +49,9 @@ public void set(final String key, final String value) {
Logger.debug(this, ()-> "Saving or Updating the key: " + key + " value: " + value);
Try.run(()-> this.systemTableFactory.saveOrUpdate(key, value))
.getOrElseThrow((e)-> new DotRuntimeException(e.getMessage(), e));

Try.run(()->HibernateUtil.addCommitListener(()->
APILocator.getLocalSystemEventsAPI().asyncNotify(new SystemTableUpdatedKeyEvent(key))));
}

@Override
Expand All @@ -56,5 +61,8 @@ public void delete(String key) {
Logger.debug(this, ()-> "Deleting the key: " + key);
Try.run(()-> this.systemTableFactory.delete(key))
.getOrElseThrow((e)-> new DotRuntimeException(e.getMessage(), e));

Try.run(()->HibernateUtil.addCommitListener(()->
APILocator.getLocalSystemEventsAPI().asyncNotify(new SystemTableUpdatedKeyEvent(key))));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package com.dotcms.business;

import com.dotcms.config.DotInitializer;
import com.dotmarketing.util.Config;

/**
*
* @author Jose Castro
* @since Oct 12th, 2023
*/
public class SystemTableInitializer implements DotInitializer {

@Override
public void init() {
Config.initSystemTableConfigSource();
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package com.dotcms.business;

import java.io.Serializable;

/**
* This is a local event to notify that a system table has been updated.
* @author jsanca
*/
public class SystemTableUpdatedKeyEvent implements Serializable {

private final String key;
public SystemTableUpdatedKeyEvent(final String key) {
this.key = key;
}

public String getKey() {
return key;
}
}

0 comments on commit 26f0c52

Please sign in to comment.