Skip to content
Permalink
Browse files

Full config save optimization

* Current config save handles multiple scenarios so there are repetitive preprocess, validation,
  cloning..., which is one of reason for slow config updates on servers
  with large config.
* This commit introduces an optimized config save which handles only full config
  saves through UI, API, config upgrade(during server restart) and config loads(timer every 5 secs)
* Entity updates through UI which do not have an API still follow the old flow.
* Config save can be toggled between new and old flow using the System
  Environment 'optimize.full.config.save', this is 'true' by default.
  • Loading branch information...
maheshp committed Dec 15, 2016
1 parent e55881c commit 2ae7fdb72e31b2868cee8a3d757402e2e0e40456
Showing with 1,739 additions and 82 deletions.
  1. +6 −0 base/src/com/thoughtworks/go/util/SystemEnvironment.java
  2. +14 −0 config/config-server/src/com/thoughtworks/go/config/GoConfigMigration.java
  3. +15 −1 config/config-server/src/com/thoughtworks/go/config/MagicalGoConfigXmlWriter.java
  4. +23 −2 server/src/com/thoughtworks/go/config/CachedGoConfig.java
  5. +152 −0 server/src/com/thoughtworks/go/config/FullConfigSaveFlow.java
  6. +114 −0 server/src/com/thoughtworks/go/config/FullConfigSaveMergeFlow.java
  7. +74 −0 server/src/com/thoughtworks/go/config/FullConfigSaveNormalFlow.java
  8. +14 −2 server/src/com/thoughtworks/go/config/GoConfigDao.java
  9. +39 −0 server/src/com/thoughtworks/go/config/GoConfigFileReader.java
  10. +74 −0 server/src/com/thoughtworks/go/config/GoConfigFileWriter.java
  11. +139 −0 server/src/com/thoughtworks/go/config/GoConfigMigrator.java
  12. +146 −38 server/src/com/thoughtworks/go/config/GoFileConfigDataSource.java
  13. +44 −0 server/src/com/thoughtworks/go/config/update/FullConfigUpdateCommand.java
  14. +1 −1 server/src/com/thoughtworks/go/server/initializers/ApplicationInitializer.java
  15. +30 −17 server/src/com/thoughtworks/go/server/service/GoConfigService.java
  16. +2 −2 server/test/common/com/thoughtworks/go/helpers/LogFileHelper.java
  17. +7 −7 server/test/common/com/thoughtworks/go/util/GoConfigFileHelper.java
  18. +2 −2 server/test/integration/com/thoughtworks/go/config/GoConfigMigrationIntegrationTest.java
  19. +215 −3 server/test/integration/com/thoughtworks/go/config/GoFileConfigDataSourceTest.java
  20. +1 −1 server/test/integration/com/thoughtworks/go/server/security/IsSecurityEnabledVoterTest.java
  21. +79 −1 server/test/unit/com/thoughtworks/go/config/CachedGoConfigTest.java
  22. +228 −0 server/test/unit/com/thoughtworks/go/config/FullConfigSaveMergeFlowTest.java
  23. +159 −0 server/test/unit/com/thoughtworks/go/config/FullConfigSaveNormalFlowTest.java
  24. +121 −0 server/test/unit/com/thoughtworks/go/config/GoConfigMigratorTest.java
  25. +1 −1 server/test/unit/com/thoughtworks/go/server/initializers/ApplicationInitializerTest.java
  26. +38 −3 server/test/unit/com/thoughtworks/go/server/service/GoConfigServiceTest.java
  27. +1 −1 server/test/unit/com/thoughtworks/go/server/service/PipelinePauseServiceTest.java
@@ -202,6 +202,8 @@

public static GoIntSystemProperty DEPENDENCY_MATERIAL_UPDATE_LISTENERS = new GoIntSystemProperty("dependency.material.check.threads", 3);

public static GoSystemProperty<Boolean> OPTIMIZE_FULL_CONFIG_SAVE = new GoBooleanSystemProperty("optimize.full.config.save", true);

private final static Map<String, String> GIT_ALLOW_PROTOCOL;

static {
@@ -800,6 +802,10 @@ public String getAgentKeyStorePassword() {
return get(SystemEnvironment.GO_AGENT_KEYSTORE_PASSWORD);
}

public boolean optimizeFullConfigSave() {
return OPTIMIZE_FULL_CONFIG_SAVE.getValue();
}

public static abstract class GoSystemProperty<T> {
private String propertyName;
private T defaultValue;
@@ -92,6 +92,8 @@ public void run() {
this.registry = registry;
}

// This method should be removed once upgrade is done using new com.thoughtworks.go.config.GoConfigMigrator#migrate()
@Deprecated
public GoConfigMigrationResult upgradeIfNecessary(File configFile, final String currentGoServerVersion) {
try {
return upgradeValidateAndVersion(configFile, true, currentGoServerVersion);
@@ -139,6 +141,18 @@ private GoConfigHolder reloadedConfig(ByteArrayOutputStream stream, String upgra
return configHolder;
}

public File revertFileToVersion(File configFile, GoConfigRevision currentConfigRevision) throws Exception {
File backupFile = getBackupFile(configFile, "invalid.");
try {
backup(configFile, backupFile);
FileUtils.writeStringToFile(configFile, currentConfigRevision.getContent());
} catch (IOException e1) {
throw new RuntimeException(String.format("Could not write to config file '%s'.", configFile.getAbsolutePath()), e1);
}

return backupFile;
}

private GoConfigMigrationResult revertFileToVersion(File configFile, GoConfigRevision currentConfigRevision, Exception e) throws Exception {
File backupFile = getBackupFile(configFile, "invalid.");
try {
@@ -29,6 +29,8 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -88,7 +90,19 @@ public void write(CruiseConfig configForEdit, OutputStream output, boolean skipP
LOGGER.debug("[Serializing Config] Finished writing config partial.");
}

private void verifyXsdValid(Document document) throws Exception {
public Document documentFrom(CruiseConfig config) {
Document document = createEmptyCruiseConfigDocument();
write(config, document.getRootElement(), configCache, registry);
return document;
}

public String toString(Document document) throws IOException {
org.apache.commons.io.output.ByteArrayOutputStream outputStream = new org.apache.commons.io.output.ByteArrayOutputStream();
XmlUtils.writeXml(document, outputStream);
return outputStream.toString(StandardCharsets.UTF_8);
}

public void verifyXsdValid(Document document) throws Exception {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
XmlUtils.writeXml(document, buffer);
InputStream content = toInputStream(buffer.toString());
@@ -17,6 +17,7 @@
package com.thoughtworks.go.config;

import com.thoughtworks.go.config.commands.EntityConfigUpdateCommand;
import com.thoughtworks.go.config.update.FullConfigUpdateCommand;
import com.thoughtworks.go.config.validation.GoConfigValidity;
import com.thoughtworks.go.domain.ConfigErrors;
import com.thoughtworks.go.listener.ConfigChangedListener;
@@ -25,6 +26,7 @@
import com.thoughtworks.go.serverhealth.HealthStateType;
import com.thoughtworks.go.serverhealth.ServerHealthService;
import com.thoughtworks.go.serverhealth.ServerHealthState;
import com.thoughtworks.go.util.SystemEnvironment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -44,6 +46,8 @@
private static final Logger LOGGER = LoggerFactory.getLogger(CachedGoConfig.class);
private final GoFileConfigDataSource dataSource;
private final CachedGoPartials cachedGoPartials;
private GoConfigMigrator goConfigMigrator;
private SystemEnvironment systemEnvironment;
private final ServerHealthService serverHealthService;
private List<ConfigChangedListener> listeners = new ArrayList<>();
private volatile CruiseConfig currentConfig;
@@ -53,11 +57,13 @@
private volatile Exception lastException;

@Autowired
public CachedGoConfig(ServerHealthService serverHealthService,
GoFileConfigDataSource dataSource, CachedGoPartials cachedGoPartials) {
public CachedGoConfig(ServerHealthService serverHealthService, GoFileConfigDataSource dataSource,
CachedGoPartials cachedGoPartials, GoConfigMigrator goConfigMigrator, SystemEnvironment systemEnvironment) {
this.serverHealthService = serverHealthService;
this.dataSource = dataSource;
this.cachedGoPartials = cachedGoPartials;
this.goConfigMigrator = goConfigMigrator;
this.systemEnvironment = systemEnvironment;
}

public static List<ConfigErrors> validate(CruiseConfig config) {
@@ -126,6 +132,21 @@ public void loadConfigIfNull() {
}
}

public synchronized ConfigSaveState writeFullConfigWithLock(FullConfigUpdateCommand updateConfigCommand) {
GoFileConfigDataSource.GoConfigSaveResult saveResult = dataSource.writeFullConfigWithLock(updateConfigCommand, this.configHolder);
saveValidConfigToCacheAndNotifyConfigChangeListeners(saveResult.getConfigHolder());
return saveResult.getConfigSaveState();
}

public synchronized void upgradeConfig() throws Exception {
if(systemEnvironment.optimizeFullConfigSave()) {
GoConfigHolder goConfigHolder = goConfigMigrator.migrate();
saveValidConfigToCacheAndNotifyConfigChangeListeners(goConfigHolder);
} else {
dataSource.upgradeIfNecessary();
}
}

public synchronized ConfigSaveState writeWithLock(UpdateConfigCommand updateConfigCommand) {
GoFileConfigDataSource.GoConfigSaveResult saveResult = dataSource.writeWithLock(updateConfigCommand, this.configHolder);
saveValidConfigToCacheAndNotifyConfigChangeListeners(saveResult.getConfigHolder());
@@ -0,0 +1,152 @@
/*
* Copyright 2016 ThoughtWorks, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.thoughtworks.go.config;

import com.rits.cloning.Cloner;
import com.thoughtworks.go.config.registry.ConfigElementImplementationRegistry;
import com.thoughtworks.go.config.remote.FileConfigOrigin;
import com.thoughtworks.go.config.remote.PartialConfig;
import com.thoughtworks.go.config.update.FullConfigUpdateCommand;
import com.thoughtworks.go.domain.GoConfigRevision;
import com.thoughtworks.go.server.util.ServerVersion;
import com.thoughtworks.go.service.ConfigRepository;
import com.thoughtworks.go.util.CachedDigestUtils;
import com.thoughtworks.go.util.SystemEnvironment;
import com.thoughtworks.go.util.TimeProvider;
import org.jdom.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.List;

public abstract class FullConfigSaveFlow {
protected final MagicalGoConfigXmlLoader loader;
protected final MagicalGoConfigXmlWriter writer;
protected final ServerVersion serverVersion;
protected final TimeProvider timeProvider;
protected final ConfigRepository configRepository;
protected final CachedGoPartials cachedGoPartials;
protected final GoConfigFileWriter fileWriter;
protected final ConfigElementImplementationRegistry configElementImplementationRegistry;
protected final Cloner cloner = new Cloner();
protected final Logger LOGGER = LoggerFactory.getLogger(getClass().getName());

public FullConfigSaveFlow(MagicalGoConfigXmlLoader loader, MagicalGoConfigXmlWriter writer,
ConfigElementImplementationRegistry configElementImplementationRegistry,
ServerVersion serverVersion, TimeProvider timeProvider,
ConfigRepository configRepository, CachedGoPartials cachedGoPartials,
GoConfigFileWriter fileWriter) {
this.loader = loader;
this.writer = writer;
this.configElementImplementationRegistry = configElementImplementationRegistry;
this.serverVersion = serverVersion;
this.timeProvider = timeProvider;
this.configRepository = configRepository;
this.cachedGoPartials = cachedGoPartials;
this.fileWriter = fileWriter;
}

public FullConfigSaveFlow(MagicalGoConfigXmlLoader loader, MagicalGoConfigXmlWriter writer,
ConfigElementImplementationRegistry configElementImplementationRegistry,
SystemEnvironment systemEnvironment, ServerVersion serverVersion,
TimeProvider timeProvider, ConfigRepository configRepository, CachedGoPartials cachedGoPartials) {
this(loader, writer, configElementImplementationRegistry, serverVersion, timeProvider, configRepository, cachedGoPartials,
new GoConfigFileWriter(systemEnvironment));
}

public abstract GoConfigHolder execute(FullConfigUpdateCommand updatingCommand, List<PartialConfig> partials, String currentUser) throws Exception;

protected void postValidationUpdates(CruiseConfig preProcessedConfig, CruiseConfig configForEdit, String xmlString, List<PartialConfig> partials) throws NoSuchFieldException, IllegalAccessException {
String md5 = CachedDigestUtils.md5Hex(xmlString);

configForEdit.setOrigins(new FileConfigOrigin());
preProcessedConfig.setOrigins(new FileConfigOrigin());

MagicalGoConfigXmlLoader.setMd5(preProcessedConfig, md5);
MagicalGoConfigXmlLoader.setMd5(configForEdit, md5);

cachedGoPartials.markAsValid(partials);
}

protected String toXmlString(CruiseConfig configForEdit) throws Exception {
Document document = documentFrom(configForEdit);

validateDocument(document);

return toXmlString(document);
}

protected void checkinToConfigRepo(String currentUser, CruiseConfig updatedConfig, String xmlString) throws Exception {
LOGGER.debug("[Config Save] Checkin updated config to git: Starting.");
configRepository.checkin(new GoConfigRevision(xmlString, updatedConfig.getMd5(), currentUser, serverVersion.version(), timeProvider));
LOGGER.debug("[Config Save] Checkin updated config to git: Done.");
}

protected void writeToConfigXml(String xmlString) {
LOGGER.debug("[Config Save] Writing config xml to file: Starting.");
this.fileWriter.writeToConfigXmlFile(xmlString);
LOGGER.debug("[Config Save] Writing config xml to file: Done.");
}

protected CruiseConfig configForEditWithPartials(FullConfigUpdateCommand updatingCommand, List<PartialConfig> partials) throws Exception {
CruiseConfig config = updatingCommand.configForEdit();
config.setPartials(partials);

return config;
}

protected CruiseConfig preprocessAndValidate(CruiseConfig configForEdit) throws Exception {
return loader.preprocessAndValidate(configForEdit);
}

protected org.jdom.Document documentFrom(CruiseConfig configForEdit) {
LOGGER.debug("[Config Save] Building Document from CruiseConfig object: Starting.");
Document document = writer.documentFrom(configForEdit);
LOGGER.debug("[Config Save] Building Document from CruiseConfig object: Done.");

return document;
}

protected void validateDocument(Document document) throws Exception {
LOGGER.debug("[Config Save] In XSD validation: Starting.");
writer.verifyXsdValid(document);
LOGGER.debug("[Config Save] In XSD validation: Done.");

LOGGER.debug("[Config Save] In DOM validation: Starting.");
MagicalGoConfigXmlLoader.validateDom(document.getRootElement(), configElementImplementationRegistry);
LOGGER.debug("[Config Save] In DOM validation: Done.");
}

protected String toXmlString(Document document) throws IOException {
LOGGER.debug("[Config Save] Serializing Document to xml: Starting.");
String xmlString = writer.toString(document);
LOGGER.debug("[Config Save] Serializing Document to xml: Done.");

return xmlString;
}

protected void setMergedConfigForEditOn(GoConfigHolder validatedConfigHolder, List<PartialConfig> partials) {
if (partials.isEmpty()) return;

LOGGER.debug("[Config Save] Updating GoConfigHolder with mergedCruiseConfigForEdit: Starting.");
CruiseConfig mergedCruiseConfigForEdit = cloner.deepClone(validatedConfigHolder.configForEdit);
mergedCruiseConfigForEdit.merge(partials, true);
validatedConfigHolder.mergedConfigForEdit = mergedCruiseConfigForEdit;
LOGGER.debug("[Config Save] Updating GoConfigHolder with mergedCruiseConfigForEdit: Done.");
}
}

0 comments on commit 2ae7fdb

Please sign in to comment.
You can’t perform that action at this time.