Skip to content

Commit

Permalink
SOLR-15258: ConfigSetService add CRUD operations, subsume ZkConfigMan…
Browse files Browse the repository at this point in the history 
…ager (apache#23)

To allow for viable alternative implementations of where ConfigSets come from.

Co-authored-by: Nazerke Seidan <nseidan@salesforce.com>
Co-authored-by: David Smiley <dsmiley@salesforce.com>
  • Loading branch information
@NazerkeBS @dsmiley
3 people committed Apr 3, 2021
1 parent 6fd6ad9 commit 1a922dc
Show file tree
Hide file tree
Showing 53 changed files with 1,068 additions and 977 deletions.
4 changes: 4 additions & 0 deletions solr/CHANGES.txt
View file
Expand Up @@ -241,6 +241,10 @@ Other Changes
* SOLR-15292: SignatureUpdateProcessorFactory will fail to initialize if it is used in a SolrCloud cluster in a way that is
known to be problematic with multiple replicas. (hossman)

* SOLR-15258: Consolidate ConfigSet handling (C.R.U.D.) into ConfigSetService to allow for viable
alternative implementations of where ConfigSets come from.
(Nazerke Seidan, David Smiley)

Bug Fixes
---------------------
* SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order execution
Expand Down
146 changes: 0 additions & 146 deletions solr/core/src/java/org/apache/solr/cloud/CloudConfigSetService.java
View file

This file was deleted.

2 changes: 1 addition & 1 deletion solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java
View file
Expand Up @@ -90,7 +90,7 @@ private static OverseerMessageHandlerSelector getOverseerMessageHandlerSelector(
final OverseerCollectionMessageHandler collMessageHandler = new OverseerCollectionMessageHandler(
zkStateReader, myId, shardHandlerFactory, adminPath, stats, overseer, overseerNodePrioritizer);
final OverseerConfigSetMessageHandler configMessageHandler = new OverseerConfigSetMessageHandler(
zkStateReader);
zkStateReader, overseer.getCoreContainer()); //coreContainer is passed instead of configSetService as configSetService is loaded late
return new OverseerMessageHandlerSelector() {
@Override
public void close() throws IOException {
Expand Down
55 changes: 17 additions & 38 deletions solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java
View file
Expand Up @@ -30,15 +30,14 @@
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkConfigManager;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ConfigSetParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.ConfigSetProperties;
import org.apache.zookeeper.CreateMode;
import org.apache.solr.core.ConfigSetService;
import org.apache.solr.core.CoreContainer;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -71,6 +70,8 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {

private ZkStateReader zkStateReader;

private CoreContainer coreContainer;

// we essentially implement a read/write lock for the ConfigSet exclusivity as follows:
// WRITE: CREATE/DELETE on the ConfigSet under operation
// READ: for the Base ConfigSet being copied in CREATE.
Expand All @@ -84,8 +85,9 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {

private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

public OverseerConfigSetMessageHandler(ZkStateReader zkStateReader) {
public OverseerConfigSetMessageHandler(ZkStateReader zkStateReader, CoreContainer coreContainer) {
this.zkStateReader = zkStateReader;
this.coreContainer = coreContainer;
this.configSetWriteWip = new HashSet<>();
this.configSetReadWip = new HashSet<>();
}
Expand Down Expand Up @@ -227,17 +229,8 @@ private String getBaseConfigSetIfCreate(ZkNodeProps message) {
}

@SuppressWarnings({"rawtypes"})
private NamedList getConfigSetProperties(String path) throws IOException {
byte[] oldPropsData = null;
try {
oldPropsData = zkStateReader.getZkClient().getData(path, null, null, true);
} catch (KeeperException.NoNodeException e) {
log.info("no existing ConfigSet properties found");
} catch (KeeperException | InterruptedException e) {
throw new IOException("Error reading old properties",
SolrZkClient.checkInterrupted(e));
}

private NamedList getConfigSetProperties(ConfigSetService configSetService, String configName, String propertyPath) throws IOException {
byte[] oldPropsData = configSetService.downloadFileFromConfig(configName, propertyPath);
if (oldPropsData != null) {
InputStreamReader reader = new InputStreamReader(new ByteArrayInputStream(oldPropsData), StandardCharsets.UTF_8);
try {
Expand Down Expand Up @@ -285,10 +278,6 @@ private byte[] getPropertyData(Map<String, Object> newProps) {
return null;
}

private String getPropertyPath(String configName, String propertyPath) {
return ZkConfigManager.CONFIGS_ZKNODE + "/" + configName + "/" + propertyPath;
}

private void createConfigSet(ZkNodeProps message) throws IOException {
String configSetName = getTaskKey(message);
if (configSetName == null || configSetName.length() == 0) {
Expand All @@ -297,13 +286,12 @@ private void createConfigSet(ZkNodeProps message) throws IOException {

String baseConfigSetName = message.getStr(BASE_CONFIGSET, DEFAULT_CONFIGSET_NAME);

ZkConfigManager configManager = new ZkConfigManager(zkStateReader.getZkClient());
if (configManager.configExists(configSetName)) {
if (coreContainer.getConfigSetService().checkConfigExists(configSetName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet already exists: " + configSetName);
}

// is there a base config that already exists
if (!configManager.configExists(baseConfigSetName)) {
if (!coreContainer.getConfigSetService().checkConfigExists(baseConfigSetName)) {
throw new SolrException(ErrorCode.BAD_REQUEST,
"Base ConfigSet does not exist: " + baseConfigSetName);
}
Expand All @@ -313,25 +301,17 @@ private void createConfigSet(ZkNodeProps message) throws IOException {
if (props != null) {
// read the old config properties and do a merge, if necessary
@SuppressWarnings({"rawtypes"})
NamedList oldProps = getConfigSetProperties(getPropertyPath(baseConfigSetName, propertyPath));
NamedList oldProps = getConfigSetProperties(coreContainer.getConfigSetService(), baseConfigSetName, propertyPath);
if (oldProps != null) {
mergeOldProperties(props, oldProps);
}
}
byte[] propertyData = getPropertyData(props);

Set<String> copiedToZkPaths = new HashSet<String>();
try {
configManager.copyConfigDir(baseConfigSetName, configSetName, copiedToZkPaths);
coreContainer.getConfigSetService().copyConfig(baseConfigSetName, configSetName);
if (propertyData != null) {
try {
zkStateReader.getZkClient().makePath(
getPropertyPath(configSetName, propertyPath),
propertyData, CreateMode.PERSISTENT, null, false, true);
} catch (KeeperException | InterruptedException e) {
throw new IOException("Error writing new properties",
SolrZkClient.checkInterrupted(e));
}
coreContainer.getConfigSetService().uploadFileToConfig(configSetName, propertyPath, propertyData, true);
}
} catch (Exception e) {
// copying the config dir or writing the properties file may have failed.
Expand All @@ -340,7 +320,7 @@ private void createConfigSet(ZkNodeProps message) throws IOException {
// the entire baseConfig set with the old properties, including immutable,
// that would make it impossible for the user to delete.
try {
if (configManager.configExists(configSetName) && copiedToZkPaths.size() > 0) {
if (coreContainer.getConfigSetService().checkConfigExists(configSetName)) {
deleteConfigSet(configSetName, true);
}
} catch (IOException ioe) {
Expand All @@ -360,8 +340,7 @@ private void deleteConfigSet(ZkNodeProps message) throws IOException {
}

private void deleteConfigSet(String configSetName, boolean force) throws IOException {
ZkConfigManager configManager = new ZkConfigManager(zkStateReader.getZkClient());
if (!configManager.configExists(configSetName)) {
if (!coreContainer.getConfigSetService().checkConfigExists(configSetName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet does not exist to delete: " + configSetName);
}

Expand All @@ -380,14 +359,14 @@ private void deleteConfigSet(String configSetName, boolean force) throws IOExcep

String propertyPath = ConfigSetProperties.DEFAULT_FILENAME;
@SuppressWarnings({"rawtypes"})
NamedList properties = getConfigSetProperties(getPropertyPath(configSetName, propertyPath));
NamedList properties = getConfigSetProperties(coreContainer.getConfigSetService(), configSetName, propertyPath);
if (properties != null) {
Object immutable = properties.get(ConfigSetProperties.IMMUTABLE_CONFIGSET_ARG);
boolean isImmutableConfigSet = immutable != null ? Boolean.parseBoolean(immutable.toString()) : false;
if (!force && isImmutableConfigSet) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Requested delete of immutable ConfigSet: " + configSetName);
}
}
configManager.deleteConfigDir(configSetName);
coreContainer.getConfigSetService().deleteConfig(configSetName);
}
}
14 changes: 7 additions & 7 deletions solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
View file
Expand Up @@ -41,7 +41,8 @@
import org.apache.commons.io.IOUtils;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkConfigManager;
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.core.ConfigSetService;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.util.CLIO;
import org.apache.zookeeper.CreateMode;
Expand Down Expand Up @@ -71,7 +72,7 @@ public class ZkCLI implements CLIO {
static final String UPCONFIG = "upconfig";
static final String EXCLUDE_REGEX_SHORT = "x";
static final String EXCLUDE_REGEX = "excluderegex";
static final String EXCLUDE_REGEX_DEFAULT = ZkConfigManager.UPLOAD_FILENAME_EXCLUDE_REGEX;
static final String EXCLUDE_REGEX_DEFAULT = ConfigSetService.UPLOAD_FILENAME_EXCLUDE_REGEX;
private static final String COLLECTION = "collection";
private static final String CLEAR = "clear";
private static final String LIST = "list";
Expand Down Expand Up @@ -206,13 +207,14 @@ public static void main(String[] args) throws InterruptedException,
}

CoreContainer cc = new CoreContainer(Paths.get(solrHome), new Properties());
cc.setCoreConfigService(new ZkConfigSetService(zkClient));

if(!ZkController.checkChrootPath(zkServerAddress, true)) {
stdout.println("A chroot was specified in zkHost but the znode doesn't exist. ");
System.exit(1);
}

ZkController.bootstrapConf(zkClient, cc);
ConfigSetService.bootstrapConf(cc);

// No need to close the CoreContainer, as it wasn't started
// up in the first place...
Expand All @@ -231,9 +233,8 @@ public static void main(String[] args) throws InterruptedException,
stdout.println("A chroot was specified in zkHost but the znode doesn't exist. ");
System.exit(1);
}
ZkConfigManager configManager = new ZkConfigManager(zkClient);
final Pattern excludePattern = Pattern.compile(excludeExpr);
configManager.uploadConfigDir(Paths.get(confDir), confName, excludePattern);
ZkMaintenanceUtils.uploadToZK(zkClient, Paths.get(confDir), ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName, excludePattern);
} else if (line.getOptionValue(CMD).equalsIgnoreCase(DOWNCONFIG)) {
if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) {
stdout.println("-" + CONFDIR + " and -" + CONFNAME
Expand All @@ -242,8 +243,7 @@ public static void main(String[] args) throws InterruptedException,
}
String confDir = line.getOptionValue(CONFDIR);
String confName = line.getOptionValue(CONFNAME);
ZkConfigManager configManager = new ZkConfigManager(zkClient);
configManager.downloadConfigDir(confName, Paths.get(confDir));
ZkMaintenanceUtils.downloadFromZK(zkClient, ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName, Paths.get(confDir));
} else if (line.getOptionValue(CMD).equalsIgnoreCase(LINKCONFIG)) {
if (!line.hasOption(COLLECTION) || !line.hasOption(CONFNAME)) {
stdout.println("-" + COLLECTION + " and -" + CONFNAME
Expand Down

0 comments on commit 1a922dc

Please sign in to comment.