Skip to content
Permalink
Browse files
[AMBARI-23975] Logsearch: do not save downloaded configuration to tmp…
… file when comparing (#1441)
  • Loading branch information
kasakrisz authored and swagle committed Jun 1, 2018
1 parent b238f56 commit f50b2d4a0d0e6deb483be448f5553b084874cf59
Showing 3 changed files with 44 additions and 54 deletions.
@@ -18,20 +18,20 @@
*/
package org.apache.ambari.logsearch.handler;

import static org.apache.solr.common.cloud.ZkConfigManager.CONFIGS_ZKNODE;

import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystems;

import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.commons.io.FileUtils;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkConfigManager;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Paths;
import java.util.UUID;

public abstract class AbstractSolrConfigHandler implements SolrZkRequestHandler<Boolean> {

private static final Logger LOG = LoggerFactory.getLogger(AbstractSolrConfigHandler.class);
@@ -46,42 +46,30 @@ public AbstractSolrConfigHandler(File configSetFolder) {
public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
boolean reloadCollectionNeeded = false;
String separator = FileSystems.getDefault().getSeparator();
String downloadFolderLocation = String.format("%s%s%s%s%s", System.getProperty("java.io.tmpdir"), separator,
UUID.randomUUID().toString(), separator, solrPropsConfig.getConfigName());
solrClient.connect();
SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient();
File tmpDir = new File(downloadFolderLocation);
try {
ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
boolean configExists = zkConfigManager.configExists(solrPropsConfig.getConfigName());
if (configExists) {
uploadMissingConfigFiles(zkClient, zkConfigManager, solrPropsConfig.getConfigName());
reloadCollectionNeeded = doIfConfigExists(solrPropsConfig, zkClient, separator, downloadFolderLocation, tmpDir);
reloadCollectionNeeded = doIfConfigExists(solrPropsConfig, zkClient, separator);
} else {
doIfConfigNotExist(solrPropsConfig, zkConfigManager);
uploadMissingConfigFiles(zkClient, zkConfigManager, solrPropsConfig.getConfigName());
}
} catch (Exception e) {
throw new RuntimeException(String.format("Cannot upload configurations to zk. (collection: %s, config set folder: %s)",
solrPropsConfig.getCollection(), solrPropsConfig.getConfigSetFolder()), e);
} finally {
if (tmpDir.exists()) {
try {
FileUtils.deleteDirectory(tmpDir);
} catch (IOException e){
LOG.error("Cannot delete temp directory.", e);
}
}
}
return reloadCollectionNeeded;
}

/**
* Update config file (like solrconfig.xml) to zookeeper znode of solr, contains a download location as well which can be
* used to determine that you need to update the configuration or not
* Update config file (like solrconfig.xml) to zookeeper znode of solr
*/
public abstract boolean updateConfigIfNeeded(SolrPropsConfig solrPropsConfig, SolrZkClient zkClient, File file,
String separator, String downloadFolderLocation) throws IOException;
String separator, byte[] content) throws IOException;

/**
* Config file name which should be uploaded to zookeeper
@@ -91,31 +79,31 @@ public abstract boolean updateConfigIfNeeded(SolrPropsConfig solrPropsConfig, So
@SuppressWarnings("unused")
public void doIfConfigNotExist(SolrPropsConfig solrPropsConfig, ZkConfigManager zkConfigManager) throws IOException {
// Do nothing
};
}

@SuppressWarnings("unused")
public void uploadMissingConfigFiles(SolrZkClient zkClient, ZkConfigManager zkConfigManager, String configName) throws IOException {
// do Nothing
}

public boolean doIfConfigExists(SolrPropsConfig solrPropsConfig, SolrZkClient zkClient, String separator, String downloadFolderLocation, File tmpDir) throws IOException {
boolean result = false;
public boolean doIfConfigExists(SolrPropsConfig solrPropsConfig, SolrZkClient zkClient, String separator) throws IOException {
LOG.info("Config set exists for '{}' collection. Refreshing it if needed...", solrPropsConfig.getCollection());
if (!tmpDir.mkdirs()) {
LOG.error("Cannot create directories for '{}'", tmpDir.getAbsolutePath());
}
ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
zkConfigManager.downloadConfigDir(solrPropsConfig.getConfigName(), Paths.get(downloadFolderLocation));
File[] listOfFiles = getConfigSetFolder().listFiles();
if (listOfFiles != null) {
try {
File[] listOfFiles = getConfigSetFolder().listFiles();
if (listOfFiles == null)
return false;
byte[] data = zkClient.getData(String.format("%s/%s/%s", CONFIGS_ZKNODE, solrPropsConfig.getConfigName(), getConfigFileName()), null, null, true);

for (File file : listOfFiles) {
if (file.getName().equals(getConfigFileName()) && updateConfigIfNeeded(solrPropsConfig, zkClient, file, separator, downloadFolderLocation)) {
result = true;
break;
if (file.getName().equals(getConfigFileName()) && updateConfigIfNeeded(solrPropsConfig, zkClient, file, separator, data)) {
return true;
}
}
return false;
} catch (KeeperException | InterruptedException e) {
throw new IOException("Error downloading files from zookeeper path " + solrPropsConfig.getConfigName(),
SolrZkClient.checkInterrupted(e));
}
return result;
}

protected File getConfigSetFolder() {
@@ -18,6 +18,14 @@
*/
package org.apache.ambari.logsearch.handler;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileSystems;
import java.util.Arrays;

import org.apache.ambari.logsearch.conf.SolrPropsConfig;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
@@ -26,12 +34,6 @@
import org.apache.zookeeper.CreateMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.FileSystems;

public class UploadConfigurationHandler extends AbstractSolrConfigHandler {

@@ -51,18 +53,17 @@ public UploadConfigurationHandler(File configSetFolder, boolean hasEnumConfig) {

@Override
public boolean updateConfigIfNeeded(SolrPropsConfig solrPropsConfig, SolrZkClient zkClient, File file,
String separator, String downloadFolderLocation) throws IOException {
boolean result = false;
if (!FileUtils.contentEquals(file, new File(String.format("%s%s%s", downloadFolderLocation, separator, file.getName())))) {
LOG.info("Solr config file differs ('{}'), upload config set to zookeeper", file.getName());
ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
zkConfigManager.uploadConfigDir(getConfigSetFolder().toPath(), solrPropsConfig.getConfigName());
String filePath = String.format("%s%s%s", getConfigSetFolder(), separator, getConfigFileName());
String configsPath = String.format("/%s/%s/%s", "configs", solrPropsConfig.getConfigName(), getConfigFileName());
uploadFileToZk(zkClient, filePath, configsPath);
result = true;
}
return result;
String separator, byte[] content) throws IOException {
if (Arrays.equals(FileUtils.readFileToByteArray(file), content))
return false;

LOG.info("Solr config file differs ('{}'), upload config set to zookeeper", file.getName());
ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
zkConfigManager.uploadConfigDir(getConfigSetFolder().toPath(), solrPropsConfig.getConfigName());
String filePath = String.format("%s%s%s", getConfigSetFolder(), separator, getConfigFileName());
String configsPath = String.format("/%s/%s/%s", "configs", solrPropsConfig.getConfigName(), getConfigFileName());
uploadFileToZk(zkClient, filePath, configsPath);
return true;
}

@Override
@@ -244,6 +244,7 @@
<exclude>**/*.txt</exclude>
<exclude>**/*.story</exclude>
<exclude>**/*.editorconfig</exclude>
<exclude>**/*.iml</exclude>
<exclude>**/src/vendor/**</exclude>
<exclude>**/yarn.lock</exclude>
<exclude>**/docker/Profile</exclude>

0 comments on commit f50b2d4

Please sign in to comment.