Skip to content
Permalink
Browse files
AMBARI-23594 - LZO Libraries Are Not Installed Correctly During Upgrade
  • Loading branch information
jonathan-hurley committed Apr 18, 2018
1 parent ae2e31e commit f3fcffbf47749d20b9071fddc37faff5d1281e83
Show file tree
Hide file tree
Showing 9 changed files with 71 additions and 95 deletions.
@@ -44,8 +44,6 @@
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.state.UpgradeContext;
import org.apache.ambari.server.state.UpgradeContext.UpgradeSummary;
import org.apache.ambari.server.state.UpgradeContextFactory;
@@ -199,10 +197,9 @@ public ExecutionCommand getExecutionCommand() {
final String componentName = executionCommand.getComponentName();

try {

if (null != componentName) {
ServiceComponent serviceComponent = service.getServiceComponent(componentName);
commandRepository = repoVersionHelper.getCommandRepository(null, serviceComponent, host);
commandRepository = repoVersionHelper.getCommandRepository(cluster, serviceComponent, host);
} else {
RepositoryVersionEntity repoVersion = service.getDesiredRepositoryVersion();
RepoOsEntity osEntity = repoVersionHelper.getOSEntityForHost(host, repoVersion);
@@ -211,7 +208,8 @@ public ExecutionCommand getExecutionCommand() {
executionCommand.setRepositoryFile(commandRepository);

} catch (SystemException e) {
throw new RuntimeException(e);
LOG.debug("Unable to find command repository with a correct operating system for host {}",
host, e);
}
}

@@ -266,10 +264,6 @@ public void setVersions(Cluster cluster) {
commandParams.put(VERSION, repositoryVersion.getVersion());
}

StackId stackId = repositoryVersion.getStackId();
StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(),
stackId.getStackVersion());

if (!commandParams.containsKey(HOOKS_FOLDER)) {
commandParams.put(HOOKS_FOLDER,configuration.getProperty(Configuration.HOOKS_FOLDER));
}
@@ -28,6 +28,7 @@
import org.apache.ambari.annotations.ExperimentalFeature;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.UpgradeContext.UpgradeSummary;
import org.apache.ambari.server.utils.StageUtils;
@@ -180,8 +181,7 @@ public ExecutionCommand() {


/**
* Provides information regarding the content of repositories. This structure replaces
* the deprecated use of {@link KeyNames#REPO_INFO}
* Provides information regarding the content of repositories.
*/
@SerializedName("repositoryFile")
private CommandRepository commandRepository;
@@ -449,14 +449,33 @@ public void setClusterId(String clusterId) {
}

/**
* Gets the repository file which was set on this command. The repository can
* be set either by the creator of the command or by the
* {@link ExecutionCommandWrapper} when it is about to execute the command.
*
* @return the repository file that is to be written.
* @see #setRepositoryFile(CommandRepository)
*/
public CommandRepository getRepositoryFile() {
return commandRepository;
}

/**
* @param repository the command repository instance.
* Sets the {@link CommandRepository} which will be sent down to the agent
* instructing it on which repository file to create on the host. In most
* cases, it is not necessary to set this file since the
* {@link ExecutionCommandWrapper} will set it in the event that it is
* missing. In fact, it is only appropriate to set this file in the following
* cases:
* <ul>
* <li>When distributing a repository to hosts in preparation for upgrade.
* This is because the service/component desired stack is not pointing to the
* new repository yet</li>
* <li>If the command does not contain a host or service/component></li>
* </ul>
*
* @param repository
* the command repository instance.
*/
public void setRepositoryFile(CommandRepository repository) {
commandRepository = repository;
@@ -521,12 +540,6 @@ public interface KeyNames {
String ORACLE_JDBC_URL = "oracle_jdbc_url";
String DB_DRIVER_FILENAME = "db_driver_filename";
String CLIENTS_TO_UPDATE_CONFIGS = "clientsToUpdateConfigs";
/**
* Keep for backward compatibility.
*/
@Deprecated
@Experimental(feature=ExperimentalFeature.PATCH_UPGRADES)
String REPO_INFO = "repo_info";

String DB_NAME = "db_name";
String GLOBAL = "global";
@@ -33,7 +33,6 @@
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MYSQL_JDBC_URL;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.STACK_NAME;
@@ -61,15 +60,13 @@
import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.actionmanager.Stage;
import org.apache.ambari.server.agent.AgentCommand.AgentCommandType;
import org.apache.ambari.server.agent.CommandRepository;
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.agent.ExecutionCommand.KeyNames;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.internal.RequestOperationLevel;
import org.apache.ambari.server.controller.internal.RequestResourceFilter;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.ambari.server.controller.spi.SystemException;
import org.apache.ambari.server.metadata.ActionMetadata;
import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
import org.apache.ambari.server.state.Cluster;
@@ -94,8 +91,6 @@
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.state.State;
import org.apache.ambari.server.state.stack.OsFamily;
import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpInProgressEvent;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.commons.lang.StringUtils;
@@ -163,15 +158,9 @@ public class AmbariCustomCommandExecutionHelper {
@Inject
private MaintenanceStateHelper maintenanceStateHelper;

@Inject
private OsFamily os_family;

@Inject
private HostRoleCommandDAO hostRoleCommandDAO;

@Inject
private RepositoryVersionHelper repoVersionHelper;

private Map<String, Map<String, Map<String, String>>> configCredentialsForService = new HashMap<>();

protected static final String SERVICE_CHECK_COMMAND_NAME = "SERVICE_CHECK";
@@ -324,7 +313,6 @@ public boolean shouldHostBeRemoved(final String hostname)

AmbariMetaInfo ambariMetaInfo = managementController.getAmbariMetaInfo();
ServiceInfo serviceInfo = ambariMetaInfo.getService(service);
StackInfo stackInfo = ambariMetaInfo.getStack(stackId);

CustomCommandDefinition customCommandDefinition = null;
ComponentInfo ci = serviceInfo.getComponentByName(componentName);
@@ -335,9 +323,6 @@ public boolean shouldHostBeRemoved(final String hostname)
long nowTimestamp = System.currentTimeMillis();

for (String hostName : candidateHosts) {

Host host = clusters.getHost(hostName);

stage.addHostRoleExecutionCommand(hostName, Role.valueOf(componentName),
RoleCommand.CUSTOM_COMMAND,
new ServiceComponentHostOpInProgressEvent(componentName, hostName, nowTimestamp),
@@ -390,11 +375,6 @@ public boolean shouldHostBeRemoved(final String hostname)
Service clusterService = cluster.getService(serviceName);
execCmd.setCredentialStoreEnabled(String.valueOf(clusterService.isCredentialStoreEnabled()));

ServiceComponent component = null;
if (StringUtils.isNotBlank(componentName)) {
component = clusterService.getServiceComponent(componentName);
}

// Get the map of service config type to password properties for the service
Map<String, Map<String, String>> configCredentials;
configCredentials = configCredentialsForService.get(clusterService.getName());
@@ -406,15 +386,6 @@ public boolean shouldHostBeRemoved(final String hostname)
execCmd.setConfigurationCredentials(configCredentials);

Map<String, String> hostLevelParams = new TreeMap<>();

// Set parameters required for re-installing clients on restart
String repoInfoString;
try {
repoInfoString = repoVersionHelper.getRepoInfoString(cluster, component, host);
} catch (SystemException e) {
throw new RuntimeException(e);
}
hostLevelParams.put(REPO_INFO, repoInfoString);
hostLevelParams.put(STACK_NAME, stackId.getStackName());
hostLevelParams.put(STACK_VERSION, stackId.getStackVersion());

@@ -523,14 +494,6 @@ public boolean shouldHostBeRemoved(final String hostname)
execCmd.setCommandParams(commandParams);
execCmd.setRoleParams(roleParams);

CommandRepository commandRepository;
try {
commandRepository = repoVersionHelper.getCommandRepository(cluster, component, host);
} catch (SystemException e) {
throw new RuntimeException(e);
}
execCmd.setRepositoryFile(commandRepository);

// perform any server side command related logic - eg - set desired states on restart
applyCustomCommandBackendLogic(cluster, serviceName, componentName, commandName, hostName);
}
@@ -733,8 +696,6 @@ public void addServiceCheckAction(Stage stage, String hostname, String smokeTest
AmbariMetaInfo ambariMetaInfo = managementController.getAmbariMetaInfo();
ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(),
stackId.getStackVersion(), serviceName);
StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(),
stackId.getStackVersion());

stage.addHostRoleExecutionCommand(hostname, Role.valueOf(smokeTestRole),
RoleCommand.SERVICE_CHECK,
@@ -45,7 +45,6 @@
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.ORACLE_JDBC_URL;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER;
@@ -360,9 +359,6 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
@Inject
private TopologyDeleteFormer topologyDeleteFormer;

@Inject
private AmbariCustomCommandExecutionHelper ambariCustomCommandExecutionHelper;

@Inject
private Provider<TopologyHolder> m_topologyHolder;

@@ -411,6 +407,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle

@Inject
private AmbariCustomCommandExecutionHelper customCommandExecutionHelper;

@Inject
private AmbariActionExecutionHelper actionExecutionHelper;

@@ -2539,7 +2536,6 @@ private void createHostAction(Cluster cluster,
}

Map<String, String> hostParams = new TreeMap<>();
hostParams.put(REPO_INFO, repoInfo);
hostParams.putAll(getRcaParameters());

if (roleCommand.equals(RoleCommand.INSTALL)) {
@@ -5603,6 +5599,7 @@ public MetadataUpdateEvent getClusterMetadata(Cluster cl) throws AmbariException
return metadataUpdateEvent;
}

@Override
public MetadataUpdateEvent getClusterMetadataOnConfigsUpdate(Cluster cl) throws AmbariException {
TreeMap<String, MetadataCluster> metadataClusters = new TreeMap<>();
StackId stackId = cl.getDesiredStackVersion();
@@ -5834,7 +5831,7 @@ public TreeMap<String, MetadataServiceInfo> getMetadataServiceLevelParams(Servic
serviceStackId.getStackVersion(), service.getName());
Long statusCommandTimeout = null;
if (serviceInfo.getCommandScript() != null) {
statusCommandTimeout = new Long(ambariCustomCommandExecutionHelper.getStatusCommandTimeout(serviceInfo));
statusCommandTimeout = new Long(customCommandExecutionHelper.getStatusCommandTimeout(serviceInfo));
}

String servicePackageFolder = serviceInfo.getServicePackageFolder();
@@ -455,25 +455,6 @@ public void addRepoInfoToHostLevelParams(final Cluster cluster, final ActionExec
hostLevelParams.put(KeyNames.STACK_NAME, stackId.getStackName());
hostLevelParams.put(KeyNames.STACK_VERSION, stackId.getStackVersion());
}

JsonObject rootJsonObject = new JsonObject();
JsonArray repositories = new JsonArray();

String hostOsFamily = cluster.getHost(hostName).getOsFamily();
for (RepoOsEntity operatingSystemEntity : repositoryVersion.getRepoOsEntities()) {
if (operatingSystemEntity.getFamily().equals(hostOsFamily)) {
for (RepoDefinitionEntity repositoryEntity : operatingSystemEntity.getRepoDefinitionEntities()) {
JsonObject repositoryInfo = new JsonObject();
repositoryInfo.addProperty("base_url", repositoryEntity.getBaseUrl());
repositoryInfo.addProperty("repo_name", repositoryEntity.getRepoName());
repositoryInfo.addProperty("repo_id", repositoryEntity.getRepoID());

repositories.add(repositoryInfo);
}
rootJsonObject.add("repositories", repositories);
}
}
hostLevelParams.put(KeyNames.REPO_INFO, rootJsonObject.toString());
}


@@ -42,6 +42,7 @@
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ConfigFactory;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.svccomphost.ServiceComponentHostStartEvent;
@@ -110,6 +111,13 @@ public static void setup() throws AmbariException {
clusters = injector.getInstance(Clusters.class);
clusters.addHost(HOST1);
clusters.addCluster(CLUSTER1, new StackId("HDP-0.1"));
clusters.mapHostToCluster(HOST1, CLUSTER1);

Map<String, String> hostAttributes = new HashMap<>();
hostAttributes.put("os_family", "redhat");
hostAttributes.put("os_release_version", "6.4");
Host host = clusters.getHost(HOST1);
host.setHostAttributes(hostAttributes);

Cluster cluster1 = clusters.getCluster(CLUSTER1);

@@ -236,6 +244,7 @@ public void testGetExecutionCommand() throws JSONException, AmbariException {

Assert.assertEquals(serviceSiteKeys.size(), serviceSiteConfig.size());

Assert.assertNotNull(processedExecutionCommand.getRepositoryFile());
}

@Test
@@ -285,7 +294,7 @@ public void testExecutionCommandHasVersionInfoWithoutCurrentClusterVersion()
Cluster cluster = clusters.getCluster(CLUSTER1);

StackId stackId = cluster.getDesiredStackVersion();

// set the repo version resolved state to verify that the version is not sent
RepositoryVersionEntity repositoryVersion = ormTestHelper.getOrCreateRepositoryVersion(stackId, "0.1-0000");
repositoryVersion.setResolved(false);
@@ -247,6 +247,7 @@ public void testRefreshQueueCustomCommand() throws Exception {
Assert.assertTrue(command.getHostLevelParams().containsKey(ExecutionCommand.KeyNames.USER_GROUPS));
Assert.assertEquals("{\"zookeeperUser\":[\"zookeeperGroup\"]}", command.getHostLevelParams().get(ExecutionCommand.KeyNames.USER_GROUPS));
Assert.assertEquals(true, command.getForceRefreshConfigTagsBeforeExecution());
Assert.assertNull(command.getRepositoryFile());
}

@Test
@@ -573,6 +574,7 @@ public void testAvailableServicesMapContainsVersions() throws Exception {
Assert.assertFalse(MapUtils.isEmpty(command.getComponentVersionMap()));
Assert.assertEquals(1, command.getComponentVersionMap().size());
Assert.assertTrue(command.getComponentVersionMap().containsKey("ZOOKEEPER"));
Assert.assertNull(command.getRepositoryFile());
}

/**
@@ -619,6 +621,7 @@ public void testAvailableServicesMapIsEmptyWhenRepositoriesNotResolved() throws
ExecutionCommand command = commands.get(0).getExecutionCommand();

Assert.assertTrue(MapUtils.isEmpty(command.getComponentVersionMap()));
Assert.assertNull(command.getRepositoryFile());
}

@Test

0 comments on commit f3fcffb

Please sign in to comment.