Skip to content
Permalink
Browse files
AMBARI-24331. Hide not restartable components from Service Auto Start
  • Loading branch information
hiveww committed Jul 24, 2018
1 parent 69426ce commit a84aa8fd07f30fe948225e7f5b83ea956025af0a
Show file tree
Hide file tree
Showing 13 changed files with 187 additions and 70 deletions.
@@ -130,7 +130,7 @@ public LogLevelFilterMap getLogLevelFilters(String clusterName) {
}
}
} catch (Exception e) {
LOG.error("Error during getting log level filters: {}", e);
LOG.error("Error during getting log level filters: {}", e.getMessage());
}
logLevelFilterMap.setFilter(logLevelFilterTreeMap);
return logLevelFilterMap;
@@ -111,6 +111,19 @@ public void setDrain(boolean drain) {
this.drain = drain;
}

public List<String> getListValue(String key) {
return getListValue(key, null);
}

public List<String> getListValue(String key, List<String> defaultValue) {
Object value = configs.get(key);
if (value != null) {
return (List<String>)value;
} else {
return defaultValue;
}
}

public String getStringValue(String property) {
return getStringValue(property, null);
}
@@ -102,5 +102,6 @@ public class LogFeederConstants {
public static final boolean MONITOR_SOLR_FILTER_STORAGE_DEFAULT = true;

public static final String SOLR_ZK_CONNECTION_STRING = "logfeeder.solr.zk_connect_string";
public static final String SOLR_URLS = "logfeeder.solr.urls";

}
@@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.ambari.logfeeder.common;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

public class LogFeederSolrClientFactory {

private static final Logger logger = LoggerFactory.getLogger(LogFeederSolrClientFactory.class);

public SolrClient createSolrClient(String zkConnectionString, String[] solrUrls, String collection) {
logger.info("Creating solr client ...");
logger.info("Using collection=" + collection);
if (solrUrls != null && solrUrls.length > 0) {
logger.info("Using lbHttpSolrClient with urls: {}",
StringUtils.join(appendTo("/" + collection, solrUrls), ","));
LBHttpSolrClient.Builder builder = new LBHttpSolrClient.Builder();
builder.withBaseSolrUrls(solrUrls);
return builder.build();
} else {
logger.info("Using zookeepr. zkConnectString=" + zkConnectionString);
CloudSolrClient.Builder builder = new CloudSolrClient.Builder();
builder.withZkHost(zkConnectionString);
CloudSolrClient solrClient = builder.build();
solrClient.setDefaultCollection(collection);
return solrClient;
}
}

private String[] appendTo(String toAppend, String... appendees) {
for (int i = 0; i < appendees.length; i++) {
appendees[i] = appendees[i] + toAppend;
}
return appendees;
}

}
@@ -19,6 +19,7 @@
package org.apache.ambari.logfeeder.conf;

import com.google.common.collect.Maps;
import org.apache.ambari.logfeeder.common.LogFeederSolrClientFactory;
import org.apache.ambari.logfeeder.docker.DockerContainerRegistry;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.input.InputConfigUploader;
@@ -38,9 +39,7 @@
import org.apache.ambari.logsearch.config.solr.LogLevelFilterManagerSolr;
import org.apache.ambari.logsearch.config.solr.LogLevelFilterUpdaterSolr;
import org.apache.ambari.logsearch.config.zookeeper.LogSearchConfigLogFeederZK;
import org.apache.commons.lang3.StringUtils;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
@@ -69,7 +68,7 @@ public LogFeederSecurityConfig logFeederSecurityConfig() {
}

@Bean
@DependsOn("logSearchConfigLogFeeder")
@DependsOn({"logSearchConfigLogFeeder", "propertyConfigurer"})
public ConfigHandler configHandler() throws Exception {
return new ConfigHandler(logSearchConfigLogFeeder());
}
@@ -95,15 +94,9 @@ public LogSearchConfigLogFeeder logSearchConfigLogFeeder() throws Exception {
@Bean
public LogLevelFilterManager logLevelFilterManager() {
if (logFeederProps.isSolrFilterStorage()) {
if (StringUtils.isNotEmpty(logFeederProps.getSolrZkConnectString())) {
CloudSolrClient.Builder builder = new CloudSolrClient.Builder();
builder.withZkHost(logFeederProps.getSolrZkConnectString());
CloudSolrClient solrClient = builder.build();
solrClient.setDefaultCollection("history");
return new LogLevelFilterManagerSolr(solrClient);
} else {
return null; // TODO: use lb http client
}
SolrClient solrClient = new LogFeederSolrClientFactory().createSolrClient(
logFeederProps.getSolrZkConnectString(), logFeederProps.getSolrUrls(), "history");
return new LogLevelFilterManagerSolr(solrClient);
} else { // no default filter manager
return null;
}
@@ -21,6 +21,7 @@
import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.plugin.common.LogFeederProperties;
import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.AbstractEnvironment;
@@ -179,6 +180,15 @@ public class LogFeederProps implements LogFeederProperties {
@Value("${" + LogFeederConstants.SOLR_ZK_CONNECTION_STRING + ":}")
private String solrZkConnectString;

@LogSearchPropertyDescription(
name = LogFeederConstants.SOLR_URLS,
description = "Comma separated solr urls (with protocol and port), override "+ LogFeederConstants.SOLR_ZK_CONNECTION_STRING + " config",
examples = {"https://localhost1:8983/solr,https://localhost2:8983"},
sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
)
@Value("${" + LogFeederConstants.SOLR_URLS + ":}")
private String solrUrlsStr;

@Inject
private LogEntryCacheConfig logEntryCacheConfig;

@@ -285,7 +295,7 @@ public void setDockerContainerRegistryEnabled(boolean dockerContainerRegistryEna
}

public boolean isUseLocalConfigs() {
return useLocalConfigs;
return this.useLocalConfigs;
}

public void setUseLocalConfigs(boolean useLocalConfigs) {
@@ -316,6 +326,21 @@ public void setSolrFilterMonitor(boolean solrFilterMonitor) {
this.solrFilterMonitor = solrFilterMonitor;
}

public String getSolrUrlsStr() {
return this.solrUrlsStr;
}

public void setSolrUrlsStr(String solrUrlsStr) {
this.solrUrlsStr = solrUrlsStr;
}

public String[] getSolrUrls() {
if (StringUtils.isNotBlank(this.solrUrlsStr)) {
return this.solrUrlsStr.split(",");
}
return null;
}

@PostConstruct
public void init() {
properties = new Properties();
@@ -331,4 +356,5 @@ public void init() {
throw new IllegalArgumentException("Cannot find logfeeder.properties on the classpath");
}
}

}
@@ -19,11 +19,13 @@

package org.apache.ambari.logfeeder.output;

import org.apache.ambari.logfeeder.common.LogFeederSolrClientFactory;
import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.plugin.input.InputMarker;
import org.apache.ambari.logfeeder.plugin.output.Output;
import org.apache.ambari.logfeeder.util.DateUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -76,6 +78,7 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> {
private String splitMode;
private int splitInterval;
private String zkConnectString;
private String[] solrUrls = null;
private int maxIntervalMS;
private int workers;
private int maxBufferSize;
@@ -121,8 +124,14 @@ private void initParams(LogFeederProps logFeederProps) throws Exception {
type = getStringValue("type");

zkConnectString = getStringValue("zk_connect_string");
if (StringUtils.isEmpty(zkConnectString)) {
throw new Exception("For solr output the zk_connect_string property need to be set");
List<String> solrUrlsList = getListValue("solr_urls");

if (StringUtils.isBlank(zkConnectString) && CollectionUtils.isEmpty(solrUrlsList)) {
throw new Exception("For solr output the zk_connect_string or solr_urls property need to be set");
}

if (CollectionUtils.isNotEmpty(solrUrlsList)) {
solrUrls = solrUrlsList.toArray(new String[0]);
}

skipLogtime = getBooleanValue("skip_logtime", DEFAULT_SKIP_LOGTIME);
@@ -176,42 +185,31 @@ private void createOutgoingBuffer() {

private void createSolrWorkers() throws Exception, MalformedURLException {
for (int count = 0; count < workers; count++) {
CloudSolrClient solrClient = getSolrClient(count);
SolrClient solrClient = getSolrClient(count);
createSolrWorkerThread(count, solrClient);
}
}

CloudSolrClient getSolrClient(int count) throws Exception, MalformedURLException {
CloudSolrClient solrClient = createSolrClient();
private SolrClient getSolrClient(int count) throws Exception, MalformedURLException {
SolrClient solrClient = new LogFeederSolrClientFactory().createSolrClient(zkConnectString, solrUrls, collection);
pingSolr(count, solrClient);

return solrClient;
}

private CloudSolrClient createSolrClient() throws Exception {
LOG.info("Using zookeepr. zkConnectString=" + zkConnectString);
LOG.info("Using collection=" + collection);

CloudSolrClient solrClient = new CloudSolrClient.Builder().withZkHost(zkConnectString).build();
solrClient.setDefaultCollection(collection);
return solrClient;
}

private void pingSolr(int count, CloudSolrClient solrClient) {
private void pingSolr(int count, SolrClient solrClient) {
try {
LOG.info("Pinging Solr server. zkConnectString=" + zkConnectString);
LOG.info("Pinging Solr server.");
SolrPingResponse response = solrClient.ping();
if (response.getStatus() == 0) {
LOG.info("Ping to Solr server is successful for worker=" + count);
} else {
LOG.warn(
String.format("Ping to Solr server failed. It would check again. worker=%d, zkConnectString=%s, collection=%s, " +
"response=%s", count, zkConnectString, collection, response));
String.format("Ping to Solr server failed. It would check again. worker=%d, collection=%s, " +
"response=%s", count, collection, response));
}
} catch (Throwable t) {
LOG.warn(String.format(
"Ping to Solr server failed. It would check again. worker=%d, zkConnectString=%s, collection=%s", count,
zkConnectString, collection), t);
"Ping to Solr server failed. It would check again. worker=%d, collection=%s", count, collection), t);
}
}

@@ -209,6 +209,7 @@ export class GraphComponent implements AfterViewInit, OnChanges, OnInit, OnDestr
this.graphContainer = this.graphContainerRef.nativeElement;
this.tooltip = this.tooltipRef.nativeElement;
this.host = d3.select(this.graphContainer);
this.createGraph();
}

ngOnChanges(changes: SimpleChanges) {
@@ -101,6 +101,8 @@

/deep/ typeahead-container .dropdown-menu {
.dropdown-list-default;
max-height: 80vh;
overflow-y: auto;

> li {
.dropdown-item-default;
@@ -58,6 +58,19 @@
<col *ngIf="isColumnDisplayed('path')" class="log-path">
<col *ngIf="isColumnDisplayed('log_message')" class="log-message">
</colgroup>
<thead>
<tr>
<th class="log-action"></th>
<th *ngIf="isColumnDisplayed('logtime')">{{getLabelForField('logtime') | translate}}</th>
<th *ngIf="isColumnDisplayed('level')">{{getLabelForField('level') | translate}}</th>
<th *ngIf="isColumnDisplayed('type')">{{getLabelForField('type') | translate}}</th>
<ng-container *ngFor="let column of displayedColumns">
<th *ngIf="customStyledColumns.indexOf(column.value) === -1">{{getLabelForField(column.value) | translate}}</th>
</ng-container>
<th *ngIf="isColumnDisplayed('path')">{{getLabelForField('path') | translate}}</th>
<th *ngIf="isColumnDisplayed('log_message')">{{getLabelForField('log_message') | translate}}</th>
</tr>
</thead>
<tbody>
<ng-container *ngFor="let log of logs; let i = index">
<tr *ngIf="i === 0 || isDifferentDates(log.logtime, logs[i - 1].logtime)" class="log-date-row">

0 comments on commit a84aa8f

Please sign in to comment.