Skip to content
Permalink
Browse files
AMBARI-24939. Refactor logsearch_metadata schama and usage for logsea…
…rch portal (#47)
  • Loading branch information
oleewere committed Nov 28, 2018
1 parent 67128e1 commit 2500713b0c24169b4eea207aead3a5e74c2aceec
Show file tree
Hide file tree
Showing 16 changed files with 174 additions and 283 deletions.
@@ -62,10 +62,10 @@ public void createLogLevelFilter(String clusterName, String logId, LogLevelFilte
if (useClusterParam) {
doc.addField("cluster_string", clusterName);
}
doc.addField("filtername", logId);
doc.addField("rowtype", "log_level_filter");
doc.addField("jsons", gson.toJson(filter));
doc.addField("username", "default");
doc.addField("name", logId);
doc.addField("type", "log_level_filter");
doc.addField("value", gson.toJson(filter));
doc.addField("username", "none");
logger.debug("Creating log level filter - logid: {}, cluster: {}", logId, clusterName);
solrClient.add(doc);
}
@@ -112,16 +112,16 @@ public LogLevelFilterMap getLogLevelFilters(String clusterName) {
if (useClusterParam) {
solrQuery.addFilterQuery("cluster_string:" + clusterName);
}
solrQuery.addFilterQuery("rowtype:log_level_filter");
solrQuery.setFields("jsons", "filtername");
solrQuery.addFilterQuery("type:log_level_filter");
solrQuery.setFields("value", "name");

final QueryResponse response = solrClient.query(solrQuery);
if (response != null) {
final SolrDocumentList documents = response.getResults();
if (documents != null && !documents.isEmpty()) {
for(SolrDocument document : documents) {
String jsons = (String) document.getFieldValue("jsons");
String logId = (String) document.getFieldValue("filtername");
String jsons = (String) document.getFieldValue("value");
String logId = (String) document.getFieldValue("name");
if (jsons != null) {
LogLevelFilter logLevelFilter = gson.fromJson(jsons, LogLevelFilter.class);
logLevelFilterTreeMap.put(logId,logLevelFilter);
@@ -149,13 +149,11 @@ public Gson getGson() {
}

private void waitForSolr(SolrClient solrClient) {
boolean solrAvailable = false;
while (!solrAvailable) {
while (true) {
try {
logger.debug("Start solr ping for log level filter collection");
SolrPingResponse pingResponse = solrClient.ping();
if (pingResponse.getStatus() == 0) {
solrAvailable = true;
break;
}
} catch (Exception e) {
@@ -31,10 +31,6 @@ public class LogFeederConstants {
public static final String SOLR_COMPONENT = "type";
public static final String SOLR_HOST = "host";

// Event History Constants History
public static final String VALUES = "jsons";
public static final String ROW_TYPE = "rowtype";

public static final String IN_MEMORY_TIMESTAMP = "in_memory_timestamp";

public static final String LOGFEEDER_PROPERTIES_FILE = "logfeeder.properties";
@@ -80,12 +80,11 @@
</fieldType>

<field name="_version_" type="long" indexed="false" stored="true"/>
<field name="filtername" type="key_lower_case" indexed="true" required="true" stored="true"/>
<field name="id" type="string" required="true"/>
<field name="jsons" type="string"/>
<field name="rowtype" type="string" required="true"/>
<field name="share_username_list" type="string" multiValued="true"/>
<field name="name" type="key_lower_case" indexed="true" required="true" stored="true"/>
<field name="username" type="key_lower_case" indexed="true" required="true" stored="true"/>
<field name="type" type="string" required="true"/>
<field name="value" type="string"/>
<field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true" stored="false"/>

<dynamicField name='ngram_*' type="n_gram" multiValued="false" stored="false"/>
@@ -133,8 +133,9 @@ public class LogSearchConstants {
public static final String REQUEST_PARAM_SOURCE_LOG_ID = "sourceLogId";
public static final String REQUEST_PARAM_KEYWORD_TYPE = "keywordType";
public static final String REQUEST_PARAM_TOKEN = "token";
public static final String REQUEST_PARAM_FILTER_NAME = "filterName";
public static final String REQUEST_PARAM_ROW_TYPE = "rowType";
public static final String REQUEST_PARAM_METADATA_NAME = "name";
public static final String REQUEST_PARAM_METADATA_TYPE = "type";
public static final String REQUEST_PARAM_METADATA_USER_NAME = "userName";
public static final String REQUEST_PARAM_UTC_OFFSET = "utcOffset";
public static final String REQUEST_PARAM_HOSTS = "hostList";
public static final String REQUEST_PARAM_USERS = "userList";
@@ -29,8 +29,8 @@
import java.util.List;

import static org.apache.ambari.logsearch.solr.SolrConstants.CommonLogConstants.CLUSTER;
import static org.apache.ambari.logsearch.solr.SolrConstants.EventHistoryConstants.FILTER_NAME;
import static org.apache.ambari.logsearch.solr.SolrConstants.EventHistoryConstants.ROW_TYPE;
import static org.apache.ambari.logsearch.solr.SolrConstants.MetadataConstants.NAME;
import static org.apache.ambari.logsearch.solr.SolrConstants.MetadataConstants.TYPE;

@Named
public class MetadataRequestQueryConverter extends AbstractConverterAware<MetadataRequest, SolrQuery> {
@@ -39,28 +39,18 @@ public class MetadataRequestQueryConverter extends AbstractConverterAware<Metada
public SolrQuery convert(MetadataRequest metadataRequest) {
SolrQuery metadataQuery = new SolrQuery();
metadataQuery.setQuery("*:*");
metadataQuery.addFilterQuery(String.format("%s:%s", TYPE, metadataRequest.getType()));
if (StringUtils.isNotBlank(metadataRequest.getName())) {
metadataQuery.addFilterQuery(String.format("%s:%s", NAME, metadataRequest.getName()));
}

int startIndex = StringUtils.isNotEmpty(metadataRequest.getStartIndex()) && StringUtils.isNumeric(metadataRequest.getStartIndex())
? Integer.parseInt(metadataRequest.getStartIndex()) : 0;
int maxRows = StringUtils.isNotEmpty(metadataRequest.getPageSize()) && StringUtils.isNumeric(metadataRequest.getPageSize())
? Integer.parseInt(metadataRequest.getPageSize()) : 10;

SolrQuery.ORDER order = metadataRequest.getSortType() != null && SolrQuery.ORDER.desc.equals(SolrQuery.ORDER.valueOf(metadataRequest.getSortType()))
? SolrQuery.ORDER.desc : SolrQuery.ORDER.asc;
String sortBy = StringUtils.isNotEmpty(metadataRequest.getSortBy()) ? metadataRequest.getSortBy() : FILTER_NAME;
String filterName = StringUtils.isBlank(metadataRequest.getFilterName()) ? "*" : "*" + metadataRequest.getFilterName() + "*";

metadataQuery.addFilterQuery(String.format("%s:%s", ROW_TYPE, metadataRequest.getRowType()));
metadataQuery.addFilterQuery(String.format("%s:%s", FILTER_NAME, SolrUtil.makeSearcableString(filterName)));
metadataQuery.setStart(startIndex);
metadataQuery.setRows(maxRows);

SolrQuery.SortClause sortOrder = SolrQuery.SortClause.create(sortBy, order);
SolrQuery.SortClause sortOrder = SolrQuery.SortClause.create(NAME, SolrQuery.ORDER.asc);
List<SolrQuery.SortClause> sort = new ArrayList<>();
sort.add(sortOrder);
metadataQuery.setRows(10000);
metadataQuery.setSorts(sort);

SolrUtil.addListFilterToSolrQuery(metadataQuery, CLUSTER, metadataRequest.getClusters());
SolrUtil.addListFilterToSolrQuery(metadataQuery, CLUSTER + "_string", metadataRequest.getClusters());

return metadataQuery;
}
@@ -89,8 +89,8 @@ public void postConstructor() {
}
}

public UpdateResponse deleteEventHistoryData(String id) {
return removeDoc("id:" + id);
public UpdateResponse deleteMetadata(String name, String type) {
return removeDoc(String.format("name:%s AND type:%s", name, type));
}

private UpdateResponse removeDoc(String query) {
@@ -113,16 +113,16 @@ public class PublicOperationDescriptions {
public static final String GET_ALL_SHIPPER_CONFIG_INFO_OD = "List all available shipper configuration element";
}

public class EventHistoryDescriptions {
public static final String FILTER_NAME_D = "The saved query as filter in Solr, search is sopprted by this param";
public static final String ROW_TYPE_D = "Row type is solr to identify as filter query";
public class MetadataDescriptions {
public static final String NAME_D = "Name identifier of the metadata";
public static final String TYPE_D = "Type of the metadata";
public static final String USER_NAME_D = "User name of the metadata";
}

public class EventHistoryOperationDescriptions {
public class MetadataOperationDescriptions {
public static final String SAVE_METADATA_OD = "Save metadata";
public static final String DELETE_METADATA_OD = "Delete metadata";
public static final String GET_METADATA_OD = "Get event metadata";
public static final String GET_ALL_USER_NAMES_OD = "Get all user names";
public static final String GET_METADATA_OD = "Get metadata";
}

public class ShipperConfigOperationDescriptions {
@@ -21,36 +21,29 @@

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.UUID;

import org.apache.ambari.logsearch.common.LogSearchContext;
import org.apache.ambari.logsearch.dao.MetadataSolrDao;
import org.apache.ambari.logsearch.model.request.impl.MetadataRequest;
import org.apache.ambari.logsearch.model.response.LogsearchMetaData;
import org.apache.ambari.logsearch.model.response.LogSearchMetaDataListResponse;
import org.apache.ambari.logsearch.util.SolrUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.springframework.core.convert.ConversionService;

import javax.inject.Inject;
import javax.inject.Named;

import static org.apache.ambari.logsearch.solr.SolrConstants.EventHistoryConstants.ID;
import static org.apache.ambari.logsearch.solr.SolrConstants.EventHistoryConstants.USER_NAME;
import static org.apache.ambari.logsearch.solr.SolrConstants.EventHistoryConstants.VALUES;
import static org.apache.ambari.logsearch.solr.SolrConstants.EventHistoryConstants.FILTER_NAME;
import static org.apache.ambari.logsearch.solr.SolrConstants.EventHistoryConstants.ROW_TYPE;
import static org.apache.ambari.logsearch.solr.SolrConstants.EventHistoryConstants.SHARE_NAME_LIST;
import static org.apache.ambari.logsearch.solr.SolrConstants.MetadataConstants.ID;
import static org.apache.ambari.logsearch.solr.SolrConstants.MetadataConstants.USER_NAME;
import static org.apache.ambari.logsearch.solr.SolrConstants.MetadataConstants.VALUE;
import static org.apache.ambari.logsearch.solr.SolrConstants.MetadataConstants.NAME;
import static org.apache.ambari.logsearch.solr.SolrConstants.MetadataConstants.TYPE;

@Named
public class MetadataManager extends JsonManagerBase {
@@ -63,120 +56,73 @@ public class MetadataManager extends JsonManagerBase {
private ConversionService conversionService;

public String saveMetadata(LogsearchMetaData metadata) {
String filterName = metadata.getFiltername();
String name = metadata.getName();

SolrInputDocument solrInputDoc = new SolrInputDocument();
if (!isValid(metadata)) {
throw new MalformedInputException("No FilterName Specified");
}

if (isNotUnique(filterName)) {
throw new AlreadyExistsException(String.format("Name '%s' already exists", metadata.getFiltername()));
}
solrInputDoc.addField(ID, metadata.getId());
solrInputDoc.addField(USER_NAME, LogSearchContext.getCurrentUsername());
solrInputDoc.addField(VALUES, metadata.getValues());
solrInputDoc.addField(FILTER_NAME, filterName);
solrInputDoc.addField(ROW_TYPE, metadata.getRowType());
List<String> shareNameList = metadata.getShareNameList();
if (CollectionUtils.isNotEmpty(shareNameList)) {
solrInputDoc.addField(SHARE_NAME_LIST, shareNameList);
if (!isValid(metadata, false)) {
throw new MalformedInputException("Name,type and value should be specified");
}
final String userName = LogSearchContext.getCurrentUsername().toLowerCase();
solrInputDoc.addField(ID, generateUniqueId(metadata, userName));
solrInputDoc.addField(USER_NAME, userName);
solrInputDoc.addField(VALUE, metadata.getValue());
solrInputDoc.addField(NAME, name);
solrInputDoc.addField(TYPE, metadata.getType());

metadataSolrDao.addDocs(solrInputDoc);
return convertObjToString(solrInputDoc);
}

private boolean isNotUnique(String filterName) {

if (filterName != null) {
SolrQuery solrQuery = new SolrQuery();
filterName = SolrUtil.makeSearcableString(filterName);
solrQuery.setQuery("*:*");
solrQuery.addFilterQuery(FILTER_NAME + ":" + filterName);
solrQuery.addFilterQuery(USER_NAME + ":" + LogSearchContext.getCurrentUsername());
SolrUtil.setRowCount(solrQuery, 0);
try {
Long numFound = metadataSolrDao.process(solrQuery).getResults().getNumFound();
if (numFound > 0) {
return true;
}
} catch (SolrException e) {
logger.error("Error while checking if metadata is unique.", e);
}
public void deleteMetadata(LogsearchMetaData metaData) {
if (!isValid(metaData, true)) {
throw new MalformedInputException("Name and type should be specified");
}
return false;
}

private boolean isValid(LogsearchMetaData mData) {
return StringUtils.isNotBlank(mData.getFiltername())
&& StringUtils.isNotBlank(mData.getRowType())
&& StringUtils.isNotBlank(mData.getValues());
}

public void deleteMetadata(String id) {
metadataSolrDao.deleteEventHistoryData(id);
metadataSolrDao.deleteMetadata(metaData.getName(), metaData.getType());
}

@SuppressWarnings("unchecked")
public LogSearchMetaDataListResponse getMetadata(MetadataRequest request) {
LogSearchMetaDataListResponse response = new LogSearchMetaDataListResponse();
String rowType = request.getRowType();
if (StringUtils.isBlank(rowType)) {
throw new MalformedInputException("Row type was not specified");
}

public Collection<LogsearchMetaData> getMetadata(MetadataRequest request) {
SolrQuery metadataQueryQuery = conversionService.convert(request, SolrQuery.class);
metadataQueryQuery.addFilterQuery(String.format("%s:%s OR %s:%s", USER_NAME, LogSearchContext.getCurrentUsername(),
SHARE_NAME_LIST, LogSearchContext.getCurrentUsername()));

final String userName;
if (StringUtils.isNotBlank(request.getUserName())) {
userName = request.getUserName();
} else {
userName = LogSearchContext.getCurrentUsername();
}
if (StringUtils.isBlank(userName)) {
throw new IllegalArgumentException("User name is not found for metadata request.");
}
metadataQueryQuery.addFilterQuery(String.format("%s:%s", USER_NAME, userName.toLowerCase()));
SolrDocumentList solrList = metadataSolrDao.process(metadataQueryQuery).getResults();

Collection<LogsearchMetaData> configList = new ArrayList<>();
Collection<LogsearchMetaData> metadataList = new ArrayList<>();

for (SolrDocument solrDoc : solrList) {
LogsearchMetaData metadata = new LogsearchMetaData();
metadata.setFiltername("" + solrDoc.get(FILTER_NAME));
metadata.setId("" + solrDoc.get(ID));
metadata.setValues("" + solrDoc.get(VALUES));
metadata.setRowType("" + solrDoc.get(ROW_TYPE));
try {
List<String> shareNameList = (List<String>) solrDoc.get(SHARE_NAME_LIST);
metadata.setShareNameList(shareNameList);
} catch (Exception e) {
// do nothing
}

metadata.setUserName("" + solrDoc.get(USER_NAME));

configList.add(metadata);
}

response.setName("metadataList");
response.setMetadataList(configList);
metadata.setName(solrDoc.get(NAME).toString());
metadata.setId(solrDoc.get(ID).toString());
metadata.setValue(solrDoc.getOrDefault(VALUE, "").toString());
metadata.setType(solrDoc.get(TYPE).toString());
metadata.setUserName(solrDoc.get(USER_NAME).toString());

response.setStartIndex(Integer.parseInt(request.getStartIndex()));
response.setPageSize(Integer.parseInt(request.getPageSize()));
metadataList.add(metadata);
}

response.setTotalCount(solrList.getNumFound());
return metadataList;

return response;
}

private String generateUniqueId(LogsearchMetaData metaData, String userName) {
String strToUUID = metaData.getName() +
metaData.getType() +
userName;
return UUID.nameUUIDFromBytes(strToUUID.getBytes()).toString();
}

public List<String> getAllUserName() {
List<String> userList = new ArrayList<>();
SolrQuery userListQuery = new SolrQuery();
userListQuery.setQuery("*:*");
SolrUtil.setFacetField(userListQuery, USER_NAME);
QueryResponse queryResponse = metadataSolrDao.process(userListQuery);
if (queryResponse == null) {
return userList;
}
List<Count> counList = queryResponse.getFacetField(USER_NAME).getValues();
for (Count cnt : counList) {
String userName = cnt.getName();
userList.add(userName);
}
return userList;
private boolean isValid(LogsearchMetaData mData, boolean skipValueCheck) {
return StringUtils.isNotBlank(mData.getName())
&& StringUtils.isNotBlank(mData.getType())
&& (skipValueCheck || StringUtils.isNotBlank(mData.getValue()));
}
}

0 comments on commit 2500713

Please sign in to comment.