Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

SOLR-14558: Record all log lines in SolrLogPostTool #1570

Merged
merged 1 commit into from
Jun 16, 2020
Merged

SOLR-14558: Record all log lines in SolrLogPostTool #1570

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 55 additions & 77 deletions solr/core/src/java/org/apache/solr/util/SolrLogPostTool.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,20 +186,25 @@ public SolrInputDocument readRecord() throws IOException {
}

if (line != null) {
SolrInputDocument lineDoc = new SolrInputDocument();
lineDoc.setField("date_dt", parseDate(line));
lineDoc.setField("line_t", line);
lineDoc.setField("type_s", "other"); // Overridden by known types below

if (line.contains("Registered new searcher")) {
return parseNewSearch(line);
parseNewSearch(lineDoc, line);
} else if (line.contains("path=/update")) {
return parseUpdate(line);
parseUpdate(lineDoc, line);
} else if (line.contains(" ERROR ")) {
this.cause = null;
return parseError(line, readTrace());
parseError(lineDoc, line, readTrace());
} else if (line.contains("start commit")) {
return parseCommit(line);
parseCommit(lineDoc, line);
} else if(line.contains("QTime=")) {
return parseQueryRecord(line);
} else {
continue;
parseQueryRecord(lineDoc, line);
}

return lineDoc;
} else {
return null;
}
Expand Down Expand Up @@ -252,94 +257,73 @@ private void setFieldIfUnset(SolrInputDocument doc, String fieldName, String fie
doc.setField(fieldName, fieldValue);
}

private SolrInputDocument parseError(String line, String trace) throws IOException {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("date_dt", parseDate(line));
doc.setField("type_s", "error");
doc.setField("line_t", line);
private void parseError(SolrInputDocument lineRecord, String line, String trace) throws IOException {
lineRecord.setField("type_s", "error");

//Don't include traces that have only the %html header.
if(trace != null && trace.length() > 6) {
doc.setField("stack_t", trace);
lineRecord.setField("stack_t", trace);
}

if(this.cause != null) {
doc.setField("root_cause_t", cause.replace("Caused by:", "").trim());
lineRecord.setField("root_cause_t", cause.replace("Caused by:", "").trim());
}

doc.setField("collection_s", parseCollection(line));
doc.setField("core_s", parseCore(line));
doc.setField("shard_s", parseShard(line));
doc.setField("replica_s", parseReplica(line));

return doc;
lineRecord.setField("collection_s", parseCollection(line));
lineRecord.setField("core_s", parseCore(line));
lineRecord.setField("shard_s", parseShard(line));
lineRecord.setField("replica_s", parseReplica(line));
}

private SolrInputDocument parseCommit(String line) throws IOException {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("date_dt", parseDate(line));
doc.setField("type_s", "commit");
doc.setField("line_t", line);
doc.setField("soft_commit_s", Boolean.toString(line.contains("softCommit=true")));

doc.setField("open_searcher_s", Boolean.toString(line.contains("openSearcher=true")));
private void parseCommit(SolrInputDocument lineRecord, String line) throws IOException {
lineRecord.setField("type_s", "commit");
lineRecord.setField("soft_commit_s", Boolean.toString(line.contains("softCommit=true")));

doc.setField("collection_s", parseCollection(line));
doc.setField("core_s", parseCore(line));
doc.setField("shard_s", parseShard(line));
doc.setField("replica_s", parseReplica(line));
lineRecord.setField("open_searcher_s", Boolean.toString(line.contains("openSearcher=true")));

return doc;
lineRecord.setField("collection_s", parseCollection(line));
lineRecord.setField("core_s", parseCore(line));
lineRecord.setField("shard_s", parseShard(line));
lineRecord.setField("replica_s", parseReplica(line));
}

private SolrInputDocument parseQueryRecord(String line) {

SolrInputDocument doc = new SolrInputDocument();
doc.setField("date_dt", parseDate(line));
doc.setField("qtime_i", parseQTime(line));
doc.setField("status_s", parseStatus(line));
private void parseQueryRecord(SolrInputDocument lineRecord, String line) {
lineRecord.setField("qtime_i", parseQTime(line));
lineRecord.setField("status_s", parseStatus(line));

String path = parsePath(line);
doc.setField("path_s", path);
lineRecord.setField("path_s", path);

if(line.contains("hits=")) {
doc.setField("hits_l", parseHits(line));
lineRecord.setField("hits_l", parseHits(line));
}

String params = parseParams(line);
doc.setField("params_t", params);
addParams(doc, params);
lineRecord.setField("params_t", params);
addParams(lineRecord, params);

doc.setField("collection_s", parseCollection(line));
doc.setField("core_s", parseCore(line));
doc.setField("node_s", parseNode(line));
doc.setField("shard_s", parseShard(line));
doc.setField("replica_s", parseReplica(line));
lineRecord.setField("collection_s", parseCollection(line));
lineRecord.setField("core_s", parseCore(line));
lineRecord.setField("node_s", parseNode(line));
lineRecord.setField("shard_s", parseShard(line));
lineRecord.setField("replica_s", parseReplica(line));


if(path != null && path.contains("/admin")) {
doc.setField("type_s", "admin");
lineRecord.setField("type_s", "admin");
} else if(path != null && params.contains("/replication")) {
doc.setField("type_s", "replication");
lineRecord.setField("type_s", "replication");
} else if (path != null && path.contains("/get")) {
doc.setField("type_s", "get");
lineRecord.setField("type_s", "get");
} else {
doc.setField("type_s", "query");
lineRecord.setField("type_s", "query");
}

return doc;
}


private SolrInputDocument parseNewSearch(String line) {

SolrInputDocument doc = new SolrInputDocument();
doc.setField("date_dt", parseDate(line));
doc.setField("core_s", parseNewSearcherCore(line));
doc.setField("type_s", "newSearcher");
doc.setField("line_t", line);

return doc;
private void parseNewSearch(SolrInputDocument lineRecord, String line) {
lineRecord.setField("core_s", parseNewSearcherCore(line));
lineRecord.setField("type_s", "newSearcher");
}

private String parseCollection(String line) {
Expand All @@ -352,25 +336,19 @@ private String parseCollection(String line) {
}
}

private SolrInputDocument parseUpdate(String line) {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("date_dt", parseDate(line));

private void parseUpdate(SolrInputDocument lineRecord, String line) {
if(line.contains("deleteByQuery=")) {
doc.setField("type_s", "deleteByQuery");
lineRecord.setField("type_s", "deleteByQuery");
} else if(line.contains("delete=")) {
doc.setField("type_s", "delete");
lineRecord.setField("type_s", "delete");
} else {
doc.setField("type_s", "update");
lineRecord.setField("type_s", "update");
}

doc.setField("collection_s", parseCollection(line));
doc.setField("core_s", parseCore(line));
doc.setField("shard_s", parseShard(line));
doc.setField("replica_s", parseReplica(line));
doc.setField("line_t", line);

return doc;
lineRecord.setField("collection_s", parseCollection(line));
lineRecord.setField("core_s", parseCore(line));
lineRecord.setField("shard_s", parseShard(line));
lineRecord.setField("replica_s", parseReplica(line));
}

private String parseNewSearcherCore(String line) {
Expand Down
16 changes: 16 additions & 0 deletions solr/core/src/test/org/apache/solr/util/SolrLogPostToolTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import java.io.BufferedReader;
import java.io.StringReader;
import java.util.Collection;
import java.util.List;
import java.util.ArrayList;

Expand Down Expand Up @@ -286,6 +287,21 @@ public void testNewSearcher() throws Exception{
assertEquals(core.getValue(), "production_cv_month_201912_shard35_replica_n1");
}

// Ensure SolrLogPostTool parses _all_ log lines into searchable records
@Test
public void testOtherRecord() throws Exception {
final String record = "2020-06-11 11:59:08.386 INFO (main) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)";
final List<SolrInputDocument> docs = readDocs(record);
assertEquals(docs.size(), 1);

SolrInputDocument doc = docs.get(0);
final Collection<String> fields = doc.getFieldNames();
assertEquals(3, fields.size());
assertEquals("2020-06-11T11:59:08.386", doc.getField("date_dt").getValue());
assertEquals("other", doc.getField("type_s").getValue());
assertEquals(record, doc.getField("line_t").getValue());
}

private List<SolrInputDocument> readDocs(String records) throws Exception {
BufferedReader bufferedReader = new BufferedReader(new StringReader(records));
ArrayList<SolrInputDocument> list = new ArrayList();
Expand Down