Skip to content

Commit

Permalink
Jvm metric test success.
Browse files Browse the repository at this point in the history
  • Loading branch information
peng-yongsheng committed Sep 2, 2017
1 parent 9b97018 commit bdf29e5
Show file tree
Hide file tree
Showing 23 changed files with 283 additions and 156 deletions.
Expand Up @@ -78,14 +78,34 @@ private static void buildMemoryMetric(JVMMetric.Builder jvmMetric) {
}

private static void buildMemoryPoolMetric(JVMMetric.Builder jvmMetric) {
MemoryPool.Builder builder_1 = MemoryPool.newBuilder();
builder_1.setType(PoolType.NEWGEN_USAGE);
builder_1.setIsHeap(true);
builder_1.setInit(20);
builder_1.setMax(100);
builder_1.setUsed(50);
builder_1.setCommited(30);
jvmMetric.addMemoryPool(builder_1.build());
jvmMetric.addMemoryPool(buildMemoryPoolMetric(PoolType.NEWGEN_USAGE, true).build());
jvmMetric.addMemoryPool(buildMemoryPoolMetric(PoolType.NEWGEN_USAGE, false).build());

jvmMetric.addMemoryPool(buildMemoryPoolMetric(PoolType.OLDGEN_USAGE, true).build());
jvmMetric.addMemoryPool(buildMemoryPoolMetric(PoolType.OLDGEN_USAGE, false).build());

jvmMetric.addMemoryPool(buildMemoryPoolMetric(PoolType.METASPACE_USAGE, true).build());
jvmMetric.addMemoryPool(buildMemoryPoolMetric(PoolType.METASPACE_USAGE, false).build());

jvmMetric.addMemoryPool(buildMemoryPoolMetric(PoolType.PERMGEN_USAGE, true).build());
jvmMetric.addMemoryPool(buildMemoryPoolMetric(PoolType.PERMGEN_USAGE, false).build());

jvmMetric.addMemoryPool(buildMemoryPoolMetric(PoolType.SURVIVOR_USAGE, true).build());
jvmMetric.addMemoryPool(buildMemoryPoolMetric(PoolType.SURVIVOR_USAGE, false).build());

jvmMetric.addMemoryPool(buildMemoryPoolMetric(PoolType.CODE_CACHE_USAGE, true).build());
jvmMetric.addMemoryPool(buildMemoryPoolMetric(PoolType.CODE_CACHE_USAGE, false).build());
}

private static MemoryPool.Builder buildMemoryPoolMetric(PoolType poolType, boolean isHeap) {
MemoryPool.Builder builder = MemoryPool.newBuilder();
builder.setType(poolType);
builder.setIsHeap(isHeap);
builder.setInit(20);
builder.setMax(100);
builder.setUsed(50);
builder.setCommited(30);
return builder;
}

private static void buildGcMetric(JVMMetric.Builder jvmMetric) {
Expand Down
@@ -1,8 +1,8 @@
package org.skywalking.apm.collector.agentstream.worker.segment.cost.define;

import org.skywalking.apm.collector.storage.define.segment.SegmentCostTable;
import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine;
import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine;
import org.skywalking.apm.collector.storage.define.segment.SegmentCostTable;

/**
* @author pengys5
Expand All @@ -27,7 +27,7 @@ public SegmentCostEsTableDefine() {

@Override public void initialize() {
addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_SEGMENT_ID, ElasticSearchColumnDefine.Type.Keyword.name()));
addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_SERVICE_NAME, ElasticSearchColumnDefine.Type.Keyword.name()));
addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_SERVICE_NAME, ElasticSearchColumnDefine.Type.Text.name()));
addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_COST, ElasticSearchColumnDefine.Type.Long.name()));
addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_START_TIME, ElasticSearchColumnDefine.Type.Long.name()));
addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_END_TIME, ElasticSearchColumnDefine.Type.Long.name()));
Expand Down
Expand Up @@ -28,9 +28,9 @@ public static void main(String[] args) throws IOException, InterruptedException,
InstanceEsDAO instanceEsDAO = new InstanceEsDAO();
instanceEsDAO.setClient(client);

InstanceDataDefine.Instance consumerInstance = new InstanceDataDefine.Instance("2", 2, "dubbox-consumer", now, 2, now, "");
InstanceDataDefine.Instance consumerInstance = new InstanceDataDefine.Instance("2", 2, "dubbox-consumer", now, 2, now, osInfo("consumer").toString());
instanceEsDAO.save(consumerInstance);
InstanceDataDefine.Instance providerInstance = new InstanceDataDefine.Instance("3", 3, "dubbox-provider", now, 3, now, "");
InstanceDataDefine.Instance providerInstance = new InstanceDataDefine.Instance("3", 3, "dubbox-provider", now, 3, now, osInfo("provider").toString());
instanceEsDAO.save(providerInstance);

ApplicationEsDAO applicationEsDAO = new ApplicationEsDAO();
Expand Down Expand Up @@ -64,10 +64,13 @@ public static void main(String[] args) throws IOException, InterruptedException,
modifyTime(provider);
HttpClientTools.INSTANCE.post("http://localhost:12800/segments", provider.toString());

diff = 0;
Thread.sleep(1000);
}
}

private static long diff = 0;

private static void modifyTime(JsonElement jsonElement) {
JsonArray segmentArray = jsonElement.getAsJsonArray();
for (JsonElement element : segmentArray) {
Expand All @@ -76,10 +79,28 @@ private static void modifyTime(JsonElement jsonElement) {
for (JsonElement span : spans) {
long startTime = span.getAsJsonObject().get("st").getAsLong();
long endTime = span.getAsJsonObject().get("et").getAsLong();
long currentTime = System.currentTimeMillis();
span.getAsJsonObject().addProperty("st", currentTime);
span.getAsJsonObject().addProperty("et", currentTime + (endTime - startTime));

if (diff == 0) {
diff = System.currentTimeMillis() - startTime;
}

span.getAsJsonObject().addProperty("st", startTime + diff);
span.getAsJsonObject().addProperty("et", endTime + diff);
}
}
}

private static JsonObject osInfo(String hostName) {
JsonObject osInfoJson = new JsonObject();
osInfoJson.addProperty("osName", "Linux");
osInfoJson.addProperty("hostName", hostName);
osInfoJson.addProperty("processId", 1);

JsonArray ipv4Array = new JsonArray();
ipv4Array.add("123.123.123.123");
ipv4Array.add("124.124.124.124");
osInfoJson.add("ipv4s", ipv4Array);

return osInfoJson;
}
}
Expand Up @@ -40,8 +40,8 @@
"tv": 0,
"lv": 2,
"ps": -1,
"st": 1501858094883,
"et": 1501858096950,
"st": 1501858094726,
"et": 1501858096804,
"ci": 3,
"cn": "",
"oi": 0,
Expand Down
Expand Up @@ -53,7 +53,12 @@ private Settings createSettingBuilder(ElasticSearchTableDefine tableDefine) {
return Settings.builder()
.put("index.number_of_shards", tableDefine.numberOfShards())
.put("index.number_of_replicas", tableDefine.numberOfReplicas())
.put("index.refresh_interval", String.valueOf(tableDefine.refreshInterval()) + "s").build();
.put("index.refresh_interval", String.valueOf(tableDefine.refreshInterval()) + "s")

.put("analysis.analyzer.collector_analyzer.tokenizer", "collector_tokenizer")
.put("analysis.tokenizer.collector_tokenizer.type", "standard")
.put("analysis.tokenizer.collector_tokenizer.max_token_length", 5)
.build();
}

private XContentBuilder createMappingBuilder(ElasticSearchTableDefine tableDefine) throws IOException {
Expand Down
Expand Up @@ -16,7 +16,6 @@
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.skywalking.apm.collector.core.util.Const;
import org.skywalking.apm.collector.storage.define.jvm.CpuMetricTable;
import org.skywalking.apm.collector.storage.define.jvm.GCMetricTable;
import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;
import org.skywalking.apm.network.proto.GCPhrase;
Expand Down Expand Up @@ -92,8 +91,8 @@ public class GCMetricEsDAO extends EsDAO implements IGCMetricDAO {
MultiGetRequestBuilder youngPrepareMultiGet = getClient().prepareMultiGet();
int i = 0;
do {
String youngId = (startTimeBucket + i) + Const.ID_SPLIT + GCPhrase.NEW_VALUE + instanceId;
youngPrepareMultiGet.add(CpuMetricTable.TABLE, CpuMetricTable.TABLE_TYPE, youngId);
String youngId = (startTimeBucket + i) + Const.ID_SPLIT + instanceId + Const.ID_SPLIT + GCPhrase.NEW_VALUE;
youngPrepareMultiGet.add(GCMetricTable.TABLE, GCMetricTable.TABLE_TYPE, youngId);
i++;
}
while (startTimeBucket + i <= endTimeBucket);
Expand All @@ -102,7 +101,7 @@ public class GCMetricEsDAO extends EsDAO implements IGCMetricDAO {
MultiGetResponse multiGetResponse = youngPrepareMultiGet.get();
for (MultiGetItemResponse itemResponse : multiGetResponse.getResponses()) {
if (itemResponse.getResponse().isExists()) {
youngArray.add(((Number)itemResponse.getResponse().getSource().get(CpuMetricTable.COLUMN_USAGE_PERCENT)).intValue());
youngArray.add(((Number)itemResponse.getResponse().getSource().get(GCMetricTable.COLUMN_COUNT)).intValue());
} else {
youngArray.add(0);
}
Expand All @@ -112,8 +111,8 @@ public class GCMetricEsDAO extends EsDAO implements IGCMetricDAO {
MultiGetRequestBuilder oldPrepareMultiGet = getClient().prepareMultiGet();
i = 0;
do {
String oldId = (startTimeBucket + i) + Const.ID_SPLIT + GCPhrase.OLD_VALUE + instanceId;
oldPrepareMultiGet.add(CpuMetricTable.TABLE, CpuMetricTable.TABLE_TYPE, oldId);
String oldId = (startTimeBucket + i) + Const.ID_SPLIT + instanceId + Const.ID_SPLIT + GCPhrase.OLD_VALUE;
oldPrepareMultiGet.add(GCMetricTable.TABLE, GCMetricTable.TABLE_TYPE, oldId);
i++;
}
while (startTimeBucket + i <= endTimeBucket);
Expand All @@ -123,7 +122,7 @@ public class GCMetricEsDAO extends EsDAO implements IGCMetricDAO {
multiGetResponse = oldPrepareMultiGet.get();
for (MultiGetItemResponse itemResponse : multiGetResponse.getResponses()) {
if (itemResponse.getResponse().isExists()) {
oldArray.add(((Number)itemResponse.getResponse().getSource().get(CpuMetricTable.COLUMN_USAGE_PERCENT)).intValue());
oldArray.add(((Number)itemResponse.getResponse().getSource().get(GCMetricTable.COLUMN_COUNT)).intValue());
} else {
oldArray.add(0);
}
Expand Down
Expand Up @@ -9,9 +9,13 @@
public interface IInstPerformanceDAO {
List<InstPerformance> getMultiple(long timeBucket, int applicationId);

int getMetric(int instanceId, long timeBucket);
int getTpsMetric(int instanceId, long timeBucket);

JsonArray getMetric(int instanceId, long startTimeBucket, long endTimeBucket);
JsonArray getTpsMetric(int instanceId, long startTimeBucket, long endTimeBucket);

int getRespTimeMetric(int instanceId, long timeBucket);

JsonArray getRespTimeMetric(int instanceId, long startTimeBucket, long endTimeBucket);

class InstPerformance {
private final int instanceId;
Expand Down
Expand Up @@ -7,5 +7,9 @@
*/
public interface ISegmentCostDAO {
JsonObject loadTop(long startTime, long endTime, long minCost, long maxCost, String operationName,
String globalTraceId, int limit, int from);
String globalTraceId, int limit, int from, Sort sort);

public enum Sort {
Cost, Time
}
}
Expand Up @@ -18,7 +18,6 @@
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.skywalking.apm.collector.core.util.Const;
import org.skywalking.apm.collector.storage.define.instance.InstPerformanceTable;
import org.skywalking.apm.collector.storage.define.jvm.CpuMetricTable;
import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;

/**
Expand Down Expand Up @@ -68,7 +67,7 @@ public class InstPerformanceEsDAO extends EsDAO implements IInstPerformanceDAO {
return instPerformances;
}

@Override public int getMetric(int instanceId, long timeBucket) {
@Override public int getTpsMetric(int instanceId, long timeBucket) {
String id = timeBucket + Const.ID_SPLIT + instanceId;
GetResponse getResponse = getClient().prepareGet(InstPerformanceTable.TABLE, id).get();

Expand All @@ -78,13 +77,13 @@ public class InstPerformanceEsDAO extends EsDAO implements IInstPerformanceDAO {
return 0;
}

@Override public JsonArray getMetric(int instanceId, long startTimeBucket, long endTimeBucket) {
@Override public JsonArray getTpsMetric(int instanceId, long startTimeBucket, long endTimeBucket) {
MultiGetRequestBuilder prepareMultiGet = getClient().prepareMultiGet();

int i = 0;
do {
String id = (startTimeBucket + i) + Const.ID_SPLIT + instanceId;
prepareMultiGet.add(CpuMetricTable.TABLE, InstPerformanceTable.TABLE_TYPE, id);
prepareMultiGet.add(InstPerformanceTable.TABLE, InstPerformanceTable.TABLE_TYPE, id);
i++;
}
while (startTimeBucket + i <= endTimeBucket);
Expand All @@ -100,4 +99,41 @@ public class InstPerformanceEsDAO extends EsDAO implements IInstPerformanceDAO {
}
return metrics;
}

@Override public int getRespTimeMetric(int instanceId, long timeBucket) {
String id = timeBucket + Const.ID_SPLIT + instanceId;
GetResponse getResponse = getClient().prepareGet(InstPerformanceTable.TABLE, id).get();

if (getResponse.isExists()) {
int callTimes = ((Number)getResponse.getSource().get(InstPerformanceTable.COLUMN_CALL_TIMES)).intValue();
int costTotal = ((Number)getResponse.getSource().get(InstPerformanceTable.COLUMN_COST_TOTAL)).intValue();
return costTotal / callTimes;
}
return 0;
}

@Override public JsonArray getRespTimeMetric(int instanceId, long startTimeBucket, long endTimeBucket) {
MultiGetRequestBuilder prepareMultiGet = getClient().prepareMultiGet();

int i = 0;
do {
String id = (startTimeBucket + i) + Const.ID_SPLIT + instanceId;
prepareMultiGet.add(InstPerformanceTable.TABLE, InstPerformanceTable.TABLE_TYPE, id);
i++;
}
while (startTimeBucket + i <= endTimeBucket);

JsonArray metrics = new JsonArray();
MultiGetResponse multiGetResponse = prepareMultiGet.get();
for (MultiGetItemResponse response : multiGetResponse.getResponses()) {
if (response.getResponse().isExists()) {
int callTimes = ((Number)response.getResponse().getSource().get(InstPerformanceTable.COLUMN_CALL_TIMES)).intValue();
int costTotal = ((Number)response.getResponse().getSource().get(InstPerformanceTable.COLUMN_COST_TOTAL)).intValue();
metrics.add(costTotal / callTimes);
} else {
metrics.add(0);
}
}
return metrics;
}
}
Expand Up @@ -13,11 +13,19 @@ public class InstPerformanceH2DAO extends H2DAO implements IInstPerformanceDAO {
return null;
}

@Override public int getMetric(int instanceId, long timeBucket) {
@Override public int getTpsMetric(int instanceId, long timeBucket) {
return 0;
}

@Override public JsonArray getMetric(int instanceId, long startTimeBucket, long endTimeBucket) {
@Override public JsonArray getTpsMetric(int instanceId, long startTimeBucket, long endTimeBucket) {
return null;
}

@Override public int getRespTimeMetric(int instanceId, long timeBucket) {
return 0;
}

@Override public JsonArray getRespTimeMetric(int instanceId, long startTimeBucket, long endTimeBucket) {
return null;
}
}
Expand Up @@ -44,9 +44,7 @@ public class MemoryMetricEsDAO extends EsDAO implements IMemoryMetricDAO {
while (startTimeBucket + i <= endTimeBucket);

JsonObject metric = new JsonObject();

JsonArray usedMetric = new JsonArray();

MultiGetResponse multiGetResponse = prepareMultiGet.get();
for (MultiGetItemResponse response : multiGetResponse.getResponses()) {
if (response.getResponse().isExists()) {
Expand All @@ -57,6 +55,7 @@ public class MemoryMetricEsDAO extends EsDAO implements IMemoryMetricDAO {
usedMetric.add(0);
}
}
metric.add("used", usedMetric);
return metric;
}
}
Expand Up @@ -45,9 +45,7 @@ public class MemoryPoolMetricEsDAO extends EsDAO implements IMemoryPoolMetricDAO
while (startTimeBucket + i <= endTimeBucket);

JsonObject metric = new JsonObject();

JsonArray usedMetric = new JsonArray();

MultiGetResponse multiGetResponse = prepareMultiGet.get();
for (MultiGetItemResponse response : multiGetResponse.getResponses()) {
if (response.getResponse().isExists()) {
Expand All @@ -58,6 +56,7 @@ public class MemoryPoolMetricEsDAO extends EsDAO implements IMemoryPoolMetricDAO
usedMetric.add(0);
}
}
metric.add("used", usedMetric);
return metric;
}
}
Expand Up @@ -15,17 +15,17 @@
import org.skywalking.apm.collector.core.util.CollectionUtils;
import org.skywalking.apm.collector.core.util.StringUtils;
import org.skywalking.apm.collector.storage.dao.DAOContainer;
import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;
import org.skywalking.apm.collector.storage.define.global.GlobalTraceTable;
import org.skywalking.apm.collector.storage.define.segment.SegmentCostTable;
import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO;

/**
* @author pengys5
*/
public class SegmentCostEsDAO extends EsDAO implements ISegmentCostDAO {

@Override public JsonObject loadTop(long startTime, long endTime, long minCost, long maxCost, String operationName,
String globalTraceId, int limit, int from) {
String globalTraceId, int limit, int from, Sort sort) {
SearchRequestBuilder searchRequestBuilder = getClient().prepareSearch(SegmentCostTable.TABLE);
searchRequestBuilder.setTypes(SegmentCostTable.TABLE_TYPE);
searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
Expand All @@ -48,7 +48,11 @@ public class SegmentCostEsDAO extends EsDAO implements ISegmentCostDAO {
mustQueryList.add(QueryBuilders.matchQuery(SegmentCostTable.COLUMN_SERVICE_NAME, operationName));
}

searchRequestBuilder.addSort(SegmentCostTable.COLUMN_COST, SortOrder.DESC);
if (Sort.Cost.equals(sort)) {
searchRequestBuilder.addSort(SegmentCostTable.COLUMN_COST, SortOrder.DESC);
} else if (Sort.Time.equals(sort)) {
searchRequestBuilder.addSort(SegmentCostTable.COLUMN_START_TIME, SortOrder.DESC);
}
searchRequestBuilder.setSize(limit);
searchRequestBuilder.setFrom(from);

Expand Down
Expand Up @@ -8,7 +8,7 @@
*/
public class SegmentCostH2DAO extends H2DAO implements ISegmentCostDAO {
@Override public JsonObject loadTop(long startTime, long endTime, long minCost, long maxCost, String operationName,
String globalTraceId, int limit, int from) {
String globalTraceId, int limit, int from, Sort sort) {
return null;
}
}

0 comments on commit bdf29e5

Please sign in to comment.