Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
CHUKWA-697. Added generic heatmap REST API and visualization. (Eric Y…
…ang)

git-svn-id: https://svn.apache.org/repos/asf/incubator/chukwa/trunk@1520826 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
macroadster committed Sep 8, 2013
1 parent 8b6ba13 commit 40b67fce7c1cec4dcc1fa9f0f56a12ef24d8d8ad
Show file tree
Hide file tree
Showing 10 changed files with 1,088 additions and 24 deletions.
@@ -12,6 +12,8 @@ Release 0.6 - Unreleased

NEW FEATURES

CHUKWA-697. Added generic heatmap REST API and visualization. (Eric Yang)

CHUKWA-695. Added ability to export widgets from graph explorer. (Eric Yang)

CHUKWA-581. Added support for custom reducer package name. (Ivy Tang via Eric Yang)
@@ -60,3 +60,6 @@ Copyright (c) 2006 JUnit.org

This product includes formalize
Copyright (c) 2012 Nathan Smith, http://formalize.me/

This product includes heatmap.js
Copyright (c) 2011-2013 Patrick Wied, http://www.patrick-wied.at/static/heatmapjs/
38 pom.xml
@@ -386,29 +386,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.6</version>
<configuration>
<encoding>UTF-8</encoding>
</configuration>
<executions>
<execution>
<id>chmod-jmx-file</id>
<phase>process-resources</phase>
<configuration>
<tasks name="setup">
<chmod file="target/conf/jmxremote.password" perm="600" />
<chmod file="target/conf/jmxremote.access" perm="600" />
</tasks>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
@@ -597,7 +574,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.4</version>
<version>1.6</version>
<executions>
<execution>
<id>setup</id>
@@ -614,6 +591,19 @@
<goal>run</goal>
</goals>
</execution>
<execution>
<id>chmod-jmx-file</id>
<phase>process-resources</phase>
<configuration>
<tasks name="setup">
<chmod file="target/conf/jmxremote.password" perm="600" />
<chmod file="target/conf/jmxremote.access" perm="600" />
</tasks>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
<execution>
<id>test-setup</id>
<phase>generate-test-resources</phase>
@@ -17,15 +17,20 @@
*/
package org.apache.hadoop.chukwa.datastore;

import java.io.IOException;
import java.util.Calendar;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.hadoop.chukwa.hicc.bean.HeatMapPoint;
import org.apache.hadoop.chukwa.hicc.bean.Heatmap;
import org.apache.hadoop.chukwa.hicc.bean.Series;
import org.apache.hadoop.chukwa.util.ExceptionUtil;

@@ -39,6 +44,7 @@
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
@@ -251,4 +257,81 @@ public static Set<String> getClusterNames(long startTime, long endTime) {
}
return clusters;
}

public static Heatmap getHeatmap(String tableName, String family, String column,
long startTime, long endTime, double max, double scale, int height) {
final long MINUTE = TimeUnit.MINUTES.toMillis(1);
Heatmap heatmap = new Heatmap();
HTableInterface table = pool.getTable(tableName);
try {
Scan scan = new Scan();
ColumnPrefixFilter cpf = new ColumnPrefixFilter(column.getBytes());
scan.addFamily(family.getBytes());
scan.setFilter(cpf);
scan.setTimeRange(startTime, endTime);
scan.setBatch(10000);
ResultScanner results = table.getScanner(scan);
Iterator<Result> it = results.iterator();
int index = 0;
// Series display in y axis
int y = 0;
HashMap<String, Integer> keyMap = new HashMap<String, Integer>();
while(it.hasNext()) {
Result result = it.next();
List<KeyValue> kvList = result.list();
for(KeyValue kv : kvList) {
String key = parseRowKey(result.getRow());
StringBuilder tmp = new StringBuilder();
tmp.append(key);
tmp.append(":");
tmp.append(new String(kv.getQualifier()));
String seriesName = tmp.toString();
long time = parseTime(result.getRow());
// Time display in x axis
int x = (int) ((time - startTime) / MINUTE);
if(keyMap.containsKey(seriesName)) {
y = keyMap.get(seriesName);
} else {
keyMap.put(seriesName, new Integer(index));
y = index;
index++;
}
double v = Double.parseDouble(new String(kv.getValue()));
heatmap.put(x, y, v);
if(v > max) {
max = v;
}
}
}
results.close();
table.close();
int radius = height / index;
// Usually scale max from 0 to 100 for visualization
heatmap.putMax(scale);
for(HeatMapPoint point : heatmap.getHeatmap()) {
double round = point.count / max * scale;
round = Math.round(round * 100.0) / 100.0;
point.put(point.x, point.y * radius, round);
}
heatmap.putRadius(radius);
heatmap.putSeries(index);
} catch (IOException e) {
log.error(ExceptionUtil.getStackTrace(e));
}
return heatmap;
}

private static String parseRowKey(byte[] row) {
String key = new String(row);
String[] parts = key.split("-", 2);
return parts[1];
}

private static long parseTime(byte[] row) {
String key = new String(row);
String[] parts = key.split("-", 2);
long time = Long.parseLong(parts[0]);
return time;
}

}
@@ -0,0 +1,31 @@
package org.apache.hadoop.chukwa.hicc.bean;

import javax.xml.bind.annotation.XmlElement;

public class HeatMapPoint {
@XmlElement
public int x;
@XmlElement
public int y;
@XmlElement
public double count;

HeatMapPoint() {
}

HeatMapPoint(int x, int y, double count) {
this.x = x;
this.y = y;
this.count = count;
}

public HeatMapPoint get() {
return this;
}

public void put(int x, int y, double count) {
this.x = x;
this.y = y;
this.count = count;
}
}
@@ -0,0 +1,60 @@
package org.apache.hadoop.chukwa.hicc.bean;

import java.util.ArrayList;

import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;

@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@XmlType(propOrder={})
public class Heatmap {
@XmlElement
private ArrayList<HeatMapPoint> data;
@XmlElement
private double max = 1.0;
@XmlElement
private int radius;
@XmlElement
private int series;

public Heatmap() {
this.data = new ArrayList<HeatMapPoint>();
}

public void put(int x, int y, double v) {
HeatMapPoint point = new HeatMapPoint(x, y, v);
data.add(point);
}

public ArrayList<HeatMapPoint> getHeatmap() {
return data;
}

public double getMax() {
return max;
}

public void putMax(double max) {
this.max = max;
}

public int getRadius() {
return radius;
}

public void putRadius(int radius) {
this.radius = radius;
}

public int getSeries() {
return series;
}

public void putSeries(int series) {
this.series = series;
}
}
@@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.chukwa.hicc.rest;

import java.text.SimpleDateFormat;

import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;

import org.apache.hadoop.chukwa.datastore.ChukwaHBaseStore;
import org.apache.hadoop.chukwa.hicc.TimeHandler;
import org.apache.hadoop.chukwa.hicc.bean.Heatmap;
import org.apache.hadoop.chukwa.util.ExceptionUtil;
import org.apache.log4j.Logger;

@Path("/heatmap")
public class HeatmapController {
static Logger log = Logger.getLogger(HeatmapController.class);

@GET
@Path("{table}/{family}/{column}")
@Produces(MediaType.APPLICATION_JSON)
public Heatmap getHeatmap(@Context HttpServletRequest request,
@PathParam("table") String table,
@PathParam("family") String family,
@PathParam("column") String column,
@QueryParam("start") String start,
@QueryParam("end") String end,
@QueryParam("max") @DefaultValue("1.0") double max,
@QueryParam("scale") @DefaultValue("100") double scale,
@QueryParam("height") @DefaultValue("400") int height) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
Heatmap heatmap = null;
long startTime = 0;
long endTime = 0;
TimeHandler time = new TimeHandler(request);
try {
if(start!=null) {
startTime = sdf.parse(start).getTime();
} else {
startTime = time.getStartTime();
}
if(end!=null) {
endTime = sdf.parse(end).getTime();
} else {
endTime = time.getEndTime();
}
heatmap = ChukwaHBaseStore.getHeatmap(table, family, column, startTime, endTime, max, scale, height);
}catch(Throwable e) {
log.error(ExceptionUtil.getStackTrace(e));
}
return heatmap;
}
}
@@ -0,0 +1,38 @@
{
"id":"system_heatmap",
"title":"System Heatmap",
"version":"0.1",
"categories":"System Metrics,Server",
"url":"iframe/jsp/heatmap.jsp",
"description":"Display system related stats",
"screendump":"\/images\/server_load.gif",
"refresh":"15",
"parameters":[
{"name":"title","type":"string","value":"CPU Utilization","edit":"1","label":"Title"},
{"name":"period","type":"custom","control":"period_control","value":"","label":"Period"},
{"name":"url","type":"select","value":"/hicc/v1/heatmap/SystemMetrics/cpu/combined.","label":"Metric","options":[
{"label":"CPU Utilization Distribution","value":"/hicc/v1/heatmap/SystemMetrics/cpu/combined.?max=1"},
{"label":"Memory Utilization Distribution","value":"/hicc/v1/heatmap/SystemMetrics/memory/ActualUsed?max=16000000000"},
{"label":"Disk Read Distribution","value":"/hicc/v1/heatmap/SystemMetrics/disk/ReadBytes.?max=125000000"},
{"label":"Disk Write Distribution","value":"/hicc/v1/heatmap/SystemMetrics/disk/WriteBytes.?max=125000000"},
{"label":"Network Transfer Distribution","value":"/hicc/v1/heatmap/SystemMetrics/network/TxBytes.?max=125000000"}
{"label":"Network Receive Distribution","value":"/hicc/v1/heatmap/SystemMetrics/network/RxBytes.?max=125000000"}
]},
{"name":"width","type":"select","value":"300","label":"Width","options":[
{"label":"300","value":"300"},
{"label":"400","value":"400"},
{"label":"500","value":"500"},
{"label":"600","value":"600"},
{"label":"800","value":"800"},
{"label":"1000","value":"1000"},
{"label":"1200","value":"1200"}
]},
{"name":"height","type":"select","value":"200","label":"Height","options":[
{"label":"200","value":"200"},
{"label":"400","value":"400"},
{"label":"600","value":"600"},
{"label":"800","value":"800"},
{"label":"1000","value":"1000"}
]}
]
}

0 comments on commit 40b67fc

Please sign in to comment.