Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
CHUKWA-207. Store HICC dashboard in HDFS. (Eric Yang)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/chukwa/trunk@885291 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
macroadster committed Nov 29, 2009
1 parent 1e44eb9 commit 8b2cb6da1b0818baad3a239e1609738573e69e7d
Show file tree
Hide file tree
Showing 85 changed files with 2,914 additions and 595 deletions.
@@ -8,6 +8,8 @@ Trunk (unreleased changes)

IMPROVEMENTS

CHUKWA-207. Store HICC dashboard data in HDFS. (Eric Yang)

CHUKWA-415. Read initial adaptors at start every time. (asrabkin)

CHUKWA-413. Improve admin guide. (asrabkin)
@@ -466,6 +466,7 @@
<target name="test-chukwa" depends="compile,compile-test,compress" description="Run Chukwa unit tests">
<mkdir dir="${test.build.dir}/var" />
<mkdir dir="${test.build.dir}/var/log" />
<mkdir dir="${test.build.dir}/var/run" />
<mkdir dir="${test.build.classes}/webapps" />
<mkdir dir="${test.build.classes}/logs" />
<copy todir="${test.build.dir}/var/log">
@@ -480,22 +481,52 @@
</copy>
<copy todir="${test.build.classes}/webapps">
<fileset dir="${build.dir}">
<include name="chukwa.war" />
<!-- include name="chukwa.war" / -->
<include name="hicc.war" />
</fileset>
</copy>
<copy file="${basedir}/conf/chukwa-demux-conf.xml" tofile="${test.build.dir}/conf/chukwa-demux-conf.xml"></copy>
<copy file="${conf.dir}/chukwa-common.xml.template" tofile="${test.build.dir}/conf/chukwa-common.xml">
<filterchain>
<replacetokens>
<token key="CHUKWA_DIR" value="/tmp/chukwa" />
</replacetokens>
</filterchain>
</copy>
<copy file="${basedir}/conf/chukwa-demux-conf.xml.template" tofile="${test.build.dir}/conf/chukwa-demux-conf.xml">
<filterchain>

<replacetokens>
<token key="TODO-DEMUX-IO-SORT-MB" value="${TODO-DEMUX-IO-SORT-MB}" />
</replacetokens>
<replacetokens>
<token key="TODO-DEMUX-FS-INMEMORY-SIZE_MB" value="${TODO-DEMUX-FS-INMEMORY-SIZE_MB}" />
</replacetokens>
<replacetokens>
<token key="TODO-DEMUX-IO-SORT-FACTOR" value="${TODO-DEMUX-IO-SORT-FACTOR}" />
</replacetokens>

</filterchain>
</copy>
<copy file="${basedir}/conf/log4j.properties" tofile="${test.build.dir}/conf/log4j.properties"></copy>
<copy file="${build.dir}/hicc.war" tofile="${test.build.classes}/webapps/hicc.war"></copy>
<copy file="${basedir}/conf/auth.conf" tofile="${test.build.dir}/var/auth.conf"></copy>
<junit showoutput="${test.output}" fork="yes" printsummary="${test.printsummary}" forkmode="${test.junit.fork.mode}" maxmemory="${test.junit.maxmemory}" dir="${test.build.dir}/classes/" timeout="${test.timeout}" errorProperty="tests.failed" failureProperty="tests.failed">
<classpath refid="testClasspath" />
<env key="CHUKWA_CONF_DIR" value="${test.build.dir}/conf" />
<env key="CHUKWA_DATA_DIR" value="${test.build.dir}/var" />
<env key="CHUKWA_LOG_DIR" value="${test.build.dir}/var/log" />
<env key="CHUKWA_PID_DIR" value="${test.build.dir}/var/run" />
<env key="CHUKWA_HDFS" value="/chukwa" />
<env key="CHUKWA_HOME" value="${test.build.classes}" />
<sysproperty key="test.src.dir" value="${test.src.dir}" />
<sysproperty key="test.build.classes" value="${test.build.classes}" />
<sysproperty key="log4j.configuration" value="log4j.properties" />
<sysproperty key="CHUKWA_DATA_DIR" value="${test.build.dir}/var" />
<sysproperty key="CHUKWA_CONF_DIR" value="${test.build.dir}/conf" />
<sysproperty key="CHUKWA_LOG_DIR" value="${test.build.dir}/var/log" />
<sysproperty key="CHUKWA_PID_DIR" value="${test.build.dir}/var/run" />
<sysproperty key="CHUKWA_HDFS" value="/chukwa" />
<sysproperty key="CHUKWA_HOME" value="${test.build.classes}" />
<formatter type="${test.junit.output.format}" />
<batchtest todir="${test.build.dir}" unless="testcase">
<fileset dir="${test.src.dir}">
@@ -1011,7 +1042,7 @@
</target>

<target name="chukwa-config" description="create real files based on templates" depends="init">

<mkdir dir="${build.dir}/conf" />
<copy todir="${build.dir}/conf">
<fileset dir="${basedir}/conf">
<exclude name="*.template" />
@@ -1021,6 +1052,14 @@
<copy file="${basedir}/conf/jdbc.conf.template" tofile="${build.dir}/conf/jdbc.conf.template">
</copy>

<copy file="${conf.dir}/chukwa-common.xml.template" tofile="${build.dir}/conf/chukwa-common.xml">
<filterchain>
<replacetokens>
<token key="CHUKWA_DIR" value="/chukwa" />
</replacetokens>
</filterchain>
</copy>

<copy file="${basedir}/conf/alert.template" tofile="${build.dir}/conf/alert">
<filterchain>
<replacetokens>
@@ -0,0 +1 @@
admin: admin, user
@@ -0,0 +1,30 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Global Chukwa Configuration file. -->

<configuration>

<property>
<name>chukwa.data.dir</name>
<value>@CHUKWA_DIR@</value>
<description>Location of Chukwa data on HDFS</description>
</property>

</configuration>
Binary file not shown.
12 ivy.xml
@@ -145,14 +145,22 @@
name="servlet-api-2.5"
rev="${jetty.version}"
conf="jetty->master"/>
<dependency org="org.apache.ant"
<!-- <dependency org="org.apache.ant"
name="ant"
rev="${jetty.ant.version}"
conf="common->default"/>
conf="common->default"/> -->
<dependency org="org.eclipse.jdt"
name="core"
rev="${jetty.eclipse.jdt.core.version}"
conf="jetty->master"/>
<!-- <dependency org="jaxb"
name="jaxb-api"
rev="${jaxb.version}"
conf="jetty->master"/>
<dependency org="jaxb"
name="jaxb-impl"
rev="${jaxb.version}"
conf="jetty->master"/> -->
<!-- <dependency org="org.apache.tomcat"
name="jasper"
rev="${jasper.version}"
@@ -47,3 +47,4 @@ rats-lib.version=0.5.1
jdiff.version=1.0.9
xmlenc.version=0.52
xerces.version=1.4.4
#jaxb.version=2.1.12
@@ -50,6 +50,9 @@ public ChukwaConfiguration(boolean loadDefaults) {

log.info("chukwaConf is " + chukwaConf);

super.addResource(new Path(chukwaConf + "/chukwa-common.xml"));
log.debug("added chukwa-agent-conf.xml to ChukwaConfiguration");

super.addResource(new Path(chukwaConf + "/chukwa-agent-conf.xml"));
log.debug("added chukwa-agent-conf.xml to ChukwaConfiguration");

@@ -0,0 +1,136 @@
package org.apache.hadoop.chukwa.datastore;

import java.io.File;
import java.io.IOException;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.json.JSONArray;
import org.json.JSONObject;

import org.apache.hadoop.chukwa.conf.ChukwaConfiguration;
import org.apache.hadoop.chukwa.hicc.HiccWebServer;
import org.apache.hadoop.chukwa.rest.bean.UserBean;
import org.apache.hadoop.chukwa.util.ExceptionUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class UserStore {
private String uid = null;
private UserBean profile = null;
private static Log log = LogFactory.getLog(UserStore.class);
private static Configuration config = new Configuration();
private static ChukwaConfiguration chukwaConf = new ChukwaConfiguration();
private static String hiccPath = config.get("fs.default.name")+File.separator+chukwaConf.get("chukwa.data.dir")+File.separator+"hicc"+File.separator+"users";

public UserStore() throws IllegalAccessException {
UserStore.config = HiccWebServer.getConfig();
}

public UserStore(String uid) throws IllegalAccessException {
this.uid = uid;
init(uid);
}

public void init(String uid) throws IllegalAccessException {
StringBuilder profilePath = new StringBuilder();
profilePath.append(hiccPath);
profilePath.append(File.separator);
profilePath.append(uid);
profilePath.append(".profile");
Path profileFile = new Path(profilePath.toString());
FileSystem fs;
try {
fs = FileSystem.get(config);
if(fs.exists(profileFile)) {
FileStatus[] fstatus = fs.listStatus(profileFile);
long size = fstatus[0].getLen();
FSDataInputStream viewStream = fs.open(profileFile);
byte[] buffer = new byte[(int)size];
viewStream.readFully(buffer);
viewStream.close();
try {
JSONObject json = new JSONObject(new String(buffer));
profile = new UserBean(json);
} catch (Exception e) {
log.error(ExceptionUtil.getStackTrace(e));
throw new IllegalAccessException("Unable to access user profile database.");
}
} else {
profile = new UserBean();
profile.setId(uid);
JSONArray ja = new JSONArray();
profile.setViews(ja);
JSONObject json = new JSONObject();
profile.setProperties(json);
}
} catch (IOException ex) {
log.error(ExceptionUtil.getStackTrace(ex));
}
}

public UserBean get() throws IllegalAccessException {
if(profile==null) {
init(uid);
}
return profile;
}

public void set(UserBean profile) throws IllegalAccessException {
StringBuilder profilePath = new StringBuilder();
profilePath.append(hiccPath);
profilePath.append(File.separator);
profilePath.append(profile.getId());
profilePath.append(".profile");
Path profileFile = new Path(profilePath.toString());
FileSystem fs;
try {
fs = FileSystem.get(config);
FSDataOutputStream out = fs.create(profileFile,true);
out.write(profile.deserialize().toString().getBytes());
out.close();
} catch (IOException ex) {
log.error(ExceptionUtil.getStackTrace(ex));
throw new IllegalAccessException("Unable to access user profile database.");
}
this.profile = profile;
}

public static JSONArray list() throws IllegalAccessException {
StringBuilder profilePath = new StringBuilder();
profilePath.append(hiccPath);
profilePath.append(File.separator);
profilePath.append("*.profile");
Path viewFile = new Path(profilePath.toString());
FileSystem fs;
JSONArray list = new JSONArray();
try {
fs = FileSystem.get(config);
FileStatus[] fstatus = fs.listStatus(viewFile);
if(fstatus!=null) {
for(int i=0;i<fstatus.length;i++) {
long size = fstatus[i].getLen();
FSDataInputStream profileStream = fs.open(fstatus[i].getPath());
byte[] buffer = new byte[(int)size];
profileStream.readFully(buffer);
profileStream.close();
try {
UserBean user = new UserBean(new JSONObject(new String(buffer)));
list.put(user.getId());
} catch (Exception e) {
log.error(ExceptionUtil.getStackTrace(e));
}
}
}
} catch (IOException ex) {
log.error(ExceptionUtil.getStackTrace(ex));
throw new IllegalAccessException("Unable to access user profile database.");
}
return list;
}
}

0 comments on commit 8b2cb6d

Please sign in to comment.