Skip to content

Commit

Permalink
Initial stats refactoring
Browse files Browse the repository at this point in the history
  • Loading branch information
mrmx committed Jan 20, 2011
1 parent 9e96d05 commit ddd06ab
Show file tree
Hide file tree
Showing 5 changed files with 152 additions and 88 deletions.
11 changes: 0 additions & 11 deletions src/main/java/org/mongoste/core/AbstractStatsEngine.java
View file
Expand Up @@ -29,17 +29,6 @@ public abstract class AbstractStatsEngine implements StatsEngine {
static final char DOT_CHR = '.';
static final char DOT_CHR_REPLACE = '_';

private boolean keepEvents = true;

@Override
public void setKeepEvents(boolean keepEvents) {
this.keepEvents = keepEvents;
}

@Override
public boolean isKeepEvents() {
return keepEvents;
}

@Override
public void setTargetOwners(String clientId, String targetType, String target, List<String> owners) throws StatsEngineException {
Expand Down
6 changes: 1 addition & 5 deletions src/main/java/org/mongoste/core/StatsEngine.java
View file
Expand Up @@ -27,13 +27,9 @@
* Stats engine interface
*/
public interface StatsEngine {

public void setKeepEvents(boolean keepEvents);
public boolean isKeepEvents();

public void init(Properties properties) throws StatsEngineException;
public void handleEvent(StatEvent event) throws StatsEngineException;
public void buildStats(TimeScope scope) throws StatsEngineException;
public void buildStats(TimeScope scope,TimeScope groupBy) throws StatsEngineException;

public List<StatAction> getActions(String clientId) throws StatsEngineException;
public List<StatBasicCounter> getTopTargets(String clientId,String targetType,String action,Integer limit) throws StatsEngineException;
Expand Down
142 changes: 93 additions & 49 deletions src/main/java/org/mongoste/core/impl/mongodb/MongoStatsEngine.java
View file
Expand Up @@ -100,9 +100,18 @@ public class MongoStatsEngine extends AbstractStatsEngine {

private boolean resetCollections = false; //For testing debug!!!!
private boolean countEvents = true;
private boolean keepEvents = true;

public MongoStatsEngine() {
}

public void setKeepEvents(boolean keepEvents) {
this.keepEvents = keepEvents;
}

public boolean isKeepEvents() {
return keepEvents;
}

public void setResetCollections(boolean resetCollections) {
this.resetCollections = resetCollections;
Expand Down Expand Up @@ -136,6 +145,8 @@ public void init(Properties properties) throws StatsEngineException {
}catch(MongoException ex) {
throw new StatsEngineException("Getting db "+dbName,ex);
}
setKeepEvents(Boolean.valueOf(properties.getProperty("events.keep", "true")));
setCountEvents(Boolean.valueOf(properties.getProperty("events.count", "true")));
initCollections();
initFunctions();
}
Expand All @@ -148,26 +159,36 @@ public void handleEvent(StatEvent event) throws StatsEngineException {
}
if(countEvents){
//Count event
countRawTarget(event);
//Global count event
countTarget(event);
//Count total actions/targets
countTargetActions(event);
if(countRawTarget(event)) {
//Global count event
countTarget(event);
//Count total actions/targets
countTargetActions(event);
} else throw new StatsEngineException("Raw count failed with event "+ event);
}
}

@Override
public void buildStats(TimeScope scope) {
String map = getFunction(FN_MAPPER_TARGETS+TimeScope.DAILY.getKey().toUpperCase());
public void buildStats(TimeScope scope,TimeScope groupBy) {
TimeScope mapperScope = groupBy;
if(mapperScope == null) {
mapperScope = TimeScope.MONTHLY;
}
switch (mapperScope) {
case GLOBAL:
case ANNUAL:
case WEEKLY:
mapperScope = TimeScope.MONTHLY;
}
String map = getFunction(FN_MAPPER_TARGETS,mapperScope);
String red = getFunction(FN_REDUCER_TARGETS);
Date now = DateUtil.getDateGMT0();
String result = getTargetScopeCollectionName(COLLECTION_STATS,now, scope);
DBCollection sourceTargetCol;
try {
sourceTargetCol = getTargetCollection(now, scope);
sourceTargetCol.mapReduce(map, red, result, EMPTY_DOC);
String statsResultCollection = getScopeCollectionName(COLLECTION_STATS,now, scope);
DBObject queryTargets = EMPTY_DOC; //TODO
try {
getTargetCollection().mapReduce(map, red, statsResultCollection, queryTargets);
} catch (StatsEngineException ex) {
log.error("Map reducing",ex);
log.error("Map reducing targets",ex);
}
}

Expand Down Expand Up @@ -345,13 +366,13 @@ public void setTargetTags(String clientId,String targetType,String target,List<S
}


protected String getTargetScopeCollectionName(String prefix,Date date, TimeScope timeScope) {
protected String getScopeCollectionName(String prefix,Date date, TimeScope timeScope) {
StatEvent event = new StatEvent();
event.setDate(date);
return getTargetScopeCollectionName(prefix,event,timeScope);
return getScopeCollectionName(prefix,event,timeScope);
}

protected String getTargetScopeCollectionName(String prefix,StatEvent event, TimeScope timeScope) {
protected String getScopeCollectionName(String prefix,StatEvent event, TimeScope timeScope) {
if(TimeScope.GLOBAL.equals(timeScope) || timeScope == null) {
return prefix;
}
Expand Down Expand Up @@ -402,34 +423,40 @@ private void saveEvent(StatEvent event) throws StatsEngineException {
log.debug("saveEvent result: {}",ws.getLastError());
}

private void countRawTarget(StatEvent event) throws StatsEngineException {
BasicDBObject q = new BasicDBObject();
q.put(EVENT_CLIENT_ID,event.getClientId());
q.put(EVENT_TARGET,event.getTarget());
q.put(EVENT_TARGET_TYPE,event.getTargetType());
q.put(EVENT_ACTION,event.getAction());
q.put(TARGET_YEAR, event.getYear());
q.put(TARGET_MONTH, event.getMonth());

BasicDBObject doc = new BasicDBObject();

//BasicDBObject docSet = new BasicDBObject();
doc.put("$addToSet", createAddToSetOwnersTagsDoc(event));
BasicDBObject incDoc = new BasicDBObject();
String dayKey = createDotPath(FIELD_DAYS , event.getDay());
String hourKey = createDotPath(dayKey ,FIELD_HOURS , event.getHour());
incDoc.put(FIELD_COUNT, 1); //Month count
incDoc.put(createDotPath(dayKey ,FIELD_COUNT),1); //Day count
incDoc.put(createDotPath(hourKey,FIELD_COUNT), 1);//Hour count
//Count metadata
Map<String,Object> metadata = event.getMetadata();
for(String metaKey : metadata.keySet()) {
incDoc.put(createDotPath(hourKey ,FIELD_META , metaKey ,metaKeyValue(metaKey, metadata.get(metaKey) )),1);
private boolean countRawTarget(StatEvent event) {
try {
BasicDBObject q = new BasicDBObject();
q.put(EVENT_CLIENT_ID,event.getClientId());
q.put(EVENT_TARGET,event.getTarget());
q.put(EVENT_TARGET_TYPE,event.getTargetType());
q.put(EVENT_ACTION,event.getAction());
q.put(TARGET_YEAR, event.getYear());
q.put(TARGET_MONTH, event.getMonth());

BasicDBObject doc = new BasicDBObject();

//BasicDBObject docSet = new BasicDBObject();
doc.put("$addToSet", createAddToSetOwnersTagsDoc(event));
BasicDBObject incDoc = new BasicDBObject();
String dayKey = createDotPath(FIELD_DAYS , event.getDay());
String hourKey = createDotPath(dayKey ,FIELD_HOURS , event.getHour());
incDoc.put(FIELD_COUNT, 1); //Month count
incDoc.put(createDotPath(dayKey ,FIELD_COUNT),1); //Day count
incDoc.put(createDotPath(hourKey,FIELD_COUNT), 1);//Hour count
//Count metadata
Map<String,Object> metadata = event.getMetadata();
for(String metaKey : metadata.keySet()) {
incDoc.put(createDotPath(hourKey ,FIELD_META , metaKey ,metaKeyValue(metaKey, metadata.get(metaKey) )),1);
}
doc.put("$inc",incDoc);
DBCollection targets = getTargetCollection(event,TimeScope.GLOBAL);
WriteResult ws = targets.update(q,doc,true,true);
//log.debug("countRawTarget result: {}",ws.getLastError());
return ws.getN() > 0;
}catch(Exception ex) {
log.error("countRawTarget failed",ex);
}
doc.put("$inc",incDoc);
DBCollection targets = getTargetCollection(event,TimeScope.GLOBAL);
WriteResult ws = targets.update(q,doc,true,true);
//log.debug("countRawTarget result: {}",ws.getLastError());
return false;
}

private void countTarget(StatEvent event) throws StatsEngineException {
Expand Down Expand Up @@ -502,8 +529,12 @@ private void initCollections() throws StatsEngineException {
}
}

protected DBCollection getStatsCollection() throws StatsEngineException {
return getStatsCollection((StatEvent)null, TimeScope.GLOBAL);
}

protected DBCollection getStatsCollection(StatEvent event, TimeScope timeScope) throws StatsEngineException {
String name = getTargetScopeCollectionName(COLLECTION_STATS, event, timeScope);
String name = getScopeCollectionName(COLLECTION_STATS, event, timeScope);
DBCollection stats = collectionMap.get(name);
if(stats == null) {
stats = db.getCollection(name);
Expand All @@ -530,7 +561,7 @@ protected DBCollection getTargetCollection(Date date, TimeScope timeScope) throw
}

protected DBCollection getTargetCollection(StatEvent event, TimeScope timeScope) throws StatsEngineException {
String name = getTargetScopeCollectionName(COLLECTION_TARGETS, event, timeScope);
String name = getScopeCollectionName(COLLECTION_TARGETS, event, timeScope);
DBCollection target = collectionMap.get(name);
if(target == null) {
target = db.getCollection(name);
Expand Down Expand Up @@ -566,7 +597,7 @@ protected DBCollection getCounterCollection() throws StatsEngineException {
}

protected DBCollection getCounterCollection(StatEvent event, TimeScope timeScope) throws StatsEngineException {
String name = getTargetScopeCollectionName(COLLECTION_COUNTERS, event, timeScope);
String name = getScopeCollectionName(COLLECTION_COUNTERS, event, timeScope);
DBCollection target = collectionMap.get(name);
if(target == null) {
target = db.getCollection(name);
Expand Down Expand Up @@ -615,13 +646,18 @@ protected void dropAllCollections() {
MongoUtil.dropCollections(db);
}

private void initFunctions() throws StatsEngineException {
addFunction(FN_MAPPER_TARGETS+TimeScope.HOURLY.getKey().toUpperCase());
addFunction(FN_MAPPER_TARGETS+TimeScope.DAILY.getKey().toUpperCase());
private void initFunctions() throws StatsEngineException {
addFunction(FN_MAPPER_TARGETS,TimeScope.MONTHLY);
addFunction(FN_MAPPER_TARGETS,TimeScope.HOURLY);
addFunction(FN_MAPPER_TARGETS,TimeScope.DAILY);
addFunction(FN_REDUCER_TARGETS);
addFunction(FN_REDUCER_PLAIN);
}

private void addFunction(String functionNamePrefix,TimeScope scopeSubfix) throws StatsEngineException {
addFunction(getFunctionName(functionNamePrefix, scopeSubfix));
}

private void addFunction(String functionName) throws StatsEngineException {
String functionBody = null;
InputStream is = null;
Expand All @@ -648,6 +684,14 @@ private void addFunction(String functionName,String body) {
functionMap.put(functionName, body);
}

private String getFunctionName(String functionNamePrefix,TimeScope scopeSubfix) {
return functionNamePrefix + scopeSubfix.getKey().toUpperCase();
}

private String getFunction(String functionNamePrefix,TimeScope scopeSubfix) {
return getFunction(getFunctionName(functionNamePrefix, scopeSubfix));
}

private String getFunction(String functionName) {
return functionMap.get(functionName);
}
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/org/mongoste/core/AbstractStatsEngineTest.java
View file
Expand Up @@ -69,7 +69,7 @@ public void handleEvent(StatEvent event) throws StatsEngineException {
}

@Override
public void buildStats(TimeScope scope) throws StatsEngineException {
public void buildStats(TimeScope scope,TimeScope groupByScope) throws StatsEngineException {
throw new UnsupportedOperationException("Not supported yet.");
}

Expand Down

0 comments on commit ddd06ab

Please sign in to comment.