From ddd06ab72d39258d68267d71f215fc432723308d Mon Sep 17 00:00:00 2001 From: mrmx Date: Thu, 20 Jan 2011 15:46:22 +0100 Subject: [PATCH] Initial stats refactoring --- .../mongoste/core/AbstractStatsEngine.java | 11 -- .../java/org/mongoste/core/StatsEngine.java | 6 +- .../core/impl/mongodb/MongoStatsEngine.java | 142 ++++++++++++------ .../core/AbstractStatsEngineTest.java | 2 +- .../impl/mongodb/MongoStatsEngineTest.java | 79 +++++++--- 5 files changed, 152 insertions(+), 88 deletions(-) diff --git a/src/main/java/org/mongoste/core/AbstractStatsEngine.java b/src/main/java/org/mongoste/core/AbstractStatsEngine.java index 0b03f30..06b3f64 100644 --- a/src/main/java/org/mongoste/core/AbstractStatsEngine.java +++ b/src/main/java/org/mongoste/core/AbstractStatsEngine.java @@ -29,17 +29,6 @@ public abstract class AbstractStatsEngine implements StatsEngine { static final char DOT_CHR = '.'; static final char DOT_CHR_REPLACE = '_'; - private boolean keepEvents = true; - - @Override - public void setKeepEvents(boolean keepEvents) { - this.keepEvents = keepEvents; - } - - @Override - public boolean isKeepEvents() { - return keepEvents; - } @Override public void setTargetOwners(String clientId, String targetType, String target, List owners) throws StatsEngineException { diff --git a/src/main/java/org/mongoste/core/StatsEngine.java b/src/main/java/org/mongoste/core/StatsEngine.java index 0844058..8e72c7c 100644 --- a/src/main/java/org/mongoste/core/StatsEngine.java +++ b/src/main/java/org/mongoste/core/StatsEngine.java @@ -27,13 +27,9 @@ * Stats engine interface */ public interface StatsEngine { - - public void setKeepEvents(boolean keepEvents); - public boolean isKeepEvents(); - public void init(Properties properties) throws StatsEngineException; public void handleEvent(StatEvent event) throws StatsEngineException; - public void buildStats(TimeScope scope) throws StatsEngineException; + public void buildStats(TimeScope scope,TimeScope groupBy) throws StatsEngineException; public List getActions(String clientId) throws StatsEngineException; public List getTopTargets(String clientId,String targetType,String action,Integer limit) throws StatsEngineException; diff --git a/src/main/java/org/mongoste/core/impl/mongodb/MongoStatsEngine.java b/src/main/java/org/mongoste/core/impl/mongodb/MongoStatsEngine.java index 9a9295a..5d7f87a 100644 --- a/src/main/java/org/mongoste/core/impl/mongodb/MongoStatsEngine.java +++ b/src/main/java/org/mongoste/core/impl/mongodb/MongoStatsEngine.java @@ -100,9 +100,18 @@ public class MongoStatsEngine extends AbstractStatsEngine { private boolean resetCollections = false; //For testing debug!!!! private boolean countEvents = true; + private boolean keepEvents = true; public MongoStatsEngine() { } + + public void setKeepEvents(boolean keepEvents) { + this.keepEvents = keepEvents; + } + + public boolean isKeepEvents() { + return keepEvents; + } public void setResetCollections(boolean resetCollections) { this.resetCollections = resetCollections; @@ -136,6 +145,8 @@ public void init(Properties properties) throws StatsEngineException { }catch(MongoException ex) { throw new StatsEngineException("Getting db "+dbName,ex); } + setKeepEvents(Boolean.valueOf(properties.getProperty("events.keep", "true"))); + setCountEvents(Boolean.valueOf(properties.getProperty("events.count", "true"))); initCollections(); initFunctions(); } @@ -148,26 +159,36 @@ public void handleEvent(StatEvent event) throws StatsEngineException { } if(countEvents){ //Count event - countRawTarget(event); - //Global count event - countTarget(event); - //Count total actions/targets - countTargetActions(event); + if(countRawTarget(event)) { + //Global count event + countTarget(event); + //Count total actions/targets + countTargetActions(event); + } else throw new StatsEngineException("Raw count failed with event "+ event); } } @Override - public void buildStats(TimeScope scope) { - String map = getFunction(FN_MAPPER_TARGETS+TimeScope.DAILY.getKey().toUpperCase()); + public void buildStats(TimeScope scope,TimeScope groupBy) { + TimeScope mapperScope = groupBy; + if(mapperScope == null) { + mapperScope = TimeScope.MONTHLY; + } + switch (mapperScope) { + case GLOBAL: + case ANNUAL: + case WEEKLY: + mapperScope = TimeScope.MONTHLY; + } + String map = getFunction(FN_MAPPER_TARGETS,mapperScope); String red = getFunction(FN_REDUCER_TARGETS); Date now = DateUtil.getDateGMT0(); - String result = getTargetScopeCollectionName(COLLECTION_STATS,now, scope); - DBCollection sourceTargetCol; - try { - sourceTargetCol = getTargetCollection(now, scope); - sourceTargetCol.mapReduce(map, red, result, EMPTY_DOC); + String statsResultCollection = getScopeCollectionName(COLLECTION_STATS,now, scope); + DBObject queryTargets = EMPTY_DOC; //TODO + try { + getTargetCollection().mapReduce(map, red, statsResultCollection, queryTargets); } catch (StatsEngineException ex) { - log.error("Map reducing",ex); + log.error("Map reducing targets",ex); } } @@ -345,13 +366,13 @@ public void setTargetTags(String clientId,String targetType,String target,List metadata = event.getMetadata(); - for(String metaKey : metadata.keySet()) { - incDoc.put(createDotPath(hourKey ,FIELD_META , metaKey ,metaKeyValue(metaKey, metadata.get(metaKey) )),1); + private boolean countRawTarget(StatEvent event) { + try { + BasicDBObject q = new BasicDBObject(); + q.put(EVENT_CLIENT_ID,event.getClientId()); + q.put(EVENT_TARGET,event.getTarget()); + q.put(EVENT_TARGET_TYPE,event.getTargetType()); + q.put(EVENT_ACTION,event.getAction()); + q.put(TARGET_YEAR, event.getYear()); + q.put(TARGET_MONTH, event.getMonth()); + + BasicDBObject doc = new BasicDBObject(); + + //BasicDBObject docSet = new BasicDBObject(); + doc.put("$addToSet", createAddToSetOwnersTagsDoc(event)); + BasicDBObject incDoc = new BasicDBObject(); + String dayKey = createDotPath(FIELD_DAYS , event.getDay()); + String hourKey = createDotPath(dayKey ,FIELD_HOURS , event.getHour()); + incDoc.put(FIELD_COUNT, 1); //Month count + incDoc.put(createDotPath(dayKey ,FIELD_COUNT),1); //Day count + incDoc.put(createDotPath(hourKey,FIELD_COUNT), 1);//Hour count + //Count metadata + Map metadata = event.getMetadata(); + for(String metaKey : metadata.keySet()) { + incDoc.put(createDotPath(hourKey ,FIELD_META , metaKey ,metaKeyValue(metaKey, metadata.get(metaKey) )),1); + } + doc.put("$inc",incDoc); + DBCollection targets = getTargetCollection(event,TimeScope.GLOBAL); + WriteResult ws = targets.update(q,doc,true,true); + //log.debug("countRawTarget result: {}",ws.getLastError()); + return ws.getN() > 0; + }catch(Exception ex) { + log.error("countRawTarget failed",ex); } - doc.put("$inc",incDoc); - DBCollection targets = getTargetCollection(event,TimeScope.GLOBAL); - WriteResult ws = targets.update(q,doc,true,true); - //log.debug("countRawTarget result: {}",ws.getLastError()); + return false; } private void countTarget(StatEvent event) throws StatsEngineException { @@ -502,8 +529,12 @@ private void initCollections() throws StatsEngineException { } } + protected DBCollection getStatsCollection() throws StatsEngineException { + return getStatsCollection((StatEvent)null, TimeScope.GLOBAL); + } + protected DBCollection getStatsCollection(StatEvent event, TimeScope timeScope) throws StatsEngineException { - String name = getTargetScopeCollectionName(COLLECTION_STATS, event, timeScope); + String name = getScopeCollectionName(COLLECTION_STATS, event, timeScope); DBCollection stats = collectionMap.get(name); if(stats == null) { stats = db.getCollection(name); @@ -530,7 +561,7 @@ protected DBCollection getTargetCollection(Date date, TimeScope timeScope) throw } protected DBCollection getTargetCollection(StatEvent event, TimeScope timeScope) throws StatsEngineException { - String name = getTargetScopeCollectionName(COLLECTION_TARGETS, event, timeScope); + String name = getScopeCollectionName(COLLECTION_TARGETS, event, timeScope); DBCollection target = collectionMap.get(name); if(target == null) { target = db.getCollection(name); @@ -566,7 +597,7 @@ protected DBCollection getCounterCollection() throws StatsEngineException { } protected DBCollection getCounterCollection(StatEvent event, TimeScope timeScope) throws StatsEngineException { - String name = getTargetScopeCollectionName(COLLECTION_COUNTERS, event, timeScope); + String name = getScopeCollectionName(COLLECTION_COUNTERS, event, timeScope); DBCollection target = collectionMap.get(name); if(target == null) { target = db.getCollection(name); @@ -615,13 +646,18 @@ protected void dropAllCollections() { MongoUtil.dropCollections(db); } - private void initFunctions() throws StatsEngineException { - addFunction(FN_MAPPER_TARGETS+TimeScope.HOURLY.getKey().toUpperCase()); - addFunction(FN_MAPPER_TARGETS+TimeScope.DAILY.getKey().toUpperCase()); + private void initFunctions() throws StatsEngineException { + addFunction(FN_MAPPER_TARGETS,TimeScope.MONTHLY); + addFunction(FN_MAPPER_TARGETS,TimeScope.HOURLY); + addFunction(FN_MAPPER_TARGETS,TimeScope.DAILY); addFunction(FN_REDUCER_TARGETS); addFunction(FN_REDUCER_PLAIN); } + private void addFunction(String functionNamePrefix,TimeScope scopeSubfix) throws StatsEngineException { + addFunction(getFunctionName(functionNamePrefix, scopeSubfix)); + } + private void addFunction(String functionName) throws StatsEngineException { String functionBody = null; InputStream is = null; @@ -648,6 +684,14 @@ private void addFunction(String functionName,String body) { functionMap.put(functionName, body); } + private String getFunctionName(String functionNamePrefix,TimeScope scopeSubfix) { + return functionNamePrefix + scopeSubfix.getKey().toUpperCase(); + } + + private String getFunction(String functionNamePrefix,TimeScope scopeSubfix) { + return getFunction(getFunctionName(functionNamePrefix, scopeSubfix)); + } + private String getFunction(String functionName) { return functionMap.get(functionName); } diff --git a/src/test/java/org/mongoste/core/AbstractStatsEngineTest.java b/src/test/java/org/mongoste/core/AbstractStatsEngineTest.java index e20a1be..d1f2196 100644 --- a/src/test/java/org/mongoste/core/AbstractStatsEngineTest.java +++ b/src/test/java/org/mongoste/core/AbstractStatsEngineTest.java @@ -69,7 +69,7 @@ public void handleEvent(StatEvent event) throws StatsEngineException { } @Override - public void buildStats(TimeScope scope) throws StatsEngineException { + public void buildStats(TimeScope scope,TimeScope groupByScope) throws StatsEngineException { throw new UnsupportedOperationException("Not supported yet."); } diff --git a/src/test/java/org/mongoste/core/impl/mongodb/MongoStatsEngineTest.java b/src/test/java/org/mongoste/core/impl/mongodb/MongoStatsEngineTest.java index 901305e..b5dc860 100644 --- a/src/test/java/org/mongoste/core/impl/mongodb/MongoStatsEngineTest.java +++ b/src/test/java/org/mongoste/core/impl/mongodb/MongoStatsEngineTest.java @@ -35,13 +35,14 @@ import org.junit.Test; import static org.junit.Assert.*; -import static org.mockito.Mockito.*; import java.util.Arrays; import java.util.Calendar; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.Properties; +import java.text.ParseException; /** * MongoStatsEngine Test @@ -52,6 +53,12 @@ public class MongoStatsEngineTest { private static class MongoStatsEngineEx extends MongoStatsEngine { public StatEvent createSampleEvent() { + return createSampleEvent(DateUtil.getDateGMT0()); + } + public StatEvent createSampleEvent(String date) throws ParseException { + return createSampleEvent(DateUtil.FORMAT_YY_MM_DD.parse(date)); + } + public StatEvent createSampleEvent(Date date) { StatEvent event = new StatEvent(); event.setClientId("client"); event.setAction("action"); @@ -59,7 +66,7 @@ public StatEvent createSampleEvent() { event.setTarget("target"); event.setTargetOwners(Arrays.asList("owner1","owner2")); event.setTargetTags(Arrays.asList("tag1","tag2")); - event.setDate(DateUtil.getDateGMT0()); + event.setDate(date); return event; } } @@ -99,15 +106,6 @@ public void testHandleEvent() throws Exception { engine.handleEvent(engine.createSampleEvent()); } - /** - * Test of buildStats method, of class MongoStatsEngine. - */ - @Test - public void testBuildStats() { - System.out.println("buildStats"); - //fail("The test case is a prototype."); - } - /** * Test of getActions method, of class MongoStatsEngine. */ @@ -131,14 +129,14 @@ public void testGetActions() throws Exception { } /** - * Test of getTargetScopeCollectionName method, of class MongoStatsEngine. + * Test of getScopeCollectionName method, of class MongoStatsEngine. */ @Test - public void testGetTargetScopeCollectionName() { - System.out.println("getTargetScopeCollectionName"); + public void testGetScopeCollectionName() { + System.out.println("getScopeCollectionName"); String collection = "collection"; String expResult = collection; - String result = engine.getTargetScopeCollectionName(collection, null, TimeScope.GLOBAL); + String result = engine.getScopeCollectionName(collection, (StatEvent)null, TimeScope.GLOBAL); System.out.println("result:"+result); assertEquals(expResult, result); Calendar cal = Calendar.getInstance(); @@ -151,27 +149,27 @@ public void testGetTargetScopeCollectionName() { event.setDate(cal.getTime()); TimeScope scope = TimeScope.ANNUAL; - result = engine.getTargetScopeCollectionName(collection, event, scope); + result = engine.getScopeCollectionName(collection, event, scope); System.out.println("result:"+result); expResult = collection + "_" + scope.getKey().toLowerCase() + year; assertEquals(expResult, result); scope = TimeScope.WEEKLY; - result = engine.getTargetScopeCollectionName(collection, event, scope); + result = engine.getScopeCollectionName(collection, event, scope); System.out.println("result:"+result); expResult = collection + "_" + scope.getKey().toLowerCase() + year; expResult = expResult + "_" + week; assertEquals(expResult, result); scope = TimeScope.MONTHLY; - result = engine.getTargetScopeCollectionName(collection, event, scope); + result = engine.getScopeCollectionName(collection, event, scope); System.out.println("result:"+result); expResult = collection + "_" + scope.getKey().toLowerCase() + year; expResult = expResult + "_" + month; assertEquals(expResult, result); scope = TimeScope.DAILY; - result = engine.getTargetScopeCollectionName(collection, event, scope); + result = engine.getScopeCollectionName(collection, event, scope); System.out.println("result:"+result); expResult = collection + "_" + scope.getKey().toLowerCase() + year; expResult = expResult + "_" + month; @@ -179,7 +177,7 @@ public void testGetTargetScopeCollectionName() { assertEquals(expResult, result); scope = TimeScope.HOURLY; - result = engine.getTargetScopeCollectionName(collection, event, scope); + result = engine.getScopeCollectionName(collection, event, scope); System.out.println("result:"+result); expResult = collection + "_" + scope.getKey().toLowerCase() + year; expResult = expResult + "_" + month; @@ -582,7 +580,7 @@ public void testTargetCollection() throws Exception { event.setDate(cal.getTime()); engine.handleEvent(event); //Events in two months: 2 docs in target collection - DBCollection targets = engine.getFullTargetCollection((StatEvent)null, TimeScope.GLOBAL); + DBCollection targets = engine.getTargetCollection((StatEvent)null, TimeScope.GLOBAL); assertNotNull(targets); assertEquals(2,targets.count()); //Add 3th event in second month: still 2 docs in target collection @@ -590,7 +588,7 @@ public void testTargetCollection() throws Exception { System.out.println("1 event for "+cal.getTime()); event.setDate(cal.getTime()); engine.handleEvent(event); - targets = engine.getFullTargetCollection((StatEvent)null, TimeScope.GLOBAL); + targets = engine.getTargetCollection((StatEvent)null, TimeScope.GLOBAL); assertNotNull(targets); assertEquals(2,targets.count()); DBCursor dbc = targets.find(); @@ -685,4 +683,41 @@ public void testGetOwnerActionCount() throws Exception { System.out.println("result:"+result); } + /** + * Test of buildStats method, of class MongoStatsEngine. + */ + @Test + public void testBuildStatsMonthly() throws Exception { + System.out.println("buildStats monthly"); + StatEvent event = engine.createSampleEvent("2011-01-01"); + engine.handleEvent(event); + event = engine.createSampleEvent("2011-02-01"); + engine.handleEvent(event); + engine.buildStats(TimeScope.GLOBAL,TimeScope.MONTHLY); + //Events in two months: 2 docs in stats collection + DBCollection stats = engine.getStatsCollection(); + assertNotNull(stats); + assertEquals(2,stats.count()); + } + + /** + * Test of getOwnerActionCount method, of class MongoStatsEngine. + */ + @Test + public void testGetStatsBetweenMonths() throws Exception { + System.out.println("testGetStatsBetweenDates"); + StatEvent event1 = engine.createSampleEvent("2011-01-01"); + engine.handleEvent(event1); + StatEvent event2 = engine.createSampleEvent("2011-02-01"); + engine.handleEvent(event2); + /* + engine.buildStats(TimeScope.GLOBAL,TimeScope.MONTHLY); + //Events in two months: 2 docs in stats collection + DBCollection stats = engine.getStatsCollection(); + assertNotNull(stats); + assertEquals(2,stats.count()); + * + */ + fail("todo"); + } } \ No newline at end of file