Skip to content
Browse files

Automatic processing of "single" mode configurations. Still need to d…

…o "interval".
  • Loading branch information...
1 parent e29d684 commit 5345805e0c44a5aa7557a81848422b9acefa628f @coryfoo committed
View
12 etc/config.json
@@ -1,19 +1,11 @@
{
"recordProcessingEnabled" : true,
- "recordStatsInterval" : 5,
"recordRetainUniqueHours" : 168,
"recordTopCount" : 1000,
"records" : [
{
- "mode" : "interval",
- "path" : "/var/log/apache/access.log",
- "format" : "combined",
- "interval" : 10
- },
- {
- "mode" : "interval",
- "path" : "/home/cory/Desktop/access_logs",
- "interval" : 30,
+ "mode" : "once",
+ "path" : "/home/cory/Desktop/access_log.txt",
"format" : "%h %l %D %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"",
"exclusions" : [ { "field" : "URL", "values" : [ "/favicon.ico", "/resources/.*" ] } ]
}
View
1 src/config/Config.jsc
@@ -5,7 +5,6 @@
"recordProcessingEnabled" : "boolean",
"recordRetainUniqueHours" : "int",
- "recordStatsInterval" : "int",
"recordTopCount" : "int",
"records" : [
{
View
54 src/config/RoninConfig.gs
@@ -6,6 +6,10 @@ uses ronin.console.*
uses java.lang.System
uses tools.stats.AccessRecordProcessor
uses java.io.File
+uses db.model.*
+uses tools.records.IntervalRecordProcessor
+uses tools.records.SingleTimeRecordProcessor
+uses java.lang.Thread
class RoninConfig extends DefaultRoninConfig {
@@ -19,15 +23,51 @@ class RoninConfig extends DefaultRoninConfig {
AdminConsole.start()
}
+ TraceEnabled = System.getProperty( "ronin.trace" ) == "true"
+ print( "Tracing Enabled: ${TraceEnabled}" )
+
// Start the log parsers now
if ( Config.RecordProcessingEnabled ?: true ) {
- print( "Starting automatic log processing..." )
- Config.Records.each( \ record -> {
- print( record.asJson().toString() )
- })
- }
+ new Thread("Log Processor") {
+ override function run() {
+ Ronin.log("Starting automatic log processing...", WARN)
+ Config.Records.each( \ record -> {
+ // I'd really love to just pass in a each "record", but goson can't handle that yet...
+ var rc = RecordConfig.find( new RecordConfig() { :Path = record.Path }).first()
+ if ( rc == null ) {
+ Ronin.log("Creating new RecordConfig for path: ${record.Path}", WARN)
+ rc = new RecordConfig() { :Path = record.Path }
- TraceEnabled = System.getProperty( "ronin.trace" ) == "true"
- print( "Tracing Enabled: ${TraceEnabled}" )
+ // Set all the values on the Record...
+ rc.Format = record.Format
+ rc.update()
+
+ record.Exclusions?.each( \ ex -> {
+ var rcf = new RecordConfigExclusion()
+ rcf.Field = ex.Field
+ rcf.RecordConfig = rc
+ rcf.update()
+
+ ex.Values.each( \ fieldValue -> {
+ rcf.RecordConfigExclusionValues.add(new RecordConfigExclusionValue() { :Value = fieldValue })
+ })
+
+ rc.update()
+ })
+
+ // TODO -- handle modified configurations
+ }
+
+ if ( record.Mode == INTERVAL ) {
+ new IntervalRecordProcessor(rc).process()
+ } else if ( record.Mode == ONCE ) {
+ new SingleTimeRecordProcessor(rc).process()
+ } else {
+ Ronin.log("Unknown record mode: ${record.Mode}", ERROR)
+ }
+ })
+ }
+ }.start()
+ }
}
}
View
13 src/controller/Stats.gs
@@ -3,13 +3,14 @@ package controller
uses ronin.RoninController
uses java.util.ArrayList
uses db.util.QueryUtil
-uses db.model.AccessRecord
+uses db.model.*
uses org.eclipse.jetty.util.ajax.JSON
uses view.Summary
uses view.Layout
uses config.UserErrorException
uses view.stats.URLs
uses view.stats.Agents
+uses tools.stats.collector.UserAgentCollector
class Stats extends RoninController {
@@ -17,8 +18,8 @@ class Stats extends RoninController {
var map = {
"total" -> AccessRecord.count(new AccessRecord()),
"urls" -> QueryUtil.countUnique("AccessRecord", "URL"),
- "hosts" -> QueryUtil.countUnique("AccessRecord", "IPAddr"),
- "agents" -> QueryUtil.countUnique("AccessRecord", "UserAgent")
+ "hosts" -> Host.count(new Host()),
+ "agents" -> UserAgent.count(new UserAgent())
}
Layout.render(Writer, \ -> Summary.render(Writer, map), Main.HEADER_SUMMARY)
@@ -59,10 +60,10 @@ class Stats extends RoninController {
function agents() {
Response.setContentType("application/json")
- Writer.write(QueryUtil.countUniqueColumns("AccessRecord", "UserAgent").map( \ p -> {
+ Writer.write(UserAgent.findSorted(new UserAgent(), UserAgent#Count, false).map( \ ua -> {
return {
- "name" -> p.First,
- "value" -> p.Second
+ "name" -> ua.AgentField,
+ "value" -> ua.Count
}
}).toJSON())
}
View
25 src/db/model.ddl
@@ -5,6 +5,31 @@ CREATE TABLE "User"(
"Salt" VARCHAR(172)
);
+CREATE TABLE "RecordConfig"(
+ "id" BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT,
+ "Path" VARCHAR(512) NOT NULL,
+ "Format" VARCHAR(256) NOT NULL
+);
+
+CREATE TABLE "RecordConfigLogFile"(
+ "id" BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT,
+ "RecordConfig_id" BIGINT NOT NULL,
+ "FileName" VARCHAR(128),
+ "LastProcessed" DATETIME
+);
+
+CREATE TABLE "RecordConfigExclusion"(
+ "id" BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT,
+ "RecordConfig_id" BIGINT NOT NULL,
+ "Field" VARCHAR(32) NOT NULL
+);
+
+CREATE TABLE "RecordConfigExclusionValue"(
+ "id" BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT,
+ "RecordConfigExclusion_id" BIGINT NOT NULL,
+ "Value" VARCHAR(256) NOT NULL
+);
+
CREATE TABLE "AccessRecord"(
"id" BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT,
"IPAddr" VARCHAR(15),
View
14 src/tools/records/IntervalRecordProcessor.gs
@@ -0,0 +1,14 @@
+package tools.records
+
+uses db.model.RecordConfig
+
+class IntervalRecordProcessor implements RecordProcessor {
+
+ construct(record : RecordConfig) {
+
+ }
+
+ override function process() {
+
+ }
+}
View
7 src/tools/records/RecordProcessor.gs
@@ -0,0 +1,7 @@
+package tools.records
+
+interface RecordProcessor {
+
+ function process()
+
+}
View
61 src/tools/records/SingleTimeRecordProcessor.gs
@@ -0,0 +1,61 @@
+package tools.records
+
+uses db.model.RecordConfig
+uses db.model.RecordConfigLogFile
+uses java.io.File
+uses ronin.Ronin
+uses tools.parser.HttpAccessLogParser
+uses java.util.ArrayList
+uses tools.parser.ParserThread
+uses tools.stats.collector.StatsCollector
+uses tools.stats.AccessRecordProcessor
+uses java.util.Date
+
+class SingleTimeRecordProcessor implements RecordProcessor {
+
+ var _record : RecordConfig
+
+ construct( record : RecordConfig ) {
+ _record = record
+ }
+
+ override function process() {
+ var path = new File( _record.Path )
+ if ( not path.exists() ) {
+ Ronin.log( "Cannot process path. Path does not exist: ${path}", WARN )
+ return
+ }
+
+ var fileList = new ArrayList<File>()
+ if ( path.File ) {
+ fileList.add( path )
+ } else if ( path.Directory ) {
+ path.eachChild( \ file -> {
+ if ( file.File ) {
+ fileList.add( file )
+ }
+ })
+ }
+
+ fileList.each( \ file -> {
+ var logFile = new RecordConfigLogFile() {
+ :FileName = file.CanonicalPath,
+ :RecordConfig = _record
+ }
+
+ var existingLog = RecordConfigLogFile.find( logFile ).first()
+ if ( existingLog == null ) {
+ // This guy hasn't been processed yet, do it now.
+ Ronin.log( "Processing file: ${file.CanonicalPath}", WARN )
+ new HttpAccessLogParser( file, _record.Format ).parse()
+
+ Ronin.log( "Updating statistics for new data", WARN )
+ new AccessRecordProcessor().startStatsCollector()
+
+ logFile.LastProcessed = new Date()
+ logFile.update()
+ }
+ })
+ }
+
+}
View
8 src/tools/stats/AccessRecordProcessor.gs
@@ -12,25 +12,19 @@ class AccessRecordProcessor {
var _hoursToRetainRawRecords : int
var _topRecordsCount : int
- var _statsIntervalMinutes : int
var _statsCollector : StatsCollector
construct() {
- _statsIntervalMinutes = RoninConfig.Config.RecordStatsInterval as int
_topRecordsCount = RoninConfig.Config.RecordTopCount as int
_hoursToRetainRawRecords = RoninConfig.Config.RecordRetainUniqueHours as int
}
function startStatsCollector() {
- _statsCollector = new StatsCollector( _statsIntervalMinutes )
+ _statsCollector = new StatsCollector( )
_statsCollector.withAction( \ -> { new TopRequestCollector( _topRecordsCount ).execute() })
.withAction( \ -> { new UserAgentCollector().execute() })
.withAction( \ -> { new HostCollector ().execute() })
.start()
}
- function stopStatsCollector() {
- _statsCollector.stop()
- }
-
}
View
18 src/tools/stats/collector/StatsCollector.gs
@@ -7,12 +7,6 @@ uses java.util.TimerTask
class StatsCollector {
var _actions = new ArrayList<block()>()
- var _timer = new Timer()
- var _interval: int
-
- construct( statsInterval: int ) {
- _interval = statsInterval
- }
function withAction( action: block() ): StatsCollector {
_actions.add( action )
@@ -22,16 +16,6 @@ class StatsCollector {
function start() {
print( "Starting ${this.IntrinsicType.Name}" )
- _timer = new Timer()
- _timer.schedule( new TimerTask() {
- override function run() {
- _actions.each( \bl -> bl() )
- }
- }, 0, _interval * 60 * 1000 )
- }
-
- function stop() {
- print( "Stopping ${this.IntrinsicType.Name}" )
- _timer.cancel()
+ _actions.each( \bl -> bl() )
}
}
View
2 src/tools/stats/collector/TopRequestCollector.gs
@@ -33,7 +33,7 @@ class TopRequestCollector {
})
// Now iterate over all the results and remove everything not in the TOP
- TopRequest.findSortedPaged(new TopRequest(), TopRequest#Duration, false /*ASC*/, _count, 1 ).each( \ tr -> tr.delete() )
+ TopRequest.findSortedPaged(new TopRequest(), TopRequest#Duration, false /*DESC sort*/, _count, _count ).each( \ tr -> tr.delete() )
}
}

0 comments on commit 5345805

Please sign in to comment.
Something went wrong with that request. Please try again.