Permalink
Browse files

Refactoring

Log parser
COnfiguration changes
  • Loading branch information...
1 parent 78c1293 commit 2fb295efa0a562b8720aaa77be4f8d288eb09939 @salerth committed Feb 3, 2012
View
Binary file not shown.
View
@@ -34,6 +34,10 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
+ <Reference Include="Graphite, Version=1.1.0.0, Culture=neutral, processorArchitecture=MSIL">
+ <SpecificVersion>False</SpecificVersion>
+ <HintPath>..\Graphite.dll</HintPath>
+ </Reference>
<Reference Include="System" />
<Reference Include="System.configuration" />
<Reference Include="System.Core" />
@@ -54,7 +58,9 @@
</ProjectReference>
</ItemGroup>
<ItemGroup>
- <None Include="App.config" />
+ <None Include="App.config">
+ <SubType>Designer</SubType>
+ </None>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
View
@@ -0,0 +1,9 @@
+using System.Collections.Generic;
+
+namespace Metrics.Parsers
+{
+ public interface IMetricParser
+ {
+ IDictionary<string, Metric> GetMetrics();
+ }
+}
@@ -0,0 +1,25 @@
+using System.Configuration;
+
+namespace Metrics.Parsers.LogTail
+{
+ [ConfigurationCollection(typeof(LogConfigurationCollection), AddItemName = "Log",
+ CollectionType = ConfigurationElementCollectionType.BasicMap)]
+ public class LogConfigurationCollection : ConfigurationElementCollection
+ {
+
+ protected override ConfigurationElement CreateNewElement()
+ {
+ return new LogConfigurationElement();
+ }
+
+ protected override object GetElementKey(ConfigurationElement element)
+ {
+ return ((LogConfigurationElement)element).GraphiteKey;
+ }
+
+ new public LogConfigurationElement this[string name]
+ {
+ get { return (LogConfigurationElement)BaseGet(name); }
+ }
+ }
+}
@@ -0,0 +1,150 @@
+using System;
+using System.Configuration;
+using System.Text.RegularExpressions;
+using System.Threading;
+
+namespace Metrics.Parsers.LogTail
+{
+ public class LogConfigurationElement : ConfigurationElement
+ {
+ private Regex regex;
+
+ internal Regex CompiledRegex
+ {
+ get
+ {
+ if (regex == null)
+ {
+ var compiled = new Regex(Expression, RegexOptions.Compiled);
+ Interlocked.CompareExchange(ref regex, compiled, null);
+ }
+
+ return regex;
+ }
+ }
+
+ [ConfigurationProperty("enabled", DefaultValue = true, IsRequired = false)]
+ public bool Enabled
+ {
+ get
+ {
+ return (Boolean)this["enabled"];
+ }
+ set
+ {
+ this["enabled"] = value;
+ }
+ }
+
+ [ConfigurationProperty("location", IsRequired = false)]
+ public string Location
+ {
+ get
+ {
+ return (String)this["location"];
+ }
+ set
+ {
+ this["location"] = value;
+ }
+ }
+
+ [ConfigurationProperty("pattern", IsRequired = false, DefaultValue = "*")]
+ public string Pattern
+ {
+ get
+ {
+ return (String)this["pattern"];
+ }
+ set
+ {
+ this["pattern"] = value;
+ }
+ }
+
+ [ConfigurationProperty("type", IsRequired = false, DefaultValue = "raw")]
+ public string AggregateType
+ {
+ get
+ {
+ return (String)this["type"];
+ }
+ set
+ {
+ this["type"] = value;
+ }
+ }
+
+ [ConfigurationProperty("value", IsRequired = true)]
+ public string Value
+ {
+ get
+ {
+ return (String)this["value"];
+ }
+ set
+ {
+ this["value"] = value;
+ }
+ }
+
+ [ConfigurationProperty("interval", IsRequired = false)]
+ public string Interval
+ {
+ get
+ {
+ return (String)this["interval"];
+ }
+ set
+ {
+ this["interval"] = value;
+ }
+ }
+
+ [ConfigurationProperty("dateFormat", IsRequired = false, DefaultValue = "yyyy-mm-dd hh:MM:ss")]
+ public string DateFormat
+ {
+ get
+ {
+ return (String)this["dateFormat"];
+ }
+ set
+ {
+ this["dateFormat"] = value;
+ }
+ }
+
+ [ConfigurationProperty("graphiteKey", IsRequired = true, IsKey = true, DefaultValue = "site")]
+ [StringValidator(InvalidCharacters = "~!@#$%^&*()[]/;'\"|\\ ", MinLength = 3, MaxLength = 50)]
+ public string GraphiteKey
+ {
+ get
+ {
+ return (String)this["graphiteKey"];
+ }
+ set
+ {
+ this["graphiteKey"] = value;
+ }
+ }
+
+ [ConfigurationProperty("regex", IsRequired = true)]
+ public string Expression
+ {
+ get
+ {
+ return (String)this["regex"];
+ }
+ set
+ {
+ this["regex"] = value;
+ }
+ }
+
+ [ConfigurationProperty("Mapping", IsRequired = true)]
+ public KeyValueConfigurationCollection Mapping
+ {
+ get { return (KeyValueConfigurationCollection)this["Mapping"]; }
+ }
+ }
+}
@@ -0,0 +1,14 @@
+using System.Configuration;
+
+namespace Metrics.Parsers.LogTail
+{
+ public class LogConfigurationSection : ConfigurationSection
+ {
+ [ConfigurationProperty("Logs", IsRequired = true)]
+ public LogConfigurationCollection Logs
+ {
+ get { return (LogConfigurationCollection)this["Logs"]; }
+ }
+
+ }
+}
View
@@ -0,0 +1,94 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Configuration;
+using System.Globalization;
+using System.IO;
+using System.Threading.Tasks;
+using Metrics.Parsers.LogTail;
+using System.Linq;
+
+namespace Metrics.Parsers
+{
+ public class LogTailParser : IMetricParser
+ {
+ private readonly LogConfigurationCollection logs;
+ private readonly ConcurrentDictionary<string, long> lastByteRead = new ConcurrentDictionary<string, long>();
+
+ public LogTailParser()
+ {
+ var section = ConfigurationManager.GetSection("LogTail") as LogConfigurationSection;
+ if (section != null)
+ {
+ logs = section.Logs;
+ }
+ }
+
+ public IDictionary<string, Metric> GetMetrics()
+ {
+ var metrics = new ConcurrentDictionary<string, Metric>();
+ Parallel.ForEach(logs.Cast<LogConfigurationElement>(), (log) => ReadTail(log, metrics));
+
+ return metrics;
+ }
+
+ private void ReadTail(LogConfigurationElement log, ConcurrentDictionary<string, Metric> metrics)
+ {
+ foreach (var file in Directory.GetFiles(log.Location, log.Pattern))
+ {
+ var offset = lastByteRead.ContainsKey(file) ? lastByteRead[file] : 0;
+
+ using (var stream = File.OpenRead(file))
+ {
+ using (var reader = new StreamReader(stream))
+ {
+ reader.BaseStream.Seek(offset, SeekOrigin.Begin);
+ while (reader.Peek() != -1)
+ {
+ var line = reader.ReadLine();
+ if (!lastByteRead.TryAdd(file, reader.BaseStream.Position))
+ {
+ lastByteRead.TryUpdate(file, reader.BaseStream.Position, offset);
+ }
+ ParseLine(log, line);
+ }
+ }
+ }
+ }
+ }
+
+ private static void ParseLine(LogConfigurationElement log, string line)
+ {
+ var matches = log.CompiledRegex.Matches(line);
+ if (matches.Count > 0)
+ {
+ string key = log.GraphiteKey;
+ //do key replacements
+ foreach (var map in log.Mapping.AllKeys)
+ {
+ if (log.Mapping[map].Value.StartsWith("?"))
+ {
+ key = key.Replace("{" + map + "}", matches[0].Groups[log.Mapping[map].Value.TrimStart('?')].Value);
+ }
+ else
+ {
+ key = key.Replace("{" + map + "}", log.Mapping[map].Value);
+ }
+ }
+
+ var timestamp = DateTime.ParseExact(matches[0].Groups[log.Interval].Value, log.DateFormat,
+ CultureInfo.InvariantCulture);
+ int value = 0;
+ switch (log.AggregateType)
+ {
+ case "avg":
+ value = Int32.Parse()
+ break;
+ case "countLine":
+ break;
+ }
+ }
+
+ }
+ }
+}
View
@@ -0,0 +1,10 @@
+using System;
+
+namespace Metrics.Parsers
+{
+ public class Metric
+ {
+ public DateTime Timestamp { get; set; }
+ public int Value { get; set; }
+ }
+}
View
@@ -41,11 +41,17 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
+ <Compile Include="LogTail\LogConfigurationCollection.cs" />
+ <Compile Include="LogTail\LogConfigurationElement.cs" />
+ <Compile Include="LogTail\LogConfigurationSection.cs" />
+ <Compile Include="IMetricParser.cs" />
+ <Compile Include="LogTailParser.cs" />
+ <Compile Include="Metric.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
- <Compile Include="WebPagetest\SiteElement.cs" />
- <Compile Include="WebPagetest\SiteCollection.cs" />
- <Compile Include="WebPagetest\SiteSection.cs" />
- <Compile Include="WebPagetest\WebPagetestXmlParser.cs" />
+ <Compile Include="WebPagetest\SiteConfigurationElement.cs" />
+ <Compile Include="WebPagetest\SiteConfigurationCollection.cs" />
+ <Compile Include="WebPagetest\SiteConfigurationSection.cs" />
+ <Compile Include="WebPagetestXmlParser.cs" />
</ItemGroup>
<ItemGroup />
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
@@ -6,24 +6,24 @@
namespace Metrics.Parsers.WebPagetest
{
- [ConfigurationCollection(typeof(SiteElement), AddItemName = "Site",
+ [ConfigurationCollection(typeof(SiteConfigurationElement), AddItemName = "Site",
CollectionType = ConfigurationElementCollectionType.BasicMap)]
- public class SiteCollection : ConfigurationElementCollection
+ public class SiteConfigurationCollection : ConfigurationElementCollection
{
protected override ConfigurationElement CreateNewElement()
{
- return new SiteElement();
+ return new SiteConfigurationElement();
}
protected override object GetElementKey(ConfigurationElement element)
{
- return ((SiteElement)element).GraphiteKey;
+ return ((SiteConfigurationElement)element).GraphiteKey;
}
- new public SiteElement this[string name]
+ new public SiteConfigurationElement this[string name]
{
- get { return (SiteElement)BaseGet(name); }
+ get { return (SiteConfigurationElement)BaseGet(name); }
}
}
}
Oops, something went wrong.

0 comments on commit 2fb295e

Please sign in to comment.