Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Support for WPT detailed metrics (i.e. size and time taken per host)

Example config file
Major refactoring
  • Loading branch information...
commit eb9aebe4d91bff9ed037d32d0554a514d9dd1159 1 parent d279320
@salerth authored
View
5 .gitignore
@@ -1,4 +1,3 @@
-
# Build Folders (you can keep bin if you'd like, to store dlls and pdbs)
bin
obj
@@ -86,4 +85,6 @@ ngit/
packages/*
*.testsettings
TestResult.xml
-*.VisualState.xml
+*.VisualState.xml
+
+metrics/App.config
View
1  metrics.sln
@@ -7,6 +7,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Parsers", "parsers\Parsers.
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{33FE5D39-2B57-498F-8C2E-65CD70C16A85}"
ProjectSection(SolutionItems) = preProject
+ .gitignore = .gitignore
README.md = README.md
EndProjectSection
EndProject
View
4 metrics/Metrics.csproj
@@ -23,6 +23,7 @@
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
+ <RunCodeAnalysis>false</RunCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' ">
<PlatformTarget>x86</PlatformTarget>
@@ -61,6 +62,9 @@
<None Include="App.config">
<SubType>Designer</SubType>
</None>
+ <None Include="example.App.config">
+ <SubType>Designer</SubType>
+ </None>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
View
10 metrics/Program.cs
@@ -10,23 +10,23 @@ class Program
{
static void Main()
{
+
//Configuration based. Config file lists logs, sizes, regex, etc
var logParser = new LogTailParser();
- SendMetricsToGraphite(logParser.GetMetrics());
+ logParser.GetMetrics(SendMetricsToGraphite);
//WPT parsers
var wptParser = new WebPagetestParser();
- SendMetricsToGraphite(wptParser.GetMetrics());
+ wptParser.GetMetrics(SendMetricsToGraphite);
}
/// <summary>
/// Sends the gathered stats to Graphite
- /// TODO : Switch to statsd if frequent interval to stop spamming
- /// TODO : - Graphite used for historical adds to support intervals > finest whisper resolution
/// </summary>
private static void SendMetricsToGraphite(IEnumerable<Metric> metrics)
{
- using (var graphiteClient = new GraphiteTcpClient(ConfigurationManager.AppSettings["GraphiteHost"], Int32.Parse(ConfigurationManager.AppSettings["GraphitePort"]), ConfigurationManager.AppSettings["GraphiteKeyPrefix"]))
+ using (var graphiteClient = new GraphiteTcpClient(ConfigurationManager.AppSettings["GraphiteHost"],
+ Int32.Parse(ConfigurationManager.AppSettings["GraphitePort"]), ConfigurationManager.AppSettings["GraphiteKeyPrefix"]))
{
foreach (var metric in metrics)
{
View
39 metrics/example.App.config
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<configuration>
+ <configSections>
+ <section name="WebPagetest" type="Metrics.Parsers.WebPagetest.SiteConfigurationSection, Metrics.Parsers"/>
+ <section name="LogTail" type="Metrics.Parsers.LogTail.LogConfigurationSection, Metrics.Parsers"/>
+ </configSections>
+ <WebPagetest wptHost="http://mywebpagetest.private.instance">
+ <Sites>
+ <Site graphiteKey="wpt.home" url="http://www.example.com/" enableDetailedMetrics="true"/>
+ <Site graphiteKey="wpt.newsHome" url="http://www.example.com/news/" enableDetailedMetrics="true"/>
+ </Sites>
+ </WebPagetest>
+ <LogTail>
+ <!--Standard IIS log with timetaken enabled -->
+ <Patterns>
+ <add key="iis" value="^(?&lt;datetime&gt;\S+\s\S+)\s\S+\s\S+\s\S+\s/(?&lt;url&gt;[^+/]*)\S*\s\S+\s\S+\s\S+\s\S+\s\S+\s(?&lt;code&gt;\S+)\s\S+\s\S+\s(?&lt;timetaken&gt;\S+)$"/>
+ </Patterns>
+ <Logs>
+ <Log pattern="*.log" maxTailMB="0" regexKey="iis" name="myWebSite">
+ <Locations>
+ <add key="server1" value="\\server1\logs\w3svc1"/>
+ <add key="server2" value="\\server2\logs\w3svc1"/>
+ </Locations>
+ <Stats>
+ <Stat graphiteKey="timers.iis.{locationKey}.myWebSite.avg" value="timetaken" type="avg" interval="datetime" dateFormat="yyyy-MM-dd HH:mm:ss" includeZeros="false"/>
+ <Stat graphiteKey="stats.iis.{locationKey}.myWebSite.{0}" type="count" interval="datetime" dateFormat="yyyy-MM-dd HH:mm:ss" />
+ </Stats>
+ <Mapping>
+ <add key="0" value="?code"/>
+ </Mapping>
+ </Log>
+ </Logs>
+ </LogTail>
+ <appSettings>
+ <add key="GraphiteKeyPrefix" value="mySite"/>
+ <add key="GraphiteHost" value="www.example.com"/>
+ <add key="GraphitePort" value="1234"/>
+ </appSettings>
+</configuration>
View
5 parsers/IMetricParser.cs
@@ -1,9 +1,10 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
namespace Metrics.Parsers
{
public interface IMetricParser
{
- IEnumerable<Metric> GetMetrics();
+ void GetMetrics(Action<IEnumerable<Metric>> sendMetrics);
}
}
View
15 parsers/LogTail/LogConfigurationCollection.cs
@@ -1,10 +1,11 @@
-using System.Configuration;
+using System.Collections.Generic;
+using System.Configuration;
namespace Metrics.Parsers.LogTail
{
[ConfigurationCollection(typeof(LogConfigurationCollection), AddItemName = "Log",
CollectionType = ConfigurationElementCollectionType.BasicMap)]
- public class LogConfigurationCollection : ConfigurationElementCollection
+ public class LogConfigurationCollection : ConfigurationElementCollection, IEnumerable<LogConfigurationElement>
{
protected override ConfigurationElement CreateNewElement()
@@ -14,12 +15,20 @@ protected override ConfigurationElement CreateNewElement()
protected override object GetElementKey(ConfigurationElement element)
{
- return ((LogConfigurationElement)element).Location + ((LogConfigurationElement)element).Pattern;
+ return ((LogConfigurationElement)element).Name + ((LogConfigurationElement)element).Pattern;
}
new public LogConfigurationElement this[string name]
{
get { return (LogConfigurationElement)BaseGet(name); }
}
+
+ public new IEnumerator<LogConfigurationElement> GetEnumerator()
+ {
+ foreach (var key in base.BaseGetAllKeys())
+ {
+ yield return BaseGet(key) as LogConfigurationElement;
+ }
+ }
}
}
View
51 parsers/LogTail/LogConfigurationElement.cs
@@ -15,8 +15,12 @@ internal Regex CompiledRegex
{
if (regex == null)
{
- var compiled = new Regex(Expression, RegexOptions.Compiled);
- Interlocked.CompareExchange(ref regex, compiled, null);
+ var section = ConfigurationManager.GetSection("LogTail") as LogConfigurationSection;
+ if (section != null)
+ {
+ var compiled = new Regex(section.Patterns[ExpressionKey].Value, RegexOptions.Compiled);
+ Interlocked.CompareExchange(ref regex, compiled, null);
+ }
}
return regex;
@@ -36,29 +40,29 @@ public bool Enabled
}
}
- [ConfigurationProperty("location", IsRequired = false)]
- public string Location
+ [ConfigurationProperty("pattern", IsRequired = false, DefaultValue = "*")]
+ public string Pattern
{
get
{
- return (String)this["location"];
+ return (String)this["pattern"];
}
set
{
- this["location"] = value;
+ this["pattern"] = value;
}
}
- [ConfigurationProperty("pattern", IsRequired = false, DefaultValue = "*")]
- public string Pattern
+ [ConfigurationProperty("name", IsRequired = true)]
+ public string Name
{
get
{
- return (String)this["pattern"];
+ return (String)this["name"];
}
set
{
- this["pattern"] = value;
+ this["name"] = value;
}
}
@@ -75,16 +79,29 @@ public int MaxTailMB
}
}
- [ConfigurationProperty("regex", IsRequired = true)]
- public string Expression
+ [ConfigurationProperty("onlyToday", IsRequired = false, DefaultValue = true)]
+ public bool OnlyToday
+ {
+ get
+ {
+ return (bool)this["onlyToday"];
+ }
+ set
+ {
+ this["onlyToday"] = value;
+ }
+ }
+
+ [ConfigurationProperty("regexKey", IsRequired = true)]
+ public string ExpressionKey
{
get
{
- return (String)this["regex"];
+ return (String)this["regexKey"];
}
set
{
- this["regex"] = value;
+ this["regexKey"] = value;
}
}
@@ -94,6 +111,12 @@ public KeyValueConfigurationCollection Mapping
get { return (KeyValueConfigurationCollection)this["Mapping"]; }
}
+ [ConfigurationProperty("Locations", IsRequired = true)]
+ public KeyValueConfigurationCollection Locations
+ {
+ get { return (KeyValueConfigurationCollection)this["Locations"]; }
+ }
+
[ConfigurationProperty("Stats", IsRequired = true)]
public LogStatConfigurationCollection Stats
{
View
7 parsers/LogTail/LogConfigurationSection.cs
@@ -4,11 +4,16 @@ namespace Metrics.Parsers.LogTail
{
public class LogConfigurationSection : ConfigurationSection
{
+ [ConfigurationProperty("Patterns", IsRequired = true)]
+ public KeyValueConfigurationCollection Patterns
+ {
+ get { return (KeyValueConfigurationCollection)this["Patterns"]; }
+ }
+
[ConfigurationProperty("Logs", IsRequired = true)]
public LogConfigurationCollection Logs
{
get { return (LogConfigurationCollection)this["Logs"]; }
}
-
}
}
View
260 parsers/LogTailParser.cs
@@ -2,7 +2,6 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Configuration;
-using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Threading.Tasks;
@@ -11,6 +10,7 @@
using System.Security.Cryptography;
using System.Text;
+[assembly: CLSCompliant(true)]
namespace Metrics.Parsers
{
public class LogTailParser : IMetricParser
@@ -26,28 +26,39 @@ public LogTailParser()
}
}
- private long GetLastByteRead(string filePath)
+ private static long GetLastByteRead(string filePath)
{
- using (var md5 = MD5.Create())
+ try
{
- string tempName = GetMd5HashFileName(md5, filePath);
- if (!File.Exists(tempName))
+ using (var md5 = MD5.Create())
{
+ string tempName = GetMD5HashFileName(md5, filePath);
+ if (!File.Exists(tempName))
+ {
+ return 0;
+ }
+
+ long offset;
+ var lines = File.ReadAllLines(tempName);
+ if (Int64.TryParse(lines[0], out offset))
+ {
+ return offset;
+ }
return 0;
}
-
- long offset;
- Int64.TryParse(File.ReadAllText(tempName), out offset);
- return offset;
+ }
+ catch
+ {
+ return 0;
}
}
- private void StoreLastByteRead(string filePath, long offset)
+ private static void StoreLastByteRead(string filePath, long offset)
{
using (var md5 = MD5.Create())
{
- string tempName = GetMd5HashFileName(md5, filePath);
- File.WriteAllText(tempName, offset.ToString());
+ var tempName = GetMD5HashFileName(md5, filePath);
+ File.WriteAllText(tempName, offset.ToString(CultureInfo.InvariantCulture) + Environment.NewLine + filePath);
}
}
@@ -55,11 +66,20 @@ private void StoreLastByteRead(string filePath, long offset)
/// Get an MD5 hased filename to store the last read byte
/// http://msdn.microsoft.com/en-us/library/system.security.cryptography.md5.aspx
/// </summary>
- public static string GetMd5HashFileName(MD5 md5Hash, string input)
+ public static string GetMD5HashFileName(HashAlgorithm hashAlgorithm, string input)
{
+ //Validate inputs
+ if (hashAlgorithm == null)
+ {
+ throw new ArgumentNullException("hashAlgorithm", "Hash algorithm cannot be null");
+ }
+ if (String.IsNullOrWhiteSpace(input))
+ {
+ throw new ArgumentNullException("input", "Input cannot be null");
+ }
// Convert the input string to a byte array and compute the hash.
- byte[] data = md5Hash.ComputeHash(Encoding.UTF8.GetBytes(input));
+ var data = hashAlgorithm.ComputeHash(Encoding.UTF8.GetBytes(input));
// Create a new Stringbuilder to collect the bytes
// and create a string.
@@ -69,7 +89,7 @@ public static string GetMd5HashFileName(MD5 md5Hash, string input)
// and format each one as a hexadecimal string.
for (int i = 0; i < data.Length; i++)
{
- sBuilder.Append(data[i].ToString("x2"));
+ sBuilder.Append(data[i].ToString("x2", CultureInfo.InvariantCulture));
}
// Return the hexadecimal string.
@@ -77,57 +97,49 @@ public static string GetMd5HashFileName(MD5 md5Hash, string input)
}
- public IEnumerable<Metric> GetMetrics()
+ public void GetMetrics(Action<IEnumerable<Metric>> sendMetrics)
{
- var metrics = new ConcurrentBag<Metric>();
+ Parallel.ForEach(logs, log =>
+ {
+ foreach (var locationKey in log.Locations.AllKeys)
+ {
+ foreach (var file in Directory.GetFiles(log.Locations[locationKey].Value, log.Pattern))
+ {
+ ReadTail(file, log, locationKey, sendMetrics);
+ }
+ }
+ });
- //Get list of log files to de-dupe in case of multiple types of parsing
- var logsByFile = new Dictionary<string, List<LogConfigurationElement>>();
- foreach (LogConfigurationElement log in logs)
- {
- foreach (var file in Directory.GetFiles(log.Location, log.Pattern))
- {
- if (!logsByFile.ContainsKey(file))
- {
- logsByFile.Add(file, new List<LogConfigurationElement>());
- }
-
- logsByFile[file].Add(log);
- }
- }
-
- //Loop through all required files and process
- Parallel.ForEach(logsByFile.Keys, key =>
- {
- var metricList = ReadTail(key, logsByFile[key]);
- Parallel.ForEach(metricList, metrics.Add);
- });
-
- return metrics;
}
-
- private IEnumerable<Metric> ReadTail(string file, IEnumerable<LogConfigurationElement> logsForFile)
+ private static void ReadTail(string file, LogConfigurationElement log, string locationKey, Action<IEnumerable<Metric>> sendMetrics)
{
var rawValues = new ConcurrentDictionary<LogStatConfigurationElement, ConcurrentBag<Metric>>();
- var metrics = new List<Metric>();
- if ((logsForFile == null) || (!logsForFile.Any()))
+ if (log == null)
{
- return metrics;
+ return;
}
//If multiple log parsers use the same file, select the maximum size (including infinite)
- long maxTailMB = logsForFile.Count(log => log.MaxTailMB == 0) > 0 ? 0 : logsForFile.Select(log => log.MaxTailMB).Max();
+ long maxTailMB = log.MaxTailMB;
//Work out the last read position
var info = new FileInfo(file);
+
+ //Skip if only today is set and file wasn't updated today
+ if ((log.OnlyToday) && (info.LastWriteTime.Date < DateTime.Now.Date))
+ {
+ return;
+ }
+
var offset = GetLastByteRead(file);
+ long lastPosition = offset;
//If file hasnt changed, don't bother opening
if (info.Length <= offset)
{
- return metrics;
+ return;
}
//If the file is greater than our maxTailMB setting, skip to the maximum and proceed
@@ -141,81 +153,94 @@ private IEnumerable<Metric> ReadTail(string file, IEnumerable<LogConfigurationEl
}
//Loop through the file doing matches
- using (var stream = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
+ var stream = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+ try
{
using (var reader = new StreamReader(stream))
{
+ stream = null;
reader.BaseStream.Seek(offset, SeekOrigin.Begin);
- long lastPosition = offset;
- try
+ long lineCount = 0;
+
+ while (reader.Peek() != -1)
{
- while (reader.Peek() != -1)
- {
- var line = reader.ReadLine();
+ var line = reader.ReadLine();
+ lineCount++;
- //Update the lastPosition counter before we try parse so any failures do not repeat the same line
- lastPosition = reader.BaseStream.Position;
- Parallel.ForEach(logsForFile, log =>
- {
- ParseLine(log, line, rawValues);
- });
+ //Update the lastPosition counter before we try parse so any failures do not repeat the same line
+ lastPosition = reader.BaseStream.Position;
+ ParseLine(log, line, rawValues, locationKey);
+ //Flush every 1000 lines to minimise memory footprint
+ if (lineCount >= 1000)
+ {
+ CollateAndSend(log, rawValues, file, lastPosition, sendMetrics);
+ rawValues.Clear();
+ lineCount = 0;
}
}
- finally
- {
- //Store where we got up to
- StoreLastByteRead(file, lastPosition);
- }
}
}
+ finally
+ {
+ if (stream != null)
+ {
+ stream.Dispose();
+ }
+ }
+
+ CollateAndSend(log, rawValues, file, lastPosition, sendMetrics);
+ }
+
+ private static void CollateAndSend(LogConfigurationElement log, IDictionary<LogStatConfigurationElement, ConcurrentBag<Metric>> rawValues,
+ string file, long lastPosition, Action<IEnumerable<Metric>> sendMetrics)
+ {
+ var metrics = new List<Metric>();
//Aggregate the metrics by their log aggregation method
- foreach (var log in logsForFile)
+ foreach (LogStatConfigurationElement stat in log.Stats)
{
- foreach (LogStatConfigurationElement stat in log.Stats)
+ if (rawValues.ContainsKey(stat))
{
- if (rawValues.ContainsKey(stat))
+ switch (stat.AggregateType)
{
- switch (stat.AggregateType)
- {
- case "count":
- metrics.AddRange(from value in rawValues[stat]
- group value by new { value.Timestamp, value.Key }
- into metricGroup
- select
- new Metric
- {
- Key = metricGroup.Key.Key,
- Timestamp = metricGroup.Key.Timestamp,
- Value = metricGroup.Count()
- });
- break;
- case "avg":
- default:
- metrics.AddRange(from value in rawValues[stat]
- group value by new { value.Timestamp, value.Key }
- into metricGroup
- select
- new Metric
- {
- Key = metricGroup.Key.Key,
- Timestamp = metricGroup.Key.Timestamp,
- Value =
- metricGroup.Sum(metric => metric.Value) /
- metricGroup.Count()
- });
- break;
- }
+ case "count":
+ metrics.AddRange(from value in rawValues[stat]
+ group value by new { value.Timestamp, value.Key }
+ into metricGroup
+ select
+ new Metric
+ {
+ Key = metricGroup.Key.Key,
+ Timestamp = metricGroup.Key.Timestamp,
+ Value = metricGroup.Count()
+ });
+ break;
+ //case "avg":
+ default:
+ metrics.AddRange(from value in rawValues[stat]
+ group value by new { value.Timestamp, value.Key }
+ into metricGroup
+ select
+ new Metric
+ {
+ Key = metricGroup.Key.Key,
+ Timestamp = metricGroup.Key.Timestamp,
+ Value =
+ metricGroup.Sum(metric => metric.Value) /
+ metricGroup.Count()
+ });
+ break;
}
}
}
- return metrics;
+ StoreLastByteRead(file, lastPosition);
+ sendMetrics(metrics);
}
- private static void ParseLine(LogConfigurationElement log, string line,
- ConcurrentDictionary<LogStatConfigurationElement, ConcurrentBag<Metric>> rawValues)
+ private static void ParseLine(LogConfigurationElement log, string line,
+ ConcurrentDictionary<LogStatConfigurationElement, ConcurrentBag<Metric>> rawValues, string locationKey)
{
var matches = log.CompiledRegex.Matches(line);
@@ -224,12 +249,12 @@ into metricGroup
{
if (matches.Count > 0)
{
- var key = stat.GraphiteKey;
+ var key = stat.GraphiteKey.Replace("{locationKey}", locationKey);
//do key replacements
foreach (var map in log.Mapping.AllKeys)
{
- if (log.Mapping[map].Value.StartsWith("?"))
+ if (log.Mapping[map].Value.StartsWith("?", StringComparison.Ordinal))
{
key = key.Replace("{" + map + "}",
matches[0].Groups[log.Mapping[map].Value.TrimStart('?')].Value);
@@ -240,21 +265,30 @@ into metricGroup
}
}
- var metric = new Metric
- {
- Key = key,
- Timestamp = DateTime.ParseExact(matches[0].Groups[stat.Interval].Value, stat.DateFormat, CultureInfo.InvariantCulture)
- };
- if (!String.IsNullOrEmpty(stat.Value))
+ try
{
- metric.Value = Int32.Parse(matches[0].Groups[stat.Value].Value);
- }
+ var metric = new Metric
+ {
+ Key = key,
+ Timestamp =
+ DateTime.ParseExact(matches[0].Groups[stat.Interval].Value,
+ stat.DateFormat, CultureInfo.InvariantCulture)
+ };
+ if (!String.IsNullOrEmpty(stat.Value))
+ {
+ metric.Value = Int32.Parse(matches[0].Groups[stat.Value].Value, CultureInfo.InvariantCulture);
+ }
- if (stat.IncludeZeros || (!stat.IncludeZeros && metric.Value > 0))
+ if (stat.IncludeZeros || (!stat.IncludeZeros && metric.Value > 0))
+ {
+ rawValues.TryAdd(stat, new ConcurrentBag<Metric>());
+ rawValues[stat].Add(metric);
+ }
+ }
+ catch
{
- rawValues.TryAdd(stat, new ConcurrentBag<Metric>());
- rawValues[stat].Add(metric);
+ //TODO : send error metric
}
}
}
View
2  parsers/Parsers.csproj
@@ -21,6 +21,8 @@
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
+ <RunCodeAnalysis>false</RunCodeAnalysis>
+ <CodeAnalysisRuleSet>AllRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
View
13 parsers/WebPagetest/SiteConfigurationElement.cs
@@ -21,6 +21,19 @@ public bool Enabled
}
}
+ [ConfigurationProperty("enableDetailedMetrics", DefaultValue = false, IsRequired = false)]
+ public bool DetailedMetrics
+ {
+ get
+ {
+ return (Boolean)this["enableDetailedMetrics"];
+ }
+ set
+ {
+ this["enableDetailedMetrics"] = value;
+ }
+ }
+
[ConfigurationProperty("allowMultipleRuns", DefaultValue = true, IsRequired = false)]
public bool AllowMultipleRuns
{
View
172 parsers/WebPagetestParser.cs
@@ -4,7 +4,7 @@
using System.IO;
using System.Net;
using System.Security.Cryptography;
-using System.Web;
+using System.Text.RegularExpressions;
using System.Xml.XPath;
using System.Configuration;
using CsvHelper;
@@ -14,8 +14,9 @@ namespace Metrics.Parsers
{
public class WebPagetestParser : IMetricParser
{
-
private static SiteConfigurationSection siteSection;
+ private static readonly Regex DetailedRegexStatement = new Regex(@"^[^,]+,[^,]+,""Cleared\sCache-Run[^,]+,[^,]+,[^,]+,""(?<host>[^,]+)"",[^,]+,[^,]+,""(?<ttl>[^,]+)"",""(?<ttfb>[^,]+)"",[^,]+,[^,]+,""(?<bytes>[^,]+)""", RegexOptions.Compiled);
+
public WebPagetestParser()
{
siteSection = ConfigurationManager.GetSection("WebPagetest") as SiteConfigurationSection;
@@ -25,10 +26,9 @@ public WebPagetestParser()
}
}
- private IEnumerable<Metric> XmlToMetrics(SiteConfigurationElement site, IXPathNavigable result)
+ private static IEnumerable<Metric> XmlToMetrics(SiteConfigurationElement site, IXPathNavigable result)
{
var metrics = new List<Metric>();
-
if (result == null)
throw new ArgumentNullException("result", "WebPagetest result was null");
@@ -37,7 +37,10 @@ private IEnumerable<Metric> XmlToMetrics(SiteConfigurationElement site, IXPathNa
var navigator = result.CreateNavigator();
foreach (XPathNavigator runNavigator in navigator.Select("response/data/run"))
{
- string run = site.AllowMultipleRuns ? "." + runNavigator.SelectSingleNode("id").Value : String.Empty;
+ var idNode = runNavigator.SelectSingleNode("id");
+ if (idNode == null) continue;
+
+ var run = site.AllowMultipleRuns ? "." + idNode.Value : String.Empty;
//firstView
DoView(runNavigator, "firstView", site, run, metrics);
@@ -49,15 +52,127 @@ private IEnumerable<Metric> XmlToMetrics(SiteConfigurationElement site, IXPathNa
break;
}
+ if (site.DetailedMetrics)
+ {
+ metrics.AddRange(GetDetailedMerics(navigator, site));
+ }
+
return metrics;
}
- private void DoView(XPathNavigator runNavigator, string view, SiteConfigurationElement site, string run, ICollection<Metric> metrics)
+ private static IEnumerable<Metric> GetDetailedMerics(XPathNavigator navigator, SiteConfigurationElement site)
+ {
+ var metrics = new List<Metric>();
+
+ //Get the test id
+ var node = navigator.SelectSingleNode("response/data/testId");
+ if (node == null) return metrics;
+ string testId = node.Value;
+
+ //Get the test url
+ node = navigator.SelectSingleNode("response/data/testUrl");
+ if (node == null) return metrics;
+ string testUrl = node.Value.Replace("http://", "");
+
+ //Get the run date
+ node = navigator.SelectSingleNode("response/data/run//date");
+ if (node == null) return metrics;
+ string dateTime = node.Value;
+
+ var request = WebRequest.Create(String.Format(CultureInfo.InvariantCulture, "{0}/result/{1}/{1}_{2}_requests.csv", siteSection.WebPagetestHost, testId, testUrl));
+
+ var response = (HttpWebResponse)request.GetResponse();
+ var stream = response.GetResponseStream();
+ try
+ {
+ if (stream == null)
+ {
+ return metrics;
+ }
+
+ using (var reader = new StreamReader(stream))
+ {
+ //null original variable to prevent double disposal
+ stream = null;
+
+ while (!reader.EndOfStream)
+ {
+ var line = reader.ReadLine();
+
+ //Ignore blank lines
+ if (String.IsNullOrWhiteSpace(line)) continue;
+
+ var matches = DetailedRegexStatement.Matches(line);
+
+ //Ignore lines that did not match
+ if (matches.Count <= 0) continue;
+
+ //Extract the host to group by
+ string host = matches[0].Groups["host"].Value.Replace(".", "_");
+ int value;
+
+ //TTFB
+ if (!Int32.TryParse(matches[0].Groups["ttfb"].Value, out value))
+ {
+ value = 0;
+ }
+ metrics.Add(new Metric
+ {
+ Key = String.Format(CultureInfo.InvariantCulture, "{0}.firstView.hosts.{1}.ttfb.avg", site.GraphiteKey,
+ host),
+ Timestamp = EpochToDateTime(dateTime),
+ Value = value
+ });
+ //TTL
+ if (!Int32.TryParse(matches[0].Groups["ttl"].Value, out value))
+ {
+ value = 0;
+ }
+
+ metrics.Add(new Metric
+ {
+ Key = String.Format(CultureInfo.InvariantCulture, "{0}.firstView.hosts.{1}.ttl.avg", site.GraphiteKey,
+ host),
+ Timestamp = EpochToDateTime(dateTime),
+ Value = value
+ });
+ //Total bytes
+ if (!Int32.TryParse(matches[0].Groups["bytes"].Value, out value))
+ {
+ value = 0;
+ }
+
+ metrics.Add(new Metric
+ {
+ Key = String.Format(CultureInfo.InvariantCulture, "{0}.firstView.hosts.{1}.bytes.count", site.GraphiteKey,
+ host),
+ Timestamp = EpochToDateTime(dateTime),
+ Value = value
+ });
+ }
+ }
+ }
+ finally
+ {
+ if (stream != null)
+ {
+ stream.Dispose();
+ }
+ }
+
+ return metrics;
+ }
+
+
+ private static void DoView(XPathNavigator runNavigator, string view, SiteConfigurationElement site, string run, List<Metric> metrics)
{
var viewNavigator = runNavigator.SelectSingleNode(view + "/results");
if (viewNavigator != null)
{
- string dateTime = viewNavigator.SelectSingleNode("date").Value;
+ var node = viewNavigator.SelectSingleNode("date");
+ if (node == null) return;
+
+ string dateTime = node.Value;
foreach (XPathNavigator metric in viewNavigator.SelectChildren(XPathNodeType.Element))
{
int numericValue;
@@ -66,7 +181,7 @@ private void DoView(XPathNavigator runNavigator, string view, SiteConfigurationE
metrics.Add(new Metric
{
Key =
- String.Format("{0}.{1}{2}.{3}", site.GraphiteKey,
+ String.Format(CultureInfo.InvariantCulture, "{0}.{1}{2}.{3}", site.GraphiteKey,
view, run, metric.Name),
Timestamp = EpochToDateTime(dateTime),
Value = numericValue
@@ -78,47 +193,60 @@ private void DoView(XPathNavigator runNavigator, string view, SiteConfigurationE
private static DateTime EpochToDateTime(string epoch)
{
- var seconds = Int64.Parse(epoch);
+ var seconds = Int64.Parse(epoch, CultureInfo.InvariantCulture);
var dt = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
dt = dt.AddSeconds(seconds);
return dt;
}
- private DateTime GetLastDateRead(string graphiteKey)
+ private static DateTime GetLastDateRead(string graphiteKey)
{
using (var md5 = MD5.Create())
{
- var tempName = "wpt_" + LogTailParser.GetMd5HashFileName(md5, graphiteKey);
+ var tempName = "wpt_" + LogTailParser.GetMD5HashFileName(md5, graphiteKey);
if (!File.Exists(tempName))
{
return DateTime.MinValue;
}
- DateTime lastRead;
- DateTime.TryParse(File.ReadAllText(tempName), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out lastRead);
- return lastRead;
+ try
+ {
+ DateTime lastRead;
+ var lines = File.ReadAllLines(tempName);
+ if (DateTime.TryParse(lines[0], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal,
+ out lastRead))
+ {
+ return lastRead;
+ }
+
+ return DateTime.MinValue;
+ }
+ catch
+ {
+ return DateTime.MinValue;
+ }
}
}
- private void StoreLastDateRead(string graphiteKey, DateTime lastRead)
+ private static void StoreLastDateRead(string graphiteKey, DateTime lastRead)
{
using (var md5 = MD5.Create())
{
- var tempName = "wpt_" + LogTailParser.GetMd5HashFileName(md5, graphiteKey);
- File.WriteAllText(tempName, lastRead.ToString(CultureInfo.InvariantCulture));
+ var tempName = "wpt_" + LogTailParser.GetMD5HashFileName(md5, graphiteKey);
+ File.WriteAllText(tempName, lastRead.ToString(CultureInfo.InvariantCulture) + Environment.NewLine + graphiteKey);
}
}
- public IEnumerable<Metric> GetMetrics()
+ public void GetMetrics(Action<IEnumerable<Metric>> sendMetrics)
{
- var metrics = new List<Metric>();
foreach (SiteConfigurationElement site in siteSection.Sites)
{
foreach (var resultUrl in GetResultUrls(site.Url, GetLastDateRead(site.GraphiteKey)))
{
try
{
- metrics.AddRange(XmlToMetrics(site, new XPathDocument(resultUrl)));
+ var metrics = XmlToMetrics(site, new XPathDocument(resultUrl));
+ sendMetrics(metrics);
}
catch
{
@@ -129,13 +257,11 @@ public IEnumerable<Metric> GetMetrics()
}
}
}
-
- return metrics;
}
private static IEnumerable<string> GetResultUrls(string url, DateTime lastRead)
{
- double difference = Math.Ceiling(DateTime.Now.Subtract(lastRead).TotalDays);
+ var difference = Math.Ceiling(DateTime.Now.Subtract(lastRead).TotalDays);
if (difference > 365)
difference = 365;
Please sign in to comment.
Something went wrong with that request. Please try again.