Permalink
Browse files

Initial commit

  • Loading branch information...
0 parents commit 0b849def7b21f92a57fc3b7e3c959268b92ae7cf @meloncholy committed Aug 22, 2012
Showing with 343 additions and 0 deletions.
  1. +22 −0 .gitattributes
  2. +25 −0 .gitignore
  3. +116 −0 README.md
  4. +30 −0 config/mt-stats-sample.json
  5. +135 −0 index.js
  6. +15 −0 package.json
22 .gitattributes
@@ -0,0 +1,22 @@
+# Auto detect text files and perform LF normalization
+* text=auto
+
+# Custom for Visual Studio
+*.cs diff=csharp
+*.sln merge=union
+*.csproj merge=union
+*.vbproj merge=union
+*.fsproj merge=union
+*.dbproj merge=union
+
+# Standard to msysgit
+*.doc diff=astextplain
+*.DOC diff=astextplain
+*.docx diff=astextplain
+*.DOCX diff=astextplain
+*.dot diff=astextplain
+*.DOT diff=astextplain
+*.pdf diff=astextplain
+*.PDF diff=astextplain
+*.rtf diff=astextplain
+*.RTF diff=astextplain
25 .gitignore
@@ -0,0 +1,25 @@
+lib-cov
+*.seed
+*.log
+*.csv
+*.dat
+*.out
+*.pid
+*.gz
+*.exe
+
+pids
+logs
+results
+
+npm-debug.log
+
+node_modules
+Stuff
+
+mt-stats.json
+mt-stats josie.json
+mt-stats katie.json
+
+*.sublime-project
+*.sublime-workspace
116 README.md
@@ -0,0 +1,116 @@
+(mt) Stats
+==========
+
+(mt) Stats is a specialised proxy server that gets your server's statistics from [MediaTemple's API servers](http://mediatemple.net/api/). It was designed to be used with the [(mt) Stats Viewer](https://github.com/meloncholy/mt-stats-viewer) front end, but may even be more generally useful.
+
+If you want to read more about how it works and some potential problems, there's a surprisingly (to me, at least) long post [here](http://meloncholy.com/blog/using-d3-for-realtime-webserver-stats).
+
+
+Setting up (mt) Stats
+---------------------
+
+- [Get an API key](https://ac.mediatemple.net/api) for your MediaTemple server. You'll also need your service ID later, which you can get by visiting `https://api.mediatemple.net/api/v1/services/ids.json?apikey=XXXXX` (I only have one service ID as I have one server, but apparently you could see more.)
+
+- Rename the config file `node_modules/mt-stats/config/mt-stats-sample.json` to `mt-stats.json` and change the service ID and API key to match your server and key.
+
+
+Using (mt) Stats with your app
+------------------------------
+
+Here's a quick example of using (mt) Stats with Express.
+
+```javascript
+var app = require("express").createServer();
+var mtStats = require("mt-stats");
+
+app.get("/api/:range?", mtStats);
+
+app.listen(3000);
+```
+
+(mt) Stats will respond differently depending on the range passed to it as part of the path.
+
+- **No range** - Server stats for current time
+- **Range is one of** `5min`, `15min`, `30min`, `1hour`, `1day`, `1week`, `1month`, `3month`, `1year` - Server stats for that past period will be returned, using the MediaTemple API server's default resolution for that period
+- **Range is [0-9]+** - Server stats for the past _number_ of seconds will be returned, using the resolution in the settings file (probably; see below)
+- **Range is [0-9]+-[0-9]+** - Server stats covering the range from the first number (time since Linux epoch in seconds) to the second number will be returned, using the resolution in the settings file
+
+
+Settings
+--------
+
+Please rename `mt-stats-sample.json` to `mt-stats.json`.
+
+- **serviceId** - Your server's service ID
+- **apiKey** - Your MediaTemple API key
+- **mode** - Doesn't do anything here, but its presence gives me a warm, comforting glow
+- **rootPath** - The root URL path for all API calls
+- **interval** - Server polling interval for client. MediaTemple's stats update every 15s
+- **ranges** - For each `range`, the `resolution` at which to request data from the API (e.g. every 15 seconds) and the maximum span (`step`) to request in one go (to stop the API server objecting). Range is the maximum timespan at which to use that resolution and step
+- **metrics** - Metrics supplied by the API. `apiKey` is the key name in JSON objects and `niceName` is the name to use on the graphs
+- **definedRanges** - MediaTemple also supports some default intervals that can be requested with these URLs, e.g. [this URL](http://bits.meloncholy.com/mt-stats/api/5min) will serve up the last 5 minutes' data
+- **currentUrl** - API server URL from which to get the current stats. `%SERVICEID` and `%APIKEY` are replaced with your service ID and API key
+- **historyUrl** - API server URL to request stats going back for the past X seconds, e.g. [this URL](http://bits.meloncholy.com/mt-stats/api/300) will also give the past 5 minutes' data
+- **rangeUrl** - API server URL to get stats covering a specified time range
+
+```javascript
+{
+ "serviceId": 000000,
+ "apiKey": "XXXXXXXX",
+ "mode": "production",
+ "rootPath": "/api/",
+ "interval": 15000,
+ "ranges": [
+ { "range": 3600, "resolution": 15, "step": 3600 },
+ { "range": 43200, "resolution": 120, "step": 28800 },
+ { "range": 86400, "resolution": 240, "step": 57600 },
+ { "range": 604800, "resolution": 1800, "step": 432000 }
+ ],
+ "metrics": [
+ { "apiKey": "cpu", "niceName": "CPU %" },
+ { "apiKey": "memory", "niceName": "Memory %" },
+ { "apiKey": "load1Min", "niceName": "Load 1 min" },
+ { "apiKey": "load5Min", "niceName": "Load 5 min" },
+ { "apiKey": "load15Min", "niceName": "Load 15 min" },
+ { "apiKey": "processes", "niceName": "Processes" },
+ { "apiKey": "diskSpace", "niceName": "Disk space" },
+ { "apiKey": "kbytesIn", "niceName": "kb in / sec" },
+ { "apiKey": "kbytesOut", "niceName": "kb out / sec" },
+ { "apiKey": "packetsIn", "niceName": "Packets in / sec" },
+ { "apiKey": "packetsOut", "niceName": "Packets out / sec" }
+ ],
+ "definedRanges": ["5min", "15min", "30min", "1hour", "1day", "1week", "1month", "3month", "1year"],
+ "currentUrl": "https://api.mediatemple.net/api/v1/stats/%SERVICEID.json?apikey=%APIKEY",
+ "historyUrl": "https://api.mediatemple.net/api/v1/stats/%SERVICEID/%RANGE.json?apikey=%APIKEY",
+ "rangeUrl": "https://api.mediatemple.net/api/v1/stats/%SERVICEID.json?start=%START&end=%END&&resolution=%RESOLUTION&apikey=%APIKEY"
+}
+```
+
+More on stats resolution
+------------------------
+
+MediaTemple may or may not respect the stats resolution you request, e.g. there's a minimum resolution of 15s and a request for 2 hour intervals will be returned at a resolution of 60 minutes.
+
+The API server will also only serve up a fairly small amount of data at a given resolution - smaller than I'd like, so the server divides up the client's range into several requests and combines them before returning. The interval at which to split a single request into multiple API queries is the `step` if this is less than the `range` for that range. So if you wanted a week's worth of data at 15 second intervals, you could add `{"range": 604800, "resolution": 15, "step": 3600 }` (but please don't as you'll hit the API server 168 times!). All numbers are seconds.
+
+The metrics bit of the config file is currently set up to return up to a week's worth of data at once; it will return more, but at a resolution that would hammer the MediaTemple server rather, so please add some more ranges if you want to do that.
+
+
+Dependencies
+------------
+
+- [Konphyg](https://github.com/pgte/konphyg)
+
+
+Legal fun
+---------
+
+Copyright © 2012 Andrew Weeks http://meloncholy.com
+
+(mt) Stats is licensed under the [MIT licence](http://meloncholy.com/licence).
+
+
+Me
+--
+
+I have a [website](http://meloncholy.com) and a [Twitter](https://twitter.com/meloncholy). Please come and say hi if you'd like or if something's not working; be lovely to hear from you.
30 config/mt-stats-sample.json
@@ -0,0 +1,30 @@
+{
+ "serviceId": 000000,
+ "apiKey": "XXXXXXXX",
+ "mode": "production",
+ "rootPath": "/api/",
+ "interval": 15000,
+ "ranges": [
+ { "range": 3600, "resolution": 15, "step": 3600 },
+ { "range": 43200, "resolution": 120, "step": 28800 },
+ { "range": 86400, "resolution": 240, "step": 57600 },
+ { "range": 604800, "resolution": 1800, "step": 432000 }
+ ],
+ "metrics": [
+ { "apiKey": "cpu", "niceName": "CPU %" },
+ { "apiKey": "memory", "niceName": "Memory %" },
+ { "apiKey": "load1Min", "niceName": "Load 1 min" },
+ { "apiKey": "load5Min", "niceName": "Load 5 min" },
+ { "apiKey": "load15Min", "niceName": "Load 15 min" },
+ { "apiKey": "processes", "niceName": "Processes" },
+ { "apiKey": "diskSpace", "niceName": "Disk space" },
+ { "apiKey": "kbytesIn", "niceName": "kb in / sec" },
+ { "apiKey": "kbytesOut", "niceName": "kb out / sec" },
+ { "apiKey": "packetsIn", "niceName": "Packets in / sec" },
+ { "apiKey": "packetsOut", "niceName": "Packets out / sec" }
+ ],
+ "definedRanges": ["5min", "15min", "30min", "1hour", "1day", "1week", "1month", "3month", "1year"],
+ "currentUrl": "https://api.mediatemple.net/api/v1/stats/%SERVICEID.json?apikey=%APIKEY",
+ "historyUrl": "https://api.mediatemple.net/api/v1/stats/%SERVICEID/%RANGE.json?apikey=%APIKEY",
+ "rangeUrl": "https://api.mediatemple.net/api/v1/stats/%SERVICEID.json?start=%START&end=%END&&resolution=%RESOLUTION&apikey=%APIKEY"
+}
135 index.js
@@ -0,0 +1,135 @@
+/*!
+* (mt) Stats
+*
+* A little library to access your MediaTemple server's stats.
+*
+* Copyright (c) 2012 Andrew Weeks http://meloncholy.com
+* Licensed under the MIT licence. See http://meloncholy.com/licence
+* Version 0.0.1
+*/
+
+"use strict";
+
+var https = require("https");
+var util = require("util");
+var url = require("url");
+var settings = require("konphyg")(__dirname + "/config/")("mt-stats");
+
+var currentOp = url.parse(settings.currentUrl.replace("%SERVICEID", settings.serviceId).replace("%APIKEY", settings.apiKey));
+var historyUrl = settings.historyUrl.replace("%SERVICEID", settings.serviceId).replace("%APIKEY", settings.apiKey);
+var rangeUrl = settings.rangeUrl.replace("%SERVICEID", settings.serviceId).replace("%APIKEY", settings.apiKey);
+
+Object.defineProperty(mtStats, "interval", { get: function () { return settings.interval; } });
+Object.defineProperty(mtStats, "metrics", { get: function () { return settings.metrics; } });
+
+function mtStats(req, res) {
+ var reqUrl = url.parse(req.url, true);
+ var start;
+ var end;
+ var range;
+ var rangeIdx;
+ var step;
+ var t;
+ var stuff = [];
+ stuff.apiOps = [];
+ // Time since Unix epoch.
+ var now = Math.round(new Date().getTime() / 1000.0);
+ // Personal epoch.
+ var epoch;
+ var loop = false;
+
+ if (reqUrl.path.length > settings.rootPath.length) {
+ range = reqUrl.path.substring(settings.rootPath.length);
+
+ if (range.indexOf("-") !== -1) {
+ t = range.split("-");
+ epoch = +t[0];
+ end = +t[1];
+ loop = true;
+ } else if (isNumber(range)) {
+ epoch = Math.floor(now - +range);
+ end = now;
+ loop = true;
+ } else if (settings.definedRanges.indexOf(range) !== -1) {
+ stuff.apiOps.push(url.parse(historyUrl.replace("%RANGE", range)));
+ } else {
+ stuff.apiOps.push(currentOp);
+ }
+
+ if (loop) {
+ rangeIdx = getRange(end - epoch);
+ // May not get the resolution specified, e.g. currently 120, 240 -> 60, 1800 -> 2220.
+ step = settings.ranges[rangeIdx].step;
+ start = Math.max(epoch, end - step + 1);
+
+ while (end > epoch) {
+ stuff.apiOps.push(url.parse(rangeUrl.replace("%START", start).replace("%END", end).replace("%RESOLUTION", settings.ranges[rangeIdx].resolution)));
+ start = Math.max(epoch, start - step);
+ end -= step;
+ }
+ }
+ } else {
+ stuff.apiOps.push(currentOp);
+ }
+
+ stuff.done = 0;
+ stuff.count = stuff.apiOps.length;
+
+ for (var i = 0, len = stuff.count; i < len; i++) {
+ stuff[stuff.count - i - 1] = "";
+ mtReq(res, stuff, i);
+ }
+}
+
+function mtReq(res, stuff, index) {
+ var apiReq;
+
+ (apiReq = https.request(stuff.apiOps[index], function (apiRes) {
+ apiRes.on("data", function (chunk) {
+ mtData(res, chunk, stuff, index);
+ });
+
+ apiRes.on("end", function () {
+ mtRender(res, stuff, index);
+ });
+ })).end();
+
+ apiReq.on("error", function (e) {
+ console.log(e);
+ });
+}
+
+function mtData(res, chunk, stuff, index) {
+ chunk = chunk.toString();
+
+ if (chunk.indexOf("DataNotAvailable") !== -1) {
+ console.error((new Date()).toString(), "Fetching range", index, "again...");
+ // Should always be "" (first chunk), but just in case.
+ stuff[stuff.count - index - 1] = "";
+ mtReq(res, stuff, index);
+ } else {
+ // Results are in reverse time order.
+ stuff[stuff.count - index - 1] += chunk;
+ }
+}
+
+function mtRender(res, stuff, index) {
+ if (stuff[stuff.count - index - 1] === "" || ++stuff.done < stuff.count) return;
+
+ res.end(stuff.join("").replace(/\]\}\}\{"statsList":.*?"stats":\[/g, ","));
+}
+
+function getRange(totalRange) {
+ for (var i = 0, len = settings.ranges.length; i < len && settings.ranges[i].range < totalRange; i++);
+ return Math.min(i, len - 1);
+}
+
+function num(v) {
+ return parseInt(v, 10) || 0;
+}
+
+function isNumber(value) {
+ return !isNaN(parseInt(value * 1));
+}
+
+module.exports = mtStats;
15 package.json
@@ -0,0 +1,15 @@
+{
+ "name": "mt-stats",
+ "description": "A little library to access your MediaTemple server's stats.",
+ "version": "0.0.1",
+ "author": "Andrew Weeks <andrew@meloncholy.com>",
+ "dependencies": {
+ "konphyg": ">= 1.0.5"
+ },
+ "keywords": ["stats", "statistics", "mediatemple", "mt"],
+ "repository": {
+ "type": "git",
+ "url": "git://github.com/meloncholy/mt-stats.git"
+ },
+ "main": "index"
+}

0 comments on commit 0b849de

Please sign in to comment.