From d1f8df6803ef054627ec1c90bebeb0b715cd0e9d Mon Sep 17 00:00:00 2001 From: Andrew Chilton Date: Mon, 10 Dec 2012 18:50:11 +1300 Subject: [PATCH] Unique approximately sortable IDs --- .gitignore | 2 ++ LICENSE | 29 +++++++++++++++++ flake.js | 39 +++++++++++++++++++++++ package.json | 38 +++++++++++++++++++++++ readme.md | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++ test/basic.js | 51 ++++++++++++++++++++++++++++++ 6 files changed, 245 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 flake.js create mode 100644 package.json create mode 100644 readme.md create mode 100644 test/basic.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a33a71f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +*~ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3a9f4ca --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +------------------------------------------------------------------------------- + +This software is published under the MIT license as published here: + +* http://opensource.org/licenses/MIT + +------------------------------------------------------------------------------- + +Copyright 2011-2012 Apps Attic Ltd. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +------------------------------------------------------------------------------- diff --git a/flake.js b/flake.js new file mode 100644 index 0000000..a1bf7c0 --- /dev/null +++ b/flake.js @@ -0,0 +1,39 @@ +var fs = require('fs'); + +function pad(str, length) { + while ( str.length < length ) { + str = '0' + str; + } + return str; +} + +module.exports = function(macInterface) { + var currentTimestamp = Date.now(); + + var pidHex = process.pid.toString(16); + pidHex = pad('' + pidHex, 4); + + var macHex = fs.readFileSync('/sys/class/net/' + macInterface + '/address').toString('utf8'); + macHex = macHex.replace(/:/g, '').replace(/\s+/g, ''); + + var counter = 0; + + return function() { + var timestamp = Date.now(); + var tsHex = timestamp.toString(16); + tsHex = pad('' + tsHex, 12); + + if ( timestamp !== currentTimestamp ) { + counter = 0; + currentTimestamp = timestamp; + } + counterHex = pad('' + counter.toString(16), 4); + + counter++; + if ( counter > 65535 ) { + counter = 0; + } + + return tsHex + '-' + counterHex + '-' + pidHex + '-' + macHex; + }; +}; diff --git a/package.json b/package.json new file mode 100644 index 0000000..9c5db0b --- /dev/null +++ b/package.json @@ -0,0 +1,38 @@ +{ + "name": "flake", + "description": "Generate unique (approximately sortable) IDs in a distributed environment.", + "version": "0.1.0", + "author": { + "name":"Andrew Chilton", + "email":"chilts@appsattic.com", + "url":"http://www.chilts.org/" + }, + "homepage": "https://github.com/appsattic/flake", + "contributors": [], + "devDependencies": { + "tap": "~0.3.x" + }, + "dependencies": {}, + "main": "./flake.js", + "engines": { + "node": ">= 0.6.0" + }, + "repository": { + "type" : "git", + "url": "git://github.com/appsattic/flake.git" + }, + "bugs" : { + "url" : "http://github.com/appsattic/flake/issues", + "mail" : "chilts@appsattic.com" + }, + "licenses": [{ + "type": "MIT", + "url": "http://appsattic.mit-license.org/2012/" + }], + "keywords": [ + "distributed", "id", "unique", "twitter", "snowflake", "flake" + ], + "scripts" : { + "test" : "tap test/*.js" + } +} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..ed255f7 --- /dev/null +++ b/readme.md @@ -0,0 +1,86 @@ +Flake - generate unique (approximately sortable) IDs in a distributed environment. + +## Usage ## + +``` +var flake = require(flake)('eth0'); +console.log(flake()); +console.log(flake()); +console.log(flake()); +``` + +Would give something like: + +``` +013b829b1520-0000-18f0-984be1b8b104 +013b829b1527-0001-18f0-984be1b8b104 +013b829b1527-0000-18f0-984be1b8b104 +``` + +Running it again might give: + +``` +013b829b9680-0000-18f3-984be1b8b104 +013b829b9685-0000-18f3-984be1b8b104 +013b829b9685-0001-18f3-984be1b8b104 +``` + +As you can see, the time has increased by a few seconds and the PID is different. The mac address is the same and the +sequence always resets. + +Note: these unique IDs are not the same as UUIDs or can be used in place of one. + +## Format of each Unique ID ## + +Each unique ID has 4 sections which are: + +``` +013b83165f6a-00f9-314b-984be1b8b104 += 013b83165f6a -> timestamp in ms since epoch += 00f9 -> counter += 314b -> PID of the current process += 984be1b8b104 -> MAC address of the interface provided (e.g. 'eth0') +``` + +Unlike Twitter's Snowflake, we also use the PID since we want to be able to generate unique IDs on the same machine at +the same time. This is because we're generating them within the program which needs them and not in a client/server +architecture. + +## Approximately Sortable ## + +If you were to generate IDs in multiple processes on the same machine and on multiple machines, then whilst we can't +guarantee that the IDs will be exactly sortable, they will be approximately sortable. In fact, to the millisecond +(assuming your clocks aren't skew from each other). + +e.g. some IDs generated on the same machine in two processes, using 'eth0' and 'wlan0' as the MAC address could be, and +printing out alternately: + +``` +013b83571852-0000-4ae8-984be1b8b104 +013b83571853-0000-4ae8-00027298bef9 +013b83571853-0000-4ae9-984be1b8b104 +013b83571853-0001-4ae8-00027298bef9 +013b83571853-0001-4ae9-984be1b8b104 +013b83571853-0002-4ae8-00027298bef9 +013b83571853-0002-4ae9-984be1b8b104 +013b83571853-0003-4ae8-00027298bef9 +013b83571853-0003-4ae9-984be1b8b104 +``` + +As you can see, the IDs are firstly sorted by timestamp, then sequence, the PID then mac address. + +## Non-Monotomic Clocks ## + +Currently there is a possibility to generate the same ID in the same process on the same machine. This is described by +[System Clock Depedency](https://github.com/twitter/snowflake#system-clock-dependency). + +This situation hasn't yet been solved but is known about. Therefore, you should run NTP in a mode which doesn't move +the clock backwards. + +## License ## + +The MIT License : http://opensource.org/licenses/MIT + +Copyright (c) 2012 AppsAttic Ltd. http://appsattic.com/ + +(Ends) diff --git a/test/basic.js b/test/basic.js new file mode 100644 index 0000000..0a3d8ac --- /dev/null +++ b/test/basic.js @@ -0,0 +1,51 @@ +var tap = require("tap"); +var flake; + +// -------------------------------------------------------------------------------------------------------------------- +// basic tests + +tap.test("load flake", function (t) { + flake = require('../flake.js'); + t.ok(flake, 'flake loaded'); + t.end(); +}); + +tap.test("using eth0", function (t) { + flakeGen = flake('eth0'); + var id = flakeGen(); + console.log(id); + t.ok(id, 'ID generated ok using eth0'); + t.end(); +}); + +// don't do this :) +tap.test("using lo", function (t) { + flakeGen = flake('lo'); + var id = flakeGen(); + console.log(id); + t.ok(id, 'ID generated OK using lo'); + t.end(); +}); + +tap.test("check ID has a particular format", function (t) { + flakeGen = flake('eth0'); + var id = flakeGen().replace(/[0-9a-f]/g, 'x'); + t.equal(id, 'xxxxxxxxxxxx-xxxx-xxxx-xxxxxxxxxxxx', 'ID format OK'); + t.end(); +}); + +tap.test("check that the PIDs are the same", function (t) { + flakeGen = flake('eth0'); + // may or may not be in the same timestamp + var id1 = flakeGen(); + var id2 = flakeGen(); + id1 = id1.split('-'); + id2 = id2.split('-'); + t.equal(id1[0], id2[0], 'timestamp the same'); + t.equal(parseInt(id1[1], 16), parseInt(id2[1], 16) - 1, 'sequence increments fine'); + t.equal(id1[2], id2[2], 'pid the same'); + t.equal(id1[3], id2[3], 'mac address the same'); + t.end(); +}); + +// --------------------------------------------------------------------------------------------------------------------