From 32a6a536436b30eab56b10a30a24fe8f25af9936 Mon Sep 17 00:00:00 2001 From: Koen Bok Date: Fri, 6 Apr 2012 11:11:59 -0700 Subject: [PATCH] Initial --- .gitignore | 3 + Makefile | 33 +++++++ README.md | 3 + package.json | 16 ++++ src/sqlbt.coffee | 229 +++++++++++++++++++++++++++++++++++++++++++++++ src/utils.coffee | 20 +++++ test/test.coffee | 164 +++++++++++++++++++++++++++++++++ 7 files changed, 468 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 package.json create mode 100644 src/sqlbt.coffee create mode 100644 src/utils.coffee create mode 100644 test/test.coffee diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bbf787b --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ + +.DS_Store +node_modules \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f2f1da4 --- /dev/null +++ b/Makefile @@ -0,0 +1,33 @@ +REPORTER = list + +all: build + +build: + @./node_modules/coffee-script/bin/coffee \ + -c \ + -o lib src + +clean: + rm -rf lib + mkdir lib + +watch: + @./node_modules/coffee-script/bin/coffee \ + -o lib \ + -cw src + +test: + @./node_modules/mocha/bin/mocha \ + --compilers coffee:coffee-script \ + --reporter $(REPORTER) \ + test/*.coffee + +testw: + @./node_modules/mocha/bin/mocha \ + --watch \ + --growl \ + --compilers coffee:coffee-script \ + --reporter $(REPORTER) \ + test/*.coffee + +.PHONY: build clean watch test \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..7425a78 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +Key value implementation on top of sql with forced indexes. A bit like AppEngines BigTable. + +make test \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..3714df1 --- /dev/null +++ b/package.json @@ -0,0 +1,16 @@ +{ + "name": "sqlbt", + "version": "0.0.1", + "description": "Key-value on top of SQL", + "author": "Koen Bok", + "dependencies": { + "coffee-script": ">=1.0.1", + "nodeunit": ">=0.5.0", + "sqlite3": ">=2.0.16", + "async": ">=0.1.10", + "underscore": ">=1.1.7", + "winston": ">=0.5.11", + "mocha": ">=1.0.1", + "should": ">=@0.6.0" + } +} diff --git a/src/sqlbt.coffee b/src/sqlbt.coffee new file mode 100644 index 0000000..29f5935 --- /dev/null +++ b/src/sqlbt.coffee @@ -0,0 +1,229 @@ +require "coffee-script" +{inspect} = require "util" + +sqlite3 = require "sqlite3" +sqlite3 = sqlite3.verbose() # Optional + +async = require "async" +_ = require "underscore" +log = require "winston" + +utils = require "./utils" + + +class Backend + + +class SQLiteBackend extends Backend + + constructor: (dsl) -> + @typeMap = + string: 'VARCHAR(255)' + text: 'TEXT' + int: 'INT' + float: 'FLOAT' + + @db = new sqlite3.Database dsl + + execute: (sql, params, callback) -> + + if _.isFunction params + callback = params + params = {} + + # log.info "[sql] #{sql} #{inspect(params)}" + + cb = (err, result) -> + throw err if err + callback(err, result) + + if sql[0..5].toLowerCase() == "select" + @db.all sql, params, cb + else + @db.run sql, params, cb + + createTable: (name, callback) -> + @execute "CREATE TABLE #{name} (key CHAR(32) NOT NULL, PRIMARY KEY (key))", callback + + createColumn: (table, name, type, callback) -> + @execute "ALTER TABLE #{table} ADD COLUMN #{name} #{@typeMap[type]}", callback + + createIndex: (table, name, columns, callback) -> + @execute "CREATE INDEX #{name} ON #{table} (#{columns.join ','})", callback + + createOrUpdateRow: (table, columns, callback) -> + + keys = _.keys columns + values = _.values columns + + @execute "INSERT OR REPLACE INTO #{table} (#{keys.join ', '}) VALUES (#{utils.oinks(values)})", + values, callback + + fetch: (table, filters, callback) -> + + values = [] + sql = "SELECT * FROM #{table}" + + if filters is not {} + + sql += " WHERE" + + for column, filter of filters + + if values.length > 0 + sql += " AND" + + operator = filter[0] + + sql += " #{column} #{operator}" + values.push filter[1] + + + # operator = filter[0] + # vals = filter[1] + # + # if not _.isArray(vals) + # vals = [vals] + # + # values.push.apply values, vals + # + # placeholders = ["?" for i in [1..vals.length]][0] + # + # + # sql += " #{column} #{operator} #{placeholders.join ', '}" + + + # if operator == "IN" + # placeholders = ["?" for i in [1..filter[1].length]][0] + # sql += " #{column} #{operator} (#{placeholders.join ', '})" + # else + # sql += " #{column} #{operator} ?" + + @execute sql, values, callback + +class Store + + constructor: (@backend, @definition) -> + # definition is list of kind, indexes, validator, model + + create: (callback) -> + + steps = [] + + for item in @definition + steps.push (cb) => @createKind item.kind, cb + steps.push (cb) => + + async.map _.keys(item.indexes), (indexName, cb) => + index = item.indexes[indexName] + @createIndex item.kind, index.type, indexName, cb + , cb + + async.series steps, -> + callback() + + + createKind: (name, callback) -> + + columns = + # key: "string" + value: "text" + + indexes = [["key"]] + + steps = [ + (cb) => @backend.createTable name, cb, + (cb) => + async.map _.keys(columns), (column, icb) => + @backend.createColumn name, column, columns[column], icb + , cb, + (cb) => + async.map indexes, (columns, icb) => + @createIndex name, "string", columns, icb + , cb + ] + + async.series steps, (error, results) -> + callback error, results + + createIndex: (kind, type, name, callback) -> + + indexName = "#{kind}_index_#{name}" + + # console.log kind, type, properties, callback + + if type not in _.keys @backend.typeMap + throw "Requested index type '#{type}' not supported by backend #{_.keys @backend.typeMap}" + + # If the property is not the object key we need to create a column + # that can hold the indexed property so we can make an sql index on + # top of that. + + steps = [] + + if name != "key" + steps.push (cb) => @backend.createColumn kind, indexName, type, cb + + steps.push (cb) => @backend.createIndex kind, indexName, [indexName], cb + + async.series steps, callback + + put: (data, callback) -> + + if not _.isArray(data) + data = [data] + + async.map data, (item, cb) => + @backend.createOrUpdateRow item.kind, @_toStore(item.kind, item), cb + , callback + + + get: (kind, key, callback) -> + if _.isArray(key) + @query kind, {"key": ["IN (#{utils.oinks(key)})", key]}, callback + else + @query kind, {"key": ["= ?", key]}, (err, result) -> + if result.length == 1 + callback err, result[0] + else + callback err, null + + query: (kind, filters, callback) -> + @backend.fetch kind, filters, (err, rows) => + callback err, rows.map (row) => + @_fromStore kind, row + + + _toStore: (kind, data) -> + + if not data.key + data.key = utils.uuid() + + key = data.key + kind = data.kind + + dataCopy = _.clone data + + delete dataCopy.key + delete dataCopy.kind + + result = + key: data.key + value: JSON.stringify(dataCopy) + + # Add the index data + for item in @definition + for name, index of item.indexes + indexName = "#{kind}_index_#{name}" + result[indexName] = index.getter(dataCopy) + + return result + + _fromStore: (kind, row) -> + result = JSON.parse(row.value) + result.key = row.key + result.kind = kind + return result + +exports.Store = Store +exports.SQLiteBackend = SQLiteBackend \ No newline at end of file diff --git a/src/utils.coffee b/src/utils.coffee new file mode 100644 index 0000000..9523be1 --- /dev/null +++ b/src/utils.coffee @@ -0,0 +1,20 @@ +exports.uuid = -> + + chars = '0123456789abcdefghijklmnopqrstuvwxyz'.split('') + output = new Array(36) + random = 0 + + for digit in [0..32] + random = 0x2000000 + (Math.random() * 0x1000000) | 0 if (random <= 0x02) + r = random & 0xf + random = random >> 4 + output[digit] = chars[if digit == 19 then (r & 0x3) | 0x8 else r] + + output.join('') + +exports.logError = (err) -> + console.log(err) if err + + +exports.oinks = (values) -> + (["?" for i in [1..values.length]][0]).join ", " \ No newline at end of file diff --git a/test/test.coffee b/test/test.coffee new file mode 100644 index 0000000..ca09965 --- /dev/null +++ b/test/test.coffee @@ -0,0 +1,164 @@ +fs = require "fs" +assert = require "assert" + +async = require "async" +should = require "should" + +sqlbt = require "../src/sqlbt" +utils = require "../src/utils" + + +path = "./test.sqlite3" + +try fs.unlinkSync path + +# backend = new sqlbt.SQLiteBackend path +backend = new sqlbt.SQLiteBackend ":memory:" + +data = + kind: "person" + name: "Koen Bok" + age: 29 + + +# models = [] +# models.push +# kind: "person" +# indexes: [ +# {type: "string", property: "name"} +# {type: "int", property: "age"} +# ] +# models.push +# kind: "product" +# indexes: [ +# {type: "string", property: "name"}, +# {type: "int", property: "price"} +# {type: "int", name: "price-plus-ten", property: (data) -> data.price + 10} +# ] +# +# store = new sqlbt.Store backend, models +# +# store.create -> +# console.log "done" + + + +models = + person: + kind: "person" + indexes: + name: + type: "string" + getter: (o) -> o.name + age: + type: "int" + getter: (o) -> o.age + product: + kind: "product" + indexes: + name: + type: "string" + getter: (o) -> o.name + price: + type: "int" + getter: (o) -> o.age + + +describe "Backend", -> + describe "#simple", -> + + backend = new sqlbt.SQLiteBackend ":memory:" + + it "should do create", (done) -> + backend.execute "CREATE TABLE man (id INTEGER NOT NULL, name TEXT, PRIMARY KEY (id))", done + + it "should do insert", (done) -> + async.map ["koen", "dirk", "hugo"], (value, cb) -> + backend.execute "INSERT INTO man (name) VALUES (?)", value, cb + , done + + it "should do select all", (done) -> + backend.execute "SELECT * FROM man", (err, result) -> + result.should.eql [ + {id:1, name:"koen"}, + {id:2, name:"dirk"}, + {id:3, name:"hugo"} + ] + done() + + it "should do select eq", (done) -> + backend.execute "SELECT * FROM man WHERE id=?", 1, (err, result) -> + result.should.eql [{id:1, name:"koen"}] + done() + + it "should do select in", (done) -> + backend.execute "SELECT * FROM man WHERE id IN (?, ?)", [1, 2], (err, result) -> + result.should.eql [ + {id:1, name:"koen"}, + {id:2, name:"dirk"} + ] + done() + + +describe "Store", -> + describe "#simple", -> + + backend = new sqlbt.SQLiteBackend ":memory:" + store = new sqlbt.Store backend, [models.person] + + data1 = + key: utils.uuid() + kind: "person" + name: "Jorn van Dijk" + age: 27 + + data2 = + key: utils.uuid() + kind: "person" + name: "Koen Bok" + age: 29 + + data3 = + key: utils.uuid() + kind: "person" + name: "Dirk Stoop" + age: 32 + + it "should create the store without error", (done) -> + store.create done + + it "should put the data without error", (done) -> + store.put data1, done + + it "should fetch the data without error", (done) -> + store.get "person", data1.key, (err, result) -> + data1.should.eql result + done() + + it "should update the data", (done) -> + data1.age = 28 + store.put data1, -> + store.get "person", data1.key, (err, result) -> + data1.should.eql result + done() + + it "should insert multiple", (done) -> + store.put [data2, data3], done + + it "should fetch all", (done) -> + store.query "person", {}, (err, result) -> + data1.should.eql result[0] + data2.should.eql result[1] + data3.should.eql result[2] + done() + + it "should fetch multiple", (done) -> + store.get "person", [data1.key, data2.key, data3.key], (err, result) -> + data1.should.eql result[0] + data2.should.eql result[1] + data3.should.eql result[2] + done() + + + +