Permalink
Browse files

splitting up, added sql update generate

  • Loading branch information...
1 parent bf81964 commit 8bb9028987a6747b30642cf292253e98dcf56897 @coolaj86 coolaj86 committed Feb 28, 2011
Showing with 294 additions and 147 deletions.
  1. +12 −98 lib/{dirpush.js → filescan.js}
  2. +15 −5 lib/filesync-db.js
  3. +83 −0 lib/processbatch.js
  4. +1 −1 lib/scanner-sqlite3.js
  5. +27 −20 lib/sql-directive.js
  6. +61 −14 lib/sqlitize3.js
  7. +21 −0 package.json
  8. +52 −9 runner.js
  9. +22 −0 tests/sqlitize3.js
@@ -8,15 +8,15 @@
return this.split("").reverse().join("");
}
- var http = require('http'),
+ var mime = require('mime'),
fs = require('fs'),
path = require('path'),
walk = require('walk'),
- filepush = require("./filepush.js"),
crypto = require('crypto'),
//scannerDb = require("./scanner-db"),
scannerDb = require('./scanner-sqlite3'),
- indexOn = require('./index-on');
+ indexOn = require('./index-on'),
+ nodate = new Date(-9999999999999999);
function sortBySizeAndTime(a, b) {
@@ -31,8 +31,8 @@
}
function create(start, options, cb) {
- cb = cb || noop;
var scanResults = [],
+ fullscan = options.fullscan || true,
updateFile = './lastupdate.json',
// inotifypp = inotify.create(true),
blacklist = [/cache/i],
@@ -72,27 +72,18 @@
return next();
}
- //"dev": 234881026,
- //"ino": 2535386,
newstat.mode = stat.mode;
- // "nlink": 1,
newstat.uid = stat.uid;
newstat.gid = stat.gid;
- //"rdev": 0,
newstat.size = stat.size;
- //"blksize": 4096,
- //"blocks": 48,
- //"atime": "2011-02-24T20:04:09.000Z",
newstat.mtime = stat.mtime.valueOf();
- //"ctime": "2011-02-24T20:04:09.000Z",
newstat.name = stat.name;
- //"type": "file",
- newstat.path = path.join(pathname, stat.name);
- newstat.ext = path.extname(newstat.path).substring(1);
- newstat.rrelpath = path.dirname(newstat.path).reverse();
+ newstat.ext = path.extname(newstat.name).substring(1);
+ newstat.rpath = pathname.reverse();
newstat.qmd5 = crypto.createHash("md5").update(newstat.mtime.toString() + newstat.size + newstat.path).digest("hex");
- newstat.basename = path.basename(newstat.name, newstat.ext);
+ newstat.basename = path.basename(newstat.name, '.' + newstat.ext);
+ newstat.type = mime.lookup(newstat.ext);
scanResults.push(newstat);
@@ -115,94 +106,17 @@
console.log("End recursion. " + scanResults.length + " files found.");
if (!scanResults.length) {
- return cb(null);
+ return cb(null, []);
}
// Insert any leftovers
- scannerDb.insert(scanResults.slice(lastBatch, scanResults.length), processScan);
+ scannerDb.insert(scanResults.slice(lastBatch, scanResults.length), function (err) {
+ cb(err, scanResults);
+ });
- function processScan(err) {
- var allStats,
- stop;
-
- if (err) {
- throw err;
- }
-
- // TODO compare against last scan
- scanResults.sort(sortBySizeAndTime);
-
- function processBatch() {
- var i,
- packet = 0,
- set,
- file;
-
- for (i = 0; i < scanResults.length; i += 1) {
- file = scanResults[i];
- packet += file.size;
- if (packet >= max_packet_size) {
- stop = i + 1;
- break;
- }
- }
-
- set = scanResults.splice(0, stop);
- allStats = indexOn(set.slice(), 'qmd5');
- if (!set.length) {
- return;
- }
- console.log('set: ');
- console.log(set);
- filepush(options, set, pushBatch);
- }
-
-
- function pushBatch(response) {
- console.log("pushBatch");
- var chunks = [];
- response.on('data', function (chunk) {
- // TODO process each chunk right away
- // TODO stream parse JSON
- chunks.push(chunk.toString());
- });
- response.on('end', function () {
- var data = JSON.parse(chunks.join(''));
-
- if (data.length !== Object.keys(allStats).length) {
- console.log(new Error("sent and receive numbers mismatch"));
- }
- data.forEach(function (item) {
- var orig = allStats[item.qmd5];
-
- allStats[item.qmd5] = undefined;
- delete allStats[item.qmd5];
-
- if (!orig) {
- console.log(new Error("got back results for a file that wasn't sent"));
- return;
- }
- orig.md5 = item.md5;
- orig.saved = true;
- });
- if (Object.keys(allStats).length) {
- console.log(new Error("files didn't sync"));
- }
-
- processBatch();
- });
- cb(response)
- }
-
- processBatch();
- }
});
}
- // TODO full scan every day
- // short scan every hour
- var fullscan = true || false,
- nodate = new Date(-9999999999999999);
fs.readFile(updateFile, function (err, data) {
var date;
View
@@ -4,22 +4,32 @@
require('noop');
var sqlite3 = require('sqlite3').verbose(),
+ future = require('futures').future(),
sqlitize = require('./sqlitize3'),
directive = require('./sql-directive'),
dbpath = require('./config').dbpath,
db = new sqlite3.Database(dbpath, createTable);
- function createTable() {
+ function createTable(err) {
+ if (err) {
+ return future.fulfill(err);
+ }
directive.tables.forEach(function (table) {
sqlitize.table(table, function (err, sql) {
- console.log(sql);
db.exec(sql, function (err) {
- throwop(err);
- console.log("Created Successfully");
+ future.fulfill(err, sql);
});
});
});
};
-
+
+ function log(err, sql) {
+ console.log(sql);
+ }
+ future.when(log);
+
+ module.exports = {
+ ready: future.when
+ };
}());
View
@@ -0,0 +1,83 @@
+(function () {
+ "use strict";
+
+ var http = require('http');
+
+ function processScan(err) {
+ var allStats,
+ stop;
+
+ if (err) {
+ throw err;
+ }
+
+ // TODO compare against last scan
+ scanResults.sort(sortBySizeAndTime);
+
+ processBatch();
+ }
+
+
+ function processBatch() {
+ var i,
+ packet = 0,
+ set,
+ file;
+
+ for (i = 0; i < scanResults.length; i += 1) {
+ file = scanResults[i];
+ packet += file.size;
+ if (packet >= max_packet_size) {
+ stop = i + 1;
+ break;
+ }
+ }
+
+ set = scanResults.splice(0, stop);
+ allStats = indexOn(set.slice(), 'qmd5');
+ if (!set.length) {
+ return;
+ }
+ console.log('set: ');
+ console.log(set);
+ filepush(options, set, pushBatch);
+ }
+
+
+ function pushBatch(response) {
+ console.log("pushBatch");
+ var chunks = [];
+ response.on('data', function (chunk) {
+ // TODO process each chunk right away
+ // TODO stream parse JSON
+ chunks.push(chunk.toString());
+ });
+ response.on('end', function () {
+ var data = JSON.parse(chunks.join(''));
+
+ if (data.length !== Object.keys(allStats).length) {
+ console.log(new Error("sent and receive numbers mismatch"));
+ }
+ data.forEach(function (item) {
+ var orig = allStats[item.qmd5];
+
+ allStats[item.qmd5] = undefined;
+ delete allStats[item.qmd5];
+
+ if (!orig) {
+ console.log(new Error("got back results for a file that wasn't sent"));
+ return;
+ }
+ orig.md5 = item.md5;
+ orig.saved = true;
+ });
+ if (Object.keys(allStats).length) {
+ console.log(new Error("files didn't sync"));
+ }
+
+ processBatch();
+ });
+ cb(response)
+ }
+
+}());
View
@@ -30,7 +30,7 @@
}
});
- statement = sqlitize3.insert_many(table, values);
+ statement = sqlitize3.insert_many(table, values, { resolve: "ignore" });
//console.log(statement);
db.exec(statement, cb);
}
View
@@ -1,7 +1,7 @@
(function () {
"use strict";
- module.exports = {
+ var directive = {
tables: [
{
name: "filequeue",
@@ -12,27 +12,25 @@
columns: [
{
name: "qmd5",
- type: "character(32)",
- notnull: true
- },
- {
- name: "path",
- type: "text", // may be more than 256 chars
- notnull: true
+ type: "char",
+ size: "32",
+ notnull: true,
+ unique: true
},
{
- name: "rrelpath",
+ name: "rpath",
type: "text",
notnull: true
},
{
- name: "name",
- type: "varchar(255)",
+ name: "basename",
+ type: "varchar",
notnull: true
},
{
name: "ext",
- type: "varchar(255)",
+ type: "varchar",
+ default: ""
},
{
name: "mtime",
@@ -45,23 +43,30 @@
notnull: true
},
{
- name: "contentType",
- type: "varchar(255)"
+ name: "type",
+ type: "varchar"
},
{
name: "tmd5",
- type: "character(32)"
+ type: "char",
+ size: 32
},
{
name: "md5",
- type: "character(32)"
+ type: "char",
+ size: 32
},
{
- name: "previousRevision",
- type: "character(32)"
+ name: "previous_revision",
+ type: "char",
+ size: 32
},
{
- name: "updateCount",
+ name: "update_count",
+ type: "integer"
+ },
+ {
+ name: "updated_on",
type: "integer"
},
{
@@ -88,7 +93,7 @@
name: "size_mtime_index"
},
{
- columns: ["path"],
+ columns: ["rpath"],
name: "path_index"
}
],
@@ -98,6 +103,8 @@
]
};
+ module.exports = directive;
+
/*
//"dev": 234881026,
//"ino": 2465719,
Oops, something went wrong.

0 comments on commit 8bb9028

Please sign in to comment.