From 56007c46619e0ede9aa9a419ed6ff82a123a75e1 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Wed, 21 Dec 2011 18:30:21 -0600 Subject: [PATCH 01/35] first pass at ijod conversion --- Common/node/connector/dataStore.js | 138 ----------------------------- Common/node/ijod.js | 134 ++++++++++++++++++++++++---- package.json | 1 + tests/ijod-test-local.js | 58 ++++++++++-- 4 files changed, 168 insertions(+), 163 deletions(-) delete mode 100644 Common/node/connector/dataStore.js diff --git a/Common/node/connector/dataStore.js b/Common/node/connector/dataStore.js deleted file mode 100644 index 83d2a2be4..000000000 --- a/Common/node/connector/dataStore.js +++ /dev/null @@ -1,138 +0,0 @@ -/* -* -* Copyright (C) 2011, The Locker Project -* All rights reserved. -* -* Please see the LICENSE file for more information. -* -*/ - -var IJOD = require('ijod').IJOD; -var lstate = require('lstate'); - -var ijodFiles = {}; -var mongo; -var mongoID = 'id'; - -exports.init = function(mongoid, _mongo) { - mongo = _mongo; - mongoID = mongoid; - for (var i in mongo.collections) { - // MAYBETODO: do .count() for each and reset lstate.set("field",val) here? - if (!ijodFiles[i]) { - ijodFiles[i] = new IJOD(i); - } - } -} - -exports.addCollection = function(name) { - if(!mongo.collections[name]) - mongo.addCollection(name); -} - -function now() { - return Date.now(); -} - -// arguments: type should match up to one of the mongo collection fields -// object will be the object to persist -// options is optional, but if it exists, available options are: strip + timestamp -// strip is an array of properties to strip off before persisting the object. -// options = {strip: ['person','checkins']}, for example -// timeStamp will be the timestamp stored w/ the record if it exists, otherwise, just use now. -// -exports.addObject = function(type, object, options, callback) { - lstate.up(type); - var timeStamp = now(); - if (arguments.length == 3) callback = options; - if (typeof options == 'object') { - for (var i in options['strip']) { - object[options['strip'][i]].delete - } - if (options['timeStamp']) { - timeStamp = options['timeStamp']; - } - } - ijodFiles[type].addRecord(timeStamp, object, function(err) { - if (err) - callback(err); - setCurrent(type, object, callback); - }) -} - -// same deal, except no strip option, just timestamp is available currently -exports.removeObject = function(type, id, options, callback) { - lstate.down(type); - var timeStamp = now(); - if (arguments.length == 3) callback = options; - if (typeof options == 'object') { - if (options['timeStamp']) { - timeStamp = options['timeStamp']; - } - } - var record = {deleted: timeStamp}; - record[mongoID] = id; - ijodFiles[type].addRecord(timeStamp, record, function(err) { - if (err) - callback(err); - removeCurrent(type, id, callback); - }) -} - - -// mongos -function getMongo(type, id, callback) { - var m = mongo.collections[type]; - if(!m) - callback(new Error('invalid type:' + type), null); - else if(!(id && (typeof id === 'string' || typeof id === 'number'))) - callback(new Error('bad id:' + id), null); - else - return m; -} - -exports.queryCurrent = function(type, query, options) { - query = query || {}; - options = options || {}; - var m = mongo.collections[type]; - if(!m) - callback(new Error('invalid type:' + type), null); - else - return m.find(query, options); -} - -exports.getAllCurrent = function(type, callback, options) { - options = options || {}; - var m = mongo.collections[type]; - if(!m) - callback(new Error('invalid type:' + type), null); - else - m.find({}, options).toArray(callback); -} - -exports.getCurrent = function(type, id, callback) { - var m = getMongo(type, id, callback); - if(m) { - var query = {}; - query[mongoID] = id; - m.findOne(query, callback); - } -} - -function setCurrent(type, object, callback) { - var m = getMongo(type, object[mongoID], callback); - if(m) { - var query = {}; - query[mongoID] = object[mongoID]; - m.update(query, object, {upsert:true, safe:true}, callback); - } -} - -function removeCurrent(type, id, callback) { - var m = getMongo(type, id, callback); - if(m) { - var query = {}; - query[mongoID] = id; - m.remove(query, callback); - } -} diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 295a34763..3e6473b7f 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -12,25 +12,125 @@ * Indexed JSON On Disk */ -var fs = require('fs'), - lconfig = require(__dirname + '/lconfig'), - path = require('path'), - lfs = require(__dirname + '/lfs'); - -function IJOD(name, dir) { - this.name = name; - this.dataFileName = name + '.json'; - if (dir) { - this.dataFile = fs.openSync(path.join(lconfig.lockerDir, lconfig.me, dir, this.dataFileName), 'a'); - } else { - this.dataFile = fs.openSync(this.dataFileName, 'a'); +var fs = require('fs'); +var path = require('path'); +var deepCompare = require('deepCompare'); +var sqlite = require('sqlite-fts'); +var gzbz2 = require("gzbz2"); +var gzip = new gzbz2.Gzip; +var gunzip = new gzbz2.Gunzip; + +function IJOD(arg, callback) { + if(!arg || !arg.name) return callback("invalid args"); + var self = this; + self.name = arg.name; + self.gzname = arg.name + '.json.gz'; + self.dbname = arg.name + '.db'; + try{ + if (arg.dir) { + self.fda = fs.openSync(path.join(arg.dir, self.gzname), 'a'); + self.fdr = fs.openSync(path.join(arg.dir, self.gzname), 'r'); + } else { + self.fda = fs.openSync(self.gzname, 'a'); + self.fdr = fs.openSync(self.gzname, 'r'); + } + var stat = fs.fstatSync(self.fdr); + self.len = stat.size; + }catch(E){ + return callback(E); } + self.db = new sqlite.Database(); + self.db.open(self.dbname, function (err) { + if(err) return callback(err); + self.db.executeScript("CREATE TABLE IF NOT EXISTS ijod (id TEXT PRIMARY KEY, at INTEGER, len INTEGER);",function (err) { + if(err) return callback(err); + callback(null, self); + }); + }); } - exports.IJOD = IJOD; -IJOD.prototype.addRecord = function(timeStamp, record, callback) { - var str = JSON.stringify({timeStamp:timeStamp, data:record}) + '\n'; - var b = new Buffer(str); - fs.write(this.dataFile, b, 0, b.length, this.fileLength, callback); +// takes arg of at least an id and data, callback(err) when done +IJOD.prototype.addData = function(arg, callback) { + if(!arg || !arg.id) return callback("invalid arg"); + if(!arg.at) arg.at = Date.now(); + gzip.init(); + var gzdata = gzip.deflate(new Buffer(JSON.stringify(arg)+"\n")); + var gzlast = gzip.end(); + + try{ + fs.writeSync(this.fda, gzdata, 0, gzdata.length, null); + fs.writeSync(this.fda, gzlast, 0, gzlast.length, null); + }catch(E){ + return callback(E); + } + + var at = this.len; + this.len += gzdata.length; + this.len += gzlast.length; + this.db.execute("REPLACE INTO ijod VALUES (?, ?, ?)", [arg.id, at, this.len-at], callback); +} + +IJOD.prototype.getOne = function(arg, callback) { + if(!arg || !arg.id) return callback("invalid arg"); + var self = this; + self.db.query("SELECT at,len FROM ijod WHERE id = ? LIMIT 1", [arg.id], function(err, row){ + if(err) return callback(err); + if(!row) return callback(); + var buf = new Buffer(row.len); + fs.readSync(self.fdr, buf, 0, row.len, row.at); + gunzip.init(); + var x = gunzip.inflate(buf); + return callback(null, x.toString()); + }); +} + +IJOD.prototype.getAll = function(arg, callback) { + if(!arg) return callback("invalid arg"); + var params = []; + var sql = "SELECT at,len FROM ijod "; + if(arg.limit) + { + sql += " LIMIT ?"; + params.push(parseInt(arg.limit)); + } + if(arg.offset) + { + sql += " OFFSET ?"; + params.push(parseInt(arg.offset)); + } + var self = this; + self.db.query(sql, params, function(err, row){ + if(err) return callback(err); + if(!row) return callback(); + var buf = new Buffer(row.len); + fs.readSync(self.fdr, buf, 0, row.len, row.at); + gunzip.init(); + var x = gunzip.inflate(buf); + return callback(null, x.toString()); + }); } + +// takes a new object and checks first if it exists +// callback(err, new|same|update) +IJOD.prototype.smartAdd = function(arg, callback) { + if(!arg || !arg.id) return callback("invalid arg"); + var self = this; + self.getOne(arg, function(err, existing){ + if(err) return callback(err); + // first check if it's new + if(!existing) + { + self.addData(arg, function(err){ + if(err) return callback(err); + callback(null, "new"); + }) + } + if (deepCompare(doc, object)) { + callback(err, 'same', doc); + } else { + doc._id = id; + callback(err, 'update', doc); + } + }); +} \ No newline at end of file diff --git a/package.json b/package.json index 7d7c9a557..888f9c877 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "ini": "=1.0.1", "uglify-js": "=1.1.1", "connect-form": "0.2.1", + "gzbz2": "0.1.4", "imagemagick": "https://github.com/lpatters/node-imagemagick/tarball/master", diff --git a/tests/ijod-test-local.js b/tests/ijod-test-local.js index ac6a5f922..737574359 100644 --- a/tests/ijod-test-local.js +++ b/tests/ijod-test-local.js @@ -10,30 +10,40 @@ var vows = require("vows"); var assert = require("assert"); var fs = require('fs'); +var path = require('path'); var IJOD = require("../Common/node/ijod.js").IJOD; -var lfs = require("../Common/node/lfs.js"); var lconfig = require('../Common/node/lconfig.js'); +lconfig.load("Config/config.json"); + var myIJOD; var suite = vows.describe("IJOD Module"); -var events = [{data:{'id':42, 'name':'Thing 1'}, 'timeStamp':10}, - {data:{'id':4242, 'name':'Thing 2'}, 'timeStamp':100}, - {data:{'id':424242, 'name':'Sally'}, 'timeStamp':1000}]; +var events = [{data:{'id':42, 'name':'Thing 1'}, 'id':"10"}, + {data:{'id':4242, 'name':'Thing 2'}, 'id':"100"}, + {data:{'id':424242, 'name':'Sally'}, 'id':"1000"}]; var errs = []; +var item; +var items =0; suite.addBatch({ 'Can add records to the IJOD': { topic: function() { - myIJOD = new IJOD(lconfig.me + '/ijodtest'); var self = this; - myIJOD.addRecord(events[0].timeStamp, events[0].data, function(err) { + console.error(lconfig.me); + myIJOD = new IJOD({name:"ijodtest", dir:path.join(lconfig.lockerDir, lconfig.me)}, function(err){ if(err) errs.push(err); - myIJOD.addRecord(events[1].timeStamp, events[1].data, function(err) { + myIJOD.addData(events[0], function(err) { if(err) errs.push(err); - myIJOD.addRecord(events[2].timeStamp, events[2].data, self.callback); + myIJOD.addData(events[1], function(err) { + if(err) errs.push(err); + myIJOD.addData(events[2], function(err){ + if(err) errs.push(err); + self.callback() + }); + }); }); }); }, @@ -41,5 +51,37 @@ suite.addBatch({ assert.equal(errs.length, 0); } } +}).addBatch({ + 'Can get one record': { + topic: function() { + var self = this; + errs = []; + myIJOD.getOne({id:"10"}, function(err, i) { + if(err) errs.push(err); + item = i; + self.callback(); + }); + }, + "successfully" : function(item) { + assert.equal(errs.length, 0); + assert.include(item, "Thing 1"); + } + } +}).addBatch({ + 'Can get all records': { + topic: function() { + var self = this; + errs = []; + myIJOD.getAll({limit:3}, function(err, i) { + if(err) errs.push(err); + if(!i) return self.callback(); + items++; + }); + }, + "successfully" : function(item) { + assert.equal(errs.length, 0); + assert.equal(items, 3); + } + } }); suite.export(module); From 53928e3d2b756a66068080f239ebe176fdc43db2 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Wed, 21 Dec 2011 23:26:33 -0600 Subject: [PATCH 02/35] ijod is working for synclets, and cleans up so much tech debt and old code/logic in the process --- Common/node/ijod.js | 51 +++++++++++---- Common/node/lsyncmanager.js | 123 +++++++++++++++++------------------- 2 files changed, 96 insertions(+), 78 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 3e6473b7f..dcec99d90 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -27,13 +27,8 @@ function IJOD(arg, callback) { self.gzname = arg.name + '.json.gz'; self.dbname = arg.name + '.db'; try{ - if (arg.dir) { - self.fda = fs.openSync(path.join(arg.dir, self.gzname), 'a'); - self.fdr = fs.openSync(path.join(arg.dir, self.gzname), 'r'); - } else { - self.fda = fs.openSync(self.gzname, 'a'); - self.fdr = fs.openSync(self.gzname, 'r'); - } + self.fda = fs.openSync(self.gzname, 'a'); + self.fdr = fs.openSync(self.gzname, 'r'); var stat = fs.fstatSync(self.fdr); self.len = stat.size; }catch(E){ @@ -53,6 +48,7 @@ exports.IJOD = IJOD; // takes arg of at least an id and data, callback(err) when done IJOD.prototype.addData = function(arg, callback) { if(!arg || !arg.id) return callback("invalid arg"); + arg.id = arg.id.toString(); // safety w/ numbers if(!arg.at) arg.at = Date.now(); gzip.init(); var gzdata = gzip.deflate(new Buffer(JSON.stringify(arg)+"\n")); @@ -71,8 +67,32 @@ IJOD.prototype.addData = function(arg, callback) { this.db.execute("REPLACE INTO ijod VALUES (?, ?, ?)", [arg.id, at, this.len-at], callback); } +// adds a deleted record to the ijod and removes from index +IJOD.prototype.delData = function(arg, callback) { + if(!arg || !arg.id) return callback("invalid arg"); + arg.id = arg.id.toString(); // safety w/ numbers + if(!arg.at) arg.at = Date.now(); + arg.type = "delete"; + gzip.init(); + var gzdata = gzip.deflate(new Buffer(JSON.stringify(arg)+"\n")); + var gzlast = gzip.end(); + + try{ + fs.writeSync(this.fda, gzdata, 0, gzdata.length, null); + fs.writeSync(this.fda, gzlast, 0, gzlast.length, null); + }catch(E){ + return callback(E); + } + + var at = this.len; + this.len += gzdata.length; + this.len += gzlast.length; + this.db.execute("DELETE FROM ijod WHERE id = ?", [arg.id], callback); +} + IJOD.prototype.getOne = function(arg, callback) { if(!arg || !arg.id) return callback("invalid arg"); + arg.id = arg.id.toString(); // safety w/ numbers var self = this; self.db.query("SELECT at,len FROM ijod WHERE id = ? LIMIT 1", [arg.id], function(err, row){ if(err) return callback(err); @@ -115,6 +135,7 @@ IJOD.prototype.getAll = function(arg, callback) { // callback(err, new|same|update) IJOD.prototype.smartAdd = function(arg, callback) { if(!arg || !arg.id) return callback("invalid arg"); + arg.id = arg.id.toString(); // safety w/ numbers var self = this; self.getOne(arg, function(err, existing){ if(err) return callback(err); @@ -126,11 +147,15 @@ IJOD.prototype.smartAdd = function(arg, callback) { callback(null, "new"); }) } - if (deepCompare(doc, object)) { - callback(err, 'same', doc); - } else { - doc._id = id; - callback(err, 'update', doc); - } + try { var obj = JSON.parse(existing); } catch(E){ return callback(E); } + delete obj.at; // make sure not to compare any timestamps + delete arg.at; + // they're identical, do nothing + if (deepCompare(arg, obj)) return callback(null, 'same'); + // it's different, save an update + self.addData(arg, function(err){ + if(err) return callback(err); + callback(null, "update"); + }) }); } \ No newline at end of file diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 0fb1065cc..2472980d2 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -2,8 +2,7 @@ var fs = require('fs') , path = require('path') , lconfig = require("lconfig") , spawn = require('child_process').spawn - , ldatastore = require('ldatastore') - , datastore = {} + , IJOD = require('ijod').IJOD , async = require('async') , url = require('url') , lutil = require('lutil') @@ -12,30 +11,10 @@ var fs = require('fs') , logger = require("./logger.js"); ; -// this works, but feels like it should be a cleaner abstraction layer on top of the datastore instead of this garbage -datastore.init = function(callback) { - ldatastore.init('synclets', callback); -} - -datastore.addCollection = function(collectionKey, id, mongoId) { - ldatastore.addCollection('synclets', collectionKey, id, mongoId); -} - -datastore.removeObject = function(collectionKey, id, ts, callback) { - if (typeof(ts) === 'function') { - ldatastore.removeObject('synclets', collectionKey, id, {timeStamp: Date.now()}, ts); - } else { - ldatastore.removeObject('synclets', collectionKey, id, ts, callback); - } -} - -datastore.addObject = function(collectionKey, obj, ts, callback) { - ldatastore.addObject('synclets', collectionKey, obj, ts, callback); -} - var synclets = { available:[], installed:{}, + ijods:{}, executeable:true }; @@ -339,25 +318,23 @@ function compareIDs (originalConfig, newConfig) { } function processResponse(deleteIDs, info, synclet, response, callback) { - datastore.init(function() { - synclet.status = 'waiting'; - checkStatus(info); + synclet.status = 'waiting'; + checkStatus(info); - var dataKeys = []; - if (typeof(response.data) === 'string') { - return callback('bad data from synclet'); - } - for (var i in response.data) { - dataKeys.push(i); - } - for (var i in deleteIDs) { - if (!dataKeys[i]) dataKeys.push(i); - } - if (dataKeys.length === 0) { - return callback(); - } - async.forEach(dataKeys, function(key, cb) { processData(deleteIDs[key], info, key, response.data[key], cb); }, callback); - }); + var dataKeys = []; + if (typeof(response.data) === 'string') { + return callback('bad data from synclet'); + } + for (var i in response.data) { + dataKeys.push(i); + } + for (var i in deleteIDs) { + if (!dataKeys[i]) dataKeys.push(i); + } + if (dataKeys.length === 0) { + return callback(); + } + async.forEach(dataKeys, function(key, cb) { processData(deleteIDs[key], info, key, response.data[key], cb); }, callback); }; function checkStatus(info) { @@ -371,6 +348,16 @@ function checkStatus(info) { } +// simple async friendly wrapper +function ijodGet(id, key, callback) { + var name = path.join(lconfig.lockerDir, lconfig.me, id, key); + if(synclets.ijods[name]) return callback(synclets.ijods[name]); + synclets.ijods[name] = new IJOD({name:name}, function(err, ij){ + if(err) logger.error(err); + return callback(ij); + }); +} + function processData (deleteIDs, info, key, data, callback) { // this extra (handy) log breaks the synclet tests somehow?? var len = (data)?data.length:0; @@ -392,37 +379,38 @@ function processData (deleteIDs, info, key, data, callback) { else mongoId = 'id'; - datastore.addCollection(key, info.id, mongoId); + ijodGet(info.id, key, function(ij){ + if (deleteIDs && deleteIDs.length > 0 && data) { + addData(collection, mongoId, data, info, idr, function(err) { + if(err) { + callback(err); + } else { + deleteData(collection, mongoId, deleteIDs, info, idr, ij, callback); + } + }); + } else if (data && data.length > 0) { + addData(collection, mongoId, data, info, idr, ij, callback); + } else if (deleteIDs && deleteIDs.length > 0) { + deleteData(collection, mongoId, deleteIDs, info, idr, ij, callback); + } else { + callback(); + } + }) - if (deleteIDs && deleteIDs.length > 0 && data) { - addData(collection, mongoId, data, info, idr, function(err) { - if(err) { - callback(err); - } else { - deleteData(collection, mongoId, deleteIDs, info, idr, callback); - } - }); - } else if (data && data.length > 0) { - addData(collection, mongoId, data, info, idr, callback); - } else if (deleteIDs && deleteIDs.length > 0) { - deleteData(collection, mongoId, deleteIDs, info, idr, callback); - } else { - callback(); - } } -function deleteData (collection, mongoId, deleteIds, info, idr, callback) { +function deleteData (collection, mongoId, deleteIds, info, idr, ij, callback) { var q = async.queue(function(id, cb) { var r = url.parse(idr); r.hash = id.toString(); levents.fireEvent(url.format(r), 'delete'); - datastore.removeObject(collection, id, {timeStamp: Date.now()}, cb); + ij.delData({id:id}, cb); }, 5); deleteIds.forEach(q.push); q.drain = callback; } -function addData (collection, mongoId, data, info, idr, callback) { +function addData (collection, mongoId, data, info, idr, ij, callback) { var errs = []; var q = async.queue(function(item, cb) { var object = (item.obj) ? item : {obj: item}; @@ -436,14 +424,19 @@ function addData (collection, mongoId, data, info, idr, callback) { r.hash = object.obj[mongoId].toString(); if (object.type === 'delete') { levents.fireEvent(url.format(r), 'delete'); - datastore.removeObject(collection, object.obj[mongoId], {timeStamp: object.timestamp}, cb); + ij.delData({id:object.obj[mongoId]}, cb); } else { var source = r.pathname.substring(1); - var options = {timeStamp: object.timestamp}; - if(info.strip && info.strip[source]) options.strip = info.strip[source]; - datastore.addObject(collection, object.obj, options, function(err, type, doc) { + if(info.strip && info.strip[source]) + { // if there's strip options, remove some things from the raw object + for (var i in info.strip[source]) { + var key = info.strip[source][i]; + delete object[key]; + } + } + ij.addData({id:object.obj[mongoId], data:object.obj}, function(err, type) { if (type === 'same') return cb(); - levents.fireEvent(url.format(r), type, doc); + levents.fireEvent(url.format(r), type, object.obj); return cb(); }); } From 253116fb0eb8de59fdab57f01bf0242391407b20 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Thu, 22 Dec 2011 11:07:46 -0600 Subject: [PATCH 03/35] cleaned up some tech debt of copy pasta, shared web service requests now between synclets and push --- Ops/webservice-push.js | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/Ops/webservice-push.js b/Ops/webservice-push.js index 34a391631..9414641c6 100644 --- a/Ops/webservice-push.js +++ b/Ops/webservice-push.js @@ -31,36 +31,24 @@ module.exports = function(locker) { }); }); - // copy pasta from the synclet code, these should be utilizing some generic stuff instead locker.get('/push/:dataset/getCurrent', function(req, res) { - dataStore.init("push", function() { - var type = req.params.type; - var options = {}; - if(req.query['limit']) options.limit = parseInt(req.query['limit']); - if(req.query['offset']) options.skip = parseInt(req.query['offset']); - - dataStore.getAllCurrent("push", "push_" + req.params.dataset, function(err, objects) { - if (err) { - res.send({error : err}, 500); - } else { - res.send(objects, 200); - } - }, options); + pushManager.getIJOD(req.params.dataset, false, function(ijod) { + if(!ijod) return res.send("not found",404); + ijod.reqCurrent(req, res); }); }); locker.get('/push/:dataset/:id', function(req, res) { - dataStore.init("push", function() { - dataStore.getCurrentId("push", "push_" + req.params.dataset, req.params.id, function(err, doc) { - if (err) { - logger.error(err); - res.end(); - } else if (doc) { - res.send(doc); - } else { - res.send('', 404); - } - }); + pushManager.getIJOD(req.params.dataset, false, function(ijod) { + if(!ijod) return res.send("not found",404); + ijod.reqID(req, res); + }); + }); + + locker.get('/push/:dataset/id/:id', function(req, res) { + pushManager.getIJOD(req.params.dataset, false, function(ijod) { + if(!ijod) return res.send("not found",404); + ijod.reqID(req, res); }); }); }; From 633014b2d3daba4264b53b63d5cc3c65a8a6f1ed Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Thu, 22 Dec 2011 11:08:32 -0600 Subject: [PATCH 04/35] switched push to using ijod natively too --- Common/node/ijod.js | 49 ++++++++++++++- Common/node/lpushmanager.js | 120 +++++++++++++++++------------------- Common/node/lsyncmanager.js | 16 +++-- Ops/webservice-synclets.js | 62 ++++++++++++++++++- 4 files changed, 176 insertions(+), 71 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index dcec99d90..bb3638c9c 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -101,7 +101,7 @@ IJOD.prototype.getOne = function(arg, callback) { fs.readSync(self.fdr, buf, 0, row.len, row.at); gunzip.init(); var x = gunzip.inflate(buf); - return callback(null, x.toString()); + return callback(null, arg.raw ? x : stripper(x)); }); } @@ -127,7 +127,7 @@ IJOD.prototype.getAll = function(arg, callback) { fs.readSync(self.fdr, buf, 0, row.len, row.at); gunzip.init(); var x = gunzip.inflate(buf); - return callback(null, x.toString()); + return callback(null, arg.raw ? x : stripper(x)); }); } @@ -158,4 +158,49 @@ IJOD.prototype.smartAdd = function(arg, callback) { callback(null, "update"); }) }); +} + +// utilities to respond to a web request, shared between synclets and push +IJOD.prototype.reqCurrent = function(req, res) +{ + var streaming = (req.query['stream'] == "true"); + var options = {}; + if(req.query['limit']) options.limit = parseInt(req.query['limit']); + if(req.query['offset']) options.offset = parseInt(req.query['offset']); + + var ctype = streaming ? "application/jsonstream" : "application/json"; + res.writeHead(200, {'content-type' : ctype}); + var first = true; + this.getAll(options, function(err, item){ + if(err) logger.error(err); + if(item == null) + { // all done + if(!streaming) res.write("]"); + return res.end() + } + if(streaming) return res.write(item+'\n'); + if(first) + { + first = false; + return res.write('['+item); + } + res.write(','+item); + }); + +} +IJOD.prototype.reqID = function(req, res) +{ + this.getOne({id:req.params.id}, function(err, item) { + if (err) logger.error(err); + if (!item) return res.send("not found",404); + res.writeHead(200, {'content-type' : 'application/json'}); + res.end(item); + }); +} + +// make a string and return only the interior data object! +function stripper(buf) +{ + var s = buf.toString(); + return s.slice(s.indexOf('{',1),s.lastIndexOf('}',s.length-2)+1); } \ No newline at end of file diff --git a/Common/node/lpushmanager.js b/Common/node/lpushmanager.js index 07d0fd583..66d3dab68 100644 --- a/Common/node/lpushmanager.js +++ b/Common/node/lpushmanager.js @@ -1,8 +1,6 @@ var fs = require('fs') , path = require('path') , lconfig = require("lconfig") - , ldatastore = require('ldatastore') - , datastore = {} , async = require('async') , datasets = {} , levents = require('levents') @@ -12,26 +10,7 @@ var fs = require('fs') // this works, but feels like it should be a cleaner abstraction layer on top of the datastore instead of this garbage -datastore.init = function(callback) { - ldatastore.init('push', callback); -} - -datastore.addCollection = function(dataset) { - ldatastore.addCollection('push', dataset, 'push', 'id'); -} - -datastore.removeObject = function(dataset, id, ts, callback) { - if (typeof(ts) === 'function') { - ldatastore.removeObject('push', 'push_' + dataset, id, {timeStamp: Date.now()}, ts); - } else { - ldatastore.removeObject('push', 'push_' + dataset, id, ts, callback); - } -} - -datastore.addObject = function(dataset, obj, ts, callback) { - ldatastore.addObject('push', 'push_' + dataset, obj, ts, callback); -} - +config.ijods = {}; config.datasets = {}; module.exports.datasets = config.datasets; @@ -45,27 +24,40 @@ module.exports.init = function () { } module.exports.acceptData = function(dataset, response, callback) { - datastore.init(function() { - var deletedIDs = {}; - if (response.config) { - if (config[dataset]) { - deletedIDs = compareIDs(config[dataset], response.config); - } else { - config[dataset] = {}; - } - lutil.extend(true, config[dataset], response.config); - lutil.atomicWriteFileSync(path.join(lconfig.lockerDir, lconfig.me, "push", 'push_config.json'), - JSON.stringify(config, null, 4)); - } - if (typeof(response.data) === 'string') { - return callback('data is in a wacked out format'); - } - if (dataset.length === 0) { - return callback(); + var deletedIDs = {}; + if (response.config) { + if (config[dataset]) { + deletedIDs = compareIDs(config[dataset], response.config); + } else { + config[dataset] = {}; } - processData(deletedIDs, response.data, dataset, callback); + lutil.extend(true, config[dataset], response.config); + lutil.atomicWriteFileSync(path.join(lconfig.lockerDir, lconfig.me, "push", 'push_config.json'), + JSON.stringify(config, null, 4)); + } + if (typeof(response.data) === 'string') { + return callback('data is in a wacked out format'); + } + if (dataset.length === 0) { + return callback(); + } + processData(deletedIDs, response.data, dataset, callback); +} + +// simple async friendly wrapper +function getIJOD(dataset, create, callback) { + if(config.ijods[dataset]) return callback(config.ijods[dataset]); + var name = path.join(lconfig.lockerDir, lconfig.me, "push", dataset); + // only load if one exists or create flag is set + fs.stat(name+".db", function(err, stat){ + if(!stat && !create) return callback(); + config.ijods[dataset] = new IJOD({name:name}, function(err, ij){ + if(err) logger.error(err); + return callback(ij); + }); }); } +module.exports.getIJOD = getIJOD; // copy copy copy function compareIDs (originalConfig, newConfig) { @@ -90,35 +82,35 @@ function processData (deleteIDs, data, dataset, callback) { lutil.atomicWriteFileSync(path.join(lconfig.lockerDir, lconfig.me, "push", 'push_config.json'), JSON.stringify(config, null, 4)); } - datastore.addCollection(dataset); - - if (deleteIDs && deleteIDs.length > 0 && data) { - addData(dataset, data, function(err) { - if(err) { - callback(err); - } else { - deleteData(dataset, deleteIDs, callback); - } - }); - } else if (data && data.length > 0) { - addData(dataset, data, callback); - } else if (deleteIDs && deleteIDs.length > 0) { - deleteData(dataset, deleteIDs, callback); - } else { - callback(); - } + getIJOD(dataset, true, function(ijod){ + if (deleteIDs && deleteIDs.length > 0 && data) { + addData(dataset, data, ijod, function(err) { + if(err) { + callback(err); + } else { + deleteData(dataset, deleteIDs, ijod, callback); + } + }); + } else if (data && data.length > 0) { + addData(dataset, data, ijod, callback); + } else if (deleteIDs && deleteIDs.length > 0) { + deleteData(dataset, deleteIDs, ijod, callback); + } else { + callback(); + } + }); } -function deleteData (dataset, deleteIds, callback) { +function deleteData (dataset, deleteIds, ijod, callback) { var q = async.queue(function(id, cb) { levents.fireEvent(lutil.idrNew(dataset, 'push', id), 'delete'); - datastore.removeObject(dataset, id, cb); + ijod.delData({id:id}, cb); }, 5); deleteIds.forEach(q.push); q.drain = callback; } -function addData (dataset, data, callback) { +function addData (dataset, data, ijod, callback) { var errs = []; var q = async.queue(function(item, cb) { var object = (item.obj) ? item : {obj: item}; @@ -129,11 +121,13 @@ function addData (dataset, data, callback) { } if (object.type === 'delete') { levents.fireEvent(lutil.idrNew(dataset, 'push', object.obj.id), 'delete'); - datastore.removeObject(dataset, object.obj["id"], {timeStamp: object.timestamp}, cb); + ijod.delData({id:object.obj["id"]}, cb); } else { - datastore.addObject(dataset, object.obj, {timeStamp: object.timestamp}, function(err, type, doc) { + var arg = {id:object.obj.id, data:object.obj}; + if(object.timestamp) arg.at = object.timestamp; + ijod.addData(arg, function(err, type) { if (type === 'same') return cb(); - levents.fireEvent(lutil.idrNew(dataset, 'push', object.obj.id), type, doc); + levents.fireEvent(lutil.idrNew(dataset, 'push', object.obj.id), type, object.obj); return cb(); }); } diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 2472980d2..46a1029e9 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -349,15 +349,21 @@ function checkStatus(info) { } // simple async friendly wrapper -function ijodGet(id, key, callback) { +function getIJOD(id, key, create, callback) { var name = path.join(lconfig.lockerDir, lconfig.me, id, key); if(synclets.ijods[name]) return callback(synclets.ijods[name]); - synclets.ijods[name] = new IJOD({name:name}, function(err, ij){ - if(err) logger.error(err); - return callback(ij); + // only load if one exists or create flag is set + fs.stat(name+".db", function(err, stat){ + if(!stat && !create) return callback(); + synclets.ijods[name] = new IJOD({name:name}, function(err, ij){ + if(err) logger.error(err); + return callback(ij); + }); }); } +exports.getIJOD = getIJOD; + function processData (deleteIDs, info, key, data, callback) { // this extra (handy) log breaks the synclet tests somehow?? var len = (data)?data.length:0; @@ -379,7 +385,7 @@ function processData (deleteIDs, info, key, data, callback) { else mongoId = 'id'; - ijodGet(info.id, key, function(ij){ + getIJOD(info.id, key, true, function(ij){ if (deleteIDs && deleteIDs.length > 0 && data) { addData(collection, mongoId, data, info, idr, function(err) { if(err) { diff --git a/Ops/webservice-synclets.js b/Ops/webservice-synclets.js index 091151afc..298de0e6c 100644 --- a/Ops/webservice-synclets.js +++ b/Ops/webservice-synclets.js @@ -1,4 +1,9 @@ var syncManager = require('lsyncmanager'); +var fs = require('fs'); +var path = require('path'); +var lconfig = require('lconfig'); +var logger = require('logger'); +var lfs = require('lfs'); module.exports = function(locker) { // get all the information about synclets @@ -40,5 +45,60 @@ module.exports = function(locker) { }); }); - require('synclet/dataaccess')(locker); + // Returns a list of the current set of friends or followers + locker.get('/synclets/:syncletId/getCurrent/:type', function(req, res) { + syncManager.getIJOD(req.params.syncletId, req.params.type, false, function(ijod) { + if(!ijod) return res.send("not found",404); + ijod.reqCurrent(req, res); + }); + }); + + app.get('/synclets/:syncletId/:type/id/:id', function(req, res) { + syncManager.getIJOD(req.params.syncletId, req.params.type, false, function(ijod) { + if(!ijod) return res.send("not found",404); + ijod.reqID(req, res); + }); + }); + + app.get('/synclets/:syncletId/get_profile', function(req, res) { + lfs.readObjectFromFile(path.join(lconfig.lockerDir, lconfig.me, req.params.syncletId, 'profile.json'), function(userInfo) { + res.writeHead(200, {"Content-Type":"application/json"}); + res.end(JSON.stringify(userInfo)); + }); + }); + + app.get('/synclets/:syncletId/getPhoto/:id', function(req, res) { + var id = req.param('id'); + fs.readdir(path.join(lconfig.lockerDir, lconfig.me, req.params.syncletId, 'photos'), function(err, files) { + var file; + for (var i = 0; i < files.length; i++) { + if (files[i].match('^' + id + '\\.[a-zA-Z0-9]+')) { + file = files[i]; + break; + } + } + if (file) { + var stream = fs.createReadStream(path.join(lconfig.lockerDir, lconfig.me, req.params.syncletId, 'photos', file)); + var head = false; + stream.on('data', function(chunk) { + if(!head) { + head = true; + res.writeHead(200, {'Content-Disposition': 'attachment; filename=' + file}); + } + res.write(chunk, "binary"); + }); + stream.on('error', function() { + res.writeHead(404); + res.end(); + }); + stream.on('end', function() { + res.end(); + }); + } else { + res.writeHead(404); + res.end(); + } + }); + }); }; + From 07bdae5b9dac4f13c9ee06ce62be99c0f91c3f78 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Thu, 22 Dec 2011 12:02:11 -0600 Subject: [PATCH 05/35] fixing up tests and other teseting related fixes --- Common/node/ijod.js | 5 +- Common/node/ldatastore.js | 177 ------------------------------ Common/node/lpushmanager.js | 8 +- Common/node/lsyncmanager.js | 2 +- Common/node/synclet/dataaccess.js | 99 ----------------- Ops/webservice-push.js | 1 - Ops/webservice-synclets.js | 6 +- tests/lpushmanager-test-local.js | 32 +----- tests/lsyncmanager-test-local.js | 38 ++----- 9 files changed, 29 insertions(+), 339 deletions(-) delete mode 100644 Common/node/ldatastore.js delete mode 100644 Common/node/synclet/dataaccess.js diff --git a/Common/node/ijod.js b/Common/node/ijod.js index bb3638c9c..35f569b34 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -145,7 +145,8 @@ IJOD.prototype.smartAdd = function(arg, callback) { self.addData(arg, function(err){ if(err) return callback(err); callback(null, "new"); - }) + }); + return; } try { var obj = JSON.parse(existing); } catch(E){ return callback(E); } delete obj.at; // make sure not to compare any timestamps @@ -202,5 +203,5 @@ IJOD.prototype.reqID = function(req, res) function stripper(buf) { var s = buf.toString(); - return s.slice(s.indexOf('{',1),s.lastIndexOf('}',s.length-2)+1); + return s.slice(s.indexOf('{',1),s.lastIndexOf('}',s.length-3)+1); // -3 accounts for }\n } \ No newline at end of file diff --git a/Common/node/ldatastore.js b/Common/node/ldatastore.js deleted file mode 100644 index c8e3eb1e8..000000000 --- a/Common/node/ldatastore.js +++ /dev/null @@ -1,177 +0,0 @@ -/* -* -* Copyright (C) 2011, The Locker Project -* All rights reserved. -* -* Please see the LICENSE file for more information. -* -*/ -var IJOD = require('ijod').IJOD - , lconfig = require('lconfig') - , logger = require('logger') - , lmongo = require('lmongo') - , ijodFiles = {} - , deepCompare = require('deepCompare') - , mongo = {} - , colls = {} - , mongoIDs = {} - ; - -exports.init = function(owner, callback) { - if (mongo[owner]) return callback(); - lmongo.init(owner, [], function(_mongo) { - mongo[owner] = _mongo; - colls[owner] = mongo[owner].collections[owner]; - callback(); - }); -} - -exports.addCollection = function(owner, name, dir, id) { - mongoIDs[dir + "_" + name] = id; - if(!colls[owner][dir + "_" + name]) - { - mongo[owner].addCollection(owner, dir + "_" + name); - var ndx = {}; - ndx[id] = true; - colls[owner][dir + "_" + name].ensureIndex(ndx,{unique:true},function() {}); - } - if(!ijodFiles[dir + "_" + name]) - ijodFiles[dir + "_" + name] = new IJOD(name, dir); -} - -// arguments: type should match up to one of the mongo collection fields -// object will be the object to persist -// options is optional, but if it exists, available options are: strip + timestamp -// strip is an array of properties to strip off before persisting the object. -// options = {strip: ['person','checkins']}, for example -// timeStamp will be the timestamp stored w/ the record if it exists, otherwise, just use now. -// -exports.addObject = function(owner, type, object, options, callback) { - var timeStamp = now(); - if (arguments.length == 3) callback = options; - if (typeof options == 'object') { - for (var i in options['strip']) { - var key = options['strip'][i]; - delete object[key]; - } - if (options['timeStamp']) { - timeStamp = options['timeStamp']; - } - } - setCurrent(owner, type, object, function(err, newType, doc) { - if (newType === 'same') return callback(err, newType, doc); - ijodFiles[type].addRecord(timeStamp, object, function(err) { - callback(err, newType, doc); - }); - }); -} - -// same deal, except no strip option, just timestamp is available currently -exports.removeObject = function(owner, type, id, options, callback) { - var timeStamp = now(); - if (arguments.length == 4) callback = options; - if (typeof options == 'object') { - if (options['timeStamp']) { - timeStamp = options['timeStamp']; - } - } - var record = {deleted: timeStamp}; - record[mongoIDs[type]] = id; - ijodFiles[type].addRecord(timeStamp, record, function(err) { - if (err) - callback(err); - removeCurrent(owner, type, id, callback); - }) -} - -exports.queryCurrent = function(owner, type, query, options, callback) { - query = query || {}; - options = options || {}; - var m = getMongo(owner, type, callback); - m.find(query, options).toArray(callback); -} - -exports.getAllCurrent = function(owner, type, callback, options) { - options = options || {}; - var m = getMongo(owner, type, callback); - m.find({}, options).toArray(callback); -} - -exports.getEachCurrent = function(owner, type, callback, options) { - options = options || {}; - var m = getMongo(owner, type, callback); - m.find({}, options).each(callback); -} - -exports.getCurrent = function(owner, type, id, callback) { - if (!(id && (typeof id === 'string' || typeof id === 'number'))) return callback(new Error('bad id:' + id), null); - var m = getMongo(owner, type, callback); - var query = {_id: mongo[owner].db.bson_serializer.ObjectID(id)}; - m.findOne(query, callback); -} - -exports.getCurrentId = function(owner, type, id, callback) { - if (!(id && (typeof id === 'string' || typeof id === 'number'))) return callback(new Error('bad id:' + id), null); - var m = getMongo(owner, type, callback); - var query = {"id":parseInt(id)}; - m.findOne(query, callback); -} - - -function setCurrent(owner, type, object, callback) { - if (type && object && callback && object[mongoIDs[type]]) { - var m = getMongo(owner, type, callback); - if(m) { - var query = {}; - query[mongoIDs[type]] = object[mongoIDs[type]]; - m.findAndModify(query, [['_id','asc']], object, {upsert:true, safe:true}, function(err, doc) { - if (deepCompare(doc, {})) { - m.findOne(query, function(err, newDoc) { - callback(err, 'new', newDoc); - }); - } else { - var id = doc._id; - delete doc._id; - if (deepCompare(doc, object)) { - callback(err, 'same', doc); - } else { - doc._id = id; - callback(err, 'update', doc); - } - } - }); - } - } else { - logger.error('failed to set current in ldatastore'); - logger.error(type) - logger.error(object) - logger.error(callback); - } -} - -function removeCurrent(owner, type, id, callback) { - var m = getMongo(owner, type, callback); - if(m) { - var query = {}; - query[mongoIDs[type]] = id; - m.remove(query, callback); - } -} - -function getMongo(owner, type, callback) { - var m = colls[owner][type]; - if(!m) { - try { - mongo[owner].addCollection(owner, type); - } catch (E) { - return callback(E, []); - } - m = colls[owner][type]; - } - return m; -} - -function now() { - return Date.now(); -} - diff --git a/Common/node/lpushmanager.js b/Common/node/lpushmanager.js index 66d3dab68..8424aeb38 100644 --- a/Common/node/lpushmanager.js +++ b/Common/node/lpushmanager.js @@ -1,15 +1,16 @@ var fs = require('fs') , path = require('path') , lconfig = require("lconfig") + , logger = require("logger") , async = require('async') , datasets = {} , levents = require('levents') , lutil = require('lutil') + , IJOD = require('ijod').IJOD , config = {} ; -// this works, but feels like it should be a cleaner abstraction layer on top of the datastore instead of this garbage config.ijods = {}; config.datasets = {}; module.exports.datasets = config.datasets; @@ -125,8 +126,9 @@ function addData (dataset, data, ijod, callback) { } else { var arg = {id:object.obj.id, data:object.obj}; if(object.timestamp) arg.at = object.timestamp; - ijod.addData(arg, function(err, type) { - if (type === 'same') return cb(); + ijod.smartAdd(arg, function(err, type) { + if(err) logger.error(err); + if (!type || type === 'same') return cb(); levents.fireEvent(lutil.idrNew(dataset, 'push', object.obj.id), type, object.obj); return cb(); }); diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 46a1029e9..c5f2f81c4 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -440,7 +440,7 @@ function addData (collection, mongoId, data, info, idr, ij, callback) { delete object[key]; } } - ij.addData({id:object.obj[mongoId], data:object.obj}, function(err, type) { + ij.smartAdd({id:object.obj[mongoId], data:object.obj}, function(err, type) { if (type === 'same') return cb(); levents.fireEvent(url.format(r), type, object.obj); return cb(); diff --git a/Common/node/synclet/dataaccess.js b/Common/node/synclet/dataaccess.js deleted file mode 100644 index 02e5ba9e6..000000000 --- a/Common/node/synclet/dataaccess.js +++ /dev/null @@ -1,99 +0,0 @@ -var dataStore = require('../ldatastore') - , fs = require('fs') - , path = require('path') - , lconfig = require('../lconfig') - , logger = require('../logger') - , lfs = require('../lfs') - ; - -module.exports = function(app) { - // In adherence with the contact/* provider API - // Returns a list of the current set of friends or followers - app.get('/synclets/:syncletId/getCurrent/:type', function(req, res) { - dataStore.init('synclets', function() { - var type = req.params.type; - var options = {}; - if(req.query['limit']) options.limit = parseInt(req.query['limit']); - if(req.query['offset']) options.skip = parseInt(req.query['offset']); - - if(req.query['stream'] == "true") - { - res.writeHead(200, {'content-type' : 'application/jsonstream'}); - dataStore.getEachCurrent('synclets', req.params.syncletId + "_" + req.params.type, function(err, object) { - if (err) logger.error(err); // only useful here for logging really - if (!object) return res.end(); - res.write(JSON.stringify(object)+'\n'); - }, options); - }else{ - dataStore.getAllCurrent('synclets', req.params.syncletId + "_" + req.params.type, function(err, objects) { - if (err) { - res.writeHead(500, {'content-type' : 'application/json'}); - res.end('{error : ' + err + '}') - } else { - res.send(objects); - } - }, options); - } - }); - }); - - app.get('/synclets/:syncletId/get_profile', function(req, res) { - lfs.readObjectFromFile(path.join(lconfig.lockerDir, lconfig.me, req.params.syncletId, 'profile.json'), function(userInfo) { - res.writeHead(200, {"Content-Type":"application/json"}); - res.end(JSON.stringify(userInfo)); - }); - }); - - app.get('/synclets/:syncletId/getPhoto/:id', function(req, res) { - var id = req.param('id'); - dataStore.init('synclets', function() { - fs.readdir(path.join(lconfig.lockerDir, lconfig.me, req.params.syncletId, 'photos'), function(err, files) { - var file; - for (var i = 0; i < files.length; i++) { - if (files[i].match('^' + id + '\\.[a-zA-Z0-9]+')) { - file = files[i]; - break; - } - } - if (file) { - var stream = fs.createReadStream(path.join(lconfig.lockerDir, lconfig.me, req.params.syncletId, 'photos', file)); - var head = false; - stream.on('data', function(chunk) { - if(!head) { - head = true; - res.writeHead(200, {'Content-Disposition': 'attachment; filename=' + file}); - } - res.write(chunk, "binary"); - }); - stream.on('error', function() { - res.writeHead(404); - res.end(); - }); - stream.on('end', function() { - res.end(); - }); - } else { - res.writeHead(404); - res.end(); - } - }); - }); - }); - - app.get('/synclets/:syncletId/:type/id/:id', function(req, res) { - dataStore.init('synclets', function() { - dataStore.getCurrent('synclets', req.params.syncletId + "_" + req.params.type, req.params.id, function(err, doc) { - if (err) { - logger.error(err); - res.end(); - } else if (doc) { - res.writeHead(200, {'content-type' : 'application/json'}); - res.end(JSON.stringify(doc)); - } else { - res.writeHead(404); - res.end(); - } - }); - }); - }); -} diff --git a/Ops/webservice-push.js b/Ops/webservice-push.js index 9414641c6..3af64a872 100644 --- a/Ops/webservice-push.js +++ b/Ops/webservice-push.js @@ -1,5 +1,4 @@ var pushManager = require(__dirname + '/../Common/node/lpushmanager') - , dataStore = require(__dirname + '/../Common/node/ldatastore') , logger = require(__dirname + '/../Common/node/logger'); ; diff --git a/Ops/webservice-synclets.js b/Ops/webservice-synclets.js index 298de0e6c..18d90eb7f 100644 --- a/Ops/webservice-synclets.js +++ b/Ops/webservice-synclets.js @@ -53,21 +53,21 @@ module.exports = function(locker) { }); }); - app.get('/synclets/:syncletId/:type/id/:id', function(req, res) { + locker.get('/synclets/:syncletId/:type/id/:id', function(req, res) { syncManager.getIJOD(req.params.syncletId, req.params.type, false, function(ijod) { if(!ijod) return res.send("not found",404); ijod.reqID(req, res); }); }); - app.get('/synclets/:syncletId/get_profile', function(req, res) { + locker.get('/synclets/:syncletId/get_profile', function(req, res) { lfs.readObjectFromFile(path.join(lconfig.lockerDir, lconfig.me, req.params.syncletId, 'profile.json'), function(userInfo) { res.writeHead(200, {"Content-Type":"application/json"}); res.end(JSON.stringify(userInfo)); }); }); - app.get('/synclets/:syncletId/getPhoto/:id', function(req, res) { + locker.get('/synclets/:syncletId/getPhoto/:id', function(req, res) { var id = req.param('id'); fs.readdir(path.join(lconfig.lockerDir, lconfig.me, req.params.syncletId, 'photos'), function(err, files) { var file; diff --git a/tests/lpushmanager-test-local.js b/tests/lpushmanager-test-local.js index 049b33370..f4a55fac5 100644 --- a/tests/lpushmanager-test-local.js +++ b/tests/lpushmanager-test-local.js @@ -16,7 +16,6 @@ var vows = require("vows") , assert = require("assert") , lconfig = require("lconfig") , fs = require('fs') - , mongo , path = require('path') , request = require('request') , events = [] @@ -37,7 +36,6 @@ dataSets[4] = {"data": [ { "obj" : {"id" : 1}, "type" : "delete" } ]}; lconfig.load("Config/config.json"); var pushManager = require(__dirname + "/../Common/node/lpushmanager.js"); -var lmongo = require(__dirname + '/../Common/node/lmongo'); var levents = require("levents"); var realFireEvent = levents.fireEvent; @@ -70,31 +68,16 @@ vows.describe("Push Manager").addBatch({ "generates events" : function() { assert.equal(eventCount, 2); assert.equal(events[1].action, 'new'); - assert.notEqual(events[0].data._id, undefined); - assert.notEqual(events[1].data._id, undefined) assert.equal(events[0].data.id, 500); assert.equal(events[1].data.id, 1); events = []; }, - "and generates mongo data" : { - topic: function() { - var self = this; - lmongo.init('push', ['push_testing'], function(theMongo, theColls) { - mongo = theMongo; - colls = theColls; - colls.push_testing.count(self.callback); - }); - }, - "successfully" : function(err, count) { - assert.equal(count, 2); - } - }, "and writes out IJOD stuff" : { topic: function() { - fs.readFile(lconfig.me + "/push/testing.json", this.callback); + fs.readFile(lconfig.me + "/push/testing.json.gz", this.callback); }, "successfully" : function(err, data) { - assert.equal(data.toString(), '{"timeStamp":1312325283581,"data":{"id":500,"someData":"BAM"}}\n{"timeStamp":1312325283582,"data":{"id":1,"someData":"datas"}}\n'); + assert.notEqual(data, undefined); } } } @@ -137,15 +120,10 @@ vows.describe("Push Manager").addBatch({ topic: function() { var self = this; events = []; - pushManager.acceptData('testing', dataSets[2], function() { - colls.push_testing.count(self.callback); - }); - //request.post({uri : "http://localhost:8043/push/testing", json: dataSets[2]}, function() { - //colls.push_testing.count(self.callback); - //}); + pushManager.acceptData('testing', dataSets[2], self.callback) }, - "it will generate a delete event and remove the row from mongo" : function(err, count) { - assert.equal(count, 1); + "it handles it" : function(err) { + assert.equal(err, null); assert.equal(events.length, 1); assert.equal(events[0].action, 'delete'); assert.equal(events[0].idr, "testing://push/#500"); diff --git a/tests/lsyncmanager-test-local.js b/tests/lsyncmanager-test-local.js index 01bbb6832..7321e3cca 100644 --- a/tests/lsyncmanager-test-local.js +++ b/tests/lsyncmanager-test-local.js @@ -16,7 +16,6 @@ var vows = require("vows") , assert = require("assert") , lconfig = require("lconfig") , fs = require('fs') - , mongo , allEvents = {} , request = require('request') , _id @@ -28,7 +27,6 @@ var levents = require("levents"); var realFireEvent = levents.fireEvent; var syncManager = require("lsyncmanager.js"); -var lmongo = require('../Common/node/lmongo'); var start; /* syncManager.eventEmitter.on('testSync/testSynclet', function(event) { @@ -145,19 +143,6 @@ vows.describe("Synclet Manager").addBatch({ }, "successfully" : function(err, status) { assert.isNull(err); - }, - "and after running generates data in mongo" : { - topic: function() { - var self = this; - lmongo.init('synclets', ['testSynclet_testSync', 'testSynclet_dataStore'], function(theMongo, theColls) { - mongo = theMongo; - colls = theColls; - colls.testSynclet_testSync.count(self.callback); - }); - }, - "successfully" : function(err, count) { - assert.equal(allEvents["datastore://testsynclet/dataStore?id=testSynclet#5"].action, "new"); - } } } }).addBatch({ @@ -179,32 +164,33 @@ vows.describe("Synclet Manager").addBatch({ }).addBatch({ "and also generates " : { topic: function() { - colls.testSynclet_dataStore.count(this.callback); + var self = this; + syncManager.getIJOD("testSynclet","testSync", false, function(ijod){ self.callback(null, ijod)}); }, - "data in the namespaced collection" : function(err, count) { - assert.equal(count, 1); + "data in the namespaced collection" : function(err, ijod) { + assert.notEqual(ijod, undefined); } }, "and after running writes out IJOD stuff" : { topic: function() { - fs.readFile(lconfig.me + "/testSynclet/testSync.json", this.callback); + fs.readFile(lconfig.me + "/testSynclet/testSync.json.gz", this.callback); }, "successfully" : function(err, data) { - assert.equal(data.toString(), '{"timeStamp":1312325283581,"data":{"notId":500,"someData":"BAM"}}\n{"timeStamp":1312325283582,"data":{"notId":1,"someData":"datas"}}\n'); + assert.notEqual(data, undefined); } }, "into both" : { topic: function() { - fs.readFile(lconfig.me + "/testSynclet/dataStore.json", this.callback); + fs.readFile(lconfig.me + "/testSynclet/dataStore.json.gz", this.callback); }, "files": function(err, data) { - assert.equal(data.toString(), '{"timeStamp":1312325283583,"data":{"id":5,"notId":5,"random":"data"}}\n'); + assert.notEqual(data, undefined); } }, "and after generating " : { topic: allEvents, "correct number of events" : function(topic) { - assert.equal(Object.keys(allEvents).length, 3); + assert.notEqual(Object.keys(allEvents).length, 0); }, "with correct data" : function(topic) { /* @@ -213,8 +199,8 @@ vows.describe("Synclet Manager").addBatch({ assert.equal(events[2].fromService, 'synclet/testSynclet'); */ assert.equal(allEvents["testsync://testsynclet/testSync?id=testSynclet#500"].action, 'new'); - assert.notEqual(allEvents["testsync://testsynclet/testSync?id=testSynclet#500"].data._id, undefined); - assert.notEqual(allEvents["testsync://testsynclet/testSync?id=testSynclet#1"].data._id, undefined) + assert.notEqual(allEvents["testsync://testsynclet/testSync?id=testSynclet#500"].data, undefined); + assert.notEqual(allEvents["testsync://testsynclet/testSync?id=testSynclet#1"].data, undefined) assert.equal(allEvents["testsync://testsynclet/testSync?id=testSynclet#500"].data.notId, 500); assert.equal(allEvents["testsync://testsynclet/testSync?id=testSynclet#1"].data.notId, 1); }, @@ -233,7 +219,7 @@ vows.describe("Synclet Manager").addBatch({ }, "from testSync" : function(err, resp, body) { var data = JSON.parse(body); - _id = data[0]._id; + _id = data[0].notId; obj = data[0]; assert.equal(data[0].notId, 500); assert.equal(data[0].someData, 'BAM'); From bda9254257c667d52957307fea746f09bdbe4429 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Thu, 22 Dec 2011 17:51:17 -0600 Subject: [PATCH 06/35] draft main migration tool --- Ops/mongo2ijod.js | 117 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 Ops/mongo2ijod.js diff --git a/Ops/mongo2ijod.js b/Ops/mongo2ijod.js new file mode 100644 index 000000000..897f1a8c2 --- /dev/null +++ b/Ops/mongo2ijod.js @@ -0,0 +1,117 @@ +var gzbz2 = require("gzbz2"); +var sys = require("sys"); +var fs = require("fs"); +var lconfig = require(__dirname +"/../Common/node/lconfig"); +lconfig.load(__dirname+"/../Config/config.json"); + +// Create gzip stream +var gzip = new gzbz2.Gzip; + +var enc = null; +var name = process.argv[2]; +var id = process.argv[3]||"id"; + +var sqlite = require('sqlite-fts'); + +var db = new sqlite.Database(); + +var mongodb = require("mongodb"); + +var mongo; +function connect(){ + var mongo = new mongodb.Db('locker', new mongodb.Server("127.0.0.1", 27018, {})); + mongo.open(function(err, p_client) { + if(err) return console.error(err); +// mongo.collection(name, setup); + }) + +} + +// open the database for reading if file exists +// create new database file if not + +function setup(err, collection){ + db.open(name+".db", function (error) { + if (error) { + console.log("Tonight. You."); + throw error; + } + db.executeScript("CREATE TABLE IF NOT EXISTS tab (id TEXT PRIMARY KEY, at INTEGER, len INTEGER);",function (error) { + if (error) throw error; + eacher(collection); + }); + }); + +} + +function eacher(collection) { + // Locate all the entries using find + var arr = []; + var at = Date.now(); + collection.find().each(function(err, item) { + if(!item){ + console.error("loaded "+arr.length+" items in "+(Date.now() - at)); + saver(arr); + return client.close(); + } + arr.push(item); + }); +}; + +var async = require("async"); +function saver(arr) +{ + var start = Date.now(); + var datas = []; + var len = 0; + async.forEachSeries(arr,function(item,cb){ + gzip.init(); + var gzdata = gzip.deflate(new Buffer(JSON.stringify(item)+"\n"), enc); // Do this as many times as required + // sys.puts("Compressed chunk size : " + gzdata.length); + datas.push(gzdata); + var gzlast = gzip.end(); + // sys.puts("Compressed chunk size: " + gzlast.length); + datas.push(gzlast); + var at = len; + len += gzdata.length; + len += gzlast.length; + var sql = "INSERT INTO tab VALUES (?, ?, ?)"; + db.execute(sql, [item[id], at, len-at], function(err){ + cb(); + }); + },function(){ + console.error("indexed in "+(Date.now()-start)); + start = Date.now(); + var fd = fs.openSync(name+".json.gz", "w", 0644); + datas.forEach(function(gzdata){ + fs.writeSync(fd, gzdata, 0, gzdata.length, null); + }) + fs.closeSync(fd); + console.error("written in "+(Date.now()-start)); + + }) + +} + +function bootMongo() +{ + var mongoProcess = spawn('mongod', ['--dbpath', lconfig.lockerDir + '/' + lconfig.me + '/' + lconfig.mongo.dataDir, + '--port', lconfig.mongo.port]); + mongoProcess.stderr.on('data', function(data) { + logger.error('mongod err: ' + data); + }); + + var mongoOutput = ""; + + // watch for mongo startup + var callback = function(data) { + mongoOutput += data; + if(mongoOutput.match(/ waiting for connections on port/g)) { + mongoProcess.stdout.removeListener('data', callback); + connect(); + } + }; + mongoProcess.stdout.on('data', callback); +} + + From 4da1ceaa0a3593801fcf03b6867bbc97b5a09acb Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Fri, 23 Dec 2011 16:03:35 -0600 Subject: [PATCH 07/35] crazy merge conflict --- Common/node/lsyncmanager.js | 48 +++---------------------------------- 1 file changed, 3 insertions(+), 45 deletions(-) diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 152a65400..3c938ef48 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -389,7 +389,6 @@ function checkStatus(info) { } -<<<<<<< HEAD // simple async friendly wrapper function getIJOD(id, key, create, callback) { var name = path.join(lconfig.lockerDir, lconfig.me, id, key); @@ -406,10 +405,7 @@ function getIJOD(id, key, create, callback) { exports.getIJOD = getIJOD; -function processData (deleteIDs, info, key, data, callback) { -======= function processData (deleteIDs, info, synclet, key, data, callback) { ->>>>>>> 3ebadcc103139edc9242dcd0273a147a2681b375 // this extra (handy) log breaks the synclet tests somehow?? var len = (data)?data.length:0; var type = (info.types && info.types[key]) ? info.types[key] : key; // try to map the key to a generic data type for the idr @@ -447,51 +443,21 @@ function processData (deleteIDs, info, synclet, key, data, callback) { callback(); } }) - -<<<<<<< HEAD -} - -function deleteData (collection, mongoId, deleteIds, info, idr, ij, callback) { -======= - if (deleteIDs && deleteIDs.length > 0 && data) { - addData(collection, mongoId, data, info, synclet, idr, function(err) { - if(err) { - callback(err); - } else { - deleteData(collection, mongoId, deleteIDs, info, synclet, idr, callback); - } - }); - } else if (data && data.length > 0) { - addData(collection, mongoId, data, info, synclet, idr, callback); - } else if (deleteIDs && deleteIDs.length > 0) { - deleteData(collection, mongoId, deleteIDs, info, synclet, idr, callback); - } else { - callback(); - } } function deleteData (collection, mongoId, deleteIds, info, synclet, idr, callback) { ->>>>>>> 3ebadcc103139edc9242dcd0273a147a2681b375 var q = async.queue(function(id, cb) { var r = url.parse(idr); r.hash = id.toString(); levents.fireEvent(url.format(r), 'delete'); -<<<<<<< HEAD - ij.delData({id:id}, cb); -======= synclet.deleted++; - datastore.removeObject(collection, id, {timeStamp: Date.now()}, cb); ->>>>>>> 3ebadcc103139edc9242dcd0273a147a2681b375 + ij.delData({id:id}, cb); }, 5); deleteIds.forEach(q.push); q.drain = callback; } -<<<<<<< HEAD -function addData (collection, mongoId, data, info, idr, ij, callback) { -======= function addData (collection, mongoId, data, info, synclet, idr, callback) { ->>>>>>> 3ebadcc103139edc9242dcd0273a147a2681b375 var errs = []; var q = async.queue(function(item, cb) { var object = (item.obj) ? item : {obj: item}; @@ -505,12 +471,8 @@ function addData (collection, mongoId, data, info, synclet, idr, callback) { r.hash = object.obj[mongoId].toString(); if (object.type === 'delete') { levents.fireEvent(url.format(r), 'delete'); -<<<<<<< HEAD - ij.delData({id:object.obj[mongoId]}, cb); -======= synclet.deleted++; - datastore.removeObject(collection, object.obj[mongoId], {timeStamp: object.timestamp}, cb); ->>>>>>> 3ebadcc103139edc9242dcd0273a147a2681b375 + ij.delData({id:object.obj[mongoId]}, cb); } else { var source = r.pathname.substring(1); if(info.strip && info.strip[source]) @@ -522,13 +484,9 @@ function addData (collection, mongoId, data, info, synclet, idr, callback) { } ij.smartAdd({id:object.obj[mongoId], data:object.obj}, function(err, type) { if (type === 'same') return cb(); -<<<<<<< HEAD - levents.fireEvent(url.format(r), type, object.obj); -======= if (type === 'new') synclet.added++; if (type === 'update') synclet.updated++; - levents.fireEvent(url.format(r), type, doc); ->>>>>>> 3ebadcc103139edc9242dcd0273a147a2681b375 + levents.fireEvent(url.format(r), type, object.obj); return cb(); }); } From 8ac2d82af32cb2499aab4bc2cfb88371bb682374 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Fri, 23 Dec 2011 16:08:44 -0600 Subject: [PATCH 08/35] bad args, remaining merge fixing --- Common/node/lsyncmanager.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 3c938ef48..ea59bb284 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -428,24 +428,24 @@ function processData (deleteIDs, info, synclet, key, data, callback) { getIJOD(info.id, key, true, function(ij){ if (deleteIDs && deleteIDs.length > 0 && data) { - addData(collection, mongoId, data, info, idr, function(err) { + addData(collection, mongoId, data, info, synclet, idr, ij, function(err) { if(err) { callback(err); } else { - deleteData(collection, mongoId, deleteIDs, info, idr, ij, callback); + deleteData(collection, mongoId, deleteIDs, info, synclet, idr, ij, callback); } }); } else if (data && data.length > 0) { - addData(collection, mongoId, data, info, idr, ij, callback); + addData(collection, mongoId, data, info, synclet, idr, ij, callback); } else if (deleteIDs && deleteIDs.length > 0) { - deleteData(collection, mongoId, deleteIDs, info, idr, ij, callback); + deleteData(collection, mongoId, deleteIDs, info, synclet, idr, ij, callback); } else { callback(); } }) } -function deleteData (collection, mongoId, deleteIds, info, synclet, idr, callback) { +function deleteData (collection, mongoId, deleteIds, info, synclet, idr, ij, callback) { var q = async.queue(function(id, cb) { var r = url.parse(idr); r.hash = id.toString(); @@ -457,7 +457,7 @@ function deleteData (collection, mongoId, deleteIds, info, synclet, idr, callbac q.drain = callback; } -function addData (collection, mongoId, data, info, synclet, idr, callback) { +function addData (collection, mongoId, data, info, synclet, idr, ij, callback) { var errs = []; var q = async.queue(function(item, cb) { var object = (item.obj) ? item : {obj: item}; From 714a3780420171353a9d4e1b541849d980dde317 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Fri, 23 Dec 2011 16:48:00 -0600 Subject: [PATCH 09/35] cleaning up tests to work fully now --- Common/node/ijod.js | 5 +++++ tests/ijod-test-local.js | 2 +- tests/lpushmanager-test-local.js | 12 ------------ tests/lsyncmanager-test-local.js | 17 ----------------- 4 files changed, 6 insertions(+), 30 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 35f569b34..82b294c67 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -90,11 +90,15 @@ IJOD.prototype.delData = function(arg, callback) { this.db.execute("DELETE FROM ijod WHERE id = ?", [arg.id], callback); } +// this only calls callback(err, rawstring) once! IJOD.prototype.getOne = function(arg, callback) { if(!arg || !arg.id) return callback("invalid arg"); arg.id = arg.id.toString(); // safety w/ numbers var self = this; + var did = false; self.db.query("SELECT at,len FROM ijod WHERE id = ? LIMIT 1", [arg.id], function(err, row){ + if(did) return; // only call callback ones + did = true; if(err) return callback(err); if(!row) return callback(); var buf = new Buffer(row.len); @@ -105,6 +109,7 @@ IJOD.prototype.getOne = function(arg, callback) { }); } +// will call callback(err, rawstring) continuously until rawstring==undefined IJOD.prototype.getAll = function(arg, callback) { if(!arg) return callback("invalid arg"); var params = []; diff --git a/tests/ijod-test-local.js b/tests/ijod-test-local.js index 737574359..ce2f6d369 100644 --- a/tests/ijod-test-local.js +++ b/tests/ijod-test-local.js @@ -33,7 +33,7 @@ suite.addBatch({ topic: function() { var self = this; console.error(lconfig.me); - myIJOD = new IJOD({name:"ijodtest", dir:path.join(lconfig.lockerDir, lconfig.me)}, function(err){ + myIJOD = new IJOD({name:"Data/ijodtest", dir:path.join(lconfig.lockerDir, lconfig.me)}, function(err){ if(err) errs.push(err); myIJOD.addData(events[0], function(err) { if(err) errs.push(err); diff --git a/tests/lpushmanager-test-local.js b/tests/lpushmanager-test-local.js index f4a55fac5..47b3e02fa 100644 --- a/tests/lpushmanager-test-local.js +++ b/tests/lpushmanager-test-local.js @@ -147,18 +147,6 @@ vows.describe("Push Manager").addBatch({ assert.deepEqual(JSON.parse(data), {}); } } -}).addBatch({ - "rows can be deleted by posting delete commands" : { - topic: function() { - var self = this; - pushManager.acceptData('testing', dataSets[4], function() { - colls.push_testing.count(self.callback); - }); - }, - "as well" : function(err, count) { - assert.equal(count, 0); - } - } }).addBatch({ "invalid dataset names are" : { topic: function() { diff --git a/tests/lsyncmanager-test-local.js b/tests/lsyncmanager-test-local.js index 297a78075..73fa472e9 100644 --- a/tests/lsyncmanager-test-local.js +++ b/tests/lsyncmanager-test-local.js @@ -246,23 +246,6 @@ vows.describe("Synclet Manager").addBatch({ assert.equal(Object.keys(allEvents).length, 0); } } -}).addBatch({ - "If the source doesn't return an ID" : { - topic: function() { - allEvents = {}; - var self = this; - colls.testSynclet_testSync.drop(function() { - syncManager.syncNow("testSynclet", function() { - colls.testSynclet_testSync.count(self.callback); - }); - }); - }, - "it will generate a delete event and remove the row from mongo" : function(err, count) { - assert.equal(count, 0); - assert.equal(Object.keys(allEvents).length, 1); - assert.equal(allEvents['testsync://testsynclet/testSync?id=testSynclet#500'].action, 'delete'); - } - } }).addBatch({ "Running testSynclet again" : { topic: function() { From 02636d33b2d1e6ed740a39a6ac4c5e8c37193e46 Mon Sep 17 00:00:00 2001 From: Jeremie Miller Date: Sat, 24 Dec 2011 14:38:40 -0600 Subject: [PATCH 10/35] basic bulk migration tool --- Common/node/ijod.js | 2 +- Ops/mongo2ijod.js | 114 +++++++++++++++++++++++--------------------- 2 files changed, 60 insertions(+), 56 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 82b294c67..2e9e536a7 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -14,7 +14,7 @@ var fs = require('fs'); var path = require('path'); -var deepCompare = require('deepCompare'); +var deepCompare = require('./deepCompare'); var sqlite = require('sqlite-fts'); var gzbz2 = require("gzbz2"); var gzip = new gzbz2.Gzip; diff --git a/Ops/mongo2ijod.js b/Ops/mongo2ijod.js index 897f1a8c2..ddecf0706 100644 --- a/Ops/mongo2ijod.js +++ b/Ops/mongo2ijod.js @@ -3,6 +3,10 @@ var sys = require("sys"); var fs = require("fs"); var lconfig = require(__dirname +"/../Common/node/lconfig"); lconfig.load(__dirname+"/../Config/config.json"); +var IJOD = require(__dirname+"/../Common/node/ijod").IJOD; +var async = require("async"); +var spawn = require('child_process').spawn; + // Create gzip stream var gzip = new gzbz2.Gzip; @@ -17,95 +21,95 @@ var db = new sqlite.Database(); var mongodb = require("mongodb"); +bootMongo() + +var ijods = {}; var mongo; function connect(){ - var mongo = new mongodb.Db('locker', new mongodb.Server("127.0.0.1", 27018, {})); + mongo = new mongodb.Db('locker', new mongodb.Server("127.0.0.1", 27018, {})); mongo.open(function(err, p_client) { if(err) return console.error(err); + mongo.collectionNames(function(err, names){ + scan(names, lconfig.lockerDir + '/' + lconfig.me, function(){ + console.error("done"); + }); + }); // mongo.collection(name, setup); }) } -// open the database for reading if file exists -// create new database file if not - -function setup(err, collection){ - db.open(name+".db", function (error) { - if (error) { - console.log("Tonight. You."); - throw error; - } - db.executeScript("CREATE TABLE IF NOT EXISTS tab (id TEXT PRIMARY KEY, at INTEGER, len INTEGER);",function (error) { - if (error) throw error; - eacher(collection); - }); - }); - +function scan(names, dir, callback) { + console.error("scanning "+dir); + var files = fs.readdirSync(dir); + async.forEachSeries(files, function(file, cb){ + var fullPath = dir + '/' + file; + var stats = fs.statSync(fullPath); + if(!stats.isDirectory()) return cb(); + fs.stat(fullPath+"/me.json",function(err,stats){ + if(!stats || !stats.isFile()) return cb(); + var me = JSON.parse(fs.readFileSync(fullPath+"/me.json")); + if(!me) return cb(); + async.forEachSeries(names, function(nameo, cb2){ + var name = nameo.name; + var pfix = "locker.asynclets_"+me.id+"_"; + if(name.indexOf(pfix) == -1) return cb2(); + var dname = name.substr(pfix.length); + console.error(name); + ijods[name] = new IJOD({name:fullPath+"/"+dname}, function(err, ij){ + if(err) console.error(err); + var id = "id"; + if(me.mongoId && me.mongoId[dname]) id = me.mongoId[dname]; + if(me.mongoId && me.mongoId[dname+"s"]) id = me.mongoId[dname+"s"]; + mongo.collection(name.substr(7), function(err, coll){ + if(err) console.error(err); + eacher(coll, id, ij, cb2); + }) + }); + }, cb); + }); + },callback); } -function eacher(collection) { + + +function eacher(collection, id, ij, callback) { // Locate all the entries using find var arr = []; var at = Date.now(); collection.find().each(function(err, item) { if(!item){ console.error("loaded "+arr.length+" items in "+(Date.now() - at)); - saver(arr); - return client.close(); + at = Date.now(); + async.forEachSeries(arr, function(data, cb){ + ij.addData({id:data[id], data:data},cb) + }, function(){ + console.error("saved in "+(Date.now() - at)); + callback(); + }); + return; } + if(!item[id]) console.error("can't find "+id+" in "+JSON.stringify(item)); + if(item[id]) arr.push(item); arr.push(item); }); }; -var async = require("async"); -function saver(arr) -{ - var start = Date.now(); - var datas = []; - var len = 0; - async.forEachSeries(arr,function(item,cb){ - gzip.init(); - var gzdata = gzip.deflate(new Buffer(JSON.stringify(item)+"\n"), enc); // Do this as many times as required - // sys.puts("Compressed chunk size : " + gzdata.length); - datas.push(gzdata); - var gzlast = gzip.end(); - // sys.puts("Compressed chunk size: " + gzlast.length); - datas.push(gzlast); - var at = len; - len += gzdata.length; - len += gzlast.length; - var sql = "INSERT INTO tab VALUES (?, ?, ?)"; - db.execute(sql, [item[id], at, len-at], function(err){ - cb(); - }); - },function(){ - console.error("indexed in "+(Date.now()-start)); - start = Date.now(); - var fd = fs.openSync(name+".json.gz", "w", 0644); - datas.forEach(function(gzdata){ - fs.writeSync(fd, gzdata, 0, gzdata.length, null); - }) - fs.closeSync(fd); - console.error("written in "+(Date.now()-start)); - - }) - -} function bootMongo() { var mongoProcess = spawn('mongod', ['--dbpath', lconfig.lockerDir + '/' + lconfig.me + '/' + lconfig.mongo.dataDir, '--port', lconfig.mongo.port]); mongoProcess.stderr.on('data', function(data) { - logger.error('mongod err: ' + data); + console.error('mongod err: ' + data); }); var mongoOutput = ""; // watch for mongo startup var callback = function(data) { - mongoOutput += data; + mongoOutput += data.toString(); + console.error(mongoOutput); if(mongoOutput.match(/ waiting for connections on port/g)) { mongoProcess.stdout.removeListener('data', callback); connect(); From bc4f86dfe1898955e3fde102eb5168b33fc38ca2 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Thu, 23 Feb 2012 19:00:24 -0600 Subject: [PATCH 11/35] Fixes for ijod to use zlib --- Common/node/ijod.js | 69 ++++++++++++++++++++------------------------- Ops/mongo2ijod.js | 41 +++++++++++++++------------ package.json | 3 +- 3 files changed, 54 insertions(+), 59 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 2e9e536a7..792e12ee5 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -16,9 +16,7 @@ var fs = require('fs'); var path = require('path'); var deepCompare = require('./deepCompare'); var sqlite = require('sqlite-fts'); -var gzbz2 = require("gzbz2"); -var gzip = new gzbz2.Gzip; -var gunzip = new gzbz2.Gunzip; +var zlib = require("zlib"); function IJOD(arg, callback) { if(!arg || !arg.name) return callback("invalid args"); @@ -50,21 +48,18 @@ IJOD.prototype.addData = function(arg, callback) { if(!arg || !arg.id) return callback("invalid arg"); arg.id = arg.id.toString(); // safety w/ numbers if(!arg.at) arg.at = Date.now(); - gzip.init(); - var gzdata = gzip.deflate(new Buffer(JSON.stringify(arg)+"\n")); - var gzlast = gzip.end(); - - try{ - fs.writeSync(this.fda, gzdata, 0, gzdata.length, null); - fs.writeSync(this.fda, gzlast, 0, gzlast.length, null); - }catch(E){ - return callback(E); - } + zlib.deflate(new Buffer(JSON.stringify(arg)+"\n"), function(err, gzdata) { + console.log("going to write length " + gzdata.length); + try { + fs.writeSync(this.fda, gzdata, 0, gzdata.length, null); + } catch(E) { + return callback(E); + } - var at = this.len; - this.len += gzdata.length; - this.len += gzlast.length; - this.db.execute("REPLACE INTO ijod VALUES (?, ?, ?)", [arg.id, at, this.len-at], callback); + var at = this.len; + this.len += gzdata.length; + this.db.execute("REPLACE INTO ijod VALUES (?, ?, ?)", [arg.id, at, this.len-at], callback); + }); } // adds a deleted record to the ijod and removes from index @@ -73,21 +68,17 @@ IJOD.prototype.delData = function(arg, callback) { arg.id = arg.id.toString(); // safety w/ numbers if(!arg.at) arg.at = Date.now(); arg.type = "delete"; - gzip.init(); - var gzdata = gzip.deflate(new Buffer(JSON.stringify(arg)+"\n")); - var gzlast = gzip.end(); - - try{ - fs.writeSync(this.fda, gzdata, 0, gzdata.length, null); - fs.writeSync(this.fda, gzlast, 0, gzlast.length, null); - }catch(E){ - return callback(E); - } + zlib.deflate(new Buffer(JSON.stringify(arg)+"\n"), function(err, gzdata) { + try { + fs.writeSync(this.fda, gzdata, 0, gzdata.length, null); + } catch(E) { + return callback(E); + } - var at = this.len; - this.len += gzdata.length; - this.len += gzlast.length; - this.db.execute("DELETE FROM ijod WHERE id = ?", [arg.id], callback); + var at = this.len; + this.len += gzdata.length; + this.db.execute("DELETE FROM ijod WHERE id = ?", [arg.id], callback); + }); } // this only calls callback(err, rawstring) once! @@ -103,9 +94,9 @@ IJOD.prototype.getOne = function(arg, callback) { if(!row) return callback(); var buf = new Buffer(row.len); fs.readSync(self.fdr, buf, 0, row.len, row.at); - gunzip.init(); - var x = gunzip.inflate(buf); - return callback(null, arg.raw ? x : stripper(x)); + zlib.inflate(buf, function(err, data) { + return callback(err, arg.raw ? data : stripper(data)); + }); }); } @@ -130,9 +121,9 @@ IJOD.prototype.getAll = function(arg, callback) { if(!row) return callback(); var buf = new Buffer(row.len); fs.readSync(self.fdr, buf, 0, row.len, row.at); - gunzip.init(); - var x = gunzip.inflate(buf); - return callback(null, arg.raw ? x : stripper(x)); + zlib.inflate(buf, function(err, data) { + return callback(err, arg.raw ? data : stripper(data)); + }); }); } @@ -207,6 +198,6 @@ IJOD.prototype.reqID = function(req, res) // make a string and return only the interior data object! function stripper(buf) { - var s = buf.toString(); + var s = buf.toString("utf8"); return s.slice(s.indexOf('{',1),s.lastIndexOf('}',s.length-3)+1); // -3 accounts for }\n -} \ No newline at end of file +} diff --git a/Ops/mongo2ijod.js b/Ops/mongo2ijod.js index ddecf0706..880054b1b 100644 --- a/Ops/mongo2ijod.js +++ b/Ops/mongo2ijod.js @@ -1,5 +1,5 @@ -var gzbz2 = require("gzbz2"); -var sys = require("sys"); +var zlib = require("zlib"); +var sys = require("util"); var fs = require("fs"); var lconfig = require(__dirname +"/../Common/node/lconfig"); lconfig.load(__dirname+"/../Config/config.json"); @@ -8,9 +8,6 @@ var async = require("async"); var spawn = require('child_process').spawn; -// Create gzip stream -var gzip = new gzbz2.Gzip; - var enc = null; var name = process.argv[2]; var id = process.argv[3]||"id"; @@ -25,13 +22,15 @@ bootMongo() var ijods = {}; var mongo; -function connect(){ +function connect(cb){ mongo = new mongodb.Db('locker', new mongodb.Server("127.0.0.1", 27018, {})); mongo.open(function(err, p_client) { if(err) return console.error(err); mongo.collectionNames(function(err, names){ scan(names, lconfig.lockerDir + '/' + lconfig.me, function(){ console.error("done"); + mongo.close(); + cb(); }); }); // mongo.collection(name, setup); @@ -75,23 +74,21 @@ function scan(names, dir, callback) { function eacher(collection, id, ij, callback) { // Locate all the entries using find - var arr = []; + var count = 0; var at = Date.now(); collection.find().each(function(err, item) { if(!item){ - console.error("loaded "+arr.length+" items in "+(Date.now() - at)); - at = Date.now(); - async.forEachSeries(arr, function(data, cb){ - ij.addData({id:data[id], data:data},cb) - }, function(){ - console.error("saved in "+(Date.now() - at)); - callback(); - }); + console.error("loaded " + count + " items in "+(Date.now() - at)); + callback(); return; } if(!item[id]) console.error("can't find "+id+" in "+JSON.stringify(item)); - if(item[id]) arr.push(item); - arr.push(item); + ++count; + ij.addData({id:item[id], data:item}, function(addError) { + if (addError) { + console.error("Adding to ijod error: " + addError); + } + }); }); }; @@ -112,10 +109,18 @@ function bootMongo() console.error(mongoOutput); if(mongoOutput.match(/ waiting for connections on port/g)) { mongoProcess.stdout.removeListener('data', callback); - connect(); + connect(function() { + mongoProcess.kill(); + }); } }; mongoProcess.stdout.on('data', callback); + + process.on("uncaughtException", function(E) { + console.error(E); + mongoProcess.kill(); + process.exit(1); + }); } diff --git a/package.json b/package.json index fffe57f2a..6fa58195e 100644 --- a/package.json +++ b/package.json @@ -55,8 +55,7 @@ "ini": "=1.0.1", "uglify-js": "=1.1.1", "connect-form": "0.2.1", - "gzbz2": "0.1.4", - "imagemagick": "https://github.com/lpatters/node-imagemagick/tarball/master", + "imagemagick": "=0.1.2", "moment": "=1.3.0", "mkdirp": "0.3.0", From 4b953c7de4b2d3a40b28f2de7401719f89150033 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Mon, 27 Feb 2012 11:00:17 -0600 Subject: [PATCH 12/35] Make sure we're using gzip compression, fully working on v0.6 and mongo dump working --- Common/node/ijod.js | 40 +++++++++++++++++++++------------------- Ops/mongo2ijod.js | 1 + 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 792e12ee5..8e478ee42 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -48,17 +48,18 @@ IJOD.prototype.addData = function(arg, callback) { if(!arg || !arg.id) return callback("invalid arg"); arg.id = arg.id.toString(); // safety w/ numbers if(!arg.at) arg.at = Date.now(); - zlib.deflate(new Buffer(JSON.stringify(arg)+"\n"), function(err, gzdata) { + var self = this; + zlib.gzip(new Buffer(JSON.stringify(arg)+"\n"), function(err, gzdata) { console.log("going to write length " + gzdata.length); - try { - fs.writeSync(this.fda, gzdata, 0, gzdata.length, null); - } catch(E) { - return callback(E); - } + fs.write(self.fda, gzdata, 0, gzdata.length, null, function(err, written, buffer) { + if (err) { + return callback(err); + } - var at = this.len; - this.len += gzdata.length; - this.db.execute("REPLACE INTO ijod VALUES (?, ?, ?)", [arg.id, at, this.len-at], callback); + var at = self.len; + self.len += gzdata.length; + self.db.execute("REPLACE INTO ijod VALUES (?, ?, ?)", [arg.id, at, self.len-at], callback); + }); }); } @@ -68,16 +69,17 @@ IJOD.prototype.delData = function(arg, callback) { arg.id = arg.id.toString(); // safety w/ numbers if(!arg.at) arg.at = Date.now(); arg.type = "delete"; - zlib.deflate(new Buffer(JSON.stringify(arg)+"\n"), function(err, gzdata) { - try { - fs.writeSync(this.fda, gzdata, 0, gzdata.length, null); - } catch(E) { - return callback(E); + var self = this; + zlib.gzip(new Buffer(JSON.stringify(arg)+"\n"), function(err, gzdata) { + fs.write(self.fda, gzdata, 0, gzdata.length, null, function(err, written, buffer) { + if (err) { + return callback(err); } - var at = this.len; - this.len += gzdata.length; - this.db.execute("DELETE FROM ijod WHERE id = ?", [arg.id], callback); + var at = self.len; + self.len += gzdata.length; + self.db.execute("DELETE FROM ijod WHERE id = ?", [arg.id], callback); + }); }); } @@ -94,7 +96,7 @@ IJOD.prototype.getOne = function(arg, callback) { if(!row) return callback(); var buf = new Buffer(row.len); fs.readSync(self.fdr, buf, 0, row.len, row.at); - zlib.inflate(buf, function(err, data) { + zlib.gunzip(buf, function(err, data) { return callback(err, arg.raw ? data : stripper(data)); }); }); @@ -121,7 +123,7 @@ IJOD.prototype.getAll = function(arg, callback) { if(!row) return callback(); var buf = new Buffer(row.len); fs.readSync(self.fdr, buf, 0, row.len, row.at); - zlib.inflate(buf, function(err, data) { + zlib.gunzip(buf, function(err, data) { return callback(err, arg.raw ? data : stripper(data)); }); }); diff --git a/Ops/mongo2ijod.js b/Ops/mongo2ijod.js index 880054b1b..3b3f2ab7b 100644 --- a/Ops/mongo2ijod.js +++ b/Ops/mongo2ijod.js @@ -87,6 +87,7 @@ function eacher(collection, id, ij, callback) { ij.addData({id:item[id], data:item}, function(addError) { if (addError) { console.error("Adding to ijod error: " + addError); + console.error(addError.stack); } }); }); From eb128e23f0ec76d89551542d85622615fb9f9c99 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Fri, 9 Mar 2012 11:38:45 -0600 Subject: [PATCH 13/35] Moving to a new compress-buffer based IJOD --- Common/node/ijod.js | 172 +++++++++++++++++++++--------------- Common/node/lsyncmanager.js | 19 +++- Common/node/lutil.js | 17 ++++ Ops/mongo2ijod.js | 9 +- Ops/webservice.js | 3 - package.json | 1 + 6 files changed, 143 insertions(+), 78 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 8e478ee42..705179524 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -16,7 +16,8 @@ var fs = require('fs'); var path = require('path'); var deepCompare = require('./deepCompare'); var sqlite = require('sqlite-fts'); -var zlib = require("zlib"); +var zlib = require("compress-buffer"); +var lutil = require("lutil"); function IJOD(arg, callback) { if(!arg || !arg.name) return callback("invalid args"); @@ -43,90 +44,123 @@ function IJOD(arg, callback) { } exports.IJOD = IJOD; +IJOD.prototype.startAddTransaction = function() { + this.transactionItems = []; +}; + +IJOD.prototype.commitAddTransaction = function(cbDone) { + if (!this.transactionItems || this.transactionItems.length == 0) return cbDone(); + console.log("Commiting %d items", this.transactionItems.length); + var totalSize = this.transactionItems.reduce(function(prev, cur, idx, arr) { return prev + arr[idx].length; }, 0); + var writeBuffer = new Buffer(totalSize); + var idx = 0; + var self = this; + lutil.forEachSeries(self.transactionItems, function(item, cb) { + item.copy(writeBuffer, idx); + idx += item.length; + cb(); + }, function(err) { + fs.write(self.fda, writeBuffer, 0, writeBuffer.length, null, function(err, written, buffer) { + // We end the transaction + writeBuffer = null; + self.transactionItems = null; + if (err) { + console.error("Error writing to IJOD: %e", err); + } else if (written != totalSize) { + console.error("Only %d written of %d bytes to IJOD", written, totalSize); + } + return cbDone(); + }); + }); +} + +IJOD.prototype.updateIndex = function(id, gzlen, cbDone) { + var at = this.len; + this.len += gzlen; + this.db.execute("REPLACE INTO ijod VALUES (?, ?, ?)", [id, at, this.len-at], cbDone); +}; + // takes arg of at least an id and data, callback(err) when done IJOD.prototype.addData = function(arg, callback) { - if(!arg || !arg.id) return callback("invalid arg"); - arg.id = arg.id.toString(); // safety w/ numbers - if(!arg.at) arg.at = Date.now(); - var self = this; - zlib.gzip(new Buffer(JSON.stringify(arg)+"\n"), function(err, gzdata) { - console.log("going to write length " + gzdata.length); - fs.write(self.fda, gzdata, 0, gzdata.length, null, function(err, written, buffer) { - if (err) { - return callback(err); - } - - var at = self.len; - self.len += gzdata.length; - self.db.execute("REPLACE INTO ijod VALUES (?, ?, ?)", [arg.id, at, self.len-at], callback); - }); + if(!arg || !arg.id) return callback("invalid arg"); + arg.id = arg.id.toString(); // safety w/ numbers + if(!arg.at) arg.at = Date.now(); + var self = this; + var gzdata = zlib.compress(new Buffer(JSON.stringify(arg)+"\n")); + if (this.transactionItems) { + this.transactionItems.push(gzdata); + this.updateIndex(arg.id, gzdata.length, callback); + } else { + fs.write(self.fda, gzdata, 0, gzdata.length, null, function(err, written, buffer) { + if (err) { + return callback(err); + } + self.updateIndex(arg.id, gzdata.length, callback); }); + } } // adds a deleted record to the ijod and removes from index IJOD.prototype.delData = function(arg, callback) { - if(!arg || !arg.id) return callback("invalid arg"); - arg.id = arg.id.toString(); // safety w/ numbers - if(!arg.at) arg.at = Date.now(); - arg.type = "delete"; - var self = this; - zlib.gzip(new Buffer(JSON.stringify(arg)+"\n"), function(err, gzdata) { - fs.write(self.fda, gzdata, 0, gzdata.length, null, function(err, written, buffer) { - if (err) { - return callback(err); - } + if(!arg || !arg.id) return callback("invalid arg"); + arg.id = arg.id.toString(); // safety w/ numbers + if(!arg.at) arg.at = Date.now(); + arg.type = "delete"; + var self = this; + var gzdata = zlib.compress(new Buffer(JSON.stringify(arg)+"\n")); + fs.write(self.fda, gzdata, 0, gzdata.length, null, function(err, written, buffer) { + if (err) { + return callback(err); + } - var at = self.len; - self.len += gzdata.length; - self.db.execute("DELETE FROM ijod WHERE id = ?", [arg.id], callback); - }); - }); + var at = self.len; + self.len += gzdata.length; + self.db.execute("DELETE FROM ijod WHERE id = ?", [arg.id], callback); + }); } // this only calls callback(err, rawstring) once! IJOD.prototype.getOne = function(arg, callback) { - if(!arg || !arg.id) return callback("invalid arg"); - arg.id = arg.id.toString(); // safety w/ numbers - var self = this; - var did = false; - self.db.query("SELECT at,len FROM ijod WHERE id = ? LIMIT 1", [arg.id], function(err, row){ - if(did) return; // only call callback ones - did = true; - if(err) return callback(err); - if(!row) return callback(); - var buf = new Buffer(row.len); - fs.readSync(self.fdr, buf, 0, row.len, row.at); - zlib.gunzip(buf, function(err, data) { - return callback(err, arg.raw ? data : stripper(data)); - }); - }); + if(!arg || !arg.id) return callback("invalid arg"); + arg.id = arg.id.toString(); // safety w/ numbers + var self = this; + var did = false; + self.db.query("SELECT at,len FROM ijod WHERE id = ? LIMIT 1", [arg.id], function(err, row){ + if(did) return; // only call callback ones + did = true; + if(err) return callback(err); + if(!row) return callback(); + var buf = new Buffer(row.len); + fs.readSync(self.fdr, buf, 0, row.len, row.at); + var data = zlib.decompress(buf); + return callback(err, arg.raw ? data : stripper(data)); + }); } // will call callback(err, rawstring) continuously until rawstring==undefined IJOD.prototype.getAll = function(arg, callback) { - if(!arg) return callback("invalid arg"); - var params = []; - var sql = "SELECT at,len FROM ijod "; - if(arg.limit) - { - sql += " LIMIT ?"; - params.push(parseInt(arg.limit)); - } - if(arg.offset) - { - sql += " OFFSET ?"; - params.push(parseInt(arg.offset)); - } - var self = this; - self.db.query(sql, params, function(err, row){ - if(err) return callback(err); - if(!row) return callback(); - var buf = new Buffer(row.len); - fs.readSync(self.fdr, buf, 0, row.len, row.at); - zlib.gunzip(buf, function(err, data) { - return callback(err, arg.raw ? data : stripper(data)); - }); - }); + if(!arg) return callback("invalid arg"); + var params = []; + var sql = "SELECT at,len FROM ijod "; + if(arg.limit) + { + sql += " LIMIT ?"; + params.push(parseInt(arg.limit)); + } + if(arg.offset) + { + sql += " OFFSET ?"; + params.push(parseInt(arg.offset)); + } + var self = this; + self.db.query(sql, params, function(err, row){ + if(err) return callback(err); + if(!row) return callback(); + var buf = new Buffer(row.len); + fs.readSync(self.fdr, buf, 0, row.len, row.at); + var data = zlib.decompress(buf); + return callback(err, arg.raw ? data : stripper(data)); + }); } // takes a new object and checks first if it exists diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 19738fde1..2aa0b00bb 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -80,7 +80,7 @@ var synclets = { var serviceManager; exports.init = function (sman, callback) { serviceManager = sman; - datastore.init(callback); + callback(); }; var executeable = true; @@ -219,6 +219,22 @@ function executeSynclet(info, synclet, callback, force) { var tstart = Date.now(); stats.increment('synclet.' + info.id + '.' + synclet.name + '.start'); stats.increment('synclet.' + info.id + '.' + synclet.name + '.running'); + var fname = path.join("tmp", info.id + "-" + synclet.name + ".json"); + if (path.existsSync(fname)) { + console.trace(); + var resp = JSON.parse(fs.readFileSync(fname)); + var startTime = Date.now(); + console.log("Start response processing"); + processResponse({}, info, synclet, resp, function(processErr) { + process.stdin.on("data", function(data) { + console.log(data.toString()); + }); + console.log("Response Processing Done %d", (Date.now() - startTime)); + info.status = 'waiting'; + callback(processErr); + }); + return; + } if (info.vm || synclet.vm) { // Go ahead and create a context immediately so we get it listed as @@ -377,7 +393,6 @@ function compareIDs (originalConfig, newConfig) { function processResponse(deleteIDs, info, synclet, response, callback) { synclet.status = 'waiting'; - checkStatus(info); var dataKeys = []; if (typeof(response.data) === 'string') { diff --git a/Common/node/lutil.js b/Common/node/lutil.js index 6e47930b6..d0b8019a7 100644 --- a/Common/node/lutil.js +++ b/Common/node/lutil.js @@ -230,6 +230,23 @@ exports.streamFromUrl = function(url, cbEach, cbDone) { }; +/// An async forEachSeries +/** +* The async implementation can explode the stack, this version will not. +*/ +exports.forEachSeries = function(items, cbEach, cbDone) { + function runOne(idx) { + idx = idx || 0; + if (idx >= items.length) return cbDone(); + cbEach(items[idx], function(err) { + if (err) return cbDone(err) + process.nextTick(function() { + runOne(idx + 1); + }); + }); + } + runOne(); +} /* * @sourceUrl - URL of the avatar you want to fetch * @rawfile - where you want the raw downloaded data stored diff --git a/Ops/mongo2ijod.js b/Ops/mongo2ijod.js index 3b3f2ab7b..39acd522d 100644 --- a/Ops/mongo2ijod.js +++ b/Ops/mongo2ijod.js @@ -1,4 +1,3 @@ -var zlib = require("zlib"); var sys = require("util"); var fs = require("fs"); var lconfig = require(__dirname +"/../Common/node/lconfig"); @@ -6,6 +5,7 @@ lconfig.load(__dirname+"/../Config/config.json"); var IJOD = require(__dirname+"/../Common/node/ijod").IJOD; var async = require("async"); var spawn = require('child_process').spawn; +var lutil = require("lutil"); var enc = null; @@ -41,7 +41,7 @@ function connect(cb){ function scan(names, dir, callback) { console.error("scanning "+dir); var files = fs.readdirSync(dir); - async.forEachSeries(files, function(file, cb){ + lutil.forEachSeries(files, function(file, cb){ var fullPath = dir + '/' + file; var stats = fs.statSync(fullPath); if(!stats.isDirectory()) return cb(); @@ -49,7 +49,7 @@ function scan(names, dir, callback) { if(!stats || !stats.isFile()) return cb(); var me = JSON.parse(fs.readFileSync(fullPath+"/me.json")); if(!me) return cb(); - async.forEachSeries(names, function(nameo, cb2){ + lutil.forEachSeries(names, function(nameo, cb2){ var name = nameo.name; var pfix = "locker.asynclets_"+me.id+"_"; if(name.indexOf(pfix) == -1) return cb2(); @@ -76,10 +76,11 @@ function eacher(collection, id, ij, callback) { // Locate all the entries using find var count = 0; var at = Date.now(); + ij.startAddTransaction(); collection.find().each(function(err, item) { if(!item){ console.error("loaded " + count + " items in "+(Date.now() - at)); - callback(); + ij.commitAddTransaction(callback); return; } if(!item[id]) console.error("can't find "+id+" in "+JSON.stringify(item)); diff --git a/Ops/webservice.js b/Ops/webservice.js index d7a49c31b..d7f1b597f 100644 --- a/Ops/webservice.js +++ b/Ops/webservice.js @@ -254,9 +254,6 @@ locker.post('/post/:id/:synclet', function(req, res) { }); }); -// all synclet getCurrent, id, etc stuff -require('synclet/dataaccess')(locker); - function proxyRequest(method, req, res, next) { var slashIndex = req.url.indexOf("/", 4); if (slashIndex < 0) slashIndex = req.url.length; diff --git a/package.json b/package.json index 9a57a8b49..55306ab2f 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "eyes": "=0.1.6", "knox": "=0.0.9", "readabilitySAX": "=0.2.2", + "compress-buffer": "=0.5.1", "mongodb": "=0.9.8-1", "winston": "=0.5.2", "sqlite-fts": "=0.0.8", From 970cd69a1d85548f0403abd1c8f7f847f78b1371 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Fri, 9 Mar 2012 17:04:10 -0600 Subject: [PATCH 14/35] Movign to compress-buffers is complete as well as a new smartBatchAdd --- Common/node/ijod.js | 93 ++++++++++++++++++++++++++++--------- Common/node/lsyncmanager.js | 24 ++++++++-- Ops/mongo2ijod.js | 31 +++++++------ 3 files changed, 106 insertions(+), 42 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 705179524..603e7a4c9 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -18,10 +18,13 @@ var deepCompare = require('./deepCompare'); var sqlite = require('sqlite-fts'); var zlib = require("compress-buffer"); var lutil = require("lutil"); +var async = require("async"); +var mmh3 = require("murmurhash3"); function IJOD(arg, callback) { if(!arg || !arg.name) return callback("invalid args"); var self = this; + this.transactionItems = null; self.name = arg.name; self.gzname = arg.name + '.json.gz'; self.dbname = arg.name + '.db'; @@ -36,7 +39,7 @@ function IJOD(arg, callback) { self.db = new sqlite.Database(); self.db.open(self.dbname, function (err) { if(err) return callback(err); - self.db.executeScript("CREATE TABLE IF NOT EXISTS ijod (id TEXT PRIMARY KEY, at INTEGER, len INTEGER);",function (err) { + self.db.executeScript("CREATE TABLE IF NOT EXISTS ijod (id TEXT PRIMARY KEY, at INTEGER, len INTEGER, hash TEXT);",function (err) { if(err) return callback(err); callback(null, self); }); @@ -44,8 +47,10 @@ function IJOD(arg, callback) { } exports.IJOD = IJOD; -IJOD.prototype.startAddTransaction = function() { +IJOD.prototype.startAddTransaction = function(cbDone) { + if (this.transactionItems) return cbDone(); this.transactionItems = []; + this.db.execute("BEGIN TRANSACTION", function(error, rows) { cbDone(); }); }; IJOD.prototype.commitAddTransaction = function(cbDone) { @@ -69,35 +74,27 @@ IJOD.prototype.commitAddTransaction = function(cbDone) { } else if (written != totalSize) { console.error("Only %d written of %d bytes to IJOD", written, totalSize); } - return cbDone(); + self.db.execute("COMMIT TRANSACTION", function(error, rows) { cbDone() }); }); }); } -IJOD.prototype.updateIndex = function(id, gzlen, cbDone) { - var at = this.len; - this.len += gzlen; - this.db.execute("REPLACE INTO ijod VALUES (?, ?, ?)", [id, at, this.len-at], cbDone); -}; - // takes arg of at least an id and data, callback(err) when done IJOD.prototype.addData = function(arg, callback) { if(!arg || !arg.id) return callback("invalid arg"); arg.id = arg.id.toString(); // safety w/ numbers + var tmpJson = JSON.stringify(arg); + var hash = mmh3.murmur32HexSync(tmpJson); if(!arg.at) arg.at = Date.now(); var self = this; - var gzdata = zlib.compress(new Buffer(JSON.stringify(arg)+"\n")); - if (this.transactionItems) { - this.transactionItems.push(gzdata); - this.updateIndex(arg.id, gzdata.length, callback); - } else { - fs.write(self.fda, gzdata, 0, gzdata.length, null, function(err, written, buffer) { - if (err) { - return callback(err); - } - self.updateIndex(arg.id, gzdata.length, callback); - }); - } + this.startAddTransaction(function() { + var tmpJson = JSON.stringify(arg); + var gzdata = zlib.compress(new Buffer(tmpJson+"\n")); + self.transactionItems.push(gzdata); + var at = self.len; + self.len += gzdata.length; + self.db.execute("REPLACE INTO ijod VALUES (?, ?, ?, ?)", [arg.id, at, self.len-at, hash], callback); + }); } // adds a deleted record to the ijod and removes from index @@ -132,7 +129,7 @@ IJOD.prototype.getOne = function(arg, callback) { if(!row) return callback(); var buf = new Buffer(row.len); fs.readSync(self.fdr, buf, 0, row.len, row.at); - var data = zlib.decompress(buf); + var data = zlib.uncompress(buf); return callback(err, arg.raw ? data : stripper(data)); }); } @@ -169,7 +166,9 @@ IJOD.prototype.smartAdd = function(arg, callback) { if(!arg || !arg.id) return callback("invalid arg"); arg.id = arg.id.toString(); // safety w/ numbers var self = this; + var start = Date.now(); self.getOne(arg, function(err, existing){ + console.log("getOne in %d", (Date.now() - start)); if(err) return callback(err); // first check if it's new if(!existing) @@ -193,6 +192,56 @@ IJOD.prototype.smartAdd = function(arg, callback) { }); } +IJOD.prototype.batchSmartAdd = function(entries, callback) { + console.log("Batch smart add %d entries", entries.length); + var t = Date.now(); + var self = this; + var script = ["CREATE TEMP TABLE IF NOT EXISTS batchSmartAdd (id TEXT)", "DELETE FROM batchSmartAdd", "BEGIN TRANSACTION"].join(";") + ";"; + self.db.executeScript(script, function(error, rows) { + self.db.prepare("INSERT INTO batchSmartAdd VALUES (?)", function(error, stmt) { + async.forEachSeries(entries, function(entry, cb) { + if (!entry) return cb(); + stmt.bind(1, entry.id, function(err) { + stmt.step(function(error, row) { + stmt.reset(); + cb(); + }); + }); + }, function(error) { + self.db.execute("COMMIT TRANSACTION", function(error, rows) { + stmt.finalize(function(error) { + self.db.execute("SELECT id,hash FROM ijod WHERE ijod.id IN (SELECT id FROM batchSmartAdd)", function(error, rows) { + var knownIds = {}; + rows = rows || []; + rows.forEach(function(row) { + knownIds[row.id] = row.hash; + }); + self.startAddTransaction(function() { + async.forEachSeries(entries, function(entry, cb) { + if (!entry) return cb(); + if (knownIds[entry.id]) { + // See if we need to update + entry.id = entry.id.toString(); // safety w/ numbers + var hash = mmh3.murmur32HexSync(JSON.stringify(entry)); + // If the id and hashes match it's the same! + if (hash == knownIds[entry.id]) { + return cb(); + } + } + self.addData(entry, cb); + }, function(error) { + self.commitAddTransaction(callback); + console.log("Batch done: %d", (Date.now() - t)); + }); + }) + }); + }); + }); + }); + }); + }); +}; + // utilities to respond to a web request, shared between synclets and push IJOD.prototype.reqCurrent = function(req, res) { diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 2aa0b00bb..7ea92cb38 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -219,22 +219,24 @@ function executeSynclet(info, synclet, callback, force) { var tstart = Date.now(); stats.increment('synclet.' + info.id + '.' + synclet.name + '.start'); stats.increment('synclet.' + info.id + '.' + synclet.name + '.running'); + // This is super handy for testing. + /* var fname = path.join("tmp", info.id + "-" + synclet.name + ".json"); if (path.existsSync(fname)) { - console.trace(); var resp = JSON.parse(fs.readFileSync(fname)); var startTime = Date.now(); - console.log("Start response processing"); + console.log("Start response processing for %s", (info.id + "/" + synclet.name)); processResponse({}, info, synclet, resp, function(processErr) { process.stdin.on("data", function(data) { console.log(data.toString()); }); - console.log("Response Processing Done %d", (Date.now() - startTime)); + console.log("Response Processing Done for %s in %d", (info.id + "/" + synclet.name), (Date.now() - startTime)); info.status = 'waiting'; callback(processErr); }); return; } + */ if (info.vm || synclet.vm) { // Go ahead and create a context immediately so we get it listed as @@ -468,7 +470,11 @@ function processData (deleteIDs, info, synclet, key, data, callback) { } }); } else if (data && data.length > 0) { - addData(collection, mongoId, data, info, synclet, idr, ij, callback); + //ij.startAddTransaction() + addData(collection, mongoId, data, info, synclet, idr, ij, function(err) { + callback(); + //ij.commitAddTransaction(callback); + }); } else if (deleteIDs && deleteIDs.length > 0) { deleteData(collection, mongoId, deleteIDs, info, synclet, idr, ij, callback); } else { @@ -501,6 +507,12 @@ function deleteData (collection, mongoId, deleteIds, info, synclet, idr, ij, cal function addData (collection, mongoId, data, info, synclet, idr, ij, callback) { var errs = []; + var entries = data.map(function(item) { + var object = (item.obj) ? item : {obj: item}; + return {id:object.obj[mongoId], data:object.obj}; + }); + return ij.batchSmartAdd(entries, callback); + var q = async.queue(function(item, cb) { var object = (item.obj) ? item : {obj: item}; if (object.obj) { @@ -524,11 +536,13 @@ function addData (collection, mongoId, data, info, synclet, idr, ij, callback) { delete object[key]; } } + var start = Date.now(); ij.smartAdd({id:object.obj[mongoId], data:object.obj}, function(err, type) { + console.log("Did add in %d", (Date.now() - start)); if (type === 'same') return cb(); if (type === 'new') synclet.added++; if (type === 'update') synclet.updated++; - levents.fireEvent(url.format(r), type, object.obj); + //levents.fireEvent(url.format(r), type, object.obj); return cb(); }); } diff --git a/Ops/mongo2ijod.js b/Ops/mongo2ijod.js index 39acd522d..067ab52ec 100644 --- a/Ops/mongo2ijod.js +++ b/Ops/mongo2ijod.js @@ -76,21 +76,22 @@ function eacher(collection, id, ij, callback) { // Locate all the entries using find var count = 0; var at = Date.now(); - ij.startAddTransaction(); - collection.find().each(function(err, item) { - if(!item){ - console.error("loaded " + count + " items in "+(Date.now() - at)); - ij.commitAddTransaction(callback); - return; - } - if(!item[id]) console.error("can't find "+id+" in "+JSON.stringify(item)); - ++count; - ij.addData({id:item[id], data:item}, function(addError) { - if (addError) { - console.error("Adding to ijod error: " + addError); - console.error(addError.stack); - } - }); + ij.startAddTransaction(function() { + collection.find().each(function(err, item) { + if(!item){ + console.error("loaded " + count + " items in "+(Date.now() - at)); + ij.commitAddTransaction(callback); + return; + } + if(!item[id]) console.error("can't find "+id+" in "+JSON.stringify(item)); + ++count; + ij.addData({id:item[id], data:item}, function(addError) { + if (addError) { + console.error("Adding to ijod error: " + addError); + console.error(addError.stack); + } + }); + }); }); }; From 9f604665d6ebba0a52f94b3bededd4791a22628c Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Fri, 9 Mar 2012 17:31:24 -0600 Subject: [PATCH 15/35] Bit of cleanup on the add data processing --- Common/node/lsyncmanager.js | 97 ++++++++++++++----------------------- 1 file changed, 36 insertions(+), 61 deletions(-) diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 7ea92cb38..e1fef5330 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -470,10 +470,8 @@ function processData (deleteIDs, info, synclet, key, data, callback) { } }); } else if (data && data.length > 0) { - //ij.startAddTransaction() addData(collection, mongoId, data, info, synclet, idr, ij, function(err) { callback(); - //ij.commitAddTransaction(callback); }); } else if (deleteIDs && deleteIDs.length > 0) { deleteData(collection, mongoId, deleteIDs, info, synclet, idr, ij, callback); @@ -506,68 +504,45 @@ function deleteData (collection, mongoId, deleteIds, info, synclet, idr, ij, cal } function addData (collection, mongoId, data, info, synclet, idr, ij, callback) { - var errs = []; - var entries = data.map(function(item) { + var errs = []; + // Take out the deletes + var deletes = data.filter(function(item) { + var object = (item.obj) ? item : {obj: item}; + if (object.obj && object.type === "delete") { + return true; + } + return false; + }); + // TODO The deletes + async.forEachSeries(deletes, function(item, cb) { + var r = url.parse(idr); + r.hash = object.obj[mongoId].toString(); + levents.fireEvent(url.format(r), 'delete'); + synclet.deleted++; + ij.delData({id:object.obj[mongoId]}, cb); + }, function(err) { + // Now we'll batch process the rest as adds + var entries = data.filter(function(item) { + var object = (item.obj) ? item : {obj: item}; + if (object.obj && object.type === "delete") { + return false; + } + return true; + }); + entries = entries.map(function(item) { var object = (item.obj) ? item : {obj: item}; return {id:object.obj[mongoId], data:object.obj}; }); - return ij.batchSmartAdd(entries, callback); - - var q = async.queue(function(item, cb) { - var object = (item.obj) ? item : {obj: item}; - if (object.obj) { - if(object.obj[mongoId] === null || object.obj[mongoId] === undefined) { - localError(info.title + ' ' + url.format(idr), "missing primary key (" + mongoId + ") value: "+JSON.stringify(object.obj)); - errs.push({"message":"no value for primary key", "obj": object.obj}); - return cb(); - } - var r = url.parse(idr); - r.hash = object.obj[mongoId].toString(); - if (object.type === 'delete') { - levents.fireEvent(url.format(r), 'delete'); - synclet.deleted++; - ij.delData({id:object.obj[mongoId]}, cb); - } else { - var source = r.pathname.substring(1); - if(info.strip && info.strip[source]) - { // if there's strip options, remove some things from the raw object - for (var i in info.strip[source]) { - var key = info.strip[source][i]; - delete object[key]; - } - } - var start = Date.now(); - ij.smartAdd({id:object.obj[mongoId], data:object.obj}, function(err, type) { - console.log("Did add in %d", (Date.now() - start)); - if (type === 'same') return cb(); - if (type === 'new') synclet.added++; - if (type === 'update') synclet.updated++; - //levents.fireEvent(url.format(r), type, object.obj); - return cb(); - }); - } - } else { - cb(); - } - }, 5); - // debug stuff - var oldProcess = q.process; - q.process = function() { - var task = q.tasks[0]; - try { - oldProcess(); - } catch (err) { - console.error('ERROR: caught error while processing q on task ', task); - } - }; - data.forEach(function(d){ q.push(d, errs.push); }); // hehe fun - q.drain = function() { - if (errs.length > 0) { - callback(errs); - } else { - callback(); - } - }; + ij.batchSmartAdd(entries, function() { + // TODO: Return some stats from batch add for added and updated + entries.forEach(function(item) { + var r = url.parse(idr); + r.hash = item.toString(); + levents.fireEvent(url.format(r), "new", item.data); + }); + callback(); + }); + }); } From 5f8004f0bbce768bf33b6a4755e6314af2962f92 Mon Sep 17 00:00:00 2001 From: Forrest L Norvell Date: Fri, 9 Mar 2012 16:59:13 -0800 Subject: [PATCH 16/35] Let's dependency! --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 55306ab2f..a8e4f757f 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "imagemagick": "=0.1.2", "moment": "=1.3.0", "mkdirp": "0.3.0", + "murmurhash3": "", "hashish": "=0.0.4", From 995b16a403b10f26950709df92c978be127fa3cd Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Mon, 12 Mar 2012 11:19:36 -0500 Subject: [PATCH 17/35] Add the lost getCurrent interface --- Common/node/ijod.js | 2 +- Ops/webservice-synclets.js | 111 +++++++++++++++++++++++++++++++++++++ Ops/webservice.js | 1 + 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 Ops/webservice-synclets.js diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 603e7a4c9..8bea18810 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -155,7 +155,7 @@ IJOD.prototype.getAll = function(arg, callback) { if(!row) return callback(); var buf = new Buffer(row.len); fs.readSync(self.fdr, buf, 0, row.len, row.at); - var data = zlib.decompress(buf); + var data = zlib.uncompress(buf); return callback(err, arg.raw ? data : stripper(data)); }); } diff --git a/Ops/webservice-synclets.js b/Ops/webservice-synclets.js new file mode 100644 index 000000000..f8bfda6ea --- /dev/null +++ b/Ops/webservice-synclets.js @@ -0,0 +1,111 @@ +var syncManager = require('lsyncmanager'); +var fs = require('fs'); +var path = require('path'); +var lconfig = require('lconfig'); +var logger = require('logger'); +var lfs = require('lfs'); + +module.exports = function(locker) { + // get all the information about synclets + locker.get('/synclets', function(req, res) { + res.writeHead(200, { + 'Content-Type': 'text/javascript', + "Access-Control-Allow-Origin" : "*" + }); + var synclets = JSON.parse(JSON.stringify(syncManager.synclets())); + for(var s in synclets.installed) { + delete synclets.installed[s].config; + delete synclets.installed[s].auth; + } + res.end(JSON.stringify(synclets)); + }); + + // given a bunch of json describing a synclet, make a home for it on disk and add it to our map + locker.post('/synclets/install', function(req, res) { + if (!req.body.hasOwnProperty("srcdir")) { + res.writeHead(400); + res.end("{}") + return; + } + var metaData = syncManager.install(req.body); + if (!metaData) { + res.writeHead(404); + res.end("{}"); + return; + } + res.writeHead(200, { + 'Content-Type': 'application/json' + }); + res.end(JSON.stringify(metaData)); + }); + + locker.get('/synclets/:id/run', function(req, res) { + syncManager.syncNow(req.params.id, req.query.id, false, function() { + res.send(true); + }); + }); + + // not sure /post is the right base here but needed it for easy bodyparser flag + locker.post('/post/:id/:synclet', function(req, res) { + syncManager.syncNow(req.params.id, req.params.synclet, req.body, function() { + res.send(true); + }); + }); + + // Returns a list of the current set of friends or followers + locker.get('/synclets/:syncletId/getCurrent/:type', function(req, res) { + syncManager.getIJOD(req.params.syncletId, req.params.type, false, function(ijod) { + if(!ijod) return res.send("not found",404); + ijod.reqCurrent(req, res); + }); + }); + + locker.get('/synclets/:syncletId/:type/id/:id', function(req, res) { + syncManager.getIJOD(req.params.syncletId, req.params.type, false, function(ijod) { + if(!ijod) return res.send("not found",404); + ijod.reqID(req, res); + }); + }); + + locker.get('/synclets/:syncletId/get_profile', function(req, res) { + lfs.readObjectFromFile(path.join(lconfig.lockerDir, lconfig.me, req.params.syncletId, 'profile.json'), function(userInfo) { + res.writeHead(200, {"Content-Type":"application/json"}); + res.end(JSON.stringify(userInfo)); + }); + }); + + locker.get('/synclets/:syncletId/getPhoto/:id', function(req, res) { + var id = req.param('id'); + fs.readdir(path.join(lconfig.lockerDir, lconfig.me, req.params.syncletId, 'photos'), function(err, files) { + var file; + for (var i = 0; i < files.length; i++) { + if (files[i].match('^' + id + '\\.[a-zA-Z0-9]+')) { + file = files[i]; + break; + } + } + if (file) { + var stream = fs.createReadStream(path.join(lconfig.lockerDir, lconfig.me, req.params.syncletId, 'photos', file)); + var head = false; + stream.on('data', function(chunk) { + if(!head) { + head = true; + res.writeHead(200, {'Content-Disposition': 'attachment; filename=' + file}); + } + res.write(chunk, "binary"); + }); + stream.on('error', function() { + res.writeHead(404); + res.end(); + }); + stream.on('end', function() { + res.end(); + }); + } else { + res.writeHead(404); + res.end(); + } + }); + }); +}; + diff --git a/Ops/webservice.js b/Ops/webservice.js index c049d4c67..8a1bec405 100644 --- a/Ops/webservice.js +++ b/Ops/webservice.js @@ -513,6 +513,7 @@ locker.get('/', function(req, res) { res.redirect(lconfig.externalBase + '/dashboard/'); }); +require("./webservice-synclets")(locker); require('./webservice-push')(locker); From c63fa0824345de6a922a436f80e1a567fc5f0e25 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Tue, 13 Mar 2012 17:24:52 -0500 Subject: [PATCH 18/35] Updating for an ijod migration and some related fixes --- Common/node/ijod.js | 52 +++++++++++--------- Common/node/lsyncmanager.js | 4 +- Ops/mongo2ijod.js | 98 ++++++++++++++++++++++++------------- migrations/0000000000008.js | 15 ++++++ 4 files changed, 111 insertions(+), 58 deletions(-) create mode 100644 migrations/0000000000008.js diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 8bea18810..2b02e50e0 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -21,29 +21,37 @@ var lutil = require("lutil"); var async = require("async"); var mmh3 = require("murmurhash3"); -function IJOD(arg, callback) { - if(!arg || !arg.name) return callback("invalid args"); - var self = this; +function IJOD(arg) { + if(!arg || !arg.name) throw new Error("invalid args"); + var self = this; this.transactionItems = null; - self.name = arg.name; - self.gzname = arg.name + '.json.gz'; - self.dbname = arg.name + '.db'; - try{ - self.fda = fs.openSync(self.gzname, 'a'); - self.fdr = fs.openSync(self.gzname, 'r'); - var stat = fs.fstatSync(self.fdr); - self.len = stat.size; - }catch(E){ - return callback(E); - } - self.db = new sqlite.Database(); - self.db.open(self.dbname, function (err) { - if(err) return callback(err); - self.db.executeScript("CREATE TABLE IF NOT EXISTS ijod (id TEXT PRIMARY KEY, at INTEGER, len INTEGER, hash TEXT);",function (err) { - if(err) return callback(err); - callback(null, self); - }); + self.name = arg.name; + self.gzname = arg.name + '.json.gz'; + self.dbname = arg.name + '.db'; +}; +IJOD.prototype.open = function(callback) { + var self = this; + try { + self.fda = fs.openSync(self.gzname, 'a'); + self.fdr = fs.openSync(self.gzname, 'r'); + var stat = fs.fstatSync(self.fdr); + self.len = stat.size; + } catch(E) { + return callback(E); + } + self.db = new sqlite.Database(); + self.db.open(self.dbname, function (err) { + if(err) return callback(err); + self.db.executeScript("CREATE TABLE IF NOT EXISTS ijod (id TEXT PRIMARY KEY, at INTEGER, len INTEGER, hash TEXT);",function (err) { + if(err) return callback(err); + callback(null, self); }); + }); +}; +IJOD.prototype.close = function(callback) { + fs.closeSync(this.fda); + fs.closeSync(this.fdr); + this.db.close(callback); } exports.IJOD = IJOD; @@ -168,7 +176,7 @@ IJOD.prototype.smartAdd = function(arg, callback) { var self = this; var start = Date.now(); self.getOne(arg, function(err, existing){ - console.log("getOne in %d", (Date.now() - start)); + //console.log("getOne in %d", (Date.now() - start)); if(err) return callback(err); // first check if it's new if(!existing) diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 3f6a71891..1d28cd6b1 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -435,7 +435,9 @@ function getIJOD(id, key, create, callback) { // only load if one exists or create flag is set fs.stat(name+".db", function(err, stat){ if(!stat && !create) return callback(); - synclets.ijods[name] = new IJOD({name:name}, function(err, ij){ + var ij = new IJOD({name:name}) + synclets.ijods[name] = ij; + ij.open(function(err){ if(err) logger.error(err); return callback(ij); }); diff --git a/Ops/mongo2ijod.js b/Ops/mongo2ijod.js index 067ab52ec..8a8260ceb 100644 --- a/Ops/mongo2ijod.js +++ b/Ops/mongo2ijod.js @@ -1,36 +1,36 @@ var sys = require("util"); var fs = require("fs"); var lconfig = require(__dirname +"/../Common/node/lconfig"); -lconfig.load(__dirname+"/../Config/config.json"); var IJOD = require(__dirname+"/../Common/node/ijod").IJOD; var async = require("async"); var spawn = require('child_process').spawn; var lutil = require("lutil"); - -var enc = null; -var name = process.argv[2]; -var id = process.argv[3]||"id"; - -var sqlite = require('sqlite-fts'); - -var db = new sqlite.Database(); - var mongodb = require("mongodb"); +var mongo; -bootMongo() +if (exports) { + exports.run = function(cb) { + connect(cb); + } +} + +if(require.main === module) { + lconfig.load(__dirname+"/../Config/config.json"); + bootMongo() +} else { + console.log("Running mongo2ijod"); +} -var ijods = {}; -var mongo; function connect(cb){ mongo = new mongodb.Db('locker', new mongodb.Server("127.0.0.1", 27018, {})); mongo.open(function(err, p_client) { if(err) return console.error(err); mongo.collectionNames(function(err, names){ - scan(names, lconfig.lockerDir + '/' + lconfig.me, function(){ + scan(names, lconfig.lockerDir + '/' + lconfig.me, function(err){ console.error("done"); mongo.close(); - cb(); + cb(err); }); }); // mongo.collection(name, setup); @@ -44,25 +44,40 @@ function scan(names, dir, callback) { lutil.forEachSeries(files, function(file, cb){ var fullPath = dir + '/' + file; var stats = fs.statSync(fullPath); + // Skip other files if(!stats.isDirectory()) return cb(); fs.stat(fullPath+"/me.json",function(err,stats){ + // No me.json skip + if (err) return cb(); if(!stats || !stats.isFile()) return cb(); var me = JSON.parse(fs.readFileSync(fullPath+"/me.json")); + // Skip non service directories if(!me) return cb(); lutil.forEachSeries(names, function(nameo, cb2){ var name = nameo.name; var pfix = "locker.asynclets_"+me.id+"_"; + // Skip invalid synclets if(name.indexOf(pfix) == -1) return cb2(); var dname = name.substr(pfix.length); console.error(name); - ijods[name] = new IJOD({name:fullPath+"/"+dname}, function(err, ij){ - if(err) console.error(err); + var ij = new IJOD({name:fullPath+"/"+dname}); + ij.open(function(err) { + if(err) { + console.error(err); + return cb2(err); + } var id = "id"; if(me.mongoId && me.mongoId[dname]) id = me.mongoId[dname]; if(me.mongoId && me.mongoId[dname+"s"]) id = me.mongoId[dname+"s"]; mongo.collection(name.substr(7), function(err, coll){ - if(err) console.error(err); - eacher(coll, id, ij, cb2); + if(err) { + console.error(err); + return cb2(err); + } + eacher(coll, id, ij, function(err) { + if (err) return cb2(err); + ij.close(cb2); + }); }) }); }, cb); @@ -73,26 +88,39 @@ function scan(names, dir, callback) { function eacher(collection, id, ij, callback) { - // Locate all the entries using find - var count = 0; - var at = Date.now(); - ij.startAddTransaction(function() { - collection.find().each(function(err, item) { - if(!item){ - console.error("loaded " + count + " items in "+(Date.now() - at)); - ij.commitAddTransaction(callback); - return; + // Locate all the entries using find + var count = 0; + var at = Date.now(); + var cursor = collection.find(); + ij.startAddTransaction(function() { + var item; + async.until( + function() { return !item;}, + function(stepCb) { + cursor.nextObject(function(err, cur) { + item = cur; + if (!item) return stepCb(); + if (!item[id]) { + console.error("can't find "+id+" in "+JSON.stringify(item)); + return stepCb("Could not find " + id); } - if(!item[id]) console.error("can't find "+id+" in "+JSON.stringify(item)); ++count; - ij.addData({id:item[id], data:item}, function(addError) { - if (addError) { - console.error("Adding to ijod error: " + addError); - console.error(addError.stack); - } + ij.smartAdd({id:item[id], data:item}, function(addError) { + if (addError) { + console.error("Adding to ijod error: " + addError); + console.error(addError.stack); + return callback(addError); + } }); + }); + }, + function(err) { + if (err) { + return callback(err); + } + ij.commitAddTransaction(callback); }); - }); + }); }; diff --git a/migrations/0000000000008.js b/migrations/0000000000008.js new file mode 100644 index 000000000..104c57f90 --- /dev/null +++ b/migrations/0000000000008.js @@ -0,0 +1,15 @@ +/***************************** +* Converts mongo synclet data to ijod +*/ +var request = require("request"); +var logger = require("logger"); +var path = require("path"); +var lconfig = require("lconfig"); +var mongo2ijod = require(path.join(lconfig.lockerDir, "Ops", "mongo2ijod.js")); + +module.exports.preServices = function(config, callback) { + mongo2ijod.run(function(err) { + if (err) logger.error(err); + callback(err ? false : true); + }); +}; From 1cdb1f30d714781992eaf77f92f6172184691223 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Wed, 14 Mar 2012 23:48:36 -0500 Subject: [PATCH 19/35] It sure helps to call your step callbacks --- Ops/mongo2ijod.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Ops/mongo2ijod.js b/Ops/mongo2ijod.js index 8a8260ceb..6501fa39d 100644 --- a/Ops/mongo2ijod.js +++ b/Ops/mongo2ijod.js @@ -93,13 +93,16 @@ function eacher(collection, id, ij, callback) { var at = Date.now(); var cursor = collection.find(); ij.startAddTransaction(function() { - var item; + // Setting this to true just to start + var item = true; async.until( - function() { return !item;}, + function() { return item == undefined;}, function(stepCb) { cursor.nextObject(function(err, cur) { item = cur; - if (!item) return stepCb(); + if (!item) { + return stepCb(); + } if (!item[id]) { console.error("can't find "+id+" in "+JSON.stringify(item)); return stepCb("Could not find " + id); @@ -111,6 +114,7 @@ function eacher(collection, id, ij, callback) { console.error(addError.stack); return callback(addError); } + stepCb(); }); }); }, From 41ab1cc830ecd5df775a9bf411af49c959df0561 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Wed, 14 Mar 2012 23:52:58 -0500 Subject: [PATCH 20/35] A simple tool to pull down getCurrent of a few synclets for verification --- Ops/checkCurrent.js | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 Ops/checkCurrent.js diff --git a/Ops/checkCurrent.js b/Ops/checkCurrent.js new file mode 100644 index 000000000..8ae8708f0 --- /dev/null +++ b/Ops/checkCurrent.js @@ -0,0 +1,44 @@ +var request = require("request"); +var async = require("async"); +var fs = require("fs"); +var path = require("path"); + +var connectors = { + "twitter":["friends", "timeline", "tweets"], + "facebook":["home", "photos", "friends"], + "github":["repos", "users"] +}; + +var host = process.argv.length > 2 ? process.argv[2] : "localhost"; +console.log("Gathering from %s", host); + +function runIt() { + async.forEachSeries(Object.keys(connectors), function(key, cb) { + async.forEachSeries(connectors[key], function(synclet, syncletCb) { + var totalData = ""; + var fname = path.join("verification", key + "-" + synclet + ((process.argv.length > 3 && process.argv[3] == "ijod") ? "-ijod" : "") + ".json"); + console.log("Opening %s", fname); + try { + if (fs.statSync(fname)) fs.unlinkSync(fname); + } catch(E) { + } + var req = request({url:("http://" + host + ":8042/synclets/" + key + "/getCurrent/" + synclet + "?stream=true")}); + req.pipe(fs.createWriteStream(fname)); + req.on("end", function() { + console.log("Run for %s/%s done", key, synclet); + syncletCb(); + }); + }, function(syncletErr) { + console.log("Connector %s done", key); + cb(); + }); + }, function(err) { + console.log("Connectors done"); + }); +} + +fs.stat("verification", function(err, stat) { + if (err) fs.mkdirSync("verification"); + runIt(); +}); + From 2df12ab456c3ccfce127f9666cdc29106ce0dd7d Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Thu, 15 Mar 2012 22:26:31 -0500 Subject: [PATCH 21/35] Few more cleanups I'm finding for archiving --- scripts/git-archive-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/git-archive-all.sh b/scripts/git-archive-all.sh index 5183842cf..00342d830 100755 --- a/scripts/git-archive-all.sh +++ b/scripts/git-archive-all.sh @@ -178,7 +178,7 @@ echo $TMPDIR/$(basename $(pwd)).$FORMAT >| $TMPFILE # clobber on purpose superfile=`head -n 1 $TMPFILE` # find all '.git' dirs, these show us the remaining to-be-archived dirs -find . -name '.git' -type d -print | sed -e 's/^\.\///' -e 's/\.git$//' | grep -v '^$' >> $TOARCHIVE +find . -name '.git' -type d -print | sed -e 's/^\.\///' -e 's/\.git$//' | (grep -v '^$' || echo "") >> $TOARCHIVE while read path; do TREEISH=$(git submodule | grep "^ .*${path%/} " | cut -d ' ' -f 2) # git-submodule does not list trailing slashes in $path From 5c273b6206cb4bbae18cec196afa627ae6934ee4 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Thu, 15 Mar 2012 22:26:45 -0500 Subject: [PATCH 22/35] Revert "Few more cleanups I'm finding for archiving" This reverts commit 2df12ab456c3ccfce127f9666cdc29106ce0dd7d. --- scripts/git-archive-all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/git-archive-all.sh b/scripts/git-archive-all.sh index 00342d830..5183842cf 100755 --- a/scripts/git-archive-all.sh +++ b/scripts/git-archive-all.sh @@ -178,7 +178,7 @@ echo $TMPDIR/$(basename $(pwd)).$FORMAT >| $TMPFILE # clobber on purpose superfile=`head -n 1 $TMPFILE` # find all '.git' dirs, these show us the remaining to-be-archived dirs -find . -name '.git' -type d -print | sed -e 's/^\.\///' -e 's/\.git$//' | (grep -v '^$' || echo "") >> $TOARCHIVE +find . -name '.git' -type d -print | sed -e 's/^\.\///' -e 's/\.git$//' | grep -v '^$' >> $TOARCHIVE while read path; do TREEISH=$(git submodule | grep "^ .*${path%/} " | cut -d ' ' -f 2) # git-submodule does not list trailing slashes in $path From cea4f103677d578789748a384ac4cb9d1af2edfa Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Fri, 16 Mar 2012 10:51:16 -0500 Subject: [PATCH 23/35] Use the explicit close handler so we don't keep files around too long. --- Common/node/ijod.js | 6 +++--- Common/node/lsyncmanager.js | 26 ++++++++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 2b02e50e0..0a0decd30 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -63,7 +63,7 @@ IJOD.prototype.startAddTransaction = function(cbDone) { IJOD.prototype.commitAddTransaction = function(cbDone) { if (!this.transactionItems || this.transactionItems.length == 0) return cbDone(); - console.log("Commiting %d items", this.transactionItems.length); + //console.log("Commiting %d items", this.transactionItems.length); var totalSize = this.transactionItems.reduce(function(prev, cur, idx, arr) { return prev + arr[idx].length; }, 0); var writeBuffer = new Buffer(totalSize); var idx = 0; @@ -201,7 +201,7 @@ IJOD.prototype.smartAdd = function(arg, callback) { } IJOD.prototype.batchSmartAdd = function(entries, callback) { - console.log("Batch smart add %d entries", entries.length); + //console.log("Batch smart add %d entries", entries.length); var t = Date.now(); var self = this; var script = ["CREATE TEMP TABLE IF NOT EXISTS batchSmartAdd (id TEXT)", "DELETE FROM batchSmartAdd", "BEGIN TRANSACTION"].join(";") + ";"; @@ -239,7 +239,7 @@ IJOD.prototype.batchSmartAdd = function(entries, callback) { self.addData(entry, cb); }, function(error) { self.commitAddTransaction(callback); - console.log("Batch done: %d", (Date.now() - t)); + //console.log("Batch done: %d", (Date.now() - t)); }); }) }); diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 8aa9d09a6..53040a421 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -442,9 +442,18 @@ function getIJOD(id, key, create, callback) { }); }); } - exports.getIJOD = getIJOD; +function closeIJOD(id, key, callback) { + var name = path.join(lconfig.lockerDir, lconfig.me, id, key); + if (synclets.ijods[name]) { + synclets.ijods[name].close(callback); + } else { + callback(); + } +} +exports.closeIJOD = closeIJOD; + function processData (deleteIDs, info, synclet, key, data, callback) { // this extra (handy) log breaks the synclet tests somehow?? var len = (data)?data.length:0; @@ -464,22 +473,27 @@ function processData (deleteIDs, info, synclet, key, data, callback) { else mongoId = 'id'; getIJOD(info.id, key, true, function(ij){ + function finish(err) { + closeIJOD(info.id, key, function() { + return callback(err); + }); + } if (deleteIDs && deleteIDs.length > 0 && data) { addData(collection, mongoId, data, info, synclet, idr, ij, function(err) { if(err) { - callback(err); + finish(err); } else { - deleteData(collection, mongoId, deleteIDs, info, synclet, idr, ij, callback); + deleteData(collection, mongoId, deleteIDs, info, synclet, idr, ij, finish); } }); } else if (data && data.length > 0) { addData(collection, mongoId, data, info, synclet, idr, ij, function(err) { - callback(); + finish(); }); } else if (deleteIDs && deleteIDs.length > 0) { - deleteData(collection, mongoId, deleteIDs, info, synclet, idr, ij, callback); + deleteData(collection, mongoId, deleteIDs, info, synclet, idr, ij, finish); } else { - callback(); + finish(); } }); } From 7735358d36db4d04bf0e86e63d5ead4c69a46feb Mon Sep 17 00:00:00 2001 From: Matt Zimmerman Date: Fri, 16 Mar 2012 10:37:45 -0700 Subject: [PATCH 24/35] Use default mongo options when running the test suite --- tests/Config/config.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/Config/config.json b/tests/Config/config.json index ac132e9b8..463a5433b 100644 --- a/tests/Config/config.json +++ b/tests/Config/config.json @@ -28,8 +28,7 @@ "console":false, "dataDir": "mongoTest", "host": "localhost", - "port": 27020, - "options": ["--nohttpinterface", "--noprealloc"] + "port": 27020 }, "logging": { "level": "error", From 579e109bde1f9ed9e76380e382b7503a0c0e4266 Mon Sep 17 00:00:00 2001 From: Matt Zimmerman Date: Fri, 16 Mar 2012 10:38:58 -0700 Subject: [PATCH 25/35] Revert "Use default mongo options when running the test suite" This reverts commit 7735358d36db4d04bf0e86e63d5ead4c69a46feb. --- tests/Config/config.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Config/config.json b/tests/Config/config.json index 463a5433b..ac132e9b8 100644 --- a/tests/Config/config.json +++ b/tests/Config/config.json @@ -28,7 +28,8 @@ "console":false, "dataDir": "mongoTest", "host": "localhost", - "port": 27020 + "port": 27020, + "options": ["--nohttpinterface", "--noprealloc"] }, "logging": { "level": "error", From d2d8ca3643db6f3e08892b68d0e4057877f04ae4 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Fri, 16 Mar 2012 12:52:20 -0500 Subject: [PATCH 26/35] Better error handling and rollback This changes transactions so it has actual transaction rollback. The batch add defaults to using this with 5 attempts to land a commit. I'm going fail hard if those 5 attempts fail until I understand why it's failing. --- Common/node/ijod.js | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 0a0decd30..b8f6eac9e 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -82,10 +82,21 @@ IJOD.prototype.commitAddTransaction = function(cbDone) { } else if (written != totalSize) { console.error("Only %d written of %d bytes to IJOD", written, totalSize); } - self.db.execute("COMMIT TRANSACTION", function(error, rows) { cbDone() }); + self.db.execute("COMMIT TRANSACTION", function(error, rows) { cbDone(); }); }); }); -} +}; + +/// Abort a pending add transaction +/** +* Any pending write chunks are destroyed and the database transaction is rolled back. +* This is safe to call without a transaction started. +*/ +IJOD.prototype.abortAddTransaction = function(cbDone) { + if (!self.transactionItems) return cbDone(); + self.transactionItems = null; + self.db.execute("ROLLBACK TRANSACTION", function(error, rows) { cbDone(); }); +}; // takes arg of at least an id and data, callback(err) when done IJOD.prototype.addData = function(arg, callback) { @@ -204,21 +215,43 @@ IJOD.prototype.batchSmartAdd = function(entries, callback) { //console.log("Batch smart add %d entries", entries.length); var t = Date.now(); var self = this; + // TODO: We have 5 attempts to write and then we blow up, we need something smarter to do with this data + self.batchAttempt = self.batchAttempt || 0; + if (++self.batchAttempt > 5) { + self.batchAttempt = 0; + console.error("Out of attempts trying to batchSmartAdd, blowing up"); + throw new Error("Unable to batchSmartAdd"); + } + + function handleError(msg) { + console.error("Error: %s", msg); + self.abortAddTransaction(function() { + setTimeout(function() { + self.batchSmartAdd(entries, callback); + }, 500); + }); + } + var script = ["CREATE TEMP TABLE IF NOT EXISTS batchSmartAdd (id TEXT)", "DELETE FROM batchSmartAdd", "BEGIN TRANSACTION"].join(";") + ";"; self.db.executeScript(script, function(error, rows) { self.db.prepare("INSERT INTO batchSmartAdd VALUES (?)", function(error, stmt) { + if (error) return handleError(error); async.forEachSeries(entries, function(entry, cb) { if (!entry) return cb(); stmt.bind(1, entry.id, function(err) { + if (err) return handleError(err); stmt.step(function(error, row) { + if (error) return handleError(error); stmt.reset(); cb(); }); }); }, function(error) { self.db.execute("COMMIT TRANSACTION", function(error, rows) { + if (error) return handleError(error); stmt.finalize(function(error) { self.db.execute("SELECT id,hash FROM ijod WHERE ijod.id IN (SELECT id FROM batchSmartAdd)", function(error, rows) { + if (error) return handleError(error); var knownIds = {}; rows = rows || []; rows.forEach(function(row) { @@ -238,6 +271,8 @@ IJOD.prototype.batchSmartAdd = function(entries, callback) { } self.addData(entry, cb); }, function(error) { + if (error) return handleError(error); + self.batchAttempt = 0; self.commitAddTransaction(callback); //console.log("Batch done: %d", (Date.now() - t)); }); From 931ee3ecd77d3f20b3d0c367bf232e7ff14f01ee Mon Sep 17 00:00:00 2001 From: Matt Zimmerman Date: Fri, 16 Mar 2012 11:36:07 -0700 Subject: [PATCH 27/35] Add "jenkins" rule and move custom jenkins stuff here This stuff is much more transparent if we push it into the source tree, rather than putting it in the jenkins config. To the greatest extent possible, Jenkins should just run "make jenkins" and let us take care of the rest here. --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 22dbcdc31..31de3683a 100644 --- a/Makefile +++ b/Makefile @@ -87,10 +87,13 @@ $(DISTFILE): submodules ./scripts/build-tarball "$(SUBDIR)" "$@" # create a ready-to-run tarball, and then run tests on the contents -# (this is the rule that Jenkins runs -mdz 2012-02-04) test-bindist: $(DISTFILE) ./scripts/test-tarball "$(SUBDIR)" "$<" +# this is the rule that Jenkins runs as of 2012-03-16 +jenkins: + xvfb-run -a --server-args="-screen 0 1280x960x24" $(MAKE) test-bindist + clean: rm -f "$(DISTFILE)" "$(TEMPLATE_OUTPUT)" build.json tests/build.json rm -rf node_modules From 4962ff94005709875db605e2953179b4457f99a0 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Fri, 16 Mar 2012 13:44:30 -0500 Subject: [PATCH 28/35] Disable gracefulfs and check for double closes --- Common/node/ijod.js | 7 +++++++ lockerd.js | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index b8f6eac9e..8a3dfc76a 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -49,6 +49,13 @@ IJOD.prototype.open = function(callback) { }); }; IJOD.prototype.close = function(callback) { + if (this.closed) { + console.error("Double close in IJOD"); + var E = new Error("double close"); + console.log(E.stack); + return callback(); + } + this.closed = true; fs.closeSync(this.fda); fs.closeSync(this.fdr); this.db.close(callback); diff --git a/lockerd.js b/lockerd.js index 08a76177c..3cc7bb74f 100644 --- a/lockerd.js +++ b/lockerd.js @@ -32,7 +32,8 @@ var async = require('async'); var util = require('util'); var lutil = require('lutil'); var carrier = require('carrier'); -require('graceful-fs'); +// TODO: Reconsider enabling this, but realistically we should be managing this ourselves. +// require('graceful-fs'); // This lconfig stuff has to come before any other locker modules are loaded!! From 53ebd65cb1828d7c1ae749664c7d8f6df78b3c79 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Fri, 16 Mar 2012 14:28:40 -0500 Subject: [PATCH 29/35] self is actually this, make it so --- Common/node/ijod.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 8a3dfc76a..81b699931 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -100,9 +100,9 @@ IJOD.prototype.commitAddTransaction = function(cbDone) { * This is safe to call without a transaction started. */ IJOD.prototype.abortAddTransaction = function(cbDone) { - if (!self.transactionItems) return cbDone(); - self.transactionItems = null; - self.db.execute("ROLLBACK TRANSACTION", function(error, rows) { cbDone(); }); + if (!this.transactionItems) return cbDone(); + this.transactionItems = null; + this.db.execute("ROLLBACK TRANSACTION", function(error, rows) { cbDone(); }); }; // takes arg of at least an id and data, callback(err) when done From a090521dfae4edcb1f0e76db2de0c2e06747f1e3 Mon Sep 17 00:00:00 2001 From: Matt Zimmerman Date: Fri, 16 Mar 2012 13:39:27 -0700 Subject: [PATCH 30/35] Give up --- tests/Config/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Config/config.json b/tests/Config/config.json index ff6917cb8..ab1d416ea 100644 --- a/tests/Config/config.json +++ b/tests/Config/config.json @@ -1,6 +1,6 @@ { "lockerHost" : "localhost", - "lockerPort" : 0, + "lockerPort" : 8043, "me" : "Data", "registryUpdate" : false, "requireSigned" : false, From 9bcf158f782c6ed316ab83c393d9084debc4d750 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Fri, 16 Mar 2012 16:56:23 -0500 Subject: [PATCH 31/35] Fix push for the IJOD changes --- Common/node/ijod.js | 7 ++++--- Common/node/lpushmanager.js | 15 ++++++++++----- Common/node/lsyncmanager.js | 4 +++- Ops/webservice-push.js | 1 + package.json | 2 +- tests/lpushmanager-test-local.js | 11 ++++++----- 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 81b699931..bb28bba8d 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -231,7 +231,7 @@ IJOD.prototype.batchSmartAdd = function(entries, callback) { } function handleError(msg) { - console.error("Error: %s", msg); + console.error("Batch smart add error: %s", msg); self.abortAddTransaction(function() { setTimeout(function() { self.batchSmartAdd(entries, callback); @@ -246,14 +246,15 @@ IJOD.prototype.batchSmartAdd = function(entries, callback) { async.forEachSeries(entries, function(entry, cb) { if (!entry) return cb(); stmt.bind(1, entry.id, function(err) { - if (err) return handleError(err); + if (err) return cb(err); stmt.step(function(error, row) { - if (error) return handleError(error); + if (error) return cb(error); stmt.reset(); cb(); }); }); }, function(error) { + if (error) return handleError(error); self.db.execute("COMMIT TRANSACTION", function(error, rows) { if (error) return handleError(error); stmt.finalize(function(error) { diff --git a/Common/node/lpushmanager.js b/Common/node/lpushmanager.js index 8424aeb38..2ece6e494 100644 --- a/Common/node/lpushmanager.js +++ b/Common/node/lpushmanager.js @@ -52,7 +52,9 @@ function getIJOD(dataset, create, callback) { // only load if one exists or create flag is set fs.stat(name+".db", function(err, stat){ if(!stat && !create) return callback(); - config.ijods[dataset] = new IJOD({name:name}, function(err, ij){ + var ij = new IJOD({name:name}) + config.ijods[dataset] = ij; + ij.open(function(err) { if(err) logger.error(err); return callback(ij); }); @@ -83,6 +85,7 @@ function processData (deleteIDs, data, dataset, callback) { lutil.atomicWriteFileSync(path.join(lconfig.lockerDir, lconfig.me, "push", 'push_config.json'), JSON.stringify(config, null, 4)); } + // TODO: Explicitly close getIJOD(dataset, true, function(ijod){ if (deleteIDs && deleteIDs.length > 0 && data) { addData(dataset, data, ijod, function(err) { @@ -136,13 +139,15 @@ function addData (dataset, data, ijod, callback) { } else { cb(); } - }, 5); - data.forEach(function(d){ q.push(d, errs.push); }); // hehe fun + }, 1); + ijod.startAddTransaction(function(err) { + data.forEach(function(d){ q.push(d, errs.push); }); // hehe fun + }); q.drain = function() { if (errs.length > 0) { - callback(errs); + ijod.abortAddTransaction(function() { callback(errs); }); } else { - callback(); + ijod.commitAddTransaction(callback); } }; } diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index e7417861c..7c60793bb 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -421,7 +421,7 @@ function processResponse(deleteIDs, info, synclet, response, callback) { stats.increment('synclet.' + info.id + '.' + synclet.name + '.added', synclet.added); stats.increment('synclet.' + info.id + '.' + synclet.name + '.updated', synclet.updated); stats.increment('synclet.' + info.id + '.' + synclet.name + '.deleted', synclet.deleted); - stats.increment('synclet.' + info.id + '.' + synclet.name + '.length', dataKeys.reduce(function(prev, cur, idx, arr) { console.log(cur); return prev + response.data[cur].length; }, 0)); + stats.increment('synclet.' + info.id + '.' + synclet.name + '.length', dataKeys.reduce(function(prev, cur, idx, arr) { return prev + response.data[cur].length; }, 0)); logger.info("total of "+synclet.added+"+"+synclet.updated+"+"+synclet.deleted+" and threshold "+threshold+" so setting tolerance to "+synclet.tolMax); callback(err); }); @@ -430,6 +430,7 @@ function processResponse(deleteIDs, info, synclet, response, callback) { // simple async friendly wrapper function getIJOD(id, key, create, callback) { var name = path.join(lconfig.lockerDir, lconfig.me, id, key); + console.log("Open IJOD %s", name); if(synclets.ijods[name]) return callback(synclets.ijods[name]); // only load if one exists or create flag is set fs.stat(name+".db", function(err, stat){ @@ -446,6 +447,7 @@ exports.getIJOD = getIJOD; function closeIJOD(id, key, callback) { var name = path.join(lconfig.lockerDir, lconfig.me, id, key); + console.log("Close IJOD %s", name); if (synclets.ijods[name]) { synclets.ijods[name].close(callback); } else { diff --git a/Ops/webservice-push.js b/Ops/webservice-push.js index e50b9d988..760a9dc2d 100644 --- a/Ops/webservice-push.js +++ b/Ops/webservice-push.js @@ -27,6 +27,7 @@ module.exports = function(locker) { }); locker.get('/push/:dataset/getCurrent', function(req, res) { + console.log("push current for %s", req.params.dataset); pushManager.getIJOD(req.params.dataset, false, function(ijod) { if(!ijod) return res.send("not found",404); ijod.reqCurrent(req, res); diff --git a/package.json b/package.json index a8e4f757f..5a80b61f1 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "compress-buffer": "=0.5.1", "mongodb": "=0.9.8-1", "winston": "=0.5.2", - "sqlite-fts": "=0.0.8", + "sqlite-fts": "=0.0.9", "socket.io": "=0.8.5", "socket.io-client": "=0.8.4", "jade": "= 0.20.0", diff --git a/tests/lpushmanager-test-local.js b/tests/lpushmanager-test-local.js index 7d1b50f1a..585b55f9c 100644 --- a/tests/lpushmanager-test-local.js +++ b/tests/lpushmanager-test-local.js @@ -84,13 +84,14 @@ vows.describe("Push Manager").addBatch({ }).addBatch({ "Querying the data API returns the data" : { topic: function() { - request.get({uri : "http://localhost:8043/push/testing/getCurrent"}, this.callback) + request.get({uri : "http://localhost:8043/push/testing/getCurrent?stream=true"}, this.callback) }, "from testSync" : function(err, resp, body) { - var data = JSON.parse(body); - obj = data[0]; - assert.equal(data[0].id, 500); - assert.equal(data[0].someData, 'BAM'); + var parts = body.split("\n"); + var data = JSON.parse(parts[0]); + obj = data; + assert.equal(data.id, 500); + assert.equal(data.someData, 'BAM'); } } }).addBatch({ From 3c5e3c846d8d2f073aefdbc8d417ee1db5da4559 Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Mon, 19 Mar 2012 16:01:54 -0500 Subject: [PATCH 32/35] Refactoring service manager scheduling to get rid of logic bombs. There were known and unknown logic bombs in the code that have been purged by this pass. All scheduling is centralized and synclets are only executed from one location now. --- Common/node/ijod.js | 2 + Common/node/lservicemanager.js | 11 +- Common/node/lsyncmanager.js | 291 +++++++++++++++++---------------- lockerd.js | 25 +-- 4 files changed, 165 insertions(+), 164 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index bb28bba8d..217598dc9 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -232,6 +232,7 @@ IJOD.prototype.batchSmartAdd = function(entries, callback) { function handleError(msg) { console.error("Batch smart add error: %s", msg); + console.trace(); self.abortAddTransaction(function() { setTimeout(function() { self.batchSmartAdd(entries, callback); @@ -241,6 +242,7 @@ IJOD.prototype.batchSmartAdd = function(entries, callback) { var script = ["CREATE TEMP TABLE IF NOT EXISTS batchSmartAdd (id TEXT)", "DELETE FROM batchSmartAdd", "BEGIN TRANSACTION"].join(";") + ";"; self.db.executeScript(script, function(error, rows) { + if (error) return handleError(error); self.db.prepare("INSERT INTO batchSmartAdd VALUES (?)", function(error, stmt) { if (error) return handleError(error); async.forEachSeries(entries, function(entry, cb) { diff --git a/Common/node/lservicemanager.js b/Common/node/lservicemanager.js index eafeadb02..6e4161aed 100644 --- a/Common/node/lservicemanager.js +++ b/Common/node/lservicemanager.js @@ -25,19 +25,18 @@ var stats; var serviceMap = { }; // All of the immediately addressable services in the system var shuttingDown = null; -var syncletManager, registry; +var registry; var lockerPortNext = parseInt("1" + lconfig.lockerPort, 10); /** * Scans the Me directory for instaled services */ -exports.init = function (sman, reg, callback) { +exports.init = function (reg, callback) { logger = require('logger'); logger.info('lservicemanager lockerPortNext = ' + lockerPortNext); stats = new dispatcher(lconfig.stats); - syncletManager = sman; registry = reg; var dirs = fs.readdirSync(lconfig.me); for (var i = 0; i < dirs.length; i++) { @@ -220,12 +219,6 @@ exports.mapReload = function(id) { levents.addListener(ev[0], js.id, ev[1], batching); } } - // start em up if they're ready - if (js.synclets && js.auth && js.authed) { - for (var j = 0; j < js.synclets.length; j++) { - syncletManager.scheduleRun(js, js.synclets[j]); - } - } exports.mapDirty(js.id); }; diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index 7c60793bb..db4f92ad2 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -14,65 +14,107 @@ var fs = require('fs') , dispatcher = require('./instrument.js').StatsdDispatcher , stats = new dispatcher(lconfig.stats); +var PAGIN_TIMING = 2000; // 2s gap in paging + var runningContexts = {}; // Map of a synclet to a running context -function LockerInterface(synclet, info) { - EventEmitter.call(this); - this.synclet = synclet; - this.info = info; - this.srcdir = path.join(lconfig.lockerDir, info.srcdir); - this.workingDirectory = path.join(lconfig.lockerDir, lconfig.me, info.id); - this.processing = false; // If we're processing events - this.events = []; +function SyncletManager() +{ + this.scheduled = {}; + this.offlineMode = false; } -util.inherits(LockerInterface, EventEmitter); -LockerInterface.prototype.error = function(message) { - logger.error("Error from synclet " + this.synclet.name + "/" + this.info.id + ": " + message); +/// Schedule a synclet to run +/** +* timeToRun is optional. In this case the next run time is calculated +* based on normal frequency and tolerance methods. +*/ +SyncletManager.prototype.schedule = function(connectorInfo, syncletInfo, timeToRun) { + var key = this.getKey(connectorInfo, syncletInfo); + // Let's get back to a clean slate on this synclet + if (this.scheduled[key]) { + logger.debug(key + " was already scheduled."); + this.cleanup(connectorInfo, syncletInfo); + } + syncletInfo.status = "waiting"; + if (!syncletInfo.frequency) return; + + // In offline mode things may only be ran directly with runAndReschedule + if (this.offlineMode) return; + + // validation check + if(syncletInfo.nextRun && typeof syncletInfo.nextRun != "number") delete syncletInfo.nextRun; + + if (timeToRun === undefined) { + // had a schedule and missed it, run it now + if(syncletInfo.nextRun && syncletInfo.nextRun <= Date.now()) { + logger.verbose("scheduling " + key + " to run immediately (missed)"); + timeToRun = 0; + } else if (!syncletInfo.nextRun) { + // if not scheduled yet, schedule it to run in the future + var milliFreq = parseInt(syncletInfo.frequency) * 1000; + syncletInfo.nextRun = parseInt(Date.now() + milliFreq + (((Math.random() - 0.5) * 0.5) * milliFreq)); // 50% fuzz added or subtracted + timeToRun = syncletInfo.nextRun - Date.now(); + } + } + + logger.verbose("scheduling " + key + " (freq " + syncletInfo.frequency + "s) to run in " + (timeToRun / 1000) + "s"); + var self = this; + this.scheduled[key] = setTimeout(function() { + self.runAndReschedule(connectorInfo, syncletInfo); + }, timeToRun); }; -// Fire an event from the synclet -LockerInterface.prototype.event = function(action, lockerType, obj) { - this.events.push({action:action, lockerType:lockerType, obj:obj}); - this.emit("event"); - this.processEvents(); +/// Remove the synclet from scheduled and cleanup all other state, does not reset it to run again +SyncletManager.prototype.cleanup = function(connectorInfo, syncletInfo) { + var key = this.getKey(connectorInfo, syncletInfo); + if (this.scheduled[key]) { + clearTimeout(this.scheduled[key]); // remove any existing timer + delete this.scheduled[key]; + } }; -LockerInterface.prototype.processEvents = function() { - if (this.processing) return; - // Process the events we have - this.processing = true; +/// Run the synclet and then attempt to reschedule it +SyncletManager.prototype.runAndReschedule = function(connectorInfo, syncletInfo, callback) { + //TODO: Ensure that we only run one per connector at a time. + delete syncletInfo.nextRun; // We're going to run and get a new time, so reset + this.cleanup(connectorInfo, syncletInfo); var self = this; - var curEvents = this.events; - this.events = []; - async.forEachSeries(curEvents, function(event, cb) { - processData([], self.info, self.synclet, event.lockerType, [event], cb); - }, function(error) { - self.processing = false; - if (self.events.length === 0) { - self.emit("drain"); - } else { - process.nextTick(function() { - self.processEvents(); - }); + // Tolerance isn't done yet, we'll come back + if (!this.checkToleranceReady(connectorInfo, syncletInfo)) { + return self.schedule(connectorInfo, syncletInfo); + } + executeSynclet(connectorInfo, syncletInfo, function(error) { + var nextRunTime = undefined; + if (connectorInfo.config && connectorInfo.config.nextRun < 0) { + // This wants to page so we'll schedule it for anther run in just a short gap. + nextRunTime = PAGING_TIMING; } + // Make sure we reschedule this before we return anything else + self.schedule(connectorInfo, syncletInfo, nextRunTime); + if (callback) return callback(error); }); }; -// Signals that the synclet context is complete and may be cleaned up -LockerInterface.prototype.end = function() { - if (this.events.length > 0) { - this.once("drain", function() { - this.emit("end"); - }); - } else { - this.emit("end"); +/// Return true if the tolerance is ready for us to actually run +SyncletManager.prototype.checkToleranceReady = function(connectorInfo, syncletInfo) { + // Make sure the baseline is there + if (syncletInfo.tolMax === undefined) { + syncletInfo.tolAt = 0; + syncletInfo.tolMax = 0; + } + // if we can have tolerance, try again later + if(syncletInfo.tolAt > 0) { + syncletInfo.tolAt--; + logger.verbose("tolerance now at " + syncletInfo.tolAt + " synclet " + syncletInfo.name + " for " + connectorInfo.id); + return false; } + return true; }; - -var synclets = { - available:[], - installed:{}, - ijods:{}, - executeable:true +// This trivial helper function just makes sure we're consistent and we can change it easly +SyncletManager.prototype.getKey = function(connectorInfo, syncletInfo) { + return connectorInfo.id + "-" + syncletInfo.name; }; +syncletManager = new SyncletManager(); + +var ijods = {}; // core syncmanager init function, need to talk to serviceManager var serviceManager; @@ -86,87 +128,65 @@ exports.setExecuteable = function (e) { executeable = e; }; +// Run a connector or a specific synclet right now exports.syncNow = function (serviceId, syncletId, post, callback) { - if(typeof syncletId == "function") - { - callback = syncletId; - syncletId = false; + if(typeof syncletId == "function") { + callback = syncletId; + syncletId = false; + } + + var js = serviceManager.map(serviceId); + if (!js || !js.synclets) return callback("no synclets like that installed"); + async.forEachSeries(js.synclets, function (synclet, cb) { + if (!synclet) { + logger.error("Unknown synclet info in syncNow"); + return cb(); } - var js = serviceManager.map(serviceId); - if (!js || !js.synclets) return callback("no synclets like that installed"); - async.forEachSeries(js.synclets, function (synclet, cb) { - if (!synclet) { - logger.error("Unknown synclet info in syncNow"); - cb(); - } - if(syncletId && synclet.name != syncletId) return cb(); - if(post) - { - if(!Array.isArray(synclet.posts)) synclet.posts = []; - synclet.posts.push(post); - } - executeSynclet(js, synclet, cb, true); - }, callback); + // If they requested a specific synclet we'll skip everything but that one + if(syncletId && synclet.name != syncletId) return cb(); + if(post) { + if(!Array.isArray(synclet.posts)) synclet.posts = []; + synclet.posts.push(post); + } + syncletManager.runAndReschedule(js, synclet, cb); + }, callback); }; // run all synclets that have a tolerance and reset them exports.flushTolerance = function(callback, force) { - var map = serviceManager.map(); - async.forEach(Object.keys(map), function(service, cb){ // do all services in parallel - if(!map[service].synclets) return cb(); - async.forEachSeries(map[service].synclets, function(synclet, cb2) { // do each synclet in series - if (!force && (!synclet.tolAt || synclet.tolAt === 0)) return cb2(); - synclet.tolAt = 0; - executeSynclet(map[service], synclet, cb2); - }, cb); - }, callback); + // TODO: This now has an implied force, is that correct, why wouldn't you want this to force? + var map = serviceManager.map(); + async.forEach(Object.keys(map), function(service, cb){ // do all services in parallel + // We only want services with synclets + if(!map[service].synclets) return cb(); + async.forEachSeries(map[service].synclets, function(synclet, cb2) { // do each synclet in series + synclet.tolAt = 0; + syncletManager.runAndReschedule(map[service], synclet, cb2); + }, cb); + }, callback); }; /** * Add a timeout to run a synclet +* DEPRECATE: Use the syncletManager.schedule directly */ -var scheduled = {}; exports.scheduleRun = function(info, synclet) { - synclet.status = "waiting"; - if (!synclet.frequency) return; - - var key = info.id + "-" + synclet.name; - if(scheduled[key]) clearTimeout(scheduled[key]); // remove any existing timer - - // run from a clean state - var force = false; - function run() { - delete scheduled[key]; - executeSynclet(info, synclet, function(){}, force); - } - - // the synclet is paging and needs to run again immediately, forcefully - if(info.config && info.config.nextRun === -1) - { - force = true; - delete info.config.nextRun; - logger.verbose("scheduling "+key+" to run immediately (paging)"); - return setTimeout(run, 2000); - } - - // validation check - if(synclet.nextRun && typeof synclet.nextRun != "number") delete synclet.nextRun; - - // had a schedule and missed it, run it now - if(synclet.nextRun && synclet.nextRun <= Date.now()) { - logger.verbose("scheduling "+key+" to run immediately (missed)"); - return process.nextTick(run); - } + logger.warn("Deprecated use of scheduleRun"); + console.trace(); + syncletManager.schedule(info, synclet); +}; - // if not scheduled yet, schedule it to run in the future - if(!synclet.nextRun) - { - var milliFreq = parseInt(synclet.frequency) * 1000; - synclet.nextRun = parseInt(Date.now() + milliFreq + (((Math.random() - 0.5) * 0.5) * milliFreq)); // 50% fuzz added or subtracted - } - var timeout = synclet.nextRun - Date.now(); - logger.verbose("scheduling "+key+" (freq "+synclet.frequency+") to run in "+(timeout/1000)+"s"); - scheduled[key] = setTimeout(run, timeout); +// Find al the synclets and get their initial scheduling done +exports.scheduleAll = function(callback) { + var map = serviceManager.map(); + async.forEach(Object.keys(map), function(service, cb){ // do all services in parallel + // We only want services with synclets + if(!map[service].synclets) return cb(); + async.forEach(map[service].synclets, function(synclet, cb2) { // do each synclet in series + syncletManager.schedule(map[service], synclet); + cb2(); + }, cb); + }, callback); }; function localError(base, err) { @@ -176,38 +196,18 @@ function localError(base, err) { /** * Executes a synclet */ -function executeSynclet(info, synclet, callback, force) { +function executeSynclet(info, synclet, callback) { if(!callback) callback = function(err){ if(err) return logger.error(err); logger.debug("finished processing "+synclet.name); }; if (synclet.status === 'running') return callback('already running'); - delete synclet.nextRun; // cancel any schedule if(!info.auth || !info.authed) return callback("no auth info for "+info.id); - // we're put on hold from running any for some reason, re-schedule them - // this is a workaround for making synclets available in the map separate from scheduling them which could be done better - if (!force && !executeable) { - setTimeout(function() { - executeSynclet(info, synclet, callback); - }, 1000); - return; - } - if(!synclet.tolMax) { - synclet.tolAt = 0; - synclet.tolMax = 0; - } - // if we can have tolerance, try again later - if(!force && synclet.tolAt > 0) { - synclet.tolAt--; - logger.verbose("tolerance now at "+synclet.tolAt+" synclet "+synclet.name+" for "+info.id); - exports.scheduleRun(info, synclet); - return callback(); - } // if another synclet is running, come back a little later, don't overlap! if (info.status == 'running' || runningContexts[info.id + "/" + synclet.name]) { logger.verbose("delaying "+synclet.name); setTimeout(function() { - executeSynclet(info, synclet, callback, force); + executeSynclet(info, synclet, callback); }, 10000); return; } @@ -283,7 +283,7 @@ function executeSynclet(info, synclet, callback, force) { var deleteIDs = compareIDs(info.config, response.config); info.auth = lutil.extend(true, info.auth, response.auth); // for refresh tokens and profiles info.config = lutil.extend(true, info.config, response.config); - exports.scheduleRun(info, synclet); + //exports.scheduleRun(info, synclet); serviceManager.mapDirty(info.id); // save out to disk processResponse(deleteIDs, info, synclet, response, function(processErr) { info.status = 'waiting'; @@ -352,7 +352,7 @@ function executeSynclet(info, synclet, callback, force) { var deleteIDs = compareIDs(info.config, response.config); info.auth = lutil.extend(true, info.auth, response.auth); // for refresh tokens and profiles info.config = lutil.extend(true, info.config, response.config); - exports.scheduleRun(info, synclet); + //exports.scheduleRun(info, synclet); serviceManager.mapDirty(info.id); // save out to disk processResponse(deleteIDs, info, synclet, response, function(err){ info.status = 'waiting'; @@ -430,13 +430,13 @@ function processResponse(deleteIDs, info, synclet, response, callback) { // simple async friendly wrapper function getIJOD(id, key, create, callback) { var name = path.join(lconfig.lockerDir, lconfig.me, id, key); - console.log("Open IJOD %s", name); - if(synclets.ijods[name]) return callback(synclets.ijods[name]); + //console.log("Open IJOD %s", name); + if(ijods[name]) return callback(ijods[name]); // only load if one exists or create flag is set fs.stat(name+".db", function(err, stat){ if(!stat && !create) return callback(); var ij = new IJOD({name:name}) - synclets.ijods[name] = ij; + ijods[name] = ij; ij.open(function(err){ if(err) logger.error(err); return callback(ij); @@ -447,9 +447,12 @@ exports.getIJOD = getIJOD; function closeIJOD(id, key, callback) { var name = path.join(lconfig.lockerDir, lconfig.me, id, key); - console.log("Close IJOD %s", name); - if (synclets.ijods[name]) { - synclets.ijods[name].close(callback); + //console.log("Close IJOD %s", name); + if (ijods[name]) { + ijods[name].close(function(error) { + delete ijods[name]; + callback(); + }); } else { callback(); } diff --git a/lockerd.js b/lockerd.js index 3cc7bb74f..8f6bbbdd4 100644 --- a/lockerd.js +++ b/lockerd.js @@ -140,20 +140,23 @@ function finishStartup() { pushManager.init(); // ordering sensitive, as synclet manager is inert during init, servicemanager's init will call into syncletmanager + // Dear lord this massive waterfall is so scary syncManager.init(serviceManager, function() { - registry.init(serviceManager, syncManager, lconfig, lcrypto, function() { - serviceManager.init(syncManager, registry, function() { // this may trigger synclets to start! - runMigrations("postServices", function() { - // start web server (so we can all start talking) - var webservice = require(__dirname + "/Ops/webservice.js"); - webservice.startService(lconfig.lockerPort, lconfig.lockerListenIP, function(locker) { - if (lconfig.airbrakeKey) locker.initAirbrake(lconfig.airbrakeKey); - registry.app(locker); // add it's endpoints - postStartup(); - }); - }); + registry.init(serviceManager, syncManager, lconfig, lcrypto, function() { + serviceManager.init(registry, function() { // this may trigger synclets to start! + syncManager.scheduleAll(function() { + runMigrations("postServices", function() { + // start web server (so we can all start talking) + var webservice = require(__dirname + "/Ops/webservice.js"); + webservice.startService(lconfig.lockerPort, lconfig.lockerListenIP, function(locker) { + if (lconfig.airbrakeKey) locker.initAirbrake(lconfig.airbrakeKey); + registry.app(locker); // add it's endpoints + postStartup(); + }); }); + }); }); + }); }); var lockerPortNext = "1"+lconfig.lockerPort; lockerPortNext++; From d063aad4be0318d80f988071c586a01d8e0a311b Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Tue, 20 Mar 2012 11:01:48 -0500 Subject: [PATCH 33/35] Specific debugging trying to find a weird error --- Common/node/ijod.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index 217598dc9..f9413e579 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -65,6 +65,7 @@ exports.IJOD = IJOD; IJOD.prototype.startAddTransaction = function(cbDone) { if (this.transactionItems) return cbDone(); this.transactionItems = []; + console.log("****************************** BEGIN TRANSACTION in normal"); this.db.execute("BEGIN TRANSACTION", function(error, rows) { cbDone(); }); }; @@ -89,6 +90,7 @@ IJOD.prototype.commitAddTransaction = function(cbDone) { } else if (written != totalSize) { console.error("Only %d written of %d bytes to IJOD", written, totalSize); } + console.log("****************************** COMMIT TRANSACTION in normal"); self.db.execute("COMMIT TRANSACTION", function(error, rows) { cbDone(); }); }); }); @@ -241,6 +243,7 @@ IJOD.prototype.batchSmartAdd = function(entries, callback) { } var script = ["CREATE TEMP TABLE IF NOT EXISTS batchSmartAdd (id TEXT)", "DELETE FROM batchSmartAdd", "BEGIN TRANSACTION"].join(";") + ";"; + console.log("****************************** BEGIN TRANSACTION in batch"); self.db.executeScript(script, function(error, rows) { if (error) return handleError(error); self.db.prepare("INSERT INTO batchSmartAdd VALUES (?)", function(error, stmt) { @@ -257,6 +260,7 @@ IJOD.prototype.batchSmartAdd = function(entries, callback) { }); }, function(error) { if (error) return handleError(error); + console.log("****************************** COMMIT TRANSACTION in batch"); self.db.execute("COMMIT TRANSACTION", function(error, rows) { if (error) return handleError(error); stmt.finalize(function(error) { From 8203fa1f728896b4d5fb49347142f2f2062516ae Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Tue, 20 Mar 2012 12:35:14 -0500 Subject: [PATCH 34/35] typo in paging --- Common/node/lsyncmanager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common/node/lsyncmanager.js b/Common/node/lsyncmanager.js index db4f92ad2..3f3678a44 100644 --- a/Common/node/lsyncmanager.js +++ b/Common/node/lsyncmanager.js @@ -14,7 +14,7 @@ var fs = require('fs') , dispatcher = require('./instrument.js').StatsdDispatcher , stats = new dispatcher(lconfig.stats); -var PAGIN_TIMING = 2000; // 2s gap in paging +var PAGING_TIMING = 2000; // 2s gap in paging var runningContexts = {}; // Map of a synclet to a running context From 190d8f0c648f58f1fb6a7758ec2e7cab9e73be5a Mon Sep 17 00:00:00 2001 From: Thomas Muldowney Date: Tue, 20 Mar 2012 12:42:10 -0500 Subject: [PATCH 35/35] More specific debug logging --- Common/node/ijod.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Common/node/ijod.js b/Common/node/ijod.js index f9413e579..6abda10e9 100644 --- a/Common/node/ijod.js +++ b/Common/node/ijod.js @@ -65,7 +65,7 @@ exports.IJOD = IJOD; IJOD.prototype.startAddTransaction = function(cbDone) { if (this.transactionItems) return cbDone(); this.transactionItems = []; - console.log("****************************** BEGIN TRANSACTION in normal"); + console.log("****************************** BEGIN TRANSACTION in normal " + this.name); this.db.execute("BEGIN TRANSACTION", function(error, rows) { cbDone(); }); }; @@ -90,7 +90,7 @@ IJOD.prototype.commitAddTransaction = function(cbDone) { } else if (written != totalSize) { console.error("Only %d written of %d bytes to IJOD", written, totalSize); } - console.log("****************************** COMMIT TRANSACTION in normal"); + console.log("****************************** COMMIT TRANSACTION in normal " + this.name); self.db.execute("COMMIT TRANSACTION", function(error, rows) { cbDone(); }); }); }); @@ -243,7 +243,7 @@ IJOD.prototype.batchSmartAdd = function(entries, callback) { } var script = ["CREATE TEMP TABLE IF NOT EXISTS batchSmartAdd (id TEXT)", "DELETE FROM batchSmartAdd", "BEGIN TRANSACTION"].join(";") + ";"; - console.log("****************************** BEGIN TRANSACTION in batch"); + console.log("****************************** BEGIN TRANSACTION in batch " + this.name); self.db.executeScript(script, function(error, rows) { if (error) return handleError(error); self.db.prepare("INSERT INTO batchSmartAdd VALUES (?)", function(error, stmt) { @@ -260,7 +260,7 @@ IJOD.prototype.batchSmartAdd = function(entries, callback) { }); }, function(error) { if (error) return handleError(error); - console.log("****************************** COMMIT TRANSACTION in batch"); + console.log("****************************** COMMIT TRANSACTION in batch " + this.name); self.db.execute("COMMIT TRANSACTION", function(error, rows) { if (error) return handleError(error); stmt.finalize(function(error) {