Merge pull request #14 from linkedconnections/development

Version 0.5.0

bin/gtfs2lc.js

@@ -29,6 +29,7 @@ program
   .option('-f, --format <format>', 'Format of the output. Possibilities: csv, ntriples, turtle, json or jsonld (default: json)')
   .option('-s, --startDate <startDate>', 'startDate in YYYYMMDD format')
   .option('-e, --endDate <endDate>', 'endDate in YYYYMMDD format')
+  .option('-S, --store <store>', 'store type: LevelStore (uses your harddisk - for if you run out of RAM) or MemStore (default)')
   .parse(process.argv);
 
 if (!program.path) {
@@ -38,7 +39,8 @@ if (!program.path) {
 
 var mapper = new gtfs2lc.Connections({
   startDate : program.startDate,
-  endDate : program.endDate
+  endDate : program.endDate,
+  store : program.store
 });
 var resultStream = null;
 mapper.resultStream(program.path, function (stream) {

lib/ConnectionsBuilder.js

@@ -22,29 +22,26 @@ ConnectionsBuilder.prototype._transform = function (connectionRule, encoding, do
   var arrivalDFM = moment.duration(connectionRule['arrival_dfm']);
   var self = this;
   this._tripsdb.get(connectionRule['trip_id'], function (error, trip) {
-    trip = JSON.parse(trip); 
-    self._servicesdb.get(trip['service_id'], function (error, service) {
-      if (error) {
-        //This may happen if -s and/or -e option have been set
-        //console.error('Error: Or you didn\'t sort the file, or there\'s an undocumented service_id in the data, or there\'s a bug in our code:', error);
-        //process.exit();
-      } else {
-        service = JSON.parse(service);
-        for (var i in service) {
-          var serviceDay = service[i];
-          var departureTime = moment(serviceDay, 'YYYYMMDD').add(departureDFM);
-          var arrivalTime = moment(serviceDay, 'YYYYMMDD').add(arrivalDFM);
-          self.push({
-            departureTime: departureTime,
-            arrivalTime: arrivalTime,
-            arrivalStop: connectionRule['arrival_stop'],
-            departureStop: connectionRule['departure_stop'],
-            trip: connectionRule['trip_id']
-          });
+    if (!error) {
+      self._servicesdb.get(trip['service_id'], function (error, service) {
+        if (!error) {
+          for (var i in service) {
+            var serviceDay = service[i];
+            var departureTime = moment(serviceDay, 'YYYYMMDD').add(departureDFM);
+            var arrivalTime = moment(serviceDay, 'YYYYMMDD').add(arrivalDFM);
+            self.push({
+              departureTime: departureTime,
+              arrivalTime: arrivalTime,
+              arrivalStop: connectionRule['arrival_stop'],
+              departureStop: connectionRule['departure_stop'],
+              trip: connectionRule['trip_id']
+            });
+          }
+
         }
-      }
-      done();
-    });
+        done();
+      });
+    }
   });
 };
 

lib/StreamIterator.js

@@ -11,6 +11,7 @@ var StreamIterator = function (stream, interval) {
   this._streamEnded = false;
   this._stream.on("end", function () {
     self._streamEnded = true;
+    self._currentObject = null;
     if (self._currentCB) {
       self._currentCB();
     }

lib/gtfs2connections.js

@@ -2,15 +2,18 @@ var csv = require('fast-csv'),
     ConnectionRules = require('./stoptimes/st2c.js'),
     ConnectionsBuilder = require('./ConnectionsBuilder.js'),
     Services = require('./services/calendar.js'),
-    DateInterval = require('./DateInterval.js');
+    DateInterval = require('./DateInterval.js'),
+    Store = require('./stores/Store.js'),
     through2 = require('through2'),
-    level = require('level'),
     moment = require('moment'),
     fs = require('fs');
 
 var Mapper = function (options) {
   this._options = options;
   this._options.interval = new DateInterval(options.startDate, options.endDate);
+  if (!this._options.store) {
+    this._options.store = 'MemStore';
+  }
 };
 
 /**
@@ -30,8 +33,8 @@ Mapper.prototype.resultStream = function (path, done) {
   var connectionRules = fs.createReadStream(path + '/stop_times.txt', {encoding:'utf8', objectMode: true}).pipe(csv({objectMode:true,headers: true})).pipe(new ConnectionRules());
 
   //Step 2 & 3: store in leveldb in 2 hidden directories
-  var tripsdb = level(path + '/.trips');
-  var servicesdb = level(path + '/.services');
+  var tripsdb = Store(path + '/.trips', this._options.store);
+  var servicesdb = Store(path + '/.services',this._options.store);
   var count = 0;
   var finished = function () {
     count ++;
@@ -44,11 +47,15 @@ Mapper.prototype.resultStream = function (path, done) {
   };
 
   services.pipe(through2.obj(function (service, encoding, doneService) {
-    servicesdb.put(service['service_id'], service['dates'], {valueEncoding: 'json'}, doneService);
+    if (service['service_id']) {
+      servicesdb.put(service['service_id'], service['dates'], doneService);
+    }
   })).on('finish', finished);
 
   trips.pipe(through2.obj(function (trip, encoding, doneTrip) {
-    tripsdb.put(trip['trip_id'], trip, {valueEncoding: 'json'}, doneTrip);
+    if (trip['trip_id']) {
+      tripsdb.put(trip['trip_id'], trip, doneTrip);
+    }
   })).on('finish', finished);
 
 };

lib/services/calendar.js

@@ -97,7 +97,7 @@ CalendarToServices.prototype._processCalendarDates = function (calendar, service
 
 CalendarToServices.prototype._transform = function (calendar, encoding, done) {
   //Step one: parse and expand the calendar in memory
-  var d = moment(calendar['start_date'],'YYYYMMDD');
+  var d = moment(calendar['start_date'], 'YYYYMMDD');
   var expanded = [];
   if (this._options.startDate && d < this._options.startDate) {
     d = this._options.startDate;

lib/stores/LevelStore.js

@@ -0,0 +1,22 @@
+var level = require('level');
+
+var LevelStore = function (name) {
+  this.name = name;
+  this._store = level(name);
+};
+
+LevelStore.prototype.get = function (key, cb) {
+  this._store.get(key, function (error, object) {
+    if (!error) {
+      cb(null, JSON.parse(object));
+    } else {
+      cb(error);
+    }
+  });
+};
+
+LevelStore.prototype.put = function (key, value, cb) {
+  this._store.put(key, value, {valueEncoding: 'json'}, cb);
+};
+
+module.exports = LevelStore;

lib/stores/MemStore.js

@@ -0,0 +1,23 @@
+var MemStore = function (name) {
+  this.name = name;
+  this._store = {};
+};
+
+MemStore.prototype.get = function (key, cb) {
+  if (this._store[key]) {
+    cb(null, this._store[key]);
+  } else {
+    cb(key + ' not found in store');
+  }
+};
+
+MemStore.prototype.put = function (key, value, cb) {
+  if (this._store[key]) {
+    cb(key + ' already exists');
+  } else {
+    this._store[key] = value;
+    cb();
+  }
+};
+
+module.exports = MemStore;

lib/stores/Store.js

@@ -0,0 +1,10 @@
+var MemStore = require('./MemStore'),
+    LevelStore = require('./LevelStore');
+
+module.exports = function (config, type) {
+  if (type === 'MemStore') {
+    return new MemStore(config);
+  } else if (type === 'LevelStore') {
+    return new LevelStore(config); 
+  }
+};

package.json

@@ -1,8 +1,8 @@
 {
   "name": "gtfs2lc",
-  "version": "0.4.0",
-  "description": "Mapping script from gtfs to linked connections",
-  "main": "lib/gtfs-csv2lc.js",
+  "version": "0.5.0",
+  "description": "Mapping script from gtfs to (linked) connections",
+  "main": "lib/gtfs2lc.js",
   "bin": {
     "gtfs2lc": "./bin/gtfs2lc.js",
     "gtfs2lc-sort": "./bin/gtfs2lc-sort.sh"