-
Notifications
You must be signed in to change notification settings - Fork 1
/
DataCrawler.js
74 lines (63 loc) · 2.15 KB
/
DataCrawler.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
var express = require('express');
var request = require('request');
var fs = require('fs');
var zlib = require("zlib");
var url = 'http://data.taipei/youbike';
var YouBike = require('./YouBike');
var mongoose = require('mongoose');
var app = express();
mongoose.connect('mongodb://localhost/youbike');
var CronJob = require('cron').CronJob;
new CronJob('*/30 * * * * *', function() {
var current_time = new Date();
console.log(current_time);
var source = request(url);
var download_file = fs.createWriteStream('./youbike.json.gz');
var out = fs.createWriteStream('youbike.json');
source.on('response', function(res) {
// console.log("source.on execute");
res.pipe(download_file);
});
download_file.on('finish', function() {
// console.log("download_file.on execute");
var inp = fs.createReadStream('youbike.json.gz');
inp.pipe(zlib.createGunzip()).pipe(out);
})
out.on('close', function() {
// console.log("out.on execute");
// var youbike_per_minute = require('./youbike.json');
fs.readFile('./youbike.json', {encoding: 'utf-8'}, function(err, data) {
if (!err) {
var youbike_per_minute = JSON.parse(data);
var length = Object.keys(youbike_per_minute.retVal).length;
var one_item = 0;
YouBike.findOne({
mday: youbike_per_minute.retVal["0001"].mday
}, function(err, youbike_data) {
if (!youbike_data) {
// do stuff here
// console.log("Find nothing!");
for (var i = 1; i <= length; i++) {
one_item = youbike_per_minute.retVal[addZero(i, 4)];
//===========================Store into Database===========================
// console.log(one_item);
var new_bike = new YouBike(one_item);
new_bike.save();
//===========================Store into Database===========================
}
}
});
} else {
console.log(err);
}
});
})
}, null, true, 'America/Los_Angeles');
function addZero(num, n) {
var len = num.toString().length;
while (len < n) {
num = "0" + num;
len++;
}
return num;
}