-
Notifications
You must be signed in to change notification settings - Fork 1
/
getNewData.js
193 lines (156 loc) · 9.95 KB
/
getNewData.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
console.log("Running this daily function...........................................");
// base dependencies for script
var mongoose = require('mongoose')
, db = require('./accessDB')
, moment = require('moment') //time library
, xml2js = require('xml2js') //xml to js
, parser = new xml2js.Parser() //xml to js
, jsdom = require("jsdom") //dom parser
, requestURL = require('request'); //gets data from outside
db.startup(process.env.MONGOLAB_URI); // start the db connection
var saved = 0;
// define the remote feed
nycGovURL= "http://www.nycgovparks.org/xml/events_300_rss.xml";
// make the request
requestURL(nycGovURL, function(error, httpResponse, data) {
//if there is an error
if (error) {
console.error(error);
response.send("uhoh there was an error");
}
// if successful HTTP 200 response
if (httpResponse.statusCode == 200) {
//cleaning the data before parsing
data = data.replace(/event:parkids/g, "eventParkids");
data = data.replace(/event:parknames/g, "eventParkNames");
data = data.replace(/event:startdate/g, "eventStartDate");
data = data.replace(/event:enddate/g, "eventEndDate");
data = data.replace(/event:endtime/g, "eventEndTime");
data = data.replace(/event:starttime/g, "eventStartTime");
data = data.replace(/event:contact_phone/g, "eventContactPhone");
data = data.replace(/event:location/g, "eventLocation");
data = data.replace(/event:categories/g, "eventCategories");
//parse the data, convert to js object
parser.parseString(data, function (err, result) {
//convert to JSON
JSON.stringify(result);
console.log(result.channel.item[0]);
var nycGovData = result.channel.item; //array of events from nycgovparks.org in JSON
console.log(nycGovData.length + " total events found");
//get data from tomorrow
var tomorrow = moment().add('days', 1);
console.log("Tomorrow is " +tomorrow);
var tomorrowStr = tomorrow.format("YYYY-MM-DD");
var nycGovDataToday = []; //variable to hold tomorrow's events
var reslog ="";
//find events that are happening tomorrow
for (i = 0; i < nycGovData.length; i++){
if (nycGovData[i].eventStartDate == tomorrowStr){
console.log("TOMORROW: " +nycGovData[i].title);
reslog += nycGovData[i].title+"<br/>";
nycGovDataToday.push(nycGovData[i]);
}
}
console.log("Found " +nycGovDataToday.length + " events happening tomorrow.");
console.log('Done with data');
nycGovDataToday.forEach(function(element, index, array){
//visit the link to get the freaking latitude and longitude
//parse the html via jsdom!
var eventLink = element.link;
var mydata = [];
jsdom.env(eventLink, ['http://code.jquery.com/jquery-1.7.min.js'], function(errors, window) {
console.log("visiting "+ eventLink);
//look for event information in the html
var $ = window.$;
$(".map_locations").each(function () {
mydata = $(this).attr('id').split('__');
//console.log("place: "+ mydata[2]);
//console.log("lat: "+mydata[0]);
//console.log("lng: "+mydata[1]);
// }); // close each on window page
// });// close visit link
var thisStartTime, thisDate, thisEndTime = "";
thisStartTime = element.eventStartTime;
thisDate = element.eventStartDate;
thisEndTime = String(element.eventEndTime);
console.log("from JSON, end time: "+ String(element.eventEndTime));
console.log("1");
var translatedStartTime = moment(thisStartTime, "h:m a");
//var translatedStartTime = moment(thisStartTime, "h:m a").add('hours',4);
console.log("start time "+ translatedStartTime.hours());
console.log("2");
console.log("thisStartTime: " + thisStartTime);
console.log("thisEndTime: " + thisEndTime);
var translatedEndTime = moment(thisEndTime, "h:m a");
//var translatedEndTime = moment(thisEndTime, "h:m a").add('hours',4);
console.log("end time "+ translatedEndTime.hours());
if (translatedEndTime.hours() == 0){
translatedEndTime = moment(translatedStartTime).add('hours', 2); //add 2 hours by default
}
console.log("3");
var translatedDate = moment(thisDate);//.format("dddd, MMMM Do YYYY, h:mm:ss a");
var startTimeStamp = moment(translatedDate);
startTimeStamp.hours(translatedStartTime.hours()).minutes(translatedStartTime.minutes());
console.log("startTimeStamp: "+ moment(startTimeStamp).calendar());
var endTimeStamp = moment(translatedDate);
endTimeStamp.hours(translatedEndTime.hours()).minutes(translatedEndTime.minutes());
console.log("endTimeStamp: "+ moment(endTimeStamp).calendar());
//not sure why I have to do this...
//translatedDate.add('days', 1);
var eventData = {
name : element.title,
urlslug : convertToSlug(element.title),
place: element.eventLocation,
desc : element.description,
link : element.link,
location: {
latitude : mydata[0]
,longitude : mydata[1]
,placename : mydata[2]
,address: mydata[4]
},
datetime : {
timestamp: new Date(translatedDate),
date: new Date(thisDate),
starttimestamp: new Date(startTimeStamp),
endtimestamp: new Date(endTimeStamp)
},
lastEdited: new Date(),
author: {
name: "NYC Parks and Recreation"
}
};//end event data
console.log("eventData: "+eventData.name);
console.log("start: "+moment(eventData.datetime.starttimestamp).calendar());
console.log("end: "+moment(new Date(eventData.datetime.endtimestamp)).calendar());
// create a new event
var thisEvent = new db.Event(eventData);
// save the event to the database
thisEvent.save(function (err, doc){
saved++;
console.error('--------------SAVED!'+ saved+ '-----------------------------');
});
});
console.log("Array: ", array.length);
//when we've saved all the elements
if (saved >= array.length){
console.log("*******CLOSING DB - SCRIPT SHOULD TERMINATE AS EXPECTED ******");
db.closeDB(); // <--- VERY IMPORTANT. MUST CLOSE DB WHEN FINISHED.
}
});//end jsdom
}); //end for each event found...
//response.redirect('/');
}); //end parser
}//end if httpResponse
}); // end of requestURL callback
function convertToSlug(Text)
{
return Text
.toLowerCase()
.replace(/[^\w ]+/g,'')
.replace(/ +/g,'-')
;
}
function getCleanURLSource(url){
return url.substring(url.charAt(0),url.indexOf('/'));
}