Permalink
Browse files

Added python data utilities.

  • Loading branch information...
1 parent 908a3ac commit fcbe6f733b123aa9b312a9a79300b48ee363f1b3 @mrbichel mrbichel committed Sep 9, 2012
View
@@ -37,4 +37,7 @@ Release*/
xcuserdata
*.xcworkspace
-DerivedData
+DerivedData
+/data-utils/db/
+/data-utils/data/
+/data-utils/.idea/
View
@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from pymongo import Connection
+from geopy import geocoders
+import time
+
+connection = Connection('127.0.0.1', 27017)
+db = connection.chaosflow
+stat_buckets = db.stat_buckets
+
+g = geocoders.Google(domain="maps.google.dk")
+
+def split(txt, seps):
+ default_sep = seps[0]
+ for sep in seps[1:]: # we skip seps[0] because that's the default seperator
+ txt = txt.replace(sep, default_sep)
+ return [i.strip() for i in txt.split(default_sep)]
+
+
+for bucket in stat_buckets.find():
+ time.sleep(1) # As to not hit googles request limit
+
+ if not bucket.get('lat') and not bucket.get('lng'):
+ while True:
+ attempt_count = 0
+ try:
+ attempt_count += 1
+
+ splits = ["ø.f.", "n.f.", "v.f.", "nø.f.", "sv.f.", "s.f.", "sydøst for", "syd for", "øst for", "nv. for"]
+ location_name = "{}, København".format(split(bucket['road'], splits)[0])
+
+ print "Attempt {} at looking up geocoordinates for {}.".format(attempt_count, location_name)
+ place, (lat, lng) = g.geocode(location_name.encode('UTF-8'))
+
+ except geocoders.google.GQueryError as e:
+ print "ERROR: {}".format(e)
+ break
+ except ValueError as e:
+ print "ERROR: {}".format(e)
+ break
+ except geocoders.google.GTooManyQueriesError as e:
+ if attempt_count < 4:
+ print "To many requests. Will retry in 10 seconds."
+ print "ERROR: {}".format(e)
+ time.sleep(10)
+ continue
+ else:
+ print "To many requests. Will not retry."
+ break
+ else:
+ print u"SUCCESS: Geocoordinates found for {}: {}, {}".format(place, lat, lng)
+ stat_buckets.update({"_id": bucket["_id"] },
+ { "$set": {"lat": lat, "lng": lng}})
+
+ break
+
View
@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+import datetime
+import os
+import glob
+
+from openpyxl.reader.excel import load_workbook
+from pymongo import Connection
+
+# see reference at http://api.mongodb.org/python/current/tutorial.html
+connection = Connection('127.0.0.1', 27017)
+db = connection.chaosflow
+stat_buckets = db.stat_buckets
+
+#stat_buckets.ensureIndex({"road": 1}, {unique: true}); # should not work like this, rather check if name is present for the same date.
+
+stat_entries = db.stat_entries
+
+# make it so that if direction_two is empty its a one way road
+# get lat lngthrough a reverse geocode lookup
+
+count = 1
+path = 'data/'
+
+def add_stat_entry(stat_bucket, hour, amount, direction, type="bike"):
+ stat_entry = {
+ "stat_bucket": stat_bucket, #todo: change this to mongodb id
+ "hour": hour,
+ "amount": amount,
+ "type": type,
+ "direction": direction
+ }
+
+ stat_entries.insert(stat_entry)
+
+
+for infile in glob.glob( os.path.join(path, '*.xlsx') ):
+
+ wb = load_workbook(filename=infile)
+
+ # get metadata sheet
+ stamopl = wb.get_sheet_by_name(name='Stamopl.')
+
+ # get traffic data sheet
+ data = wb.get_sheet_by_name(name='DATA')
+
+ stat_bucket = {}
+ for i in range(8):
+ # Interpret metadata
+ label = stamopl.cell("A{}".format(i+1)).value
+ val = stamopl.cell("B{}".format(i+1)).value
+
+ if label == "NUMMER":
+ stat_bucket["number"] = val
+ elif label == "VEJ":
+ stat_bucket["road"] = val
+ elif label == "DATO":
+ stat_bucket["date"] = val
+ elif label == "VEJRET":
+ stat_bucket["weather"] = val
+ elif label == "VEJRFAKTOR":
+ stat_bucket["weather_factor"] = val
+ elif label == "ANM:":
+ stat_bucket["comments"] = val
+ elif label == "RETN. 1":
+ stat_bucket["direction_one"] = val
+ elif label == "RETN. 2":
+ stat_bucket["direction_two"] = val
+
+ current_stat_bucket = stat_buckets.insert(stat_bucket)
+
+ stat_entry = {}
+
+ dir_one = []
+ dir_two = []
+
+ # interpret traffic data, for now we only fetch bikes
+ if data.cell("H2").value == 62:
+ # this is the bike data for direction #1
+ for i in range(11):
+ dir_one.append(data.cell("H{}".format(3+i)).value)
+
+ if data.cell("H33").value == 62:
+ # this is the bike data for direction #2
+ for i in range(11): # step through 12 hours of data
+ dir_two.append(data.cell("H{}".format(34+i)).value)
+
+ elif data.cell("M5").value == "_x0001_" or data.cell("M5").value == 62:
+ # dir 1 is in m6-m17
+ for i in range(11):
+ dir_one.append(data.cell("M{}".format(6+i)).value)
+
+ if data.cell("M31").value == "_x0001_" or data.cell("M31").value == 62:
+ # dir 2 is in m32-m43
+ for i in range(11):
+ dir_two.append(data.cell("M{}".format(32+i)).value)
+
+ if dir_one:
+ for i in range(11):
+ add_stat_entry(current_stat_bucket, 7+i, dir_one[i], 1)
+
+ if dir_two:
+ for i in range(11):
+ add_stat_entry(current_stat_bucket, 7+i, dir_two[i], 2)
+
+ count+=1
+
+
+
+
+
+
+
No changes.
@@ -0,0 +1,37 @@
+from bson.objectid import ObjectId
+from flask import Flask, render_template
+
+from bson.json_util import dumps
+import pymongo
+
+# configuration
+DEBUG = True
+
+app = Flask(__name__)
+app.config.from_object(__name__)
+
+connection = pymongo.Connection("localhost", 27017)
+db = connection.chaosflow
+
+stat_buckets = db.stat_buckets
+stat_entries = db.stat_entries
+
+
+@app.route("/")
+def index():
+ return render_template('index.html')
+
+@app.route("/locations")
+def locations():
+ return dumps(stat_buckets.find())
+
+@app.route("/locations/<location_id>")
+def location(location_id):
+ return dumps(stat_buckets.find_one({"_id": ObjectId(location_id)}))
+
+@app.route("/locations/<location_id>/entries")
+def entries_for_location(location_id):
+ return dumps(stat_entries.find({"stat_bucket": ObjectId(location_id)}))
+
+if __name__ == "__main__":
+ app.run()
@@ -0,0 +1,39 @@
+<!DOCTYPE html>
+<!--[if lt IE 7 ]> <html lang="en" class="no-js ie6"> <![endif]-->
+<!--[if IE 7 ]> <html lang="en" class="no-js ie7"> <![endif]-->
+<!--[if IE 8 ]> <html lang="en" class="no-js ie8"> <![endif]-->
+<!--[if IE 9 ]> <html lang="en" class="no-js ie9"> <![endif]-->
+<!--[if (gt IE 9)|!(IE)]><!--> <html dir="ltr" lang="DA_dk" class="no-js"> <!--<![endif]-->
+<head>
+ <title>Chaotic Flow</title>
+
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+
+ <meta name=viewport content="width=device-width, minimum-scale=1.0, maximum-scale=1.0" />
+
+ <meta name=description content="">
+ <meta name=author content="">
+
+ <link rel=stylesheet href="{{ url_for('static', filename='css/screen.css') }}" media="screen, projection" type="text/css" />
+
+
+ <!--[if lt IE 9]><script src="https://html5shim.googlecode.com/svn/trunk/html5.js"></script><![endif]-->
+
+
+ <!-- backbone here -->
+ <!--<script src="http://d3js.org/d3.v2.js"></script>-->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js"></script>
+ <script src="{{ url_for('static', filename='js/main.js') }}"></script>
+
+
+</head>
+ <body>
+
+ <h1>Chaotic Bike Flow Visualization</h1>
+ <div id="content">
+
+ </div>
+
+ </body>
+</html>

0 comments on commit fcbe6f7

Please sign in to comment.