Skip to content
This repository
Browse code

gradient polyline added

  • Loading branch information...
commit 0a2e153d2fda8c58abcd3c8d7826a2c180832c78 1 parent de5f071
Chris Johnson-Roberson authored
2  chrome/content/papermachines/processors/geoparser.py
@@ -76,7 +76,7 @@ def run_geoparser(self):
76 76
 					if not os.path.exists(xml_filename):
77 77
 						annotation = self.annotate(str_to_parse)
78 78
 						with codecs.open(xml_filename, 'w', encoding='utf8') as xml_file:
79  
-							xml_file.write(annotation)
  79
+							xml_file.write(annotation.decode('utf-8'))
80 80
 					else:
81 81
 						with codecs.open(xml_filename, 'r', encoding='utf8') as xml_file:
82 82
 							annotation = xml_file.read()
3  chrome/content/papermachines/processors/geoparser_export.py
@@ -23,7 +23,7 @@ def process(self):
23 23
 
24 24
 		self.run_geoparser()
25 25
 
26  
-		header = ["name", "lat", "lng", "itemID", "context"]
  26
+		header = ["name", "lat", "lng", "entityURI", "itemID", "context"]
27 27
 
28 28
 		csv_output_filename = os.path.join(self.out_dir, self.name + self.collection + '.csv')
29 29
 		with open(csv_output_filename, 'wb') as f:
@@ -51,6 +51,7 @@ def process(self):
51 51
 						row_dict = {}
52 52
 						row_dict["name"] = name
53 53
 						row_dict["itemID"] = itemID
  54
+						row_dict["entityURI"] = entityURI
54 55
 						row_dict["lat"] = place["coordinates"][1]
55 56
 						row_dict["lng"] = place["coordinates"][0]
56 57
 
108  chrome/content/papermachines/processors/geoparser_flightpaths.py
@@ -12,72 +12,92 @@ def _basic_params(self):
12 12
 		self.name = "geoparser_flightpaths"
13 13
 		self.dry_run = False
14 14
 		self.require_stopwords = False
  15
+		self.template_filename = os.path.join(self.cwd, "templates", "geoparser_flightpaths_gmaps.html")
15 16
 
16 17
 	def process(self):
17 18
 		"""
18 19
 		create a JSON file with geographical data extracted from texts
19 20
 		"""
20 21
 
21  
-		self.run_geoparser()
22  
-
23  
-		max_country_weight = 0
24  
-
25  
-		for place in sorted(self.places.keys()):
26  
-			if self.places[place]["type"] == "Country":
27  
-				country_sum = sum(self.places[place]["weight"].values())
28  
-				if country_sum > max_country_weight:
29  
-					max_country_weight = country_sum
30  
-
31  
-		placeIDsToNames = dict((k, v["name"]) for k, v in self.places_by_entityURI.iteritems())
32  
-		placeIDsToCoords = dict((k, v["coordinates"]) for k, v in self.places_by_entityURI.iteritems())
  22
+		csv_input = os.path.join(self.out_dir, 'geoparser_export' + self.collection + '.csv')
  23
+		if not os.path.exists(csv_input):
  24
+			import geoparser_export
  25
+			subprocessor = geoparser_export.GeoparserExport()
  26
+			subprocessor.process()
33 27
 
34 28
 		linksByYear = {}
35  
-		sources = {}
  29
+		itemIDToYear = {}
  30
+		places = {}
36 31
 
37 32
 		for filename in self.files:
38  
-			if self.metadata[filename].get('city') is None or len(self.geo_parsed[filename]) < 2:
  33
+			file_geoparsed = filename.replace(".txt", "_geoparse.json")
  34
+			if os.path.exists(file_geoparsed):
  35
+				try:
  36
+					geoparse_obj = json.load(file(file_geoparsed))
  37
+				except:
  38
+					logging.error("File " + file_geoparsed + " could not be read.")
  39
+					continue
  40
+
  41
+			if geoparse_obj.get('city') is None:
39 42
 				continue
40 43
 			try:
41 44
 				title = os.path.basename(filename)
42 45
 				itemID = self.metadata[filename]['itemID']
43  
-				year = self.metadata[filename]['year']
  46
+				if not self.metadata[filename]['year'].isdigit():
  47
+					continue
  48
+				year = int(self.metadata[filename]['year'])
  49
+				if year < 100:
  50
+					year += 1900
  51
+				elif year < 200:
  52
+					year += 1800
  53
+
  54
+				itemIDToYear[itemID] = year
  55
+
44 56
 				if year not in linksByYear:
45 57
 					linksByYear[year] = {}
46  
-				source = self.metadata[filename]['city']
47  
-				if source != None:
48  
-					if source not in sources:
49  
-						sources[source] = {}
50  
-					if year not in sources[source]:
51  
-						sources[source][year] = 0
52  
-					sources[source][year] += 1
53  
-				targets = self.geo_parsed[filename]
54  
-				for target in targets:
55  
-					edge = str(source) + ',' + str(target)
  58
+				source = geoparse_obj.get('city')
  59
+				places[source] = geoparse_obj["places_by_entityURI"][source]
  60
+				for target in geoparse_obj.get('places'):
  61
+					places[target] = geoparse_obj["places_by_entityURI"][target]
  62
+					edge = ','.join([source, target])
56 63
 					if edge not in linksByYear[year]:
57  
-						linksByYear[year][edge] = 0
58  
-					linksByYear[year][edge] += 1
  64
+						linksByYear[year][edge] = {}
  65
+					if itemID not in linksByYear[year][edge]:
  66
+						linksByYear[year][edge][itemID] = 0
  67
+					linksByYear[year][edge][itemID] += 1
59 68
 			except:
60 69
 				logging.info(traceback.format_exc())
61 70
 
62 71
 		years = sorted(linksByYear.keys())
63  
-		groupedLinksByYear = []
64  
-
  72
+		groupedLinksByYear = {}
65 73
 		for year in years:
66  
-			groupedLinksByYear.append([])
67  
-			for edge in linksByYear[year]:
68  
-				weight = linksByYear[year][edge]
69  
-				source, target = [int(x) for x in edge.split(',')]
70  
-				groupedLinksByYear[-1].append({'source': source, 'target': target, 'year': year, 'weight': weight})
71  
-
72  
-
73  
-		params = {"PLACEIDSTOCOORDS": json.dumps(placeIDsToCoords),
74  
-			"PLACEIDSTONAMES": json.dumps(placeIDsToNames),
75  
-			"PLACESMENTIONED": json.dumps(dict((k, v["weight"]) for k, v in self.places.iteritems() if v["type"] != "Country")),
76  
-			"TEXTSFROMPLACE": json.dumps(sources),
77  
-			"COUNTRIES": json.dumps(dict((v["name"], v["weight"]) for k, v in self.places.iteritems() if v["type"] == "Country")),
78  
-			"MAX_COUNTRY_WEIGHT": str(max_country_weight),
79  
-			"STARTDATE": str(min([int(x["year"]) for x in self.metadata.values() if x["year"].isdigit() and x["year"] != "0000"])),
80  
-			"ENDDATE": str(max([int(x["year"]) for x in self.metadata.values() if x["year"].isdigit()])),
  74
+			groupedLinksByYear[year] = []
  75
+			for edge, text_weights in linksByYear[year].iteritems():
  76
+				weight = sum(text_weights.values())
  77
+				texts = text_weights.keys()
  78
+				source, target = edge.split(',')
  79
+				groupedLinksByYear[year].append({'edge': [source, target], 'texts': texts, 'year': year, 'weight': weight})
  80
+
  81
+		# max_count = 0
  82
+		counts = {}
  83
+		for rowdict in self.parse_csv(csv_input):
  84
+			# coords = ','.join([rowdict["lat"], rowdict["lng"]])
  85
+			itemID = rowdict["itemID"]
  86
+			if itemID not in itemIDToYear:
  87
+				continue
  88
+
  89
+			year = itemIDToYear[itemID]
  90
+			entityURI = rowdict["entityURI"]
  91
+			if "counts" not in places[entityURI]:
  92
+				places[entityURI]["counts"] = {}
  93
+			if year not in places[entityURI]["counts"]:
  94
+				places[entityURI]["counts"][year] = 0			
  95
+			places[entityURI]["counts"][year] += 1
  96
+
  97
+		params = {"STARTDATE": str(min(linksByYear.keys())),
  98
+			"ENDDATE": str(max(linksByYear.keys())),
  99
+			"ENTITYURIS": json.dumps(places),
  100
+			"YEARS": json.dumps(years),
81 101
 			"LINKS_BY_YEAR": json.dumps(groupedLinksByYear)
82 102
 		}
83 103
 		self.write_html(params)
207  chrome/content/papermachines/processors/support/flightpaths_gmaps.js
... ...
@@ -1,8 +1,76 @@
1 1
 
2 2
 var map;
3  
-var heatmap;
  3
+var heatmap, heatmapData;
  4
+var animating;
  5
+var animationInterval = 250;
  6
+var linksByText = {};
4 7
 
5  
-var link_polylines;
  8
+var link_polylines = {};
  9
+
  10
+var playPause = function () {
  11
+  if (animating) {
  12
+    clearInterval(animating);
  13
+    animating = false;
  14
+    document.getElementById("playPause").textContent = "\u25b6";
  15
+  } else {
  16
+    var searchTime = document.getElementById("searchTime"),
  17
+      now = parseInt(searchTime.value),
  18
+      min = parseInt(searchTime.min),
  19
+      max = parseInt(searchTime.max);
  20
+    if (now == max) {
  21
+      searchTime.value = min;
  22
+    }
  23
+    document.getElementById("playPause").textContent = "\u2759\u2759"; //.style("letter-spacing", "-2px")
  24
+    animating = setInterval(incrementTime, animationInterval);
  25
+  }
  26
+};
  27
+
  28
+var incrementTime = function () {
  29
+  var searchTime = document.getElementById("searchTime"),
  30
+    now = parseInt(searchTime.value),
  31
+    max = parseInt(searchTime.max);
  32
+  if (now > (max - 1)) {
  33
+    playPause();
  34
+  } else {
  35
+    searchTime.value = (now + 1);
  36
+    timeAction();   
  37
+  }
  38
+};
  39
+
  40
+var timeAction = function () {
  41
+  var queryTime = document.getElementById("searchTime").value;
  42
+  document.getElementById("timeDisplay").textContent = queryTime;
  43
+  endDate = parseInt(queryTime);
  44
+  updateHeatmapData();
  45
+  for (var year in link_polylines) {
  46
+    (function (displaying) {
  47
+      link_polylines[year].forEach(function (d) { d.setVisible(displaying); });
  48
+    })(year >= startDate && year < endDate);
  49
+  }
  50
+  // max_country_weight = d3.max(Object.keys(countries).map(function (d) { return getCountryWeight(d);}))
  51
+  // feature.style("fill", countryColor);
  52
+  // layers.style("stroke-opacity", fadeOldConnections).style("display", timeFilter);
  53
+  // data.attr("r", function(d) {return valueToRadius(d) * currentScale / height;});
  54
+};
  55
+
  56
+var updateHeatmapData = function () {
  57
+  var data = [], max = 0;
  58
+  for (var uri in entityURIs) {
  59
+    var d = entityURIs[uri];
  60
+    var sum = 0;
  61
+    for (var year in d.counts) {
  62
+      if (year >= startDate && year < endDate) {
  63
+        sum += d.counts[year];
  64
+      }
  65
+    }
  66
+    if (sum > max) { max = sum; }
  67
+    if (sum > 0) {
  68
+      data.push({"lat": d["coordinates"][1], "lng": d["coordinates"][0], "count": sum});      
  69
+    }
  70
+  }
  71
+  heatmapData = {"max": max, "data": data};
  72
+  heatmap.setDataSet(heatmapData);
  73
+};
6 74
 
7 75
 window.onload = function(){
8 76
   var searchTime = document.getElementById("searchTime");
@@ -17,8 +85,8 @@ window.onload = function(){
17 85
   playPauseButton.onclick = playPause;
18 86
 
19 87
   document.getElementById("searches").appendChild(playPauseButton);
20  
-/*  d3.select("#playPause").text("\u25b6").style("margin-left", "1em");
21  
-  d3.select("#timeDisplay").text(endDate); */
  88
+  document.getElementById("playPause").textContent = "\u25b6";
  89
+  document.getElementById("timeDisplay").textContent = endDate;
22 90
 
23 91
   var myLatlng = new google.maps.LatLng(-15.6778, -47.4384);
24 92
   var myOptions = {
@@ -46,12 +114,135 @@ window.onload = function(){
46 114
     var legend = heatmap.heatmap.get("legend").get("element");
47 115
     legend.hidden = !legend.hidden;
48 116
   };
49  
-
50  
-    var myData = INTENSITY;
51 117
   
52 118
   // this is important, because if you set the data set too early, the latlng/pixel projection doesn't work
53 119
   google.maps.event.addListenerOnce(map, "idle", function(){
54  
-    heatmap.setDataSet(myData);
  120
+    updateHeatmapData();
  121
+    for (var year in linksByYear) {
  122
+        for (var i in linksByYear[year]) {
  123
+          var link = linksByYear[year][i];
  124
+          if (!(year in link_polylines)) {
  125
+            link_polylines[year] = [];
  126
+          }
  127
+          // var line = new google.maps.Polyline({
  128
+          //     "map": map, 
  129
+          //     "geodesic": true,
  130
+          //     "strokeOpacity": 0.1,
  131
+          //     "strokeWidth": 1,
  132
+          //     "path": link.edge.map(function (d) { var coords = entityURIs[d]["coordinates"]; return new google.maps.LatLng(coords[1], coords[0]); })
  133
+          // });
  134
+          var line = new GradientPolyline({
  135
+              "map": map,
  136
+              "strokeOpacity": 0.1,
  137
+              "strokeWidth": 1,
  138
+              "colors": ["#ff0000", "#0000ff"],
  139
+              "path": link.edge.map(function (d) { var coords = entityURIs[d]["coordinates"]; return new google.maps.LatLng(coords[1], coords[0]); })
  140
+          });
  141
+
  142
+          link_polylines[year].push(line);
  143
+
  144
+          link.texts.forEach(function (text) {
  145
+            if (!(text in linksByText)) { linksByText[text] = [];}
  146
+            linksByText[text].push(line);            
  147
+          });
  148
+        }
  149
+    }
55 150
   });
56 151
 };
57  
-    </div>
  152
+
  153
+
  154
+function colorComponents(color_str) {
  155
+  var hex;
  156
+  if (color_str.length >= 6) {
  157
+    hex = parseInt(color_str.replace("#",""), 16);
  158
+  } else {
  159
+    hex = 0xffffff;
  160
+  }
  161
+  var r = (hex & 0xff0000) >> 16;
  162
+  var g = (hex & 0x00ff00) >> 8;
  163
+  var b = hex & 0x0000ff;
  164
+  return [r,g,b];
  165
+}
  166
+
  167
+function GradientPolyline(obj) {
  168
+  var colors = obj["colors"];
  169
+  var map = obj["map"];
  170
+  var path = obj["path"];
  171
+  this.strokeOpacity = obj["strokeOpacity"] || 0.1;
  172
+
  173
+  this.set('map', map);
  174
+  this.set('visible', true);
  175
+  var points = 50;
  176
+
  177
+  this.segments = this.generateSegments(path[0], path[1], points);
  178
+  this.colors = this.generateColors(colors[0], colors[1], points);
  179
+  this.segments_polylines = new Array();
  180
+  for (var i = 0; i < this.segments.length - 1; i++) {
  181
+    var new_poly = new google.maps.Polyline({
  182
+      "path": [this.segments[i], this.segments[i+1]],
  183
+      "strokeColor": this.colors[i],
  184
+      "strokeOpacity": this.strokeOpacity
  185
+    });
  186
+    new_poly.bindTo('map', this);
  187
+    new_poly.bindTo('visible', this);
  188
+    this.segments_polylines.push(new_poly);
  189
+  }
  190
+}
  191
+
  192
+GradientPolyline.prototype = new google.maps.MVCObject();
  193
+
  194
+GradientPolyline.prototype.setVisible = function (visible) {
  195
+  this.set('visible', visible);
  196
+};
  197
+
  198
+GradientPolyline.prototype.generateColors = function (start_color, end_color, points) {
  199
+  var colors = new Array();
  200
+  var start_rgb = colorComponents(start_color);
  201
+  var end_rgb = colorComponents(end_color);
  202
+
  203
+  for (var i = 0, n = points - 1; i < n; i++) {
  204
+    var p = (i*1.0)/n;
  205
+    var new_color = new Array(3);
  206
+    for (var j = 0; j < 3; j++) {
  207
+      new_color[j] = Math.round((start_rgb[j] * (1-p)) + (end_rgb[j] * (p)));
  208
+    }
  209
+    var new_rgb = new_color.reduce(function (a, b) {
  210
+      var b_hex = b.toString(16);
  211
+      if (b_hex.length == 1) {
  212
+        b_hex = "0" + b_hex;
  213
+      }
  214
+      return a + b_hex;
  215
+    }, "#");
  216
+    colors.push(new_rgb);
  217
+  }
  218
+  return colors;
  219
+};
  220
+
  221
+GradientPolyline.prototype.generateSegments = function (start, end, points) {
  222
+
  223
+  var geodesicPoints = new Array();
  224
+  with (Math) {
  225
+    var lat1 = start.lat() * (PI/180);
  226
+    var lon1 = start.lng() * (PI/180);
  227
+    var lat2 = end.lat() * (PI/180);
  228
+    var lon2 = end.lng() * (PI/180);
  229
+
  230
+    var d = 2*asin(sqrt( pow((sin((lat1-lat2)/2)),2) + cos(lat1)*cos(lat2)*pow((sin((lon1-lon2)/2)),2)));
  231
+
  232
+    for (var n = 0 ; n < points+1 ; n++ ) {
  233
+      var f = (1/points) * n;
  234
+      // f = f.toFixed(6);
  235
+      var A = sin((1-f)*d)/sin(d)
  236
+      var B = sin(f*d)/sin(d)
  237
+      var x = A*cos(lat1)*cos(lon1) +  B*cos(lat2)*cos(lon2)
  238
+      var y = A*cos(lat1)*sin(lon1) +  B*cos(lat2)*sin(lon2)
  239
+      var z = A*sin(lat1)           +  B*sin(lat2)
  240
+
  241
+      var latN = atan2(z,sqrt(pow(x,2)+pow(y,2)))
  242
+      var lonN = atan2(y,x)
  243
+      var p = new google.maps.LatLng(latN/(PI/180), lonN/(PI/180));
  244
+      geodesicPoints.push(p);
  245
+    }
  246
+  }
  247
+  return geodesicPoints;
  248
+};
8  chrome/content/papermachines/processors/support/stream.js
@@ -27,7 +27,8 @@ var timeRanges;
27 27
 var searchN = 0;
28 28
 var graphColors = d3.scale.category10().domain(d3.range(10));
29 29
 
30  
-var gradientOpacity = d3.scale.log().clamp(true).range([1,0]);
  30
+var gradientExponentScale = 0.3;
  31
+var gradientOpacity = d3.scale.pow().exponent(gradientExponentScale).clamp(true).range([1,0]);
31 32
 
32 33
 var legend, showLegend = true;
33 34
 var startDate, endDate;
@@ -1205,6 +1206,11 @@ function findTopicProportionAndStdev(i, contributingDocs, data) {
1205 1206
   topicStdevs[i] = Math.sqrt((1.0 / (vals.length - 1.0)) * d3.sum(variances));
1206 1207
 }
1207 1208
 
  1209
+function changeGradientScale(val) {
  1210
+  gradientExponentScale = val;
  1211
+  gradientOpacity.exponent(gradientExponentScale);
  1212
+  updateGradient();
  1213
+}
1208 1214
 function saveSVG() {
1209 1215
   var xml = "<svg xmlns='http://www.w3.org/2000/svg'><style>";
1210 1216
     for (i in document.styleSheets)
51  chrome/content/papermachines/processors/templates/geoparser_flightpaths_gmaps.html
@@ -5,6 +5,50 @@
5 5
     <title>Geoparser: COLLECTION_NAME</title>
6 6
     <script type="text/javascript" src="support/html5slider.js"></script>
7 7
     <link type="text/css" rel="text/stylesheet" href="support/button.css"/>
  8
+    <style type="text/css">
  9
+      body, html {
  10
+              margin:0;
  11
+              padding:0;
  12
+              font-family:Arial;
  13
+            }
  14
+      h1 {
  15
+        margin-bottom:10px;
  16
+      }
  17
+      #main {
  18
+        position:relative;
  19
+        width:1020px;
  20
+        padding:20px;
  21
+        margin:auto;
  22
+      }
  23
+      #heatmapArea {
  24
+        position:relative;
  25
+        float:left;
  26
+        width:960px;
  27
+        height:600px;
  28
+        border:1px dashed black;
  29
+      }
  30
+      #configArea {
  31
+        position:relative;
  32
+        float:left;
  33
+        width:200px;
  34
+        padding:15px;
  35
+        padding-top:0;
  36
+        padding-right:0;
  37
+      }
  38
+      .btn {
  39
+        margin-top:25px;
  40
+        padding:10px 20px 10px 20px;
  41
+        -moz-border-radius:15px;
  42
+        -o-border-radius:15px;
  43
+        -webkit-border-radius:15px;
  44
+        border-radius:15px;
  45
+        border:2px solid black;
  46
+        cursor:pointer;
  47
+        color:white;
  48
+        background-color:black;
  49
+      }
  50
+    </style>
  51
+
8 52
     <link type="text/css" rel="text/stylesheet" href="support/flightpaths.css"/>
9 53
     <script type="text/javascript" src="http://maps.google.com/maps/api/js?sensor=false"></script>
10 54
   </head>
@@ -24,12 +68,7 @@
24 68
     </div>
25 69
     <script type="text/javascript">
26 70
     var linksByYear = LINKS_BY_YEAR;
27  
-    var placeIDsToCoords = PLACEIDSTOCOORDS;
28  
-    var placeIDsToNames = PLACEIDSTONAMES;
29  
-    var placesMentioned = PLACESMENTIONED;
30  
-    var textsFromPlace = TEXTSFROMPLACE;
31  
-    var countries = COUNTRIES;
32  
-    var max_country_weight = MAX_COUNTRY_WEIGHT;
  71
+    var entityURIs = ENTITYURIS;
33 72
     var startDate = STARTDATE;
34 73
     var endDate = ENDDATE;
35 74
     </script>
44  chrome/content/papermachines/processors/templates/geoparser_heatmap.html
@@ -4,6 +4,50 @@
4 4
 <title>Heatmap: COLLECTION_NAME</title>
5 5
 	<meta name="viewport" content="width=device-width, initial-scale=1.0">
6 6
     <link type="text/css" rel="text/stylesheet" href="support/heatmap.css"/>
  7
+
  8
+<style type="text/css">
  9
+      body, html {
  10
+              margin:0;
  11
+              padding:0;
  12
+              font-family:Arial;
  13
+            }
  14
+      h1 {
  15
+        margin-bottom:10px;
  16
+      }
  17
+      #main {
  18
+        position:relative;
  19
+        width:1020px;
  20
+        padding:20px;
  21
+        margin:auto;
  22
+      }
  23
+      #heatmapArea {
  24
+        position:relative;
  25
+        float:left;
  26
+        width:960px;
  27
+        height:600px;
  28
+        border:1px dashed black;
  29
+      }
  30
+      #configArea {
  31
+        position:relative;
  32
+        float:left;
  33
+        width:200px;
  34
+        padding:15px;
  35
+        padding-top:0;
  36
+        padding-right:0;
  37
+      }
  38
+      .btn {
  39
+        margin-top:25px;
  40
+        padding:10px 20px 10px 20px;
  41
+        -moz-border-radius:15px;
  42
+        -o-border-radius:15px;
  43
+        -webkit-border-radius:15px;
  44
+        border-radius:15px;
  45
+        border:2px solid black;
  46
+        cursor:pointer;
  47
+        color:white;
  48
+        background-color:black;
  49
+      }
  50
+    </style>
7 51
 <script type="text/javascript" src="http://maps.google.com/maps/api/js?sensor=false"></script>
8 52
 
9 53
 </head>

0 notes on commit 0a2e153

Please sign in to comment.
Something went wrong with that request. Please try again.