⚙ Batch saving via Gearman

bin/gearman_worker.php

@@ -0,0 +1,33 @@
+<?php
+	include(__DIR__ . '/../www/include/init.php');
+	loadlib('github_users');
+	loadlib('wof_save');
+
+	$worker = new GearmanWorker();
+	$worker->addServer();
+	$worker->addFunction('save_to_github', 'gearman_save_to_github');
+	$worker->addFunction('update_search_index', 'gearman_update_search_index');
+	while ($worker->work());
+
+	function gearman_save_to_github($job) {
+		$details = $job->workload();
+		$details = unserialize($details);
+
+		$github_user = github_users_get_by_user_id($details['user_id']);
+		if (! $github_user) {
+			// No user found
+			return;
+		}
+
+		$oauth_token = $github_user['oauth_token'];
+		wof_save_to_github($details['geojson'], $details['geojson_data'], $oauth_token);
+	}
+
+	function gearman_update_search_index($job) {
+		$details = $job->workload();
+		$details = unserialize($details);
+
+		wof_elasticsearch_update_document($details['geojson_data']);
+	}
+
+?>

services/geojson-server/wof-geojson-server.py

@@ -74,7 +74,7 @@ def geojson_save():
 
     # Repeat back the file we just wrote
     gf = open(path, 'r')
-    return gf.read()
+    return jsonify(ok=1, geojson=gf.read())
 
 @app.route('/pip', methods=['GET'])
 def geojson_hierarchy():

www/include/lib_wof_geojson.php

@@ -38,16 +38,21 @@ function wof_geojson_save($geojson) {
 			'geojson' => $geojson
 		));
 
+		// Check for connection errors
 		if (! $rsp['ok']) {
-			if ($rsp['body']) {
-				$rsp['error'] = "Error from GeoJSON service: {$rsp['body']}";
-			}
+			$rsp['error'] = "Error connecting to GeoJSON service: {$rsp['body']}";
+			return;
+		}
+
+		$rsp = json_decode($rsp['body'], true);
+		if (! $rsp['ok']) {
+			$rsp['error'] = "Error with saving via GeoJSON service: {$rsp['error']}";
 			return $rsp;
 		}
 
 		return array(
 			'ok' => 1,
-			'geojson' => $rsp['body']
+			'geojson' => $rsp['geojson']
 		);
 	}
 

www/include/lib_wof_save.php

@@ -32,6 +32,14 @@ function wof_save_file($input_path) {
 
 	function wof_save_string($geojson) {
 
+		if (! $GLOBALS['gearman_client']) {
+			$gearman_client = new GearmanClient();
+			$gearman_client->addServer();
+			$GLOBALS['gearman_client'] = $gearman_client;
+		} else {
+			$gearman_client = $GLOBALS['gearman_client'];
+		}
+
 		$geojson_data = json_decode($geojson, true);
 		if (! $geojson_data) {
 			return array(
@@ -78,37 +86,78 @@ function wof_save_string($geojson) {
 
 		$wof_id = $geojson_data['properties']['wof:id'];
 
-		$rsp = wof_save_to_github($geojson, $geojson_data);
-		if (! $rsp['ok']) {
-			$rsp['error'] = "Error saving to GitHub: {$rsp['error']}";
-			return $rsp;
-		}
+		$github_details = serialize(array(
+			'geojson' => $geojson,
+			'geojson_data' => $geojson_data,
+			'user_id' => $GLOBALS['cfg']['user']['id']
+		));
+		$gearman_client->doBackground("save_to_github", $github_details);
 
-		register_shutdown_function('wof_elasticsearch_update_document', $geojson_data);
+		$search_details = serialize(array(
+			'geojson_data' => $geojson_data
+		));
+		$gearman_client->doBackground("update_search_index", $search_details);
 
 		return array(
 			'ok' => 1,
-			'wof_id' => $wof_id
+			'wof_id' => $wof_id,
+			'geojson' => $geojson_data
 		);
 	}
 
-	function wof_save_batch($ids, $properties) {
-		// TODO: save all the things.
-		return array(
-			'ok' => 1,
-			'placeholder' => "Hello from wof_save_batch."
-		);
+	function wof_save_batch($batch_ids, $batch_properties) {
+		foreach ($batch_ids as $wof_id) {
+			$geojson_path = wof_utils_id2abspath($GLOBALS['cfg']['wof_data_dir'], $wof_id);
+			$errors = array();
+			$saved = array();
+			if (file_exists($geojson_path)) {
+				$existing_geojson = file_get_contents($geojson_path);
+				$existing_feature = json_decode($existing_geojson, true);
+				$existing_feature['properties'] = array_merge(
+					$existing_feature['properties'],
+					$batch_properties
+				);
+				$updated_geojson = json_encode($existing_feature);
+				$rsp = wof_save_string($updated_geojson);
+
+				if (! $rsp['ok']) {
+					$errors[$wof_id] = $rsp['error'];
+				} else {
+					$saved[$wof_id] = $rsp['geojson'];
+				}
+			} else {
+				$errors[$wof_id] = "Could not find WOF GeoJSON file.";
+			}
+		}
+
+		if (! $errors) {
+			return array(
+				'ok' => 1,
+				'properties' => $batch_properties,
+				'saved' => $saved
+			);
+		} else {
+			return array(
+				'ok' => 0,
+				'properties' => $batch_properties,
+				'error' => 'Error batch saving WOF documents.',
+				'details' => $errors,
+				'saved' => $saved
+			);
+		}
 	}
 
-	function wof_save_to_github($geojson, $geojson_data) {
+	function wof_save_to_github($geojson, $geojson_data, $oauth_token = null) {
 
-		// Get the GitHub oauth token
-		$rsp = github_users_curr_oauth_token();
-		if (! $rsp['ok']) {
-			return $rsp;
+		if (! $oauth_token) {
+			// Get the GitHub oauth token if none was specified
+			$rsp = github_users_curr_oauth_token();
+			if (! $rsp['ok']) {
+				return $rsp;
+			}
+			$oauth_token = $rsp['oauth_token'];
 		}
 
-		$oauth_token = $rsp['oauth_token'];
 		$owner = $GLOBALS['cfg']['wof_github_owner'];
 		$repo = $GLOBALS['cfg']['wof_github_repo'];
 		$wof_id = $geojson_data['properties']['wof:id'];

www/javascript/mapzen.whosonfirst.boundaryissues.search.js

@@ -108,30 +108,31 @@ mapzen.whosonfirst.boundaryissues.search = (function() {
 						ids: batch_update_ids.join(',')
 					};
 					var status = $(e.target).data('status');
+
 					var today = mapzen.whosonfirst.boundaryissues.edit.get_edtf_date(new Date());
 					if (status == 'current') {
 						data.properties = {
-							"wof:is_current": 1
+							"mz:is_current": 1
 						};
 					} else if (status == 'closed') {
 						data.properties = {
-							"wof:is_current": 0,
+							"mz:is_current": 0,
 							"edtf:cessation": today
 						};
 					} else if (status == 'deprecated') {
 						data.properties = {
-							"wof:is_current": 0,
+							"mz:is_current": 0,
 							"edtf:deprecated": today
 						};
 					} else if (status == 'funky') {
 						data.properties = {
-							"wof:is_funky": 1
+							"mz:is_funky": 1
 						};
 					}
 					data.properties = JSON.stringify(data.properties);
 
 					var onsuccess = function(rsp) {
-						alert(rsp.placeholder);
+						console.log(rsp);
 					};
 					var onerror = function(rsp) {
 						mapzen.whosonfirst.log.debug("error with batch saving.");