collector: implemented cache mode, getting a general summary file and…

… the geoJson-Output, fixes #21

apps/collector/collectCommunities.py

@@ -4,87 +4,158 @@
 import urllib
 from optparse import OptionParser
 from urllib.request import urlopen
+from datetime import tzinfo, timedelta, datetime
 
 #log helper function
 def log(logLevel, message):
 	if logLevel <= options.logLevel:
 		print("Message from engine room (level " + str(logLevel) + "): " + message)
 
-#define some constants
-ffDirUrl = "https://raw.github.com/freifunk/api.freifunk.net/master/directory/directory.json"
-ffMapOutput = "/var/www/ffmap/ffMap.json"
-
-#read some command line arguments
-parser = OptionParser()
-parser.add_option("-l", "--loglevel", dest="logLevel", default=1, type=int, help="define logleel")
-(options, args) = parser.parse_args()
-
 #load directory
-try:
-	ffDirectoryRaw = urlopen(ffDirUrl)
-except BaseException as e:
-	log(0, "error reading directory " + str(e))
-	exit(1)
+def loadDirectory(url):
+	try:
+		ffDirectoryRaw = urlopen(url)
+	except BaseException as e:
+		log(0, "error reading directory " + str(e))
+		exit(1)
 
-ffDirectory = json.loads(ffDirectoryRaw.readall().decode('utf-8'))
-log(4, "our directory: " + str(ffDirectory))
-features=[]
-#prepare GeoJSON format
-ffMapData = { "type" : "FeatureCollection", "features" : features }
+	return json.loads(ffDirectoryRaw.readall().decode('utf-8'))
 
-for community in ffDirectory:
-	properties=dict()
-	log(3, "working on community: " + ffDirectory[community])
+#create a summarized json file, works as cache
+def summarizedJson(ffDir, path):
+	time = datetime.now().isoformat(' ')
+	summary = dict() 
+	#open summarized file first
 	try:
-		ffApi = json.loads(urlopen(ffDirectory[community]).readall().decode('utf-8'))
-	except UnicodeError as e:
+		summaryFile = open(path, "r")
+	except IOError as e:
+		if e.errno == 2:
+			summaryFile = open(path, "w")
+		else:
+			log(0, "error opening summary file " +str(e))
+			exit(1)
+	except BaseException as e:
+		log(0, "error opening summary file " +str(e))
+		exit(1)
+
+	if summaryFile.mode == "r":
+		content = summaryFile.read()
+		if content != "":
+			log(4, "cache file " +content)
+			summary = json.loads(content)
+		#close and reopen file
+		summaryFile.close()
+		summaryFile = open(path, "w")
+	if summary is None:
+		summary = dict()
+
+	for community in ffDir:
+		log(3, "working on community: " + ffDir[community])
 		try:
-			ffApi = json.loads(urlopen(ffDirectory[community]).readall().decode('iso8859_2'))
-			log(0, "Unicode Error: " + ffDirectory[community] + ": " + str(e) + ", try iso8859_2 instead")
-			pass
+			ffApi = json.loads(urlopen(ffDir[community]).readall().decode('utf-8'))
+		except UnicodeError as e:
+			try:
+				ffApi = json.loads(urlopen(ffDir[community]).readall().decode('iso8859_2'))
+				log(0, "Unicode Error: " + ffDir[community] + ": " + str(e) + ", try iso8859_2 instead")
+				pass
+			except BaseException as e:
+				log(0, "Error reading community api file " + ffDir[community] + ": " + str(e))
+				continue
 		except BaseException as e:
-			log(0, "Error reading community api file " + ffDirectory[community] + ": " + str(e))
+			log(0, "Error reading community api file " + ffDir[community] + ": " + str(e))
 			continue
-	except BaseException as e:
-		log(0, "Error reading community api file " + ffDirectory[community] + ": " + str(e))
-		continue
+
+		ffApi['mtime'] = time
+		summary[community] = ffApi
+	log(4, "our summary: " + str(summary))
+	summaryResult = json.dumps(summary, indent=4)
+
+	try:
+		summaryFile.write(str(summaryResult))
+		summaryFile.flush()
+	finally:
+		summaryFile.close()
+
+#create geojson output
+def geoJson(summary, geoJsonPath):
+
+	features=[]
+	#prepare GeoJSON format
+	ffGeoJson = { "type" : "FeatureCollection", "features" : features }
+
+	for community in summary:
+		properties=dict()
+		details = summary[community]
+		log(3, "working on community: " + str(summary[community]))
 
-	#add data according to http://wiki.freifunk.net/Fields_we_should_use
-	try: 
-		for contacts in ffApi['contact']:
-			properties[contacts] = ffApi['contact'][contacts] 
-		geometry = { "type" : "Point", "coordinates" : [ ffApi['location']['lon'], ffApi['location']['lat']] }
-		properties['name'] = ffApi['name']
-		properties['city'] = ffApi['location']['city']
-		if 'address' in ffApi['location']:
-			properties['address'] = ffApi['location']['address']
-		properties['url'] = ffApi['url']
-		if 'metacommunity' in ffApi:
-			properties['metacommunity'] = ffApi['metacommunity']
-		if 'feeds' in ffApi:
-			properties['feeds'] = ffApi['feeds']
-		if 'events' in ffApi:
-			properties['events'] = ffApi['events']
-		if 'nodes' in ffApi['state']:
-			properties['nodes'] = ffApi['state']['nodes']
+		#add data according to http://wiki.freifunk.net/Fields_we_should_use
+		try: 
+			for contacts in details['contact']:
+				properties[contacts] = details['contact'][contacts] 
+			geometry = { "type" : "Point", "coordinates" : [ details['location']['lon'], details['location']['lat']] }
+			properties['name'] = details['name']
+			properties['city'] = details['location']['city']
+			if 'address' in details['location']:
+				properties['address'] = details['location']['address']
+			properties['url'] = details['url']
+			if 'metadetails' in details:
+				properties['metadetails'] = details['metadetails']
+			if 'feeds' in details:
+				properties['feeds'] = details['feeds']
+			if 'events' in details:
+				properties['events'] = details['events']
+			if 'nodes' in details['state']:
+				properties['nodes'] = details['state']['nodes']
 
-		properties['mtime'] = ffApi['state']['lastchange']
-	except BaseException as e:
-		log(1, "There's something wrong with the JSON file: " + str(e))
-		continue
+			properties['mtime'] = details['state']['lastchange']
+		except BaseException as e:
+			log(1, "There's something wrong with the JSON file: " + str(e))
+			continue
 
 
-	features.append({ "type" : "Feature", "geometry" : geometry, "properties" : properties })
-	properties=""
+		features.append({ "type" : "Feature", "geometry" : geometry, "properties" : properties })
+		properties=""
 
-result = json.dumps(ffMapData)
-log(3, "our result: " + result)
-#write summary to bin directory
-try:
-	f = open(ffMapOutput, "w")
+	result = json.dumps(ffGeoJson, indent=4)
+	log(3, "our result: " + result)
+	#write summary to bin directory
 	try:
-		f.write(str(result))
-	finally:
-		f.close()
-except IOError:
-	pass
+		f = open(geoJsonPath, "w")
+		try:
+			f.write(str(result))
+		finally:
+			f.close()
+	except IOError:
+		pass
+
+#define some constants
+ffDirUrl = "https://raw.github.com/freifunk/api.freifunk.net/master/directory/directory.json"
+ffGeoJson = "/var/www/ffmap/ffGeoJson.json"
+ffSummarizedJson = "/var/www/ffmap/ffSummarizedDir.json"
+ffHtmlTable = "/var/www/ffmap/ffHtmlTable.html"
+
+#read some command line arguments
+parser = OptionParser()
+parser.add_option("-l", "--loglevel", dest="logLevel", default=1, type=int, help="define logleel")
+parser.add_option("-g", "--geojson", dest="geoJSON", default=True, action="store_true", help="Output format: geoJSON")
+(options, args) = parser.parse_args()
+
+#first step: load directory
+ffDirectory = loadDirectory(ffDirUrl) 
+log(4, "our directory: " + str(ffDirectory))
+
+#second step: write all information to a summarized json
+summarizedJson(ffDirectory, ffSummarizedJson)
+
+#now create all other formats
+#open summary file
+try:
+	summaryFile = open(ffSummarizedJson)
+	summary = json.loads(summaryFile.read())
+	summaryFile.close()
+except IOError as e:
+	log(0, "error working on summary file " +str(e))
+
+if options.geoJSON:
+	geoJson(summary, ffGeoJson)
+