performance tweaks, t0 blank png, archive/compression

api/MapArchiver/Archiver.py

@@ -5,6 +5,7 @@
 import fileinput, os, sys
 import time
 import subprocess
+import shutil
 #from logger import *
 
 class StoppableThread(threading.Thread):
@@ -83,9 +84,10 @@ def _convert(self, inputFile, outputFile):
         with open(os.devnull, 'w') as FNULL:
             cmd = 'gdalwarp -s_srs "EPSG:4326" -t_srs "EPSG:3857" -overwrite -co COMPRESS=DEFLATE ' + inputFile + " " + outputFile
             status = subprocess.call(cmd, shell=True, stdout=FNULL)
+        #shutil.copyfile(inputFile,outputFile)
 
         t1 = time.time()
-        #print t1-t0
+        #print "Archived in: " + str(t1-t0)
 
         #cleanup
         self.cleanup(inputFile)

api/Model.py

@@ -76,7 +76,9 @@ def onMessage(self, func):
 	def __runModelJob(self, ids):
 		log.info('Model Kicked off')
 		path = Config.runPath + '/' + ids["run"]
-		outputPath = path + Config.aggregatesPath
+		#outputPath = Config.rasterPath + '/' + ids["run"]
+		outputPath = path + "/aggs/"
+		print "outputpath: " + outputPath
 		inputPath = path + '/'
 		paramPath = path + '/params.txt'
 
@@ -87,6 +89,8 @@ def __runModelJob(self, ids):
 		if (self.local):
 			cmd = "stdbuf -oL " + Config.migExecutable + " " + paramPath + " " + inputPath + " " + outputPath
 
+			print cmd
+
 			#run process
 			proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=0, shell=True)
 
@@ -118,7 +122,7 @@ def __runModelJob(self, ids):
 							self.emit('{"event":"timeline_state","data":{"speciesID":"'+speciesID+'","timelineID":"'+timelineID+'","state":'+state+'}}')
 
 						if result[0] == self.STEPWRITTEN:
-							runFile = inputPath + "aggs/agg" + time + ".asc"
+							runFile = inputPath + "aggs/agg" + time + ".tif"
 							renderedFile = Config.rasterPath + ids["run"] + "/agg" + time + ".tif"
 
 							#render map
@@ -145,9 +149,10 @@ def __runModelJob(self, ids):
 			log.info(job.stdout)
 
 	def cleanupDirectory(self,path):
-		os.remove(path + "/ma.asc")
-		os.remove(path + "/max_pre1.asc")
-		os.remove(path + "/dist_p.asc")
+		pass
+		os.remove(path + "/ma.tif")
+		os.remove(path + "/max_pre1.tif")
+		os.remove(path + "/dist_p.tif")
 
 	def processOutput(self,output):
 		line = output.lower()
@@ -232,24 +237,24 @@ def __writeInputFiles(self, ids):
 		dmDest = dest + '/dist_p.asc'
 
 		#run async, export current distribution
-		distTifPath = dest+'/dist.tif'
+		distTifPath = dest+'/dist_p.tif'
 		distAscPath = dest+"/dist_p.asc"
 		distCmd = 'gdal_rasterize -burn 1 -a_nodata 0 -init 0 -te 143.9165675170000043 -20.0251072599999986 147.0005675170000075 -14.9801072599999987 -ts 3084 5045 -a_srs "EPSG:3857" PG:"host=localhost user=postgres dbname=nyc password=password1" -sql "SELECT * FROM distribution WHERE species = \'siam\'" -of GTiff '+distTifPath
-		distCmd += " ; "
-		distCmd += "gdal_translate -a_nodata -9999 -of AAIGrid "+distTifPath+" "+distAscPath
+		#distCmd += " ; "
+		#distCmd += "gdal_translate -a_nodata -9999 -of AAIGrid "+distTifPath+" "+distAscPath
 		proc = subprocess.Popen(distCmd, stdout=subprocess.PIPE, bufsize=0, shell=True)
 
 		#rasterize Control Mechanisms
 		adjustTifPath = dest+'/ma.tif'
 		status = subprocess.call('gdal_rasterize -a "controlMechanism" -a_nodata -9999 -init -9999 -te 143.9165675170000043 -20.0251072599999986 147.0005675170000075 -14.9801072599999987 -ts 3084 5045 -a_srs "EPSG:3857" PG:"host=localhost user=postgres dbname=nyc password=password1" -sql "SELECT * FROM nyc_buildings WHERE timeline = \''+ids['timeline']+'\'" -of GTiff '+adjustTifPath, shell=True)
 
 		#translate to AAIGrid
-		maxPath = dest+'/ma.asc'
-		status = subprocess.call('gdal_translate -a_nodata -9999 -of AAIGrid '+adjustTifPath+' '+maxPath, shell=True)
+		#maxPath = dest+'/ma.asc'
+		#status = subprocess.call('gdal_translate -a_nodata -9999 -of AAIGrid '+adjustTifPath+' '+maxPath, shell=True)
 		#
 		# #cleanup intermediary files
 		# os.remove(mergedTifPath)
-		os.remove(adjustTifPath)
+		#os.remove(adjustTifPath)
 
 		#wait for distribution to finish
 		while True:
@@ -261,11 +266,11 @@ def __writeInputFiles(self, ids):
 				print output
 
 		#cleanup intermediary file
-		os.remove(distTifPath)
+		#os.remove(distTifPath)
 
 		#copy initial distribution map over
-		src = Config.initialFiles + '/max_pre1.asc'
-		dmDest = dest + '/max_pre1.asc'
+		src = Config.initialFiles + '/max_pre1.tif'
+		dmDest = dest + '/max_pre1.tif'
 		shutil.copyfile(src,dmDest)
 
 

api/WMS/WMS.py

@@ -4,8 +4,9 @@
 import mapscript
 import fileinput, os, sys
 import bencode, hashlib
-import StringIO
+from StringIO import StringIO
 import math
+import time
 #from logger import *
 
 class StoppableThread(threading.Thread):
@@ -98,7 +99,7 @@ def __processParams(self,params):
     #public method to add a file to the render queue
     def getMap(self,wmsParams=[]):
 
-        output = StringIO.StringIO()
+        output = StringIO()
 
         params = self.__processParams(wmsParams)
 
@@ -118,18 +119,21 @@ def getMap(self,wmsParams=[]):
             noCache = True
 
         if noCache:
-            time = int(params['time'])
-            print time
-            times = range(0,31)
-            if time in times:
-                times.remove(time)
+            layerTime = int(params['time'])
+            #print time
+
+            if layerTime >= 1:
+                #self.renderQueue.put({"params":params,"range":times})
+                mapImg = self.__generateMap(params)
+                mapImg.write(output)
+                del mapImg
             else:
-                return False
+                try:
+                    with open("WMS/blank.png", 'r') as f:
+                        output.write(f.read())
+                except:
+                    print "file doesn't exist"
 
-            #self.renderQueue.put({"params":params,"range":times})
-            mapImg = self.__generateMap(params)
-            mapImg.write(output)
-            del mapImg
 
         imgOutput = output.getvalue()
         del output
@@ -162,6 +166,7 @@ def __generateMap(self,params):
         layer.data = layerParams[0]
         layer.setProjection(layerParams[1])
 
+
         savepath = self.__getFilename(params)
         mapImg = m.draw()
         #mapImg.save(savepath)
@@ -170,11 +175,11 @@ def __generateMap(self,params):
         del layer
         del layerParams
 
-        print "Rendered: " + str(identifier) + ", " + str(i)
+        #print "Rendered: " + str(identifier) + ", " + str(i)
         return mapImg
 
     def _findLayerParams(self,identifier,i):
-        ascPath = "/media/ramfs/runs/"+identifier+"/aggs/agg"+str(i)+".asc"
+        ascPath = "/media/ramfs/runs/"+identifier+"/aggs/agg"+str(i)+".tif"
         tifPath = "/media/scratch/imgs/"+identifier+"/agg"+str(i)+".tif"
 
         if os.path.isfile(ascPath):

api/WMS/static.map

@@ -1,34 +1,16 @@
 MAP
     NAME "WMS-Mapserver"
     STATUS ON
-    SIZE 1542 2523
-    #EXTENT 143.91 -20 148 -15
-    #EXTENT -180 -90 180 90
-    EXTENT 16078473 -2203421 16365455 -1723647
-    UNITS DD
-    SHAPEPATH "/media/ramfs"
 
-    TRANSPARENT ON
-    IMAGECOLOR 0 255 255
-    IMAGEQUALITY 95
+    #TRANSPARENT ON
+    #IMAGECOLOR 0 255 255
+    IMAGEQUALITY 50
     IMAGETYPE png
 
     PROJECTION
         "init=epsg:3857"
     END
 
-    #
-    # Start of web interface definition
-    #
-    WEB
-        METADATA
-            "wms_title"           "WMS-MapServer"
-            "wms_onlineresource"  ""
-            "wms_enable_request" "*"
-            "wms_srs"             "EPSG:4269 EPSG:4326 EPSG:3857"
-        END
-    END # WEB
-
     #
     # Start of layer definitions
     #
@@ -38,34 +20,33 @@ MAP
         STATUS DEFAULT
         DATA "/media/ramfs/runs/siam-1/agg0.asc"
         PROJECTION
-            "init=epsg:4326"
+            "init=epsg:3857"
         END
         CLASSITEM "[pixel]"
         OFFSITE 0 0 0
         TRANSPARENCY 100
-        PROCESSING "CLOSE_CONNECTION=ALWAYS" 
+        PROCESSING "CLOSE_CONNECTION=ALWAYS"
 
           # class using an EXPRESSION using only [pixel].
           CLASS
-            EXPRESSION (([pixel] > 0.1) AND ([pixel] < 2))
+            EXPRESSION (([pixel] > 2.4))
             STYLE
-              OPACITY 50
-              COLOR 255 250 100
+              OPACITY 100
+              COLOR 255 30 30
             END
           END
-
           CLASS
-            EXPRESSION (([pixel] >= 2) AND ([pixel] < 2.5))
+            EXPRESSION (([pixel] > 1.9))
             STYLE
               OPACITY 80
               COLOR 255 200 60
             END
           END
           CLASS
-            EXPRESSION (([pixel] >= 2.5) AND ([pixel] < 50))
+            EXPRESSION (([pixel] > 0.1))
             STYLE
-              OPACITY 100
-              COLOR 255 30 30
+              OPACITY 50
+              COLOR 255 250 100
             END
           END