Skip to content

Commit

Permalink
performance tweaks, t0 blank png, archive/compression
Browse files Browse the repository at this point in the history
  • Loading branch information
pumped committed Nov 28, 2015
1 parent 60001ab commit 1748622
Show file tree
Hide file tree
Showing 5 changed files with 53 additions and 60 deletions.
4 changes: 3 additions & 1 deletion api/MapArchiver/Archiver.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import fileinput, os, sys
import time
import subprocess
import shutil
#from logger import *

class StoppableThread(threading.Thread):
Expand Down Expand Up @@ -83,9 +84,10 @@ def _convert(self, inputFile, outputFile):
with open(os.devnull, 'w') as FNULL:
cmd = 'gdalwarp -s_srs "EPSG:4326" -t_srs "EPSG:3857" -overwrite -co COMPRESS=DEFLATE ' + inputFile + " " + outputFile
status = subprocess.call(cmd, shell=True, stdout=FNULL)
#shutil.copyfile(inputFile,outputFile)

t1 = time.time()
#print t1-t0
#print "Archived in: " + str(t1-t0)

#cleanup
self.cleanup(inputFile)
Expand Down
33 changes: 19 additions & 14 deletions api/Model.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,9 @@ def onMessage(self, func):
def __runModelJob(self, ids):
log.info('Model Kicked off')
path = Config.runPath + '/' + ids["run"]
outputPath = path + Config.aggregatesPath
#outputPath = Config.rasterPath + '/' + ids["run"]
outputPath = path + "/aggs/"
print "outputpath: " + outputPath
inputPath = path + '/'
paramPath = path + '/params.txt'

Expand All @@ -87,6 +89,8 @@ def __runModelJob(self, ids):
if (self.local):
cmd = "stdbuf -oL " + Config.migExecutable + " " + paramPath + " " + inputPath + " " + outputPath

print cmd

#run process
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=0, shell=True)

Expand Down Expand Up @@ -118,7 +122,7 @@ def __runModelJob(self, ids):
self.emit('{"event":"timeline_state","data":{"speciesID":"'+speciesID+'","timelineID":"'+timelineID+'","state":'+state+'}}')

if result[0] == self.STEPWRITTEN:
runFile = inputPath + "aggs/agg" + time + ".asc"
runFile = inputPath + "aggs/agg" + time + ".tif"
renderedFile = Config.rasterPath + ids["run"] + "/agg" + time + ".tif"

#render map
Expand All @@ -145,9 +149,10 @@ def __runModelJob(self, ids):
log.info(job.stdout)

def cleanupDirectory(self,path):
os.remove(path + "/ma.asc")
os.remove(path + "/max_pre1.asc")
os.remove(path + "/dist_p.asc")
pass
os.remove(path + "/ma.tif")
os.remove(path + "/max_pre1.tif")
os.remove(path + "/dist_p.tif")

def processOutput(self,output):
line = output.lower()
Expand Down Expand Up @@ -232,24 +237,24 @@ def __writeInputFiles(self, ids):
dmDest = dest + '/dist_p.asc'

#run async, export current distribution
distTifPath = dest+'/dist.tif'
distTifPath = dest+'/dist_p.tif'
distAscPath = dest+"/dist_p.asc"
distCmd = 'gdal_rasterize -burn 1 -a_nodata 0 -init 0 -te 143.9165675170000043 -20.0251072599999986 147.0005675170000075 -14.9801072599999987 -ts 3084 5045 -a_srs "EPSG:3857" PG:"host=localhost user=postgres dbname=nyc password=password1" -sql "SELECT * FROM distribution WHERE species = \'siam\'" -of GTiff '+distTifPath
distCmd += " ; "
distCmd += "gdal_translate -a_nodata -9999 -of AAIGrid "+distTifPath+" "+distAscPath
#distCmd += " ; "
#distCmd += "gdal_translate -a_nodata -9999 -of AAIGrid "+distTifPath+" "+distAscPath
proc = subprocess.Popen(distCmd, stdout=subprocess.PIPE, bufsize=0, shell=True)

#rasterize Control Mechanisms
adjustTifPath = dest+'/ma.tif'
status = subprocess.call('gdal_rasterize -a "controlMechanism" -a_nodata -9999 -init -9999 -te 143.9165675170000043 -20.0251072599999986 147.0005675170000075 -14.9801072599999987 -ts 3084 5045 -a_srs "EPSG:3857" PG:"host=localhost user=postgres dbname=nyc password=password1" -sql "SELECT * FROM nyc_buildings WHERE timeline = \''+ids['timeline']+'\'" -of GTiff '+adjustTifPath, shell=True)

#translate to AAIGrid
maxPath = dest+'/ma.asc'
status = subprocess.call('gdal_translate -a_nodata -9999 -of AAIGrid '+adjustTifPath+' '+maxPath, shell=True)
#maxPath = dest+'/ma.asc'
#status = subprocess.call('gdal_translate -a_nodata -9999 -of AAIGrid '+adjustTifPath+' '+maxPath, shell=True)
#
# #cleanup intermediary files
# os.remove(mergedTifPath)
os.remove(adjustTifPath)
#os.remove(adjustTifPath)

#wait for distribution to finish
while True:
Expand All @@ -261,11 +266,11 @@ def __writeInputFiles(self, ids):
print output

#cleanup intermediary file
os.remove(distTifPath)
#os.remove(distTifPath)

#copy initial distribution map over
src = Config.initialFiles + '/max_pre1.asc'
dmDest = dest + '/max_pre1.asc'
src = Config.initialFiles + '/max_pre1.tif'
dmDest = dest + '/max_pre1.tif'
shutil.copyfile(src,dmDest)


Expand Down
33 changes: 19 additions & 14 deletions api/WMS/WMS.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@
import mapscript
import fileinput, os, sys
import bencode, hashlib
import StringIO
from StringIO import StringIO
import math
import time
#from logger import *

class StoppableThread(threading.Thread):
Expand Down Expand Up @@ -98,7 +99,7 @@ def __processParams(self,params):
#public method to add a file to the render queue
def getMap(self,wmsParams=[]):

output = StringIO.StringIO()
output = StringIO()

params = self.__processParams(wmsParams)

Expand All @@ -118,18 +119,21 @@ def getMap(self,wmsParams=[]):
noCache = True

if noCache:
time = int(params['time'])
print time
times = range(0,31)
if time in times:
times.remove(time)
layerTime = int(params['time'])
#print time

if layerTime >= 1:
#self.renderQueue.put({"params":params,"range":times})
mapImg = self.__generateMap(params)
mapImg.write(output)
del mapImg
else:
return False
try:
with open("WMS/blank.png", 'r') as f:
output.write(f.read())
except:
print "file doesn't exist"

#self.renderQueue.put({"params":params,"range":times})
mapImg = self.__generateMap(params)
mapImg.write(output)
del mapImg

imgOutput = output.getvalue()
del output
Expand Down Expand Up @@ -162,6 +166,7 @@ def __generateMap(self,params):
layer.data = layerParams[0]
layer.setProjection(layerParams[1])


savepath = self.__getFilename(params)
mapImg = m.draw()
#mapImg.save(savepath)
Expand All @@ -170,11 +175,11 @@ def __generateMap(self,params):
del layer
del layerParams

print "Rendered: " + str(identifier) + ", " + str(i)
#print "Rendered: " + str(identifier) + ", " + str(i)
return mapImg

def _findLayerParams(self,identifier,i):
ascPath = "/media/ramfs/runs/"+identifier+"/aggs/agg"+str(i)+".asc"
ascPath = "/media/ramfs/runs/"+identifier+"/aggs/agg"+str(i)+".tif"
tifPath = "/media/scratch/imgs/"+identifier+"/agg"+str(i)+".tif"

if os.path.isfile(ascPath):
Expand Down
Binary file added api/WMS/blank.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
43 changes: 12 additions & 31 deletions api/WMS/static.map
Original file line number Diff line number Diff line change
@@ -1,34 +1,16 @@
MAP
NAME "WMS-Mapserver"
STATUS ON
SIZE 1542 2523
#EXTENT 143.91 -20 148 -15
#EXTENT -180 -90 180 90
EXTENT 16078473 -2203421 16365455 -1723647
UNITS DD
SHAPEPATH "/media/ramfs"

TRANSPARENT ON
IMAGECOLOR 0 255 255
IMAGEQUALITY 95
#TRANSPARENT ON
#IMAGECOLOR 0 255 255
IMAGEQUALITY 50
IMAGETYPE png

PROJECTION
"init=epsg:3857"
END

#
# Start of web interface definition
#
WEB
METADATA
"wms_title" "WMS-MapServer"
"wms_onlineresource" ""
"wms_enable_request" "*"
"wms_srs" "EPSG:4269 EPSG:4326 EPSG:3857"
END
END # WEB

#
# Start of layer definitions
#
Expand All @@ -38,34 +20,33 @@ MAP
STATUS DEFAULT
DATA "/media/ramfs/runs/siam-1/agg0.asc"
PROJECTION
"init=epsg:4326"
"init=epsg:3857"
END
CLASSITEM "[pixel]"
OFFSITE 0 0 0
TRANSPARENCY 100
PROCESSING "CLOSE_CONNECTION=ALWAYS"
PROCESSING "CLOSE_CONNECTION=ALWAYS"

# class using an EXPRESSION using only [pixel].
CLASS
EXPRESSION (([pixel] > 0.1) AND ([pixel] < 2))
EXPRESSION (([pixel] > 2.4))
STYLE
OPACITY 50
COLOR 255 250 100
OPACITY 100
COLOR 255 30 30
END
END

CLASS
EXPRESSION (([pixel] >= 2) AND ([pixel] < 2.5))
EXPRESSION (([pixel] > 1.9))
STYLE
OPACITY 80
COLOR 255 200 60
END
END
CLASS
EXPRESSION (([pixel] >= 2.5) AND ([pixel] < 50))
EXPRESSION (([pixel] > 0.1))
STYLE
OPACITY 100
COLOR 255 30 30
OPACITY 50
COLOR 255 250 100
END
END

Expand Down

0 comments on commit 1748622

Please sign in to comment.