Skip to content

Commit

Permalink
working on exporting osm files, version version of review script.
Browse files Browse the repository at this point in the history
  • Loading branch information
@jremillard
jremillard committed Dec 23, 2017
1 parent 0cda7cb commit 1f6c07c
Show file tree
Hide file tree
Showing 4 changed files with 180 additions and 75 deletions.
12 changes: 6 additions & 6 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,10 @@ The entire training workflow is in the trainall.py script, which calls the follo

## Configuration

1. Ubuntu 17.10
2. A Bing key, create a secrets.py file, add in bingKey ="your key"
3. Create a virtual environment python 3.6
4. In the virtual environment, run "pip install -r requirements.txt"
5. TensorFlow 1.3+
6. Keras 2.0.8+.
- Ubuntu 17.10
- A Bing key, create a secrets.py file, add in bingKey ="your key"
- Create a virtual environment python 3.6
- In the virtual environment, run "pip install -r requirements.txt"
- TensorFlow 1.3+
- Keras 2.0.8+.

133 changes: 65 additions & 68 deletions createosmanomaly.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@
from skimage import draw
from skimage import io

showFigures = False

def toDegrees(rad):
return rad * 180/math.pi

def writeOSM( osmFileName,featureName, simpleContour,tilePixel, qkRoot) :
with open(osmFileName,"wt",encoding="ascii") as f:
f.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
Expand All @@ -45,31 +50,44 @@ def writeOSM( osmFileName,featureName, simpleContour,tilePixel, qkRoot) :
f.write("</osm>\n")
f.close

def writeShape(wayNumber, finalShape, image, bbTop,bbHeight,bbLeft,bbWidth) :
nPts = int(finalShape.length)
if ( nPts > 5000) :
nPts = 5000
fitContour = np.zeros((nPts,1,2), dtype=np.int32)

if ( nPts > 3):

for t in range(0,nPts) :
pt = finalShape.interpolate(t)
fitContour[t,0,0] = pt.x
fitContour[t,0,1] = pt.y

fitContour = [ fitContour ]
fitContour = [ cv2.approxPolyDP(cnt,2,True) for cnt in fitContour]

image = np.copy(imageNoMasks)
cv2.drawContours(image, fitContour,-1, (0,255,0), 2)
if ( showFigures ):
fig.add_subplot(2,2,3)
plt.title(featureName + " " + str(r['scores'][i]) + " Fit")
plt.imshow(image[bbTop:bbTop+bbHeight,bbLeft:bbLeft+bbWidth])

while ( os.path.exists( "anomaly/add/{0:06d}.osm".format(wayNumber) )) :
wayNumber += 1

def _find_getch():
try:
import termios
except ImportError:
# Non-POSIX. Return msvcrt's (Windows') getch.
import msvcrt
return msvcrt.getch
debugFileName = os.path.join( inference_config.ROOT_DIR, "anomaly","add","{0:06d}.jpg".format(wayNumber))
io.imsave(debugFileName,image[bbTop:bbTop+bbHeight,bbLeft:bbLeft+bbWidth],quality=100)

# POSIX system. Create and return a getch that manipulates the tty.
import sys, tty
def _getch():
fd = sys.stdin.fileno()
old_settings = termios.tcgetattr(fd)
try:
tty.setraw(fd)
ch = sys.stdin.read(1)
finally:
termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
return ch
osmFileName = os.path.join( inference_config.ROOT_DIR, "anomaly","add","{0:06d}.osm".format(wayNumber))
writeOSM( osmFileName,featureName, fitContour[0],tilePixel, qkRoot)

return _getch
if (showFigures ):
plt.show(block=False)
plt.pause(0.05)

getch = _find_getch()
return wayNumber



ROOT_DIR_ = os.path.dirname(os.path.realpath(sys.argv[0]))
Expand Down Expand Up @@ -153,7 +171,6 @@ class InferenceConfig(osmmodelconfig.OsmModelConfig):
if ( not os.path.isdir("anomaly/overlap")) :
os.mkdir("anomaly/overlap")

showFigures = False
fig = {}
if ( showFigures):
fig = plt.figure()
Expand Down Expand Up @@ -316,14 +333,16 @@ class InferenceConfig(osmmodelconfig.OsmModelConfig):

simpleContour = simpleContour[0]

if ( featureName == "baseball" and isinstance(simpleContour,list) ):
while ( os.path.exists( "anomaly/add/{}.osm".format(wayNumber) )) :
print(" {}".format(featureName))
if ( featureName == "baseball" and isinstance(simpleContour,np.ndarray) ):

while ( os.path.exists( "anomaly/add/{0:06d}.osm".format(wayNumber) )) :
wayNumber += 1

debugFileName = os.path.join( inference_config.ROOT_DIR, "anomaly","add","{}.jpg".format(wayNumber))
debugFileName = os.path.join( inference_config.ROOT_DIR, "anomaly","add","{0:06d}.jpg".format(wayNumber))
io.imsave(debugFileName,image[bbTop:bbTop+bbHeight,bbLeft:bbLeft+bbWidth],quality=100)

osmFileName = os.path.join( inference_config.ROOT_DIR, "anomaly","add","{}.osm".format(wayNumber))
osmFileName = os.path.join( inference_config.ROOT_DIR, "anomaly","add","{0:06d}.osm".format(wayNumber))
writeOSM( osmFileName,featureName, simpleContour,tilePixel, qkRoot)

fitContour = simpleContour
Expand Down Expand Up @@ -352,7 +371,7 @@ def makePie(paramsX):
fitShape = geometry.LineString(pts)

fitShape = affinity.translate(fitShape, -width/2,-width/2 )
fitShape = affinity.rotate(fitShape,angle,use_radians=True )
fitShape = affinity.rotate(fitShape,angle )
fitShape = affinity.translate(fitShape, centerX,centerY )

return fitShape
Expand All @@ -377,29 +396,38 @@ def fitPie(paramsX):

cm = np.mean( rawContours[0],axis=0)

result = {}
results = []
angleStepCount = 8
for angleI in range(angleStepCount):

centerX = cm[0,0]
centerY = cm[0,1]
width = math.sqrt(cv2.contourArea(rawContours[0]))
angle = 2*math.pi * float(angleI)/angleStepCount
angle = 360 * float(angleI)/angleStepCount
x0 = np.array([centerX,centerY,width,angle ])

resultR = scipy.optimize.minimize(fitPie, x0, method='nelder-mead', options={'xtol': 1e-6,'maxiter':50 })

if ( angleI == 0):
result = resultR
results.append(resultR)

if ( resultR.fun < result.fun):
result = resultR
#print("{} {}".format(angle * 180.0 / math.pi,resultR.fun ))
bestScore = 1e100
bestResult = {}
for result in results:
if result.fun < bestScore :
bestScore = result.fun
bestResult = result

resultR = scipy.optimize.minimize(fitPie, resultR.x, method='nelder-mead', options={'xtol': 1e-6 })
bestResult = scipy.optimize.minimize(fitPie, bestResult.x, method='nelder-mead', options={'xtol': 1e-6 })
finalShape = makePie(bestResult.x)
wayNumber = writeShape(wayNumber, finalShape, image, bbTop,bbHeight,bbLeft,bbWidth)

#print(result)
finalShape = makePie(result.x)
for result in results:
angle = result.x[3]
angleDelta = int(math.fabs(result.x[3]-bestResult.x[3])) % 360
if result.fun < 1.2*bestScore and angleDelta > 45 :
result = scipy.optimize.minimize(fitPie, result.x, method='nelder-mead', options={'xtol': 1e-6 })
finalShape = makePie(result.x)
wayNumber = writeShape(wayNumber, finalShape, image, bbTop,bbHeight,bbLeft,bbWidth)

else:

Expand Down Expand Up @@ -457,41 +485,10 @@ def fitRect(paramsX):

#print(result)
finalShape = makeRect(result.x)

nPts = int(finalShape.length)
if ( nPts > 5000) :
nPts = 5000
fitContour = np.zeros((nPts,1,2), dtype=np.int32)

if ( nPts > 3):

for t in range(0,nPts) :
pt = finalShape.interpolate(t)
fitContour[t,0,0] = pt.x
fitContour[t,0,1] = pt.y

fitContour = [ fitContour ]
fitContour = [ cv2.approxPolyDP(cnt,2,True) for cnt in fitContour]

image = np.copy(imageNoMasks)
cv2.drawContours(image, fitContour,-1, (0,255,0), 2)
if ( showFigures ):
fig.add_subplot(2,2,3)
plt.title(featureName + " " + str(r['scores'][i]) + " Fit")
plt.imshow(image[bbTop:bbTop+bbHeight,bbLeft:bbLeft+bbWidth])

while ( os.path.exists( "anomaly/add/{}.osm".format(wayNumber) )) :
wayNumber += 1

debugFileName = os.path.join( inference_config.ROOT_DIR, "anomaly","add","{}.jpg".format(wayNumber))
io.imsave(debugFileName,image[bbTop:bbTop+bbHeight,bbLeft:bbLeft+bbWidth],quality=100)
wayNumber = writeShape(wayNumber, finalShape, image, bbTop,bbHeight,bbLeft,bbWidth)

osmFileName = os.path.join( inference_config.ROOT_DIR, "anomaly","add","{}.osm".format(wayNumber))
writeOSM( osmFileName,featureName, fitContour[0],tilePixel, qkRoot)

if (showFigures ):
plt.show(block=False)
plt.pause(0.05)



Expand Down
8 changes: 7 additions & 1 deletion getdatafromosm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,14 @@ def saveOsmData(query) :
# - bridges
# - solar panels farms
# - wind turbines
# - rail road crossings.
# - railroad crossings.
# - active rail roads
# - water tanks
# - wafer/lakes/rivers
# - parking lots
# - driveways
# - gas stations
# - building (Microsoft has already done this)
# - Running track

print(summary)
102 changes: 102 additions & 0 deletions reviewosmanomaly.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
import os
import sys
import glob
import imagestoosm.config as osmcfg
import xml.etree.ElementTree as ET
import QuadKey.quadkey as quadkey
import shapely.geometry as geometry
import matplotlib.pyplot as plt
import skimage.io

def _find_getch():
try:
Expand All @@ -27,3 +33,99 @@ def _getch():
getch = _find_getch()


# read in OSM files, convert to pixels z18, make them shapely polygons

addDirectory = os.path.join( "anomaly","add","*.osm")

newWays = {}
for osmFileName in glob.glob(addDirectory):

(path,filename) = os.path.split(osmFileName);
wayNumber = os.path.splitext(filename)[0]

newEntry = {
"imageName" : os.path.join( path,str(wayNumber) + ".jpg") ,
"osmFile" : osmFileName,
"tags" : {},
"status" : ""
}

tree = ET.parse(osmFileName)
root = tree.getroot()

for tag in root.findall('./way/tag'):
key = tag.attrib["k"]
val = tag.attrib["v"]

newEntry['tags'][key] = val

pts =[]
for node in root.iter('node'):
pt = ( float(node.attrib['lat']),float(node.attrib['lon']))
pixel = quadkey.TileSystem.geo_to_pixel(pt,osmcfg.tileZoom)
pts.append(pixel)

if ( len(pts ) > 2 ):
newEntry["geometry"] = geometry.Polygon(pts)
#print(newEntry )
newWays[osmFileName] = newEntry

# read in review file, file status (accepted or rejected), path to osm

# for each add way that

fig = plt.figure()


for wayKey in sorted(newWays):
# if reviewed skip
way = newWays[wayKey]

if ( len(way['status']) == 0 ) :

# for each way, check for overlap, same tags, add to review cluster
subPlotCols = 2
subPlotRows = 1
maxSubPlots = subPlotCols*subPlotRows

reviewSet = [way]
for otherKey in sorted(newWays):
other = newWays[otherKey]
if ( other != way and len(other['status']) == 0 and way['tags'] == other['tags'] and other['geometry'].intersects( way['geometry'])):
reviewSet.append(other)

viewSet = []
for wayIndex in range(len(reviewSet)):
viewSet.append(reviewSet[wayIndex])

if ( len(viewSet) == maxSubPlots or wayIndex+1 >= len(reviewSet)):

for plotIndex in range(maxSubPlots):
sb = fig.add_subplot(subPlotRows,subPlotCols,plotIndex+1)
sb.cla()

for wayIndex in range(len(viewSet)):
fig.add_subplot(subPlotRows,subPlotCols,wayIndex+1)
plt.title(viewSet[wayIndex]['osmFile'])
image = skimage.io.imread( viewSet[wayIndex]['imageName'])
plt.imshow(image)
viewSet[wayIndex]['status'] = 'rejected'

plt.show(block=False)
plt.pause(0.05)

getch()

viewSet = [ reviewSet[0] ]


# read in diagnostic images from review cluster
# display up to 9 images
# get choice (r reject call, 1-9 to accept, ask again if other, q quit)
# if > 9 images and an image was selected bring it forward, keep looping
# update review file, reject and accept flags for each way in review cluster





0 comments on commit 1f6c07c

Please sign in to comment.