Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
sqlite added
  • Loading branch information
Calvin Metcalf committed Apr 10, 2013
1 parent dbb26a0 commit bdda2c4
Show file tree
Hide file tree
Showing 9 changed files with 268 additions and 41 deletions.
Binary file modified esri2open.tbx
Binary file not shown.
74 changes: 61 additions & 13 deletions esri2open/esri2open.py
Expand Up @@ -13,11 +13,12 @@
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
#imports #imports
from arcpy import AddMessage, GetCount_management from arcpy import AddMessage, GetCount_management
from utilities import getExt,listFields,getShp, parseProp from utilities import getExt, listFields,getShp, parseProp, parseFieldType
from parseRow import parse from parseRow import parse
from csv import DictWriter from csv import DictWriter
from json import dump from json import dump

from sqlite3 import Connection
from os.path import splitext, split


#---- #----
#prepare files #prepare files
Expand All @@ -30,42 +31,84 @@ def prepareCSV(outJSON,featureClass,fileType,includeGeometry):
for field in fields: for field in fields:
if (fields[field] != u'OID') and field.lower() not in ('shape_length','shape_area','shape.len','shape.length','shape_len','shape.area',shp.lower()): if (fields[field] != u'OID') and field.lower() not in ('shape_length','shape_area','shape.len','shape.length','shape_len','shape.area',shp.lower()):
fieldNames.append(field) fieldNames.append(field)
if includeGeometry: if includeGeometry!="none":
fieldNames.append("geometry") fieldNames.append("geometry")
outCSV=DictWriter(out,fieldNames,extrasaction='ignore') outCSV=DictWriter(out,fieldNames,extrasaction='ignore')
fieldObject = {} fieldObject = {}
for fieldName in fieldNames: for fieldName in fieldNames:
fieldObject[fieldName]=fieldName fieldObject[fieldName]=fieldName
outCSV.writerow(fieldObject) outCSV.writerow(fieldObject)
return [outCSV,out] return [outCSV,out]

def prepareSqlite(out,featureClass,fileType,includeGeometry):
[shp,shpType]=getShp(featureClass)
if shpType == "point":
gType = 1
elif shpType == "multipoint":
gType = 4
elif shpType == "polyline":
gType = 5
elif shpType == "polygon":
gType = 6
fields=listFields(featureClass)
fieldNames = []
for field in fields:
if (fields[field] != u'OID') and field.lower() not in ('shape_length','shape_area','shape.len','shape.length','shape_len','shape.area',shp.lower()):
fieldNames.append(parseFieldType(field,fields[field]))
if includeGeometry:
fieldNames.append("GEOMETRY blob")
conn=Connection(out.name)
c=conn.cursor()
c.execute("""CREATE TABLE spatial_ref_sys (
srid INTEGER UNIQUE,
auth_name TEXT,
auth_srid INTEGER,
srtext TEXT )""")
c.execute("insert into spatial_ref_sys(srid ,auth_name ,auth_srid ,srtext) values(?,?,?,?)",(4326, u'EPSG', 4326, u'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]'))
name = splitext(split(out.name)[1])[0]
c.execute("""CREATE TABLE geometry_columns (
f_table_name TEXT,
f_geometry_column TEXT,
geometry_type INTEGER,
coord_dimension INTEGER,
srid INTEGER,
geometry_format TEXT )""")
c.execute("""insert into geometry_columns( f_table_name, f_geometry_column, geometry_type, coord_dimension, srid,geometry_format) values(?,?,?,?,?,?)""",(name,"GEOMETRY",gType,2,4326))
c.execute("create table {0}({1})".format(name,", ".join(fieldNames)))
return [name,c,conn]
def prepareGeoJSON(outJSON,*args): def prepareGeoJSON(outJSON,*args):
out = open(outJSON,"wb") out = open(outJSON,"wb")
out.write("""{"type":"FeatureCollection","features":[""") out.write("""{"type":"FeatureCollection","features":[""")
return [out] return out


def prepareJSON(outJSON,*args): def prepareJSON(outJSON,*args):
out = open(outJSON,"wb") out = open(outJSON,"wb")
out.write("""{"rows":[""") out.write("""{"rows":[""")
return [out] return out


def prepareFile(outJSON,featureClass,fileType,includeGeometry): def prepareFile(outJSON,featureClass,fileType,includeGeometry):
if fileType == "geojson": if fileType == "geojson":
return prepareGeoJSON(outJSON,featureClass,fileType,includeGeometry) return prepareGeoJSON(outJSON,featureClass,fileType,includeGeometry)
elif fileType == "csv": elif fileType == "csv":
return prepareCSV(outJSON,featureClass,fileType,includeGeometry) return prepareCSV(outJSON,featureClass,fileType,includeGeometry)
if fileType == "json": elif fileType == "json":
return prepareJSON(outJSON,featureClass,fileType,includeGeometry) return prepareJSON(outJSON,featureClass,fileType,includeGeometry)
elif fileType == "sqlite":
return prepareSqlite(outJSON,featureClass,fileType,includeGeometry)
else: else:
return False return False
#---- #----
#close file #close file
#---- #----
def closeJSON(out): def closeJSON(out):
out[0].write("""]}""") out.write("""]}""")
out[0].close() out.close()
return True return True


def closeSqlite(out):
out[2].commit()
out[1].close()
return True

def closeCSV(out): def closeCSV(out):
out[1].close() out[1].close()
return True return True
Expand Down Expand Up @@ -96,7 +139,7 @@ def writeFile(outArray,featureClass,fileType,includeGeometry, first=True):
return parser.cleanUp(row) return parser.cleanUp(row)


#this is the main entry point into the module #this is the main entry point into the module
def toOpen(featureClass, outJSON, includeGeometry="true"): def toOpen(featureClass, outJSON, includeGeometry="geojson"):
#check the file type based on the extention #check the file type based on the extention
fileType=getExt(outJSON) fileType=getExt(outJSON)
#some sanity checking #some sanity checking
Expand All @@ -107,8 +150,13 @@ def toOpen(featureClass, outJSON, includeGeometry="true"):
elif not fileType: elif not fileType:
AddMessage("this filetype doesn't make sense") AddMessage("this filetype doesn't make sense")
return return
#to deal with esri giving "true" instead of True #geojson needs geometry
includeGeometry = (includeGeometry=="true") if fileType=="geojson":
includeGeometry="geojson"
elif fileType=="sqlite":
includeGeometry="well know binary"
else:
includeGeometry=includeGeometry.lower()
#open up the file #open up the file
outFile=prepareFile(outJSON,featureClass,fileType,includeGeometry) outFile=prepareFile(outJSON,featureClass,fileType,includeGeometry)
#outFile will be false if the format isn't defined #outFile will be false if the format isn't defined
Expand Down
44 changes: 29 additions & 15 deletions esri2open/parseGeometry.py
@@ -1,12 +1,17 @@
from wkt import getWKTFunc
from wkb import getWKBFunc

def getPoint(pt):
return [pt.X,pt.Y]
def parseLineGeom(line): def parseLineGeom(line):
out=[] out=[]
lineCount=line.count lineCount=line.count
if lineCount ==1: if lineCount ==1:
return ["Point",[line[0].X,line[0].Y]] return ["Point",getPoint(line[0])]
i=0 i=0
while i<lineCount: while i<lineCount:
pt=line[i] pt=line[i]
out.append([pt.X,pt.Y]) out.append(getPoint(pt))
i+=1 i+=1
if len(out)==2 and out[0]==out[1]: if len(out)==2 and out[0]==out[1]:
return ["Point",out[0]] return ["Point",out[0]]
Expand All @@ -19,7 +24,7 @@ def parsePolyGeom(poly):
while i<polyCount: while i<polyCount:
pt=poly[i] pt=poly[i]
if pt: if pt:
out.append([pt.X,pt.Y]) out.append(getPoint(pt))
else: else:
polys.append(out) polys.append(out)
out=[] out=[]
Expand All @@ -33,7 +38,7 @@ def parsePolyGeom(poly):
def parsePoint(geometry): def parsePoint(geometry):
geo=dict() geo=dict()
geo["type"]="Point" geo["type"]="Point"
geo["coordinates"]=[geometry.firstPoint.X,geometry.firstPoint.Y] geo["coordinates"]=getPoint(geometry.firstPoint)
return geo return geo
def parseMultiPoint(geometry): def parseMultiPoint(geometry):
if not geometry.partCount: if not geometry.partCount:
Expand All @@ -48,7 +53,7 @@ def parseMultiPoint(geometry):
i=0 i=0
while i<pointCount: while i<pointCount:
point=geometry.getPart(i) point=geometry.getPart(i)
points.append([point.X,point.Y]) points.append(getPoint(point))
i+=1 i+=1
geo["coordinates"]=points geo["coordinates"]=points
return geo return geo
Expand Down Expand Up @@ -185,14 +190,23 @@ def parseMultiPatch():
return {} return {}


#this should probobly be a class #this should probobly be a class
def getParseFunc(shpType): def getParseFunc(shpType, geo):
if shpType == "point": if geo == "none":
return parsePoint return False
elif shpType == "multipoint": elif geo=="well known binary":
return parseMultiPoint return getWKBFunc(shpType)
elif shpType == "polyline":
return parseMultiLineString
elif shpType == "polygon":
return parseMultiPolygon
else: else:
return parseMultiPatch if shpType == "point":
fun = parsePoint
elif shpType == "multipoint":
fun = parseMultiPoint
elif shpType == "polyline":
fun = parseMultiLineString
elif shpType == "polygon":
fun = parseMultiPolygon
else:
fun = parseMultiPatch
if geo=="geojson":
return fun
elif geo=="well known text":
return getWKTFunc(fun)
36 changes: 26 additions & 10 deletions esri2open/parseRow.py
@@ -1,4 +1,4 @@
from utilities import listFields, getShp, getOID, statusMessage, parseProp from utilities import listFields, getShp, getOID, statusMessage, parseProp, makeInter
from arcpy import SpatialReference, SearchCursor from arcpy import SpatialReference, SearchCursor
from parseGeometry import getParseFunc from parseGeometry import getParseFunc
from json import dump from json import dump
Expand All @@ -7,8 +7,8 @@
wgs84="GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]];-400 -400 1000000000;-100000 10000;-100000 10000;8.98315284119522E-09;0.001;0.001;IsHighPrecision" wgs84="GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]];-400 -400 1000000000;-100000 10000;-100000 10000;8.98315284119522E-09;0.001;0.001;IsHighPrecision"


class parse: class parse:
def __init__(self,outArray,featureClass,fileType,includeGeometry, first=True): def __init__(self,outFile,featureClass,fileType,includeGeometry, first=True):
self.outFile = outArray[0] self.outFile = outFile
self.fileType = fileType self.fileType = fileType
#first we set put the local variables we'll need #first we set put the local variables we'll need
[self.shp,self.shpType]=getShp(featureClass) [self.shp,self.shpType]=getShp(featureClass)
Expand All @@ -23,16 +23,15 @@ def __init__(self,outArray,featureClass,fileType,includeGeometry, first=True):
self.first=first self.first=first
self.status = statusMessage(featureClass) self.status = statusMessage(featureClass)
#define the correct geometry function if we're exporting geometry #define the correct geometry function if we're exporting geometry
if fileType=="geojson" or includeGeometry: self.parseGeo = getParseFunc(self.shpType,includeGeometry)
self.parseGeo = getParseFunc(self.shpType)
else:
self.parseGeo = False
if fileType=="geojson": if fileType=="geojson":
self.parse = self.parseGeoJSON self.parse = self.parseGeoJSON
elif fileType=="csv": elif fileType=="csv":
self.parse = self.parseCSV self.parse = self.parseCSV
elif fileType=="json": elif fileType=="json":
self.parse = self.parseJSON self.parse = self.parseJSON
elif fileType=="sqlite":
self.parse = self.parseSqlite


def cleanUp(self,row): def cleanUp(self,row):
del row del row
Expand All @@ -48,7 +47,7 @@ def parseCSV(self,row):
fc["geometry"]=self.parseGeo(row.getValue(self.shp)) fc["geometry"]=self.parseGeo(row.getValue(self.shp))
except: except:
return return
self.outFile.writerow(fc) self.outFile[0].writerow(fc)


def parseGeoJSON(self,row): def parseGeoJSON(self,row):
#more messages #more messages
Expand All @@ -72,7 +71,7 @@ def parseGeoJSON(self,row):
#if it isn't the first feature, add a comma #if it isn't the first feature, add a comma
self.outFile.write(",") self.outFile.write(",")
dump(fc,self.outFile) dump(fc,self.outFile)

def parseJSON(self,row): def parseJSON(self,row):
#more messages #more messages
self.status.update() self.status.update()
Expand All @@ -87,4 +86,21 @@ def parseJSON(self,row):
dump(fc,self.outFile) dump(fc,self.outFile)
else: else:
self.outFile.write(",") self.outFile.write(",")
dump(fc,self.outFile) dump(fc,self.outFile)

def parseSqlite(self,row):
#more messages
self.status.update()
fc=parseProp(row,self.fields, self.shp)
if self.parseGeo:
try:
fc["GEOMETRY"]=self.parseGeo(row.getValue(self.shp))
except:
return
keys = fc.keys()
values = fc.values()
[name,c,conn]=outFile
c.execute("""insert into {0}({1})
values({2})
""".format(name,", ".join(keys),makeInter(len(values))),values)
conn.commit()
24 changes: 23 additions & 1 deletion esri2open/utilities.py
Expand Up @@ -60,4 +60,26 @@ def getExt(fileName):
if split[1]: if split[1]:
return split[1][1:].lower() return split[1][1:].lower()
else: else:
return False return False

def parseFieldType(name, esriType):
if esriType.lower() in ("text","string","date"):
return name+" text"
elif esriType.lower() in ("short","long"):
return name+" integer"
elif esriType.lower() in ("float","single","double"):
return name+" real"
else:
return name+" blob"

def zm(shp):
desc = Describe(shp)
return [desc.hasZ,desc.hasM]

def makeInter(n):
out = []
i = 0
while i<n:
i+=1
out.append("?")
return ",".join(out)
64 changes: 64 additions & 0 deletions esri2open/wkb.py
@@ -0,0 +1,64 @@
from struct import pack
from sqlite3 import Binary
def pts(c):
return ["dd",[c.X,c.Y]]
def linearRing(coordinates):
partCount=coordinates.partCount
i=0
values =[0]
outnum = "I"
out = ["I",[0]]
while i<partCount:
pt = coordinates.getPart(i)
if pt:
[ptrn,c]=pts(pt)
outnum+=ptrn
values[0]+=1
values.extend(c)
else:
out[0]+=outnum
out[1][0]+=1
out[1].extend(values)
values =[0]
outnum = "I"
i+=1
out[0]+=outnum
out[1][0]+=1
out[1].extend(values)
return out
def makePoint(c):
values = ["<BI",1,1]
[ptrn,coords] = pts(c)
values[0]+=ptrn
values.extend(coords)
return Binary(pack(*values))
def makeMultiPoint(c):
values = ["<BI",1,4]
[ptrn,coords]=linearRing(c)
values[0]+=ptrn
values.extend(coords)
return Binary(pack(*values))
def makeMultiLineString(c):
values = ["<BI",1,5]
[ptrn,coords]=linearRing(c)
values[0]+=ptrn
values.extend(coords)
return Binary(pack(*values))
def makeMultiPolygon(c):
values = ["<BI",1,6]
[ptrn,coords]=linearRing(c)
values[0]+=ptrn
values.extend(coords)
return Binary(pack(*values))



def getWKBFunc(type):
if type == "point":
return makePoint
elif type == "multipoint":
return makeMultiPoint
elif type == "polyline":
return makeMultiLineString
elif type == "polygon":
return makeMultiPolygon

0 comments on commit bdda2c4

Please sign in to comment.