Permalink
Browse files

sqlite added

  • Loading branch information...
Calvin Metcalf
Calvin Metcalf committed Apr 10, 2013
1 parent dbb26a0 commit bdda2c40f749efbc3cd2d0932fedecaa03b8ea32
Showing with 268 additions and 41 deletions.
  1. BIN esri2open.tbx
  2. +61 −13 esri2open/esri2open.py
  3. +29 −15 esri2open/parseGeometry.py
  4. +26 −10 esri2open/parseRow.py
  5. +23 −1 esri2open/utilities.py
  6. +64 −0 esri2open/wkb.py
  7. +63 −0 esri2open/wkt.py
  8. +1 −1 merge.py
  9. +1 −1 multiple.py
View
Binary file not shown.
View
@@ -13,11 +13,12 @@
# ---------------------------------------------------------------------------
#imports
from arcpy import AddMessage, GetCount_management
-from utilities import getExt,listFields,getShp, parseProp
+from utilities import getExt, listFields,getShp, parseProp, parseFieldType
from parseRow import parse
from csv import DictWriter
from json import dump
-
+from sqlite3 import Connection
+from os.path import splitext, split
#----
#prepare files
@@ -30,42 +31,84 @@ def prepareCSV(outJSON,featureClass,fileType,includeGeometry):
for field in fields:
if (fields[field] != u'OID') and field.lower() not in ('shape_length','shape_area','shape.len','shape.length','shape_len','shape.area',shp.lower()):
fieldNames.append(field)
- if includeGeometry:
+ if includeGeometry!="none":
fieldNames.append("geometry")
outCSV=DictWriter(out,fieldNames,extrasaction='ignore')
fieldObject = {}
for fieldName in fieldNames:
fieldObject[fieldName]=fieldName
outCSV.writerow(fieldObject)
return [outCSV,out]
-
+def prepareSqlite(out,featureClass,fileType,includeGeometry):
+ [shp,shpType]=getShp(featureClass)
+ if shpType == "point":
+ gType = 1
+ elif shpType == "multipoint":
+ gType = 4
+ elif shpType == "polyline":
+ gType = 5
+ elif shpType == "polygon":
+ gType = 6
+ fields=listFields(featureClass)
+ fieldNames = []
+ for field in fields:
+ if (fields[field] != u'OID') and field.lower() not in ('shape_length','shape_area','shape.len','shape.length','shape_len','shape.area',shp.lower()):
+ fieldNames.append(parseFieldType(field,fields[field]))
+ if includeGeometry:
+ fieldNames.append("GEOMETRY blob")
+ conn=Connection(out.name)
+ c=conn.cursor()
+ c.execute("""CREATE TABLE spatial_ref_sys (
+ srid INTEGER UNIQUE,
+ auth_name TEXT,
+ auth_srid INTEGER,
+ srtext TEXT )""")
+ c.execute("insert into spatial_ref_sys(srid ,auth_name ,auth_srid ,srtext) values(?,?,?,?)",(4326, u'EPSG', 4326, u'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]'))
+ name = splitext(split(out.name)[1])[0]
+ c.execute("""CREATE TABLE geometry_columns (
+ f_table_name TEXT,
+ f_geometry_column TEXT,
+ geometry_type INTEGER,
+ coord_dimension INTEGER,
+ srid INTEGER,
+ geometry_format TEXT )""")
+ c.execute("""insert into geometry_columns( f_table_name, f_geometry_column, geometry_type, coord_dimension, srid,geometry_format) values(?,?,?,?,?,?)""",(name,"GEOMETRY",gType,2,4326))
+ c.execute("create table {0}({1})".format(name,", ".join(fieldNames)))
+ return [name,c,conn]
def prepareGeoJSON(outJSON,*args):
out = open(outJSON,"wb")
out.write("""{"type":"FeatureCollection","features":[""")
- return [out]
+ return out
def prepareJSON(outJSON,*args):
out = open(outJSON,"wb")
out.write("""{"rows":[""")
- return [out]
+ return out
def prepareFile(outJSON,featureClass,fileType,includeGeometry):
if fileType == "geojson":
return prepareGeoJSON(outJSON,featureClass,fileType,includeGeometry)
elif fileType == "csv":
return prepareCSV(outJSON,featureClass,fileType,includeGeometry)
- if fileType == "json":
+ elif fileType == "json":
return prepareJSON(outJSON,featureClass,fileType,includeGeometry)
+ elif fileType == "sqlite":
+ return prepareSqlite(outJSON,featureClass,fileType,includeGeometry)
else:
return False
#----
#close file
#----
def closeJSON(out):
- out[0].write("""]}""")
- out[0].close()
+ out.write("""]}""")
+ out.close()
return True
-
+
+def closeSqlite(out):
+ out[2].commit()
+ out[1].close()
+ return True
+
def closeCSV(out):
out[1].close()
return True
@@ -96,7 +139,7 @@ def writeFile(outArray,featureClass,fileType,includeGeometry, first=True):
return parser.cleanUp(row)
#this is the main entry point into the module
-def toOpen(featureClass, outJSON, includeGeometry="true"):
+def toOpen(featureClass, outJSON, includeGeometry="geojson"):
#check the file type based on the extention
fileType=getExt(outJSON)
#some sanity checking
@@ -107,8 +150,13 @@ def toOpen(featureClass, outJSON, includeGeometry="true"):
elif not fileType:
AddMessage("this filetype doesn't make sense")
return
- #to deal with esri giving "true" instead of True
- includeGeometry = (includeGeometry=="true")
+ #geojson needs geometry
+ if fileType=="geojson":
+ includeGeometry="geojson"
+ elif fileType=="sqlite":
+ includeGeometry="well know binary"
+ else:
+ includeGeometry=includeGeometry.lower()
#open up the file
outFile=prepareFile(outJSON,featureClass,fileType,includeGeometry)
#outFile will be false if the format isn't defined
View
@@ -1,12 +1,17 @@
+from wkt import getWKTFunc
+from wkb import getWKBFunc
+
+def getPoint(pt):
+ return [pt.X,pt.Y]
def parseLineGeom(line):
out=[]
lineCount=line.count
if lineCount ==1:
- return ["Point",[line[0].X,line[0].Y]]
+ return ["Point",getPoint(line[0])]
i=0
while i<lineCount:
pt=line[i]
- out.append([pt.X,pt.Y])
+ out.append(getPoint(pt))
i+=1
if len(out)==2 and out[0]==out[1]:
return ["Point",out[0]]
@@ -19,7 +24,7 @@ def parsePolyGeom(poly):
while i<polyCount:
pt=poly[i]
if pt:
- out.append([pt.X,pt.Y])
+ out.append(getPoint(pt))
else:
polys.append(out)
out=[]
@@ -33,7 +38,7 @@ def parsePolyGeom(poly):
def parsePoint(geometry):
geo=dict()
geo["type"]="Point"
- geo["coordinates"]=[geometry.firstPoint.X,geometry.firstPoint.Y]
+ geo["coordinates"]=getPoint(geometry.firstPoint)
return geo
def parseMultiPoint(geometry):
if not geometry.partCount:
@@ -48,7 +53,7 @@ def parseMultiPoint(geometry):
i=0
while i<pointCount:
point=geometry.getPart(i)
- points.append([point.X,point.Y])
+ points.append(getPoint(point))
i+=1
geo["coordinates"]=points
return geo
@@ -185,14 +190,23 @@ def parseMultiPatch():
return {}
#this should probobly be a class
-def getParseFunc(shpType):
- if shpType == "point":
- return parsePoint
- elif shpType == "multipoint":
- return parseMultiPoint
- elif shpType == "polyline":
- return parseMultiLineString
- elif shpType == "polygon":
- return parseMultiPolygon
+def getParseFunc(shpType, geo):
+ if geo == "none":
+ return False
+ elif geo=="well known binary":
+ return getWKBFunc(shpType)
else:
- return parseMultiPatch
+ if shpType == "point":
+ fun = parsePoint
+ elif shpType == "multipoint":
+ fun = parseMultiPoint
+ elif shpType == "polyline":
+ fun = parseMultiLineString
+ elif shpType == "polygon":
+ fun = parseMultiPolygon
+ else:
+ fun = parseMultiPatch
+ if geo=="geojson":
+ return fun
+ elif geo=="well known text":
+ return getWKTFunc(fun)
View
@@ -1,4 +1,4 @@
-from utilities import listFields, getShp, getOID, statusMessage, parseProp
+from utilities import listFields, getShp, getOID, statusMessage, parseProp, makeInter
from arcpy import SpatialReference, SearchCursor
from parseGeometry import getParseFunc
from json import dump
@@ -7,8 +7,8 @@
wgs84="GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]];-400 -400 1000000000;-100000 10000;-100000 10000;8.98315284119522E-09;0.001;0.001;IsHighPrecision"
class parse:
- def __init__(self,outArray,featureClass,fileType,includeGeometry, first=True):
- self.outFile = outArray[0]
+ def __init__(self,outFile,featureClass,fileType,includeGeometry, first=True):
+ self.outFile = outFile
self.fileType = fileType
#first we set put the local variables we'll need
[self.shp,self.shpType]=getShp(featureClass)
@@ -23,16 +23,15 @@ def __init__(self,outArray,featureClass,fileType,includeGeometry, first=True):
self.first=first
self.status = statusMessage(featureClass)
#define the correct geometry function if we're exporting geometry
- if fileType=="geojson" or includeGeometry:
- self.parseGeo = getParseFunc(self.shpType)
- else:
- self.parseGeo = False
+ self.parseGeo = getParseFunc(self.shpType,includeGeometry)
if fileType=="geojson":
self.parse = self.parseGeoJSON
elif fileType=="csv":
self.parse = self.parseCSV
elif fileType=="json":
self.parse = self.parseJSON
+ elif fileType=="sqlite":
+ self.parse = self.parseSqlite
def cleanUp(self,row):
del row
@@ -48,7 +47,7 @@ def parseCSV(self,row):
fc["geometry"]=self.parseGeo(row.getValue(self.shp))
except:
return
- self.outFile.writerow(fc)
+ self.outFile[0].writerow(fc)
def parseGeoJSON(self,row):
#more messages
@@ -72,7 +71,7 @@ def parseGeoJSON(self,row):
#if it isn't the first feature, add a comma
self.outFile.write(",")
dump(fc,self.outFile)
-
+
def parseJSON(self,row):
#more messages
self.status.update()
@@ -87,4 +86,21 @@ def parseJSON(self,row):
dump(fc,self.outFile)
else:
self.outFile.write(",")
- dump(fc,self.outFile)
+ dump(fc,self.outFile)
+
+ def parseSqlite(self,row):
+ #more messages
+ self.status.update()
+ fc=parseProp(row,self.fields, self.shp)
+ if self.parseGeo:
+ try:
+ fc["GEOMETRY"]=self.parseGeo(row.getValue(self.shp))
+ except:
+ return
+ keys = fc.keys()
+ values = fc.values()
+ [name,c,conn]=outFile
+ c.execute("""insert into {0}({1})
+ values({2})
+ """.format(name,", ".join(keys),makeInter(len(values))),values)
+ conn.commit()
View
@@ -60,4 +60,26 @@ def getExt(fileName):
if split[1]:
return split[1][1:].lower()
else:
- return False
+ return False
+
+def parseFieldType(name, esriType):
+ if esriType.lower() in ("text","string","date"):
+ return name+" text"
+ elif esriType.lower() in ("short","long"):
+ return name+" integer"
+ elif esriType.lower() in ("float","single","double"):
+ return name+" real"
+ else:
+ return name+" blob"
+
+def zm(shp):
+ desc = Describe(shp)
+ return [desc.hasZ,desc.hasM]
+
+def makeInter(n):
+ out = []
+ i = 0
+ while i<n:
+ i+=1
+ out.append("?")
+ return ",".join(out)
View
@@ -0,0 +1,64 @@
+from struct import pack
+from sqlite3 import Binary
+def pts(c):
+ return ["dd",[c.X,c.Y]]
+def linearRing(coordinates):
+ partCount=coordinates.partCount
+ i=0
+ values =[0]
+ outnum = "I"
+ out = ["I",[0]]
+ while i<partCount:
+ pt = coordinates.getPart(i)
+ if pt:
+ [ptrn,c]=pts(pt)
+ outnum+=ptrn
+ values[0]+=1
+ values.extend(c)
+ else:
+ out[0]+=outnum
+ out[1][0]+=1
+ out[1].extend(values)
+ values =[0]
+ outnum = "I"
+ i+=1
+ out[0]+=outnum
+ out[1][0]+=1
+ out[1].extend(values)
+ return out
+def makePoint(c):
+ values = ["<BI",1,1]
+ [ptrn,coords] = pts(c)
+ values[0]+=ptrn
+ values.extend(coords)
+ return Binary(pack(*values))
+def makeMultiPoint(c):
+ values = ["<BI",1,4]
+ [ptrn,coords]=linearRing(c)
+ values[0]+=ptrn
+ values.extend(coords)
+ return Binary(pack(*values))
+def makeMultiLineString(c):
+ values = ["<BI",1,5]
+ [ptrn,coords]=linearRing(c)
+ values[0]+=ptrn
+ values.extend(coords)
+ return Binary(pack(*values))
+def makeMultiPolygon(c):
+ values = ["<BI",1,6]
+ [ptrn,coords]=linearRing(c)
+ values[0]+=ptrn
+ values.extend(coords)
+ return Binary(pack(*values))
+
+
+
+def getWKBFunc(type):
+ if type == "point":
+ return makePoint
+ elif type == "multipoint":
+ return makeMultiPoint
+ elif type == "polyline":
+ return makeMultiLineString
+ elif type == "polygon":
+ return makeMultiPolygon
Oops, something went wrong.

0 comments on commit bdda2c4

Please sign in to comment.