Skip to content
Permalink
Browse files

[sextante] fixes in gdal/ogr algorithms and more tests

  • Loading branch information
volaya committed Apr 12, 2013
1 parent e8650aa commit 1a4ce3ecd5cdbf4cf88b514fdece152155ae46f9
@@ -47,10 +47,7 @@ def ogrConnectionString(self, uri):
layer = QGisLayers.getObjectFromUri(uri, False)
if layer == None:
return uri;
provider = layer.dataProvider().name()
qDebug("inputLayer provider '%s'" % provider)
#qDebug("inputLayer layer '%s'" % layer.providerType())
qDebug("inputLayer.source '%s'" % layer.source())
provider = layer.dataProvider().name()
if provider == 'spatialite':
#dbname='/geodata/osm_ch.sqlite' table="places" (Geometry) sql=
regex = re.compile("dbname='(.+)'")
@@ -23,37 +23,30 @@
# This will get replaced with a git SHA1 when you do a git archive
__revision__ = '$Format:%H$'

import os
import re
import string
from string import Template
import tempfile

from PyQt4.QtCore import *
from PyQt4.QtGui import *

from qgis.core import *

from sextante.core.SextanteLog import SextanteLog
from sextante.core.QGisLayers import QGisLayers

from sextante.parameters.ParameterVector import ParameterVector
from sextante.parameters.ParameterString import ParameterString
from sextante.outputs.OutputVector import OutputVector

from sextante.core.GeoAlgorithmExecutionException import GeoAlgorithmExecutionException
from sextante.parameters.ParameterSelection import ParameterSelection
from sextante.gdal.OgrAlgorithm import OgrAlgorithm
from sextante.gdal.pyogr.ogr2ogr import *

from sextante.gdal.GdalUtils import GdalUtils

try:
from osgeo import gdal, ogr, osr
gdalAvailable = True
from osgeo import gdal, ogr, osr
gdalAvailable = True
except:
gdalAvailable = False
gdalAvailable = False

GeomOperation = Enum(["NONE", "SEGMENTIZE", "SIMPLIFY_PRESERVE_TOPOLOGY"])

FORMATS = ['ESRI Shapefile','GeoJSON',' GeoRSS','SQLite','Generic Mapping Tools','Mapinfo TAB','ESRI Shapefile','INTERLIS 1',
'Geography Markup Language','Geoconcept','AutoCAD DXF','INTERLIS 2','','Microstation DGN',
'Comma Separated Value','Atlas BNAGPS eXchange Format','S-57 Base file','Keyhole Markup Language']
EXTS = ["shp",'geojson','.xml','.sqlite','.gmt','.tab','.shp','.ili','.gml','.txt','.dxf','.ili','.dgn','.csv','.bna','.gpx','.000','.kml']

class Ogr2Ogr(OgrAlgorithm):

@@ -69,63 +62,48 @@ def defineCharacteristics(self):

#we add the input vector layer. It can have any kind of geometry
#It is a mandatory (not optional) one, hence the False argument
self.addParameter(ParameterVector(self.INPUT_LAYER, "Input layer", ParameterVector.VECTOR_TYPE_ANY, False))
#self.addParameter(ParameterString(self.DEST_DS, "Output DS", "/tmp/out.sqlite"))
self.addParameter(ParameterString(self.DEST_FORMAT, "Destination Format", "ESRI Shapefile")) #SQLite
self.addParameter(ParameterString(self.DEST_DSCO, "Creation Options", "")) #SPATIALITE=YES
self.addParameter(ParameterVector(self.INPUT_LAYER, "Input layer", ParameterVector.VECTOR_TYPE_ANY, False))
self.addParameter(ParameterSelection(self.DEST_FORMAT, "Destination Format", FORMATS))
self.addParameter(ParameterString(self.DEST_DSCO, "Creation Options", ""))

self.addOutput(OutputVector(self.OUTPUT_LAYER, "Output layer"))

def processAlgorithm(self, progress):
'''Here is where the processing itself takes place'''

if not gdalAvailable:
SextanteLog.addToLog(SextanteLog.LOG_ERROR, "GDAL bindings not installed." )
return
raise GeoAlgorithmExecutionException("GDAL bindings not installed.")

input = self.getParameterValue(self.INPUT_LAYER)
ogrLayer = self.ogrConnectionString(input)
output = self.getOutputValue(self.OUTPUT_LAYER)

#dst_ds = self.getParameterValue(self.DEST_DS)
dst_ds = self.ogrConnectionString(output)
dst_format = self.getParameterValue(self.DEST_FORMAT)
ogr_dsco = [self.getParameterValue(self.DEST_DSCO)] #TODO: split
#dst_ds = "PG:dbname='glarus_np' options='-c client_encoding=LATIN9'"
#dst_format ="PostgreSQL"

qDebug("Opening data source '%s'" % ogrLayer)

output = self.getOutputFromName(self.OUTPUT_LAYER)
outfile = output.value

formatIdx = self.getParameterValue(self.DEST_FORMAT)

ext = EXTS[formatIdx]
if not outfile.endswith(ext):
outfile = outfile + ext;
output.value = outfile

dst_ds = self.ogrConnectionString(outfile)
dst_format = FORMATS[formatIdx]
ogr_dsco = [self.getParameterValue(self.DEST_DSCO)]

poDS = ogr.Open( ogrLayer, False )
if poDS is None:
SextanteLog.addToLog(SextanteLog.LOG_ERROR, self.failure(ogrLayer))
return

srs = osr.SpatialReference()
srs.ImportFromEPSG( 21781 ) #FIXME
qDebug("Creating output '%s'" % dst_ds)
raise GeoAlgorithmExecutionException(self.failure(ogrLayer))

if dst_format == "SQLite" and os.path.isfile(dst_ds):
os.remove(dst_ds)
qDebug("Using driver '%s'" % dst_format)
driver = ogr.GetDriverByName(dst_format)
os.remove(dst_ds)
driver = ogr.GetDriverByName(str(dst_format))
poDstDS = driver.CreateDataSource(dst_ds, options = ogr_dsco)
if poDstDS is None:
SextanteLog.addToLog(SextanteLog.LOG_ERROR, "Error creating %s" % dst_ds)
raise GeoAlgorithmExecutionException("Error creating %s" % dst_ds)
return
self.ogrtransform(poDS, poDstDS, bOverwrite = True)
#ogr2ogr(pszFormat = dst_format, pszDataSource = poDS, pszDestDataSource = poDstDS, bOverwrite = True)

def transformed_template(self, template, substitutions):
vrt_templ = Template(open(template).read())
vrt_xml = vrt_templ.substitute(substitutions)
vrt = tempfile.mktemp( '.vrt', 'ogr_', '/vsimem')
# Create in-memory file
gdal.FileFromMemBuffer(vrt, vrt_xml)
return vrt

def transformed_datasource(self, template, substitutions):
vrt = transformed_template(template, substitutions)
ds = ogr.Open(vrt)
return ds
self.ogrtransform(poDS, poDstDS, bOverwrite = True)


def ogrtransform(self,
poSrcDS,
@@ -163,8 +141,7 @@ def ogrtransform(self,
poLayer = poSrcDS.GetLayer(iLayer)

if poLayer is None:
SextanteLog.addToLog(SextanteLog.LOG_ERROR, "FAILURE: Couldn't fetch advertised layer %d!" % iLayer)
return False
raise GeoAlgorithmExecutionException( "FAILURE: Couldn't fetch advertised layer %d!" % iLayer)

papoLayers[iLayer] = poLayer
iLayer = iLayer + 1
@@ -179,16 +156,12 @@ def ogrtransform(self,
poLayer = poSrcDS.GetLayerByName(layername)

if poLayer is None:
SextanteLog.addToLog(SextanteLog.LOG_ERROR, "FAILURE: Couldn't fetch advertised layer %s!" % layername)
return False
raise GeoAlgorithmExecutionException("FAILURE: Couldn't fetch advertised layer %s!" % layername)

papoLayers[iLayer] = poLayer
iLayer = iLayer + 1

for poSrcLayer in papoLayers:
qDebug(poSrcLayer.GetLayerDefn().GetName())
#TODO: poDstDS.GetLayerByName for VRT layer fails if name is not lower case

for poSrcLayer in papoLayers:
ok = TranslateLayer( poSrcDS, poSrcLayer, poDstDS, papszLCO, pszNewLayerName, \
bTransform, poOutputSRS, poSourceSRS, papszSelFields, \
bAppend, eGType, bOverwrite, eGeomOp, dfGeomOpParam, \
@@ -16,7 +16,6 @@
* *
***************************************************************************
"""
from sextante.parameters.ParameterString import ParameterString

__author__ = 'Victor Olaya'
__date__ = 'August 2012'
@@ -25,11 +24,9 @@
__revision__ = '$Format:%H$'

import os

from PyQt4 import QtGui

from sextante.core.GeoAlgorithm import GeoAlgorithm

from sextante.parameters.ParameterString import ParameterString
from sextante.parameters.ParameterRaster import ParameterRaster
from sextante.parameters.ParameterNumber import ParameterNumber
from sextante.parameters.ParameterBoolean import ParameterBoolean
@@ -107,7 +104,7 @@ def processAlgorithm(self, progress):
if srs is not None:
arguments.append("-a_srs")
arguments.append(str(srs))
self.crs = srs
self.crs = QgsCoordinateReferenceSystem(crsId)
if sds:
arguments.append("-sds")
if len(extra) > 0:
@@ -25,18 +25,14 @@

import os
from PyQt4 import QtGui

from qgis.core import *

from sextante.core.GeoAlgorithm import GeoAlgorithm

from sextante.parameters.ParameterRaster import ParameterRaster
from sextante.parameters.ParameterSelection import ParameterSelection
from sextante.parameters.ParameterCrs import ParameterCrs
from sextante.parameters.ParameterNumber import ParameterNumber
from sextante.parameters.ParameterString import ParameterString
from sextante.outputs.OutputRaster import OutputRaster

from sextante.gdal.GdalUtils import GdalUtils

class warp(GeoAlgorithm):
@@ -70,7 +66,9 @@ def processAlgorithm(self, progress):
arguments.append("-s_srs")
arguments.append(str(self.getParameterValue(warp.SOURCE_SRS)))
arguments.append("-t_srs")
arguments.append(str(self.getParameterValue(warp.DEST_SRS)))
crsId = self.getParameterValue(warp.DEST_SRS)
self.crs = QgsCoordinateReferenceSystem(crsId)
arguments.append(str(crsId))
arguments.append("-r")
arguments.append(warp.METHOD_OPTIONS[self.getParameterValue(warp.METHOD)])
arguments.append("-of")
@@ -24,7 +24,65 @@ def test_gdalogrsieveWithUnsupportedOutputFormat(self):
self.assertTrue(os.path.isfile(output))
dataset=gdal.Open(output, GA_ReadOnly)
strhash=hash(str(dataset.ReadAsArray(0).tolist()))
self.assertEqual(strhash,-1353696889)
self.assertEqual(strhash,-1353696889)

def test_gdalogrwarpreproject(self):
outputs=sextante.runalg("gdalogr:warpreproject",raster(),"EPSG:23030","EPSG:4326",0,0,"",None)
output=outputs['OUTPUT']
self.assertTrue(os.path.isfile(output))
dataset=gdal.Open(output, GA_ReadOnly)
strhash=hash(str(dataset.ReadAsArray(0).tolist()))
self.assertEqual(strhash,-2021328784)

def test_gdalogrmerge(self):
outputs=sextante.runalg("gdalogr:merge",raster(),False,False,None)
output=outputs['OUTPUT']
self.assertTrue(os.path.isfile(output))
dataset=gdal.Open(output, GA_ReadOnly)
strhash=hash(str(dataset.ReadAsArray(0).tolist()))
self.assertEqual(strhash,-1353696889)

def test_gdalogrogr2ogr(self):
outputs=sextante.runalg("gdalogr:ogr2ogr",union(),3,"",None)
output=outputs['OUTPUT_LAYER']
layer=QGisLayers.getObjectFromUri(output, True)
fields=layer.pendingFields()
expectednames=['id','poly_num_a','poly_st_a','id_2','poly_num_b','poly_st_b']
expectedtypes=['Integer','Real','String','Integer','Real','String']
names=[str(f.name()) for f in fields]
types=[str(f.typeName()) for f in fields]
self.assertEqual(expectednames, names)
self.assertEqual(expectedtypes, types)
features=sextante.getfeatures(layer)
self.assertEqual(8, len(features))
feature=features.next()
attrs=feature.attributes()
expectedvalues=["1","1.1","string a","2","1","string a"]
values=[str(attr.toString()) for attr in attrs]
self.assertEqual(expectedvalues, values)
wkt='POLYGON((270807.08580285 4458940.1594565,270798.42294527 4458914.62661676,270780.81854858 4458914.21983449,270763.52289518 4458920.715993,270760.3449542 4458926.6570575,270763.78234766 4458958.22561242,270794.30290024 4458942.16424502,270807.08580285 4458940.1594565))'
self.assertEqual(wkt, str(feature.geometry().exportToWkt()))

def test_gdalogrogr2ogrWrongExtension(self):
outputs=sextante.runalg("gdalogr:ogr2ogr",union(),3,"",SextanteUtils.getTempFilename("wrongext"))
output=outputs['OUTPUT_LAYER']
layer=QGisLayers.getObjectFromUri(output, True)
fields=layer.pendingFields()
expectednames=['id','poly_num_a','poly_st_a','id_2','poly_num_b','poly_st_b']
expectedtypes=['Integer','Real','String','Integer','Real','String']
names=[str(f.name()) for f in fields]
types=[str(f.typeName()) for f in fields]
self.assertEqual(expectednames, names)
self.assertEqual(expectedtypes, types)
features=sextante.getfeatures(layer)
self.assertEqual(8, len(features))
feature=features.next()
attrs=feature.attributes()
expectedvalues=["1","1.1","string a","2","1","string a"]
values=[str(attr.toString()) for attr in attrs]
self.assertEqual(expectedvalues, values)
wkt='POLYGON((270807.08580285 4458940.1594565,270798.42294527 4458914.62661676,270780.81854858 4458914.21983449,270763.52289518 4458920.715993,270760.3449542 4458926.6570575,270763.78234766 4458958.22561242,270794.30290024 4458942.16424502,270807.08580285 4458940.1594565))'
self.assertEqual(wkt, str(feature.geometry().exportToWkt()))

def suite():
suite = unittest.makeSuite(GdalTest, 'test')
@@ -3,9 +3,30 @@
from sextante.tests.TestData import points, points2, polygons, polygons2, lines, union,\
table, polygonsGeoJson, raster
from sextante.core.QGisLayers import QGisLayers
from sextante.core.SextanteUtils import SextanteUtils

class GeoAlgorithmTest(unittest.TestCase):
pass
class GeoAlgorithmTest(unittest.TestCase):

def testWrongformat(self):
outputs=sextante.runalg("qgis:countpointsinpolygon",polygons(),points(),"NUMPOINTS",SextanteUtils.getTempFilename("wrongext"))
output=outputs['OUTPUT']
self.assertTrue(output.endswith('shp'))
layer=QGisLayers.getObjectFromUri(output, True)
fields=layer.pendingFields()
expectednames=['ID','POLY_NUM_A','POLY_ST_A','NUMPOINTS']
expectedtypes=['Integer','Real','String','Real']
names=[str(f.name()) for f in fields]
types=[str(f.typeName()) for f in fields]
self.assertEqual(expectednames, names)
self.assertEqual(expectedtypes, types)
features=sextante.getfeatures(layer)
self.assertEqual(2, len(features))
feature=features.next()
attrs=feature.attributes()
expectedvalues=["1","1.1","string a","6"]
values=[str(attr.toString()) for attr in attrs]
self.assertEqual(expectedvalues, values)


def suite():
suite = unittest.makeSuite(GeoAlgorithmTest, 'test')
@@ -767,24 +767,24 @@ def test_qgisaddautoincrementalfield(self):
self.assertEqual(wkt, str(feature.geometry().exportToWkt()))

def test_qgisaddfieldtoattributestable(self):
outputs=sextante.runalg("qgis:addfieldtoattributestable",points(),"newfield",1,None)
outputs=sextante.runalg("qgis:addfieldtoattributestable",lines(),"field",0,10,0,None)
output=outputs['OUTPUT_LAYER']
layer=QGisLayers.getObjectFromUri(output, True)
fields=layer.pendingFields()
expectednames=['ID','PT_NUM_A','PT_ST_A','newfield']
expectedtypes=['Integer','Real','String','Real']
expectednames=['ID','LINE_NUM_A','LINE_ST_A','field']
expectedtypes=['Integer','Real','String','Integer']
names=[str(f.name()) for f in fields]
types=[str(f.typeName()) for f in fields]
self.assertEqual(expectednames, names)
self.assertEqual(expectedtypes, types)
features=sextante.getfeatures(layer)
self.assertEqual(12, len(features))
self.assertEqual(3, len(features))
feature=features.next()
attrs=feature.attributes()
expectedvalues=["1","1.1","a",""]
expectedvalues=["1","11.1","string a",""]
values=[str(attr.toString()) for attr in attrs]
self.assertEqual(expectedvalues, values)
wkt='POINT(270839.65586926 4458983.16267036)'
wkt='LINESTRING(270818.44773413 4458997.23886624, 270833.27466046 4458983.16267036, 270830.83478651 4458975.28000067, 270822.38906898 4458967.20964836, 270823.32748204 4458959.70234389, 270822.7644342 4458958.01320039)'
self.assertEqual(wkt, str(feature.geometry().exportToWkt()))

def test_qgiscreateequivalentnumericalfield(self):

0 comments on commit 1a4ce3e

Please sign in to comment.
You can’t perform that action at this time.