Skip to content
Permalink
Browse files

Merge pull request #3076 from rouault/delimitedtext_qt5_fix

Delimited text provider: fix parsing of subset URL parameter with QT 5 and fix tests
  • Loading branch information
rouault committed May 17, 2016
2 parents 8ee697b + 902953c commit aa66bc6a208368b81f942dc676e663d8f4ad34bf
@@ -1,7 +1,6 @@
ProcessingGdalAlgorithmsTest
ProcessingQgisAlgorithmsTest
PyQgsComposerPicture
PyQgsDelimitedTextProvider
PyQgsGeometryTest
PyQgsJSONUtils
PyQgsLocalServer
@@ -26,6 +26,9 @@
#include <QSettings>
#include <QRegExp>
#include <QUrl>
#if QT_VERSION >= 0x050000
#include <QUrlQuery>
#endif

#include "qgsapplication.h"
#include "qgsdataprovider.h"
@@ -152,7 +155,12 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( const QString& uri )

if ( url.hasQueryItem( "subset" ) )
{
#if QT_VERSION < 0x050000
subset = url.queryItemValue( "subset" );
#else
// We need to specify FullyDecoded so that %25 is decoded as %
subset = QUrlQuery( url ).queryItemValue( "subset" , QUrl::FullyDecoded );
#endif
QgsDebugMsg( "subset is: " + subset );
}

@@ -29,6 +29,7 @@
import re
import tempfile
import inspect
import sys
import time
import test_qgsdelimitedtextprovider_wanted as want # NOQA

@@ -57,6 +58,53 @@
geomkey = "#geometry"
fidkey = "#fid"

try:
# Qt 5
from qgis.PyQt.QtCore import QUrlQuery

class MyUrl:

def __init__(self, url):
self.url = url
self.query = QUrlQuery()

@classmethod
def fromLocalFile(cls, filename):
return cls(QUrl.fromLocalFile(filename))

def addQueryItem(self, k, v):
self.query.addQueryItem(k, v)

def toString(self):
urlstr = self.url.toString()
querystr = self.query.toString(QUrl.FullyDecoded)
if querystr != '':
urlstr += '?'
urlstr += querystr
return urlstr
except:
MyUrl = QUrl


def normalize_query_items_order(s):
splitted_url = s.split('?')
urlstr = splitted_url[0]
if len(splitted_url) == 2:
items_list = splitted_url[1].split('&')
items_map = {}
for item in items_list:
splitted_item = item.split('=')
items_map[splitted_item[0]] = splitted_item[1]
first_arg = True
for k in sorted(items_map.keys()):
if first_arg:
urlstr += '?'
first_arg = False
else:
urlstr += '&'
urlstr += k + '=' + items_map[k]
return urlstr

# Thought we could connect to messageReceived signal but doesn't seem to be available
# in python :-( Not sure why?

@@ -77,7 +125,10 @@ def __exit__(self, type, value, traceback):

def logMessage(self, msg, tag, level):
if tag == self.tag or not self.tag:
self.log.append(unicode(msg))
if sys.version_info.major == 2:
self.log.append(unicode(msg))
else:
self.log.append(str(msg))

def messages(self):
return self.log
@@ -92,7 +143,7 @@ def setUpClass(cls):
srcpath = os.path.join(TEST_DATA_DIR, 'provider')
cls.basetestfile = os.path.join(srcpath, 'delimited_xy.csv')

url = QUrl.fromLocalFile(cls.basetestfile)
url = MyUrl.fromLocalFile(cls.basetestfile)
url.addQueryItem("crs", "epsg:4326")
url.addQueryItem("type", "csv")
url.addQueryItem("xField", "X")
@@ -119,7 +170,7 @@ def setUpClass(cls):
srcpath = os.path.join(TEST_DATA_DIR, 'provider')
cls.basetestfile = os.path.join(srcpath, 'delimited_wkt.csv')

url = QUrl.fromLocalFile(cls.basetestfile)
url = MyUrl.fromLocalFile(cls.basetestfile)
url.addQueryItem("crs", "epsg:4326")
url.addQueryItem("type", "csv")
url.addQueryItem("wktField", "wkt")
@@ -133,7 +184,7 @@ def setUpClass(cls):

cls.basetestpolyfile = os.path.join(srcpath, 'delimited_wkt_poly.csv')

url = QUrl.fromLocalFile(cls.basetestpolyfile)
url = MyUrl.fromLocalFile(cls.basetestpolyfile)
url.addQueryItem("crs", "epsg:4326")
url.addQueryItem("type", "csv")
url.addQueryItem("wktField", "wkt")
@@ -188,7 +239,10 @@ def layerData(self, layer, request={}, offset=0):
for field in f.fields():
fields.append(str(field.name()))
fieldTypes.append(str(field.typeName()))
fielddata = dict((name, unicode(f[name])) for name in fields)
if sys.version_info.major == 2:
fielddata = dict((name, unicode(f[name])) for name in fields)
else:
fielddata = dict((name, str(f[name])) for name in fields)
g = f.constGeometry()
if g:
fielddata[geomkey] = str(g.exportToWkt())
@@ -216,7 +270,7 @@ def delimitedTextData(self, testname, filename, requests, verbose, **params):
# and return the data for the layer (fields, data)

filepath = os.path.join(unitTestDataPath("delimitedtext"), filename)
url = QUrl.fromLocalFile(filepath)
url = MyUrl.fromLocalFile(filepath)
if not requests:
requests = [{}]
for k in params.keys():
@@ -227,7 +281,7 @@ def delimitedTextData(self, testname, filename, requests, verbose, **params):
if verbose:
print(testname)
layer = QgsVectorLayer(urlstr, 'test', 'delimitedtext')
uri = unicode(layer.dataProvider().dataSourceUri())
uri = layer.dataProvider().dataSourceUri()
if verbose:
print(uri)
basename = os.path.basename(filepath)
@@ -318,20 +372,20 @@ def runTest(self, file, requests, **params):
print("Running test:", testname)
result = self.delimitedTextData(testname, file, requests, verbose, **params)
if rebuildTests:
printWanted(testname, result)
self.printWanted(testname, result)
assert False, "Test not run - being rebuilt"
try:
wanted = eval('want.{0}()'.format(testname))
except:
printWanted(testname, result)
self.printWanted(testname, result)
assert False, "Test results not available for {0}".format(testname)

data = result['data']
log = result['log']
failures = []
if result['uri'] != wanted['uri']:
if normalize_query_items_order(result['uri']) != normalize_query_items_order(wanted['uri']):
msg = "Layer Uri ({0}) doesn't match expected ({1})".format(
result['uri'], wanted['uri'])
normalize_query_items_order(result['uri']), normalize_query_items_order(wanted['uri']))
print(' ' + msg)
failures.append(msg)
if result['fieldTypes'] != wanted['fieldTypes']:
@@ -381,7 +435,7 @@ def runTest(self, file, requests, **params):
print(' Message log correct: Passed')

if failures:
printWanted(testname, result)
self.printWanted(testname, result)

assert len(failures) == 0, "\n".join(failures)

@@ -604,14 +658,14 @@ def test_029_file_watcher(self):
f.write("id,name\n1,rabbit\n2,pooh\n")

def appendfile(layer):
with file(filename, 'a') as f:
with open(filename, 'a') as f:
f.write('3,tigger\n')
# print "Appended to file - sleeping"
time.sleep(1)
QCoreApplication.instance().processEvents()

def rewritefile(layer):
with file(filename, 'w') as f:
with open(filename, 'w') as f:
f.write("name,size,id\ntoad,small,5\nmole,medium,6\nbadger,big,7\n")
# print "Rewritten file - sleeping"
time.sleep(1)

0 comments on commit aa66bc6

Please sign in to comment.
You can’t perform that action at this time.