Skip to content
Permalink
Browse files

Merge pull request #37164 from suricactus/virt_csv

FIX virtual layers with CSV layers opened with "delimitedtext" provider
  • Loading branch information
m-kuhn committed Jun 22, 2020
2 parents e4718d7 + 0b16654 commit 28e50c12db2beca51d39878293450201b74e647f
@@ -61,6 +61,15 @@ QRegExp QgsDelimitedTextProvider::sCrdDmsRegexp( "^\\s*(?:([-+nsew])\\s*)?(\\d{1
QgsDelimitedTextProvider::QgsDelimitedTextProvider( const QString &uri, const ProviderOptions &options )
: QgsVectorDataProvider( uri, options )
{
// uri should be in the form of "file:///path/to/file.csv?query=params", if not, enforce it in that format
// first read the already encoded url to get the query string
QUrl url = QUrl::fromEncoded( uri.toLatin1() );
// temporarily store the query string
const QString tmpUrlQuery = url.query();
// make sure that the url is actually prefixed with "file://". However, this breaks the query part ("?" char gets encoded), so discard the query string
url = QUrl::fromLocalFile( url.path() );
// finally restore the query part
url.setQuery( tmpUrlQuery );

// Add supported types to enable creating expression fields in field calculator
setNativeTypes( QList< NativeType >()
@@ -77,7 +86,6 @@ QgsDelimitedTextProvider::QgsDelimitedTextProvider( const QString &uri, const Pr

QgsDebugMsgLevel( "Delimited text file uri is " + uri, 2 );

const QUrl url = QUrl::fromEncoded( uri.toLatin1() );
mFile = qgis::make_unique< QgsDelimitedTextFile >();
mFile->setFromUrl( url );

@@ -317,7 +317,10 @@ bool QgsVirtualLayerProvider::createIt()
provider.replace( QLatin1String( "'" ), QLatin1String( "''" ) );
QString source = mLayers.at( i ).source;
source.replace( QLatin1String( "'" ), QLatin1String( "''" ) );
QString encoding = mLayers.at( i ).encoding;
// the encoding might be an empty string, which breaks the SQL query below
QString encoding = mLayers.at( i ).encoding.isEmpty()
? QStringLiteral( "System" )
: mLayers.at( i ).encoding;
QString createStr = QStringLiteral( "DROP TABLE IF EXISTS \"%1\"; CREATE VIRTUAL TABLE \"%1\" USING QgsVLayer('%2','%4',%3)" )
.arg( vname,
provider,
@@ -555,11 +558,17 @@ QgsRectangle QgsVirtualLayerProvider::extent() const
void QgsVirtualLayerProvider::updateStatistics() const
{
bool hasGeometry = mDefinition.geometryWkbType() != QgsWkbTypes::NoGeometry;
QString subset = mSubset.isEmpty() ? QString() : " WHERE " + mSubset;
QString subset = mSubset.isEmpty() ? QString() : QStringLiteral( " WHERE %1" ).arg( mSubset );

// `mTableName` might be a null string if the layer creation failed. Assert such situations at least during development.
Q_ASSERT( ! mTableName.isNull() );

QString sql = QStringLiteral( "SELECT Count(*)%1 FROM %2%3" )
.arg( hasGeometry ? QStringLiteral( ",Min(MbrMinX(%1)),Min(MbrMinY(%1)),Max(MbrMaxX(%1)),Max(MbrMaxY(%1))" ).arg( quotedColumn( mDefinition.geometryField() ) ) : QString(),
mTableName,
subset );
.arg(
hasGeometry ? QStringLiteral( ",Min(MbrMinX(%1)),Min(MbrMinY(%1)),Max(MbrMaxX(%1)),Max(MbrMaxY(%1))" ).arg( quotedColumn( mDefinition.geometryField() ) ) : QString(),
mTableName,
subset
);
Sqlite::Query q( mSqlite.get(), sql );
if ( q.step() == SQLITE_ROW )
{
@@ -63,7 +63,7 @@
class MyUrl:

def __init__(self, url):
self.url = url
self.url = QUrl(url)
self.query = QUrlQuery()

@classmethod
@@ -129,6 +129,29 @@ def messages(self):
return self.log


class TestQgsDelimitedTextProviderLoading(unittest.TestCase):

def test_open_filepath_with_file_prefix(self):
srcpath = os.path.join(TEST_DATA_DIR, 'provider')
basetestfile = os.path.join(srcpath, 'delimited_xy.csv')

url = MyUrl.fromLocalFile(basetestfile)
url.addQueryItem("type", "csv")

vl = QgsVectorLayer(url.toString(), 'test', 'delimitedtext')
assert vl.isValid(), "{} is invalid".format(basetestfile)

def test_treat_open_filepath_without_file_prefix(self):
srcpath = os.path.join(TEST_DATA_DIR, 'provider')
basetestfile = os.path.join(srcpath, 'delimited_xy.csv')

url = MyUrl(basetestfile)
url.addQueryItem("type", "csv")

vl = QgsVectorLayer(url.toString(), 'test', 'delimitedtext')
assert vl.isValid(), "{} is invalid".format(basetestfile)


class TestQgsDelimitedTextProviderXY(unittest.TestCase, ProviderTestCase):

@classmethod

0 comments on commit 28e50c1

Please sign in to comment.
You can’t perform that action at this time.