Fix text encoding issues in virtual layers (fixes #14350)

Author: Hugo Mercier

src/providers/virtual/qgsvirtuallayersqlitemodule.cpp

@@ -613,7 +613,7 @@ int vtableFilter( sqlite3_vtab_cursor * cursor, int idxNum, const char *idxStr,
       {
         int n = sqlite3_value_bytes( argv[0] );
         const char* t = reinterpret_cast<const char*>( sqlite3_value_text( argv[0] ) );
-        QString str( QByteArray::fromRawData( t, n ) );
+        QString str = QString::fromUtf8( t, n );
         expr += "'" + str.replace( "'", "''" ) + "'";
         break;
       }
@@ -738,7 +738,7 @@ void qgisFunctionWrapper( sqlite3_context* ctxt, int nArgs, sqlite3_value** args
       {
         int n = sqlite3_value_bytes( args[i] );
         const char* t = reinterpret_cast<const char*>( sqlite3_value_text( args[i] ) );
-        QString str( QByteArray::fromRawData( t, n ) ); // don't copy data
+        QString str = QString::fromUtf8( t, n );
         variants << QVariant( str );
         break;
       }

tests/src/python/test_provider_virtual.py

@@ -334,7 +334,6 @@ def test_recursiveLayer(self):
         QgsMapLayerRegistry.instance().removeMapLayer(l.id())
 
     def test_no_geometry(self):
-        source = QUrl.toPercentEncoding(os.path.join(self.testDataDir, "france_parts.shp"))
         df = QgsVirtualLayerDefinition()
         df.addSource("vtab", os.path.join(self.testDataDir, "france_parts.shp"), "ogr")
         df.setGeometryWkbType(QgsWKBTypes.NoGeometry)
@@ -720,6 +719,24 @@ def test_qgisExpressionFunctions(self):
         for f in l.getFeatures():
             self.assertEqual(f.attributes(), ['hello world', 2016, u'This', u'project'])
 
+    def test_query_with_accents(self):
+        # shapefile with accents and latin1 encoding
+        df = QgsVirtualLayerDefinition()
+        df.addSource("vtab", os.path.join(self.testDataDir, "france_parts.shp"), "ogr", "ISO-8859-1")
+        df.setQuery(u"SELECT * FROM vtab WHERE TYPE_1 = 'Région'")
+        vl = QgsVectorLayer(df.toString(), "testq", "virtual")
+        self.assertEqual(vl.isValid(), True)
+        ids = [f.id() for f in vl.getFeatures()]
+        self.assertEqual(len(ids), 4)
+
+        # the same shapefile with a wrong encoding
+        df.addSource("vtab", os.path.join(self.testDataDir, "france_parts.shp"), "ogr", "UTF-8")
+        df.setQuery(u"SELECT * FROM vtab WHERE TYPE_1 = 'Région'")
+        vl2 = QgsVectorLayer(df.toString(), "testq", "virtual")
+        self.assertEqual(vl2.isValid(), True)
+        ids = [f.id() for f in vl2.getFeatures()]
+        self.assertEqual(ids, [])
+
 
 if __name__ == '__main__':
     unittest.main()