Merge pull request #34874 from elpaso/bugfix-gh34823-pgraster-schema-…

…and-pk-3_12

[backport] PG raster, bugfixes and SQL filter

.ci/travis/linux/scripts/test_blacklist.txt

@@ -15,9 +15,6 @@ PyQgsAuthManagerPKIPostgresTest
 PyQgsAuthManagerOgrPostgresTest
 PyQgsDbManagerPostgis
 
-# Broken on master builds due to failure while loading pg raster test data
-PyQgsPostgresRasterProvider
-
 # Needs an OpenCL device, the library is not enough
 qgis_openclutilstest
 

src/core/qgsdatasourceuri.cpp

@@ -86,7 +86,6 @@ QgsDataSourceUri::QgsDataSourceUri( const QString &u )
         }
         else
         {
-          mSchema.clear();
           mTable = pval;
         }
 
@@ -118,6 +117,10 @@ QgsDataSourceUri::QgsDataSourceUri( const QString &u )
           mGeometryColumn = QString();
         }
       }
+      else if ( pname == QLatin1String( "schema" ) )
+      {
+        mSchema = pval;
+      }
       else if ( pname == QLatin1String( "key" ) )
       {
         mKeyColumn = pval;

src/providers/postgres/qgspgtablemodel.cpp

@@ -174,7 +174,6 @@ void QgsPgTableModel::addTableEntry( const QgsPostgresLayerProperty &layerProper
       selItem->setCheckState( Qt::Unchecked );
       checkPkUnicityItem->setFlags( checkPkUnicityItem->flags() & ~ Qt::ItemIsUserCheckable );
       checkPkUnicityItem->setCheckState( Qt::Unchecked );
-      sqlItem->setEnabled( false );
     }
 
     QList<QStandardItem *> childItemList;
@@ -396,15 +395,21 @@ QString QgsPgTableModel::layerURI( const QModelIndex &index, const QString &conn
   {
     if ( isRaster )
     {
-      // GDAL connection string
+      // GDAL/PG connection string
       const QString schemaName = index.sibling( index.row(), DbtmSchema ).data( Qt::DisplayRole ).toString();
       const QString tableName = index.sibling( index.row(), DbtmTable ).data( Qt::DisplayRole ).toString();
       const QString geomColumnName = index.sibling( index.row(), DbtmGeomCol ).data( Qt::DisplayRole ).toString();
-      return QStringLiteral( "PG:  %1 mode=2 schema='%2' column='%3' table='%4'" )
-             .arg( connInfo )
-             .arg( schemaName )
-             .arg( geomColumnName )
-             .arg( tableName );
+      QString connString  { QStringLiteral( "PG:  %1 mode=2 schema='%2' column='%3' table='%4'" )
+                            .arg( connInfo )
+                            .arg( schemaName )
+                            .arg( geomColumnName )
+                            .arg( tableName ) };
+      const QString sql { index.sibling( index.row(), DbtmSql ).data( Qt::DisplayRole ).toString() };
+      if ( ! sql.isEmpty() )
+      {
+        connString.append( QStringLiteral( " sql=%1" ).arg( sql ) );
+      }
+      return connString;
     }
     else
     {

src/providers/postgres/raster/qgspostgresrasterprovider.cpp

@@ -385,7 +385,7 @@ bool QgsPostgresRasterProvider::readBlock( int bandNo, const QgsRectangle &viewE
       bandNo,
       rasterExtent,
       overviewFactor,
-      quotedIdentifier( pkSql() ),
+      pkSql(),  // already quoted
       quotedIdentifier( mRasterColumn ),
       tableToQuery,
       QString::number( mCrs.postgisSrid() ),
@@ -864,9 +864,22 @@ bool QgsPostgresRasterProvider::init()
         const QByteArray hexAscii { result.PQgetvalue( 0, 5 ).toLatin1().mid( 2 ) };
         QgsConstWkbPtr ptr { QByteArray::fromHex( hexAscii ) };
 
-        if ( ! p.fromWkb( ptr ) )
+        if ( hexAscii.isEmpty() || ! p.fromWkb( ptr ) )
         {
-          throw QgsPostgresRasterProviderException( tr( "Cannot get extent from raster_columns" ) );
+          // Try to determine extent from raster
+          const QString extentSql { QStringLiteral( "SELECT ST_Envelope( %1 ) "
+                                    "FROM %2 WHERE %3" )
+                                    .arg( quotedIdentifier( mRasterColumn ) )
+                                    .arg( mQuery )
+                                    .arg( mSqlWhereClause.isEmpty() ? "'t'" : mSqlWhereClause ) };
+
+          QgsPostgresResult extentResult( connectionRO()->PQexec( extentSql ) );
+          const QByteArray extentHexAscii { extentResult.PQgetvalue( 0, 0 ).toLatin1() };
+          QgsConstWkbPtr extentPtr { QByteArray::fromHex( extentHexAscii ) };
+          if ( extentHexAscii.isEmpty() || ! p.fromWkb( extentPtr ) )
+          {
+            throw QgsPostgresRasterProviderException( tr( "Cannot get extent from raster" ) );
+          }
         }
 
         mExtent = p.boundingBox();
@@ -902,8 +915,8 @@ bool QgsPostgresRasterProvider::init()
         }
 
         // Compute raster size
-        mHeight = static_cast<long>( mExtent.height() / std::abs( mScaleY ) );
-        mWidth = static_cast<long>( mExtent.width() / std::abs( mScaleX ) );
+        mHeight = static_cast<long>( std::round( mExtent.height() / std::abs( mScaleY ) ) );
+        mWidth = static_cast<long>( std::round( mExtent.width() / std::abs( mScaleX ) ) );
         // is tiled?
         mIsTiled = ( mWidth != mTileWidth ) || ( mHeight != mTileHeight );
 
@@ -1344,6 +1357,10 @@ bool QgsPostgresRasterProvider::determinePrimaryKey()
         {
           pkType = QgsPostgresPrimaryKeyType::PktUint64;
         }
+        else if ( fieldTypeName == QLatin1String( "text" ) )
+        {
+          pkType = QgsPostgresPrimaryKeyType::PktFidMap;
+        }
         // Always use PktFidMap for multi-field keys
         mPrimaryKeyType = i ? QgsPostgresPrimaryKeyType::PktFidMap : pkType;
         mPrimaryKeyAttrs << name;
@@ -1410,9 +1427,14 @@ QString QgsPostgresRasterProvider::pkSql()
   Q_ASSERT( ! mPrimaryKeyAttrs.isEmpty() );
   if ( mPrimaryKeyAttrs.count( ) > 1 )
   {
-    return mPrimaryKeyAttrs.join( ',' ).prepend( '(' ).append( ')' );
+    QStringList pkeys;
+    for ( const auto &k : qgis::as_const( mPrimaryKeyAttrs ) )
+    {
+      pkeys.push_back( quotedIdentifier( k ) );
+    }
+    return pkeys.join( ',' ).prepend( '(' ).append( ')' );
   }
-  return mPrimaryKeyAttrs.first();
+  return quotedIdentifier( mPrimaryKeyAttrs.first() );
 }
 
 

src/providers/postgres/raster/qgspostgresrasterprovider.h

@@ -181,7 +181,7 @@ class QgsPostgresRasterProvider : public QgsRasterDataProvider
     void determinePrimaryKeyFromUriKeyColumn();
 
     /**
-     * Returns the SQL frament to retrieve the PK from the raster table
+     * Returns the quoted SQL frament to retrieve the PK from the raster table
      */
     QString pkSql();
 

tests/src/core/testqgsdatasourceuri.cpp

@@ -50,6 +50,7 @@ void TestQgsDataSourceUri::checkparser_data()
   QTest::addColumn<QgsDataSourceUri::SslMode>( "sslmode" );
   QTest::addColumn<QString>( "sql" );
   QTest::addColumn<QString>( "myparam" );
+  QTest::addColumn<QString>( "schema" );
 
 
   QTest::newRow( "oci" )
@@ -71,6 +72,7 @@ void TestQgsDataSourceUri::checkparser_data()
       << QgsDataSourceUri::SslPrefer // sslmode
       << "" // sql
       << "myvalue" // myparam
+      << "myschema"
       ;
 
   QTest::newRow( "pgrast" )
@@ -92,8 +94,10 @@ void TestQgsDataSourceUri::checkparser_data()
       << QgsDataSourceUri::SslPrefer // sslmode
       << "" // sql
       << "" // myparam
+      << "public"
       ;
 
+
   QTest::newRow( "pgmlsz" )
       << "PG: dbname=mydb host=myhost user=myname password=mypasswd port=5432 mode=2 schema=public column=geom table=mytable type=MultiLineStringZ"
       << "mytable" // table
@@ -113,6 +117,7 @@ void TestQgsDataSourceUri::checkparser_data()
       << QgsDataSourceUri::SslPrefer // sslmode
       << "" // sql
       << "" // myparam
+      << "public"
       ;
 
   QTest::newRow( "DB2" )
@@ -134,6 +139,7 @@ void TestQgsDataSourceUri::checkparser_data()
       << QgsDataSourceUri::SslPrefer // sslmode
       << "" // sql
       << "myvalue" // myparam
+      << "TEST"
       ;
 }
 
@@ -157,6 +163,7 @@ void TestQgsDataSourceUri::checkparser()
   QFETCH( QgsDataSourceUri::SslMode, sslmode );
   QFETCH( QString, sql );
   QFETCH( QString, myparam );
+  QFETCH( QString, schema );
 
   QgsDataSourceUri ds( uri );
   QCOMPARE( ds.table(), table );

tests/src/python/test_provider_postgresraster.py

@@ -32,23 +32,41 @@
     QgsRasterLayer,
     QgsPointXY,
     QgsRaster,
+    QgsProviderRegistry,
 )
 from qgis.testing import start_app, unittest
-from utilities import unitTestDataPath
+from utilities import unitTestDataPath, compareWkt
 
 QGISAPP = start_app()
 TEST_DATA_DIR = unitTestDataPath()
 
 
 class TestPyQgsPostgresRasterProvider(unittest.TestCase):
 
+    @classmethod
+    def _load_test_table(cls, schemaname, tablename, basename=None):
+
+        postgres_conn = cls.dbconn + " sslmode=disable "
+        md = QgsProviderRegistry.instance().providerMetadata('postgres')
+        conn = md.createConnection(postgres_conn, {})
+
+        if basename is None:
+            basename = tablename
+
+        if tablename not in [n.tableName() for n in conn.tables(schemaname)]:
+            with open(os.path.join(TEST_DATA_DIR, 'provider', 'postgresraster', basename + '.sql'), 'r') as f:
+                sql = f.read()
+                conn.executeSql(sql)
+            assert (tablename in [n.tableName() for n in conn.tables(schemaname)])
+
     @classmethod
     def setUpClass(cls):
         """Run before all tests"""
         cls.dbconn = 'service=qgis_test'
         if 'QGIS_PGTEST_DB' in os.environ:
             cls.dbconn = os.environ['QGIS_PGTEST_DB']
         # Create test layers
+        cls._load_test_table('public', 'raster_tiled_3035')
         cls.rl = QgsRasterLayer(cls.dbconn + ' sslmode=disable key=\'rid\' srid=3035  table="public"."raster_tiled_3035" sql=', 'test', 'postgresraster')
         assert cls.rl.isValid()
         cls.source = cls.rl.dataProvider()
@@ -99,18 +117,21 @@ def testBlockTiled(self):
     def testNoConstraintRaster(self):
         """Read unconstrained raster layer"""
 
+        self._load_test_table('public', 'raster_3035_no_constraints')
         rl = QgsRasterLayer(self.dbconn + ' sslmode=disable key=\'pk\' srid=3035  table="public"."raster_3035_no_constraints" sql=', 'test', 'postgresraster')
         self.assertTrue(rl.isValid())
 
     def testPkGuessing(self):
         """Read raster layer with no pkey in uri"""
 
+        self._load_test_table('public', 'raster_tiled_3035')
         rl = QgsRasterLayer(self.dbconn + ' sslmode=disable srid=3035  table="public"."raster_tiled_3035" sql=', 'test', 'postgresraster')
         self.assertTrue(rl.isValid())
 
     def testWhereCondition(self):
         """Read raster layer with where condition"""
 
+        self._load_test_table('public', 'raster_3035_tiled_no_overviews')
         rl_nowhere = QgsRasterLayer(self.dbconn + ' sslmode=disable srid=3035  table="public"."raster_3035_tiled_no_overviews"' +
                                     'sql=', 'test', 'postgresraster')
         self.assertTrue(rl_nowhere.isValid())
@@ -130,18 +151,22 @@ def testWhereCondition(self):
     def testNoPk(self):
         """Read raster with no PK"""
 
+        self._load_test_table('public', 'raster_3035_tiled_no_pk')
         rl = QgsRasterLayer(self.dbconn + ' sslmode=disable srid=3035  table="public"."raster_3035_tiled_no_pk"' +
                             'sql=', 'test', 'postgresraster')
         self.assertTrue(rl.isValid())
 
     def testCompositeKey(self):
         """Read raster with composite pks"""
 
+        self._load_test_table('public', 'raster_3035_tiled_composite_pk')
         rl = QgsRasterLayer(self.dbconn + ' sslmode=disable srid=3035  table="public"."raster_3035_tiled_composite_pk"' +
                             'sql=', 'test', 'postgresraster')
         self.assertTrue(rl.isValid())
+        data = rl.dataProvider().block(1, rl.extent(), 3, 3)
+        self.assertEqual(int(data.value(0, 0)), 142)
 
-    @unittest.skipIf(os.environ.get('TRAVIS', '') == 'true', 'Performance test is disabled in Travis environment')
+    @unittest.skip('Performance test is disabled in Travis environment')
     def testSpeed(self):
         """Compare speed with GDAL provider, this test was used during development"""
 
@@ -151,7 +176,7 @@ def testSpeed(self):
             speed_db='qgis_test'
         )
 
-        table = 'eu_dem_tiled'
+        table = 'basic_map_tiled'
         schema = 'public'
 
         def _speed_check(schema, table, width, height):
@@ -195,6 +220,39 @@ def _speed_check(schema, table, width, height):
 
         _speed_check(schema, table, 1000, 1000)
 
+    def testOtherSchema(self):
+        """Test that a layer in a different schema than public can be loaded
+        See: GH #34823"""
+
+        self._load_test_table('idro', 'cosmo_i5_snow', 'bug_34823_pg_raster')
+
+        rl = QgsRasterLayer(self.dbconn + " sslmode=disable table={table} schema={schema}".format(table='cosmo_i5_snow', schema='idro'), 'pg_layer', 'postgresraster')
+        self.assertTrue(rl.isValid())
+        self.assertTrue(compareWkt(rl.extent().asWktPolygon(), 'POLYGON((-64.79286766849691048 -77.26689086732433509, -62.18292922825105506 -77.26689086732433509, -62.18292922825105506 -74.83694818157819384, -64.79286766849691048 -74.83694818157819384, -64.79286766849691048 -77.26689086732433509))'))
+
+    def testUntiledMultipleRows(self):
+        """Test multiple rasters (one per row)"""
+
+        self._load_test_table('public', 'raster_3035_untiled_multiple_rows')
+
+        rl = QgsRasterLayer(self.dbconn + " sslmode=disable table={table} schema={schema} sql=\"pk\" = 1".format(table='raster_3035_untiled_multiple_rows', schema='public'), 'pg_layer', 'postgresraster')
+        self.assertTrue(rl.isValid())
+        block = rl.dataProvider().block(1, rl.extent(), 2, 2)
+        data = []
+        for i in range(2):
+            for j in range(2):
+                data.append(int(block.value(i, j)))
+        self.assertEqual(data, [136, 142, 145, 153])
+
+        rl = QgsRasterLayer(self.dbconn + " sslmode=disable table={table} schema={schema} sql=\"pk\" = 2".format(table='raster_3035_untiled_multiple_rows', schema='public'), 'pg_layer', 'postgresraster')
+        self.assertTrue(rl.isValid())
+        block = rl.dataProvider().block(1, rl.extent(), 2, 2)
+        data = []
+        for i in range(2):
+            for j in range(2):
+                data.append(int(block.value(i, j)))
+        self.assertEqual(data, [136, 142, 161, 169])
+
 
 if __name__ == '__main__':
     unittest.main()

tests/testdata/provider/postgresraster/raster_3035_no_constraints.sql

@@ -0,0 +1,8 @@
+
+--
+-- in-db float 32 raster with no constraints
+--
+CREATE TABLE "raster_3035_no_constraints" ("rid" serial PRIMARY KEY,"rast" raster);
+INSERT INTO "raster_3035_no_constraints" ("rast") VALUES ('0100000100000000000000394000000000000039C000000000D9204F41000000008F8B424100000000000000000000000000000000DB0B0000060005004A003C1CC66A610843880B0E431CC2194306342543B7633C43861858436E0A1143BBAD194359612743A12B334317BE4343DECE59432B621B43F0E42843132B3843AC824043E6CF48436E465A435C4D2D430FA63D43F87A4843B5494A4349454E4374F35B43906E41433AB54C43B056504358575243B1EC574322615F43'::raster);
+
+

tests/testdata/provider/postgresraster/raster_3035_tiled_composite_pk.sql

@@ -0,0 +1,21 @@
+
+--
+-- in-db float32 tiled raster with composite PK
+--
+
+CREATE TABLE "public"."raster_3035_tiled_composite_pk" ("pk1" INT NOT NULL, "pk2" INT NOT NULL, "rast" raster,"category" text,
+  CONSTRAINT unique_pk UNIQUE ("pk1", "pk2")
+);
+INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C000000000D9204F41000000008F8B424100000000000000000000000000000000DB0B0000020002004A003C1CC66A610843880B0E436E0A1143BBAD1943'::raster,'cat1', 1, 1);
+INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C000000000F2204F41000000008F8B424100000000000000000000000000000000DB0B0000020002004A003C1CC61CC219430634254359612743A12B3343'::raster,'cat1', 1, 2);
+INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C0000000000B214F41000000008F8B424100000000000000000000000000000000DB0B0000020002004A003C1CC6B7633C438618584317BE4343DECE5943'::raster,'cat1', 1, 3);
+INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C000000000D9204F4100000000768B424100000000000000000000000000000000DB0B0000020002004A003C1CC62B621B43F0E428435C4D2D430FA63D43'::raster,'cat1', 2, 1);
+INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C000000000F2204F4100000000768B424100000000000000000000000000000000DB0B0000020002004A003C1CC6132B3843AC824043F87A4843B5494A43'::raster,'cat1', 2, 2);
+INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C0000000000B214F4100000000768B424100000000000000000000000000000000DB0B0000020002004A003C1CC6E6CF48436E465A4349454E4374F35B43'::raster,'cat1', 2, 3);
+INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C000000000D9204F41000000005D8B424100000000000000000000000000000000DB0B0000020001004A003C1CC6906E41433AB54C43'::raster,'cat1', 3, 1);
+INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C000000000F2204F41000000005D8B424100000000000000000000000000000000DB0B0000020001004A003C1CC6B056504358575243'::raster,'cat1', 3, 2);
+INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C0000000000B214F41000000005D8B424100000000000000000000000000000000DB0B0000020001004A003C1CC6B1EC574322615F43'::raster,'cat2', 3, 3);
+CREATE INDEX ON "public"."raster_3035_tiled_composite_pk" USING gist (st_convexhull("rast"));
+ANALYZE "public"."raster_3035_tiled_composite_pk";
+SELECT AddRasterConstraints('public','raster_3035_tiled_composite_pk','rast',TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,FALSE,TRUE,TRUE,TRUE,TRUE,TRUE);
+

tests/testdata/provider/postgresraster/raster_3035_tiled_no_overviews.sql

@@ -0,0 +1,17 @@
+
+--
+-- in-db float32 tiled raster with no overviews (for WHERE testing)
+--
+CREATE TABLE "public"."raster_3035_tiled_no_overviews" ("rid" serial PRIMARY KEY,"rast" raster,"category" text);
+INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C000000000D9204F41000000008F8B424100000000000000000000000000000000DB0B0000020002004A003C1CC66A610843880B0E436E0A1143BBAD1943'::raster,'cat1');
+INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C000000000F2204F41000000008F8B424100000000000000000000000000000000DB0B0000020002004A003C1CC61CC219430634254359612743A12B3343'::raster,'cat1');
+INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C0000000000B214F41000000008F8B424100000000000000000000000000000000DB0B0000020002004A003C1CC6B7633C438618584317BE4343DECE5943'::raster,'cat1');
+INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C000000000D9204F4100000000768B424100000000000000000000000000000000DB0B0000020002004A003C1CC62B621B43F0E428435C4D2D430FA63D43'::raster,'cat1');
+INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C000000000F2204F4100000000768B424100000000000000000000000000000000DB0B0000020002004A003C1CC6132B3843AC824043F87A4843B5494A43'::raster,'cat1');
+INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C0000000000B214F4100000000768B424100000000000000000000000000000000DB0B0000020002004A003C1CC6E6CF48436E465A4349454E4374F35B43'::raster,'cat1');
+INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C000000000D9204F41000000005D8B424100000000000000000000000000000000DB0B0000020001004A003C1CC6906E41433AB54C43'::raster,'cat1');
+INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C000000000F2204F41000000005D8B424100000000000000000000000000000000DB0B0000020001004A003C1CC6B056504358575243'::raster,'cat1');
+INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C0000000000B214F41000000005D8B424100000000000000000000000000000000DB0B0000020001004A003C1CC6B1EC574322615F43'::raster,'cat2');
+CREATE INDEX ON "public"."raster_3035_tiled_no_overviews" USING gist (st_convexhull("rast"));
+ANALYZE "public"."raster_3035_tiled_no_overviews";
+SELECT AddRasterConstraints('public','raster_3035_tiled_no_overviews','rast',TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,FALSE,TRUE,TRUE,TRUE,TRUE,TRUE);