Skip to content
Permalink
Browse files

Merge pull request #34846 from elpaso/bugfix-gh34823-pgraster-schema-…

…and-pk

PG raster, bugfixes and SQL filter
  • Loading branch information
elpaso committed Mar 4, 2020
2 parents fa6958f + a6679ee commit d6821a94cff8c917315f5494fff6daab8e01f5f5
@@ -15,9 +15,6 @@ PyQgsAuthManagerPKIPostgresTest
PyQgsAuthManagerOgrPostgresTest
PyQgsDbManagerPostgis

# Broken on master builds due to failure while loading pg raster test data
PyQgsPostgresRasterProvider

# Needs an OpenCL device, the library is not enough
qgis_openclutilstest

@@ -86,7 +86,6 @@ QgsDataSourceUri::QgsDataSourceUri( const QString &u )
}
else
{
mSchema.clear();
mTable = pval;
}

@@ -118,6 +117,10 @@ QgsDataSourceUri::QgsDataSourceUri( const QString &u )
mGeometryColumn = QString();
}
}
else if ( pname == QLatin1String( "schema" ) )
{
mSchema = pval;
}
else if ( pname == QLatin1String( "key" ) )
{
mKeyColumn = pval;
@@ -174,7 +174,6 @@ void QgsPgTableModel::addTableEntry( const QgsPostgresLayerProperty &layerProper
selItem->setCheckState( Qt::Unchecked );
checkPkUnicityItem->setFlags( checkPkUnicityItem->flags() & ~ Qt::ItemIsUserCheckable );
checkPkUnicityItem->setCheckState( Qt::Unchecked );
sqlItem->setEnabled( false );
}

QList<QStandardItem *> childItemList;
@@ -396,15 +395,21 @@ QString QgsPgTableModel::layerURI( const QModelIndex &index, const QString &conn
{
if ( isRaster )
{
// GDAL connection string
// GDAL/PG connection string
const QString schemaName = index.sibling( index.row(), DbtmSchema ).data( Qt::DisplayRole ).toString();
const QString tableName = index.sibling( index.row(), DbtmTable ).data( Qt::DisplayRole ).toString();
const QString geomColumnName = index.sibling( index.row(), DbtmGeomCol ).data( Qt::DisplayRole ).toString();
return QStringLiteral( "PG: %1 mode=2 schema='%2' column='%3' table='%4'" )
.arg( connInfo )
.arg( schemaName )
.arg( geomColumnName )
.arg( tableName );
QString connString { QStringLiteral( "PG: %1 mode=2 schema='%2' column='%3' table='%4'" )
.arg( connInfo )
.arg( schemaName )
.arg( geomColumnName )
.arg( tableName ) };
const QString sql { index.sibling( index.row(), DbtmSql ).data( Qt::DisplayRole ).toString() };
if ( ! sql.isEmpty() )
{
connString.append( QStringLiteral( " sql=%1" ).arg( sql ) );
}
return connString;
}
else
{
@@ -385,7 +385,7 @@ bool QgsPostgresRasterProvider::readBlock( int bandNo, const QgsRectangle &viewE
bandNo,
rasterExtent,
overviewFactor,
quotedIdentifier( pkSql() ),
pkSql(), // already quoted
quotedIdentifier( mRasterColumn ),
tableToQuery,
QString::number( mCrs.postgisSrid() ),
@@ -864,9 +864,22 @@ bool QgsPostgresRasterProvider::init()
const QByteArray hexAscii { result.PQgetvalue( 0, 5 ).toLatin1().mid( 2 ) };
QgsConstWkbPtr ptr { QByteArray::fromHex( hexAscii ) };

if ( ! p.fromWkb( ptr ) )
if ( hexAscii.isEmpty() || ! p.fromWkb( ptr ) )
{
throw QgsPostgresRasterProviderException( tr( "Cannot get extent from raster_columns" ) );
// Try to determine extent from raster
const QString extentSql { QStringLiteral( "SELECT ST_Envelope( %1 ) "
"FROM %2 WHERE %3" )
.arg( quotedIdentifier( mRasterColumn ) )
.arg( mQuery )
.arg( mSqlWhereClause.isEmpty() ? "'t'" : mSqlWhereClause ) };

QgsPostgresResult extentResult( connectionRO()->PQexec( extentSql ) );
const QByteArray extentHexAscii { extentResult.PQgetvalue( 0, 0 ).toLatin1() };
QgsConstWkbPtr extentPtr { QByteArray::fromHex( extentHexAscii ) };
if ( extentHexAscii.isEmpty() || ! p.fromWkb( extentPtr ) )
{
throw QgsPostgresRasterProviderException( tr( "Cannot get extent from raster" ) );
}
}

mExtent = p.boundingBox();
@@ -902,8 +915,8 @@ bool QgsPostgresRasterProvider::init()
}

// Compute raster size
mHeight = static_cast<long>( mExtent.height() / std::abs( mScaleY ) );
mWidth = static_cast<long>( mExtent.width() / std::abs( mScaleX ) );
mHeight = static_cast<long>( std::round( mExtent.height() / std::abs( mScaleY ) ) );
mWidth = static_cast<long>( std::round( mExtent.width() / std::abs( mScaleX ) ) );
// is tiled?
mIsTiled = ( mWidth != mTileWidth ) || ( mHeight != mTileHeight );

@@ -1344,6 +1357,10 @@ bool QgsPostgresRasterProvider::determinePrimaryKey()
{
pkType = QgsPostgresPrimaryKeyType::PktUint64;
}
else if ( fieldTypeName == QLatin1String( "text" ) )
{
pkType = QgsPostgresPrimaryKeyType::PktFidMap;
}
// Always use PktFidMap for multi-field keys
mPrimaryKeyType = i ? QgsPostgresPrimaryKeyType::PktFidMap : pkType;
mPrimaryKeyAttrs << name;
@@ -1410,9 +1427,14 @@ QString QgsPostgresRasterProvider::pkSql()
Q_ASSERT( ! mPrimaryKeyAttrs.isEmpty() );
if ( mPrimaryKeyAttrs.count( ) > 1 )
{
return mPrimaryKeyAttrs.join( ',' ).prepend( '(' ).append( ')' );
QStringList pkeys;
for ( const auto &k : qgis::as_const( mPrimaryKeyAttrs ) )
{
pkeys.push_back( quotedIdentifier( k ) );
}
return pkeys.join( ',' ).prepend( '(' ).append( ')' );
}
return mPrimaryKeyAttrs.first();
return quotedIdentifier( mPrimaryKeyAttrs.first() );
}


@@ -181,7 +181,7 @@ class QgsPostgresRasterProvider : public QgsRasterDataProvider
void determinePrimaryKeyFromUriKeyColumn();

/**
* Returns the SQL frament to retrieve the PK from the raster table
* Returns the quoted SQL frament to retrieve the PK from the raster table
*/
QString pkSql();

@@ -50,6 +50,7 @@ void TestQgsDataSourceUri::checkparser_data()
QTest::addColumn<QgsDataSourceUri::SslMode>( "sslmode" );
QTest::addColumn<QString>( "sql" );
QTest::addColumn<QString>( "myparam" );
QTest::addColumn<QString>( "schema" );


QTest::newRow( "oci" )
@@ -71,6 +72,7 @@ void TestQgsDataSourceUri::checkparser_data()
<< QgsDataSourceUri::SslPrefer // sslmode
<< "" // sql
<< "myvalue" // myparam
<< "myschema"
;

QTest::newRow( "pgrast" )
@@ -92,8 +94,10 @@ void TestQgsDataSourceUri::checkparser_data()
<< QgsDataSourceUri::SslPrefer // sslmode
<< "" // sql
<< "" // myparam
<< "public"
;


QTest::newRow( "pgmlsz" )
<< "PG: dbname=mydb host=myhost user=myname password=mypasswd port=5432 mode=2 schema=public column=geom table=mytable type=MultiLineStringZ"
<< "mytable" // table
@@ -113,6 +117,7 @@ void TestQgsDataSourceUri::checkparser_data()
<< QgsDataSourceUri::SslPrefer // sslmode
<< "" // sql
<< "" // myparam
<< "public"
;

QTest::newRow( "DB2" )
@@ -134,6 +139,7 @@ void TestQgsDataSourceUri::checkparser_data()
<< QgsDataSourceUri::SslPrefer // sslmode
<< "" // sql
<< "myvalue" // myparam
<< "TEST"
;
}

@@ -157,6 +163,7 @@ void TestQgsDataSourceUri::checkparser()
QFETCH( QgsDataSourceUri::SslMode, sslmode );
QFETCH( QString, sql );
QFETCH( QString, myparam );
QFETCH( QString, schema );

QgsDataSourceUri ds( uri );
QCOMPARE( ds.table(), table );
@@ -32,23 +32,41 @@
QgsRasterLayer,
QgsPointXY,
QgsRaster,
QgsProviderRegistry,
)
from qgis.testing import start_app, unittest
from utilities import unitTestDataPath
from utilities import unitTestDataPath, compareWkt

QGISAPP = start_app()
TEST_DATA_DIR = unitTestDataPath()


class TestPyQgsPostgresRasterProvider(unittest.TestCase):

@classmethod
def _load_test_table(cls, schemaname, tablename, basename=None):

postgres_conn = cls.dbconn + " sslmode=disable "
md = QgsProviderRegistry.instance().providerMetadata('postgres')
conn = md.createConnection(postgres_conn, {})

if basename is None:
basename = tablename

if tablename not in [n.tableName() for n in conn.tables(schemaname)]:
with open(os.path.join(TEST_DATA_DIR, 'provider', 'postgresraster', basename + '.sql'), 'r') as f:
sql = f.read()
conn.executeSql(sql)
assert (tablename in [n.tableName() for n in conn.tables(schemaname)])

@classmethod
def setUpClass(cls):
"""Run before all tests"""
cls.dbconn = 'service=qgis_test'
if 'QGIS_PGTEST_DB' in os.environ:
cls.dbconn = os.environ['QGIS_PGTEST_DB']
# Create test layers
cls._load_test_table('public', 'raster_tiled_3035')
cls.rl = QgsRasterLayer(cls.dbconn + ' sslmode=disable key=\'rid\' srid=3035 table="public"."raster_tiled_3035" sql=', 'test', 'postgresraster')
assert cls.rl.isValid()
cls.source = cls.rl.dataProvider()
@@ -99,18 +117,21 @@ def testBlockTiled(self):
def testNoConstraintRaster(self):
"""Read unconstrained raster layer"""

self._load_test_table('public', 'raster_3035_no_constraints')
rl = QgsRasterLayer(self.dbconn + ' sslmode=disable key=\'pk\' srid=3035 table="public"."raster_3035_no_constraints" sql=', 'test', 'postgresraster')
self.assertTrue(rl.isValid())

def testPkGuessing(self):
"""Read raster layer with no pkey in uri"""

self._load_test_table('public', 'raster_tiled_3035')
rl = QgsRasterLayer(self.dbconn + ' sslmode=disable srid=3035 table="public"."raster_tiled_3035" sql=', 'test', 'postgresraster')
self.assertTrue(rl.isValid())

def testWhereCondition(self):
"""Read raster layer with where condition"""

self._load_test_table('public', 'raster_3035_tiled_no_overviews')
rl_nowhere = QgsRasterLayer(self.dbconn + ' sslmode=disable srid=3035 table="public"."raster_3035_tiled_no_overviews"' +
'sql=', 'test', 'postgresraster')
self.assertTrue(rl_nowhere.isValid())
@@ -130,18 +151,22 @@ def testWhereCondition(self):
def testNoPk(self):
"""Read raster with no PK"""

self._load_test_table('public', 'raster_3035_tiled_no_pk')
rl = QgsRasterLayer(self.dbconn + ' sslmode=disable srid=3035 table="public"."raster_3035_tiled_no_pk"' +
'sql=', 'test', 'postgresraster')
self.assertTrue(rl.isValid())

def testCompositeKey(self):
"""Read raster with composite pks"""

self._load_test_table('public', 'raster_3035_tiled_composite_pk')
rl = QgsRasterLayer(self.dbconn + ' sslmode=disable srid=3035 table="public"."raster_3035_tiled_composite_pk"' +
'sql=', 'test', 'postgresraster')
self.assertTrue(rl.isValid())
data = rl.dataProvider().block(1, rl.extent(), 3, 3)
self.assertEqual(int(data.value(0, 0)), 142)

@unittest.skipIf(os.environ.get('TRAVIS', '') == 'true', 'Performance test is disabled in Travis environment')
@unittest.skip('Performance test is disabled in Travis environment')
def testSpeed(self):
"""Compare speed with GDAL provider, this test was used during development"""

@@ -151,7 +176,7 @@ def testSpeed(self):
speed_db='qgis_test'
)

table = 'eu_dem_tiled'
table = 'basic_map_tiled'
schema = 'public'

def _speed_check(schema, table, width, height):
@@ -195,6 +220,39 @@ def _speed_check(schema, table, width, height):

_speed_check(schema, table, 1000, 1000)

def testOtherSchema(self):
"""Test that a layer in a different schema than public can be loaded
See: GH #34823"""

self._load_test_table('idro', 'cosmo_i5_snow', 'bug_34823_pg_raster')

rl = QgsRasterLayer(self.dbconn + " sslmode=disable table={table} schema={schema}".format(table='cosmo_i5_snow', schema='idro'), 'pg_layer', 'postgresraster')
self.assertTrue(rl.isValid())
self.assertTrue(compareWkt(rl.extent().asWktPolygon(), 'POLYGON((-64.79286766849691048 -77.26689086732433509, -62.18292922825105506 -77.26689086732433509, -62.18292922825105506 -74.83694818157819384, -64.79286766849691048 -74.83694818157819384, -64.79286766849691048 -77.26689086732433509))'))

def testUntiledMultipleRows(self):
"""Test multiple rasters (one per row)"""

self._load_test_table('public', 'raster_3035_untiled_multiple_rows')

rl = QgsRasterLayer(self.dbconn + " sslmode=disable table={table} schema={schema} sql=\"pk\" = 1".format(table='raster_3035_untiled_multiple_rows', schema='public'), 'pg_layer', 'postgresraster')
self.assertTrue(rl.isValid())
block = rl.dataProvider().block(1, rl.extent(), 2, 2)
data = []
for i in range(2):
for j in range(2):
data.append(int(block.value(i, j)))
self.assertEqual(data, [136, 142, 145, 153])

rl = QgsRasterLayer(self.dbconn + " sslmode=disable table={table} schema={schema} sql=\"pk\" = 2".format(table='raster_3035_untiled_multiple_rows', schema='public'), 'pg_layer', 'postgresraster')
self.assertTrue(rl.isValid())
block = rl.dataProvider().block(1, rl.extent(), 2, 2)
data = []
for i in range(2):
for j in range(2):
data.append(int(block.value(i, j)))
self.assertEqual(data, [136, 142, 161, 169])


if __name__ == '__main__':
unittest.main()

Large diffs are not rendered by default.

@@ -0,0 +1,8 @@

--
-- in-db float 32 raster with no constraints
--
CREATE TABLE "raster_3035_no_constraints" ("rid" serial PRIMARY KEY,"rast" raster);
INSERT INTO "raster_3035_no_constraints" ("rast") VALUES ('0100000100000000000000394000000000000039C000000000D9204F41000000008F8B424100000000000000000000000000000000DB0B0000060005004A003C1CC66A610843880B0E431CC2194306342543B7633C43861858436E0A1143BBAD194359612743A12B334317BE4343DECE59432B621B43F0E42843132B3843AC824043E6CF48436E465A435C4D2D430FA63D43F87A4843B5494A4349454E4374F35B43906E41433AB54C43B056504358575243B1EC574322615F43'::raster);


@@ -0,0 +1,21 @@

--
-- in-db float32 tiled raster with composite PK
--

CREATE TABLE "public"."raster_3035_tiled_composite_pk" ("pk1" INT NOT NULL, "pk2" INT NOT NULL, "rast" raster,"category" text,
CONSTRAINT unique_pk UNIQUE ("pk1", "pk2")
);
INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C000000000D9204F41000000008F8B424100000000000000000000000000000000DB0B0000020002004A003C1CC66A610843880B0E436E0A1143BBAD1943'::raster,'cat1', 1, 1);
INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C000000000F2204F41000000008F8B424100000000000000000000000000000000DB0B0000020002004A003C1CC61CC219430634254359612743A12B3343'::raster,'cat1', 1, 2);
INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C0000000000B214F41000000008F8B424100000000000000000000000000000000DB0B0000020002004A003C1CC6B7633C438618584317BE4343DECE5943'::raster,'cat1', 1, 3);
INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C000000000D9204F4100000000768B424100000000000000000000000000000000DB0B0000020002004A003C1CC62B621B43F0E428435C4D2D430FA63D43'::raster,'cat1', 2, 1);
INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C000000000F2204F4100000000768B424100000000000000000000000000000000DB0B0000020002004A003C1CC6132B3843AC824043F87A4843B5494A43'::raster,'cat1', 2, 2);
INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C0000000000B214F4100000000768B424100000000000000000000000000000000DB0B0000020002004A003C1CC6E6CF48436E465A4349454E4374F35B43'::raster,'cat1', 2, 3);
INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C000000000D9204F41000000005D8B424100000000000000000000000000000000DB0B0000020001004A003C1CC6906E41433AB54C43'::raster,'cat1', 3, 1);
INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C000000000F2204F41000000005D8B424100000000000000000000000000000000DB0B0000020001004A003C1CC6B056504358575243'::raster,'cat1', 3, 2);
INSERT INTO "public"."raster_3035_tiled_composite_pk" ("rast","category", "pk1", "pk2") VALUES ('0100000100000000000000394000000000000039C0000000000B214F41000000005D8B424100000000000000000000000000000000DB0B0000020001004A003C1CC6B1EC574322615F43'::raster,'cat2', 3, 3);
CREATE INDEX ON "public"."raster_3035_tiled_composite_pk" USING gist (st_convexhull("rast"));
ANALYZE "public"."raster_3035_tiled_composite_pk";
SELECT AddRasterConstraints('public','raster_3035_tiled_composite_pk','rast',TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,FALSE,TRUE,TRUE,TRUE,TRUE,TRUE);

@@ -0,0 +1,17 @@

--
-- in-db float32 tiled raster with no overviews (for WHERE testing)
--
CREATE TABLE "public"."raster_3035_tiled_no_overviews" ("rid" serial PRIMARY KEY,"rast" raster,"category" text);
INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C000000000D9204F41000000008F8B424100000000000000000000000000000000DB0B0000020002004A003C1CC66A610843880B0E436E0A1143BBAD1943'::raster,'cat1');
INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C000000000F2204F41000000008F8B424100000000000000000000000000000000DB0B0000020002004A003C1CC61CC219430634254359612743A12B3343'::raster,'cat1');
INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C0000000000B214F41000000008F8B424100000000000000000000000000000000DB0B0000020002004A003C1CC6B7633C438618584317BE4343DECE5943'::raster,'cat1');
INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C000000000D9204F4100000000768B424100000000000000000000000000000000DB0B0000020002004A003C1CC62B621B43F0E428435C4D2D430FA63D43'::raster,'cat1');
INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C000000000F2204F4100000000768B424100000000000000000000000000000000DB0B0000020002004A003C1CC6132B3843AC824043F87A4843B5494A43'::raster,'cat1');
INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C0000000000B214F4100000000768B424100000000000000000000000000000000DB0B0000020002004A003C1CC6E6CF48436E465A4349454E4374F35B43'::raster,'cat1');
INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C000000000D9204F41000000005D8B424100000000000000000000000000000000DB0B0000020001004A003C1CC6906E41433AB54C43'::raster,'cat1');
INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C000000000F2204F41000000005D8B424100000000000000000000000000000000DB0B0000020001004A003C1CC6B056504358575243'::raster,'cat1');
INSERT INTO "public"."raster_3035_tiled_no_overviews" ("rast","category") VALUES ('0100000100000000000000394000000000000039C0000000000B214F41000000005D8B424100000000000000000000000000000000DB0B0000020001004A003C1CC6B1EC574322615F43'::raster,'cat2');
CREATE INDEX ON "public"."raster_3035_tiled_no_overviews" USING gist (st_convexhull("rast"));
ANALYZE "public"."raster_3035_tiled_no_overviews";
SELECT AddRasterConstraints('public','raster_3035_tiled_no_overviews','rast',TRUE,TRUE,TRUE,TRUE,TRUE,TRUE,FALSE,TRUE,TRUE,TRUE,TRUE,TRUE);

0 comments on commit d6821a9

Please sign in to comment.
You can’t perform that action at this time.