From b270ff1da7a16bb420141eeada6b2d8c36f38faf Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Mon, 17 Feb 2020 11:35:10 +0100 Subject: [PATCH 1/5] Recover pointcloud tests This reverts commit bf6915633cf12a556cd807b136e63509b6cdb33f while improving predictability of results --- .../test_qgsproviderconnection_postgres.py | 23 ++++-- tests/testdata/Dockerfile-postgis | 11 +++ tests/testdata/provider/testdata_pg.sh | 1 + .../provider/testdata_pg_pointcloud.sql | 70 +++++++++++++++++++ 4 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 tests/testdata/provider/testdata_pg_pointcloud.sql diff --git a/tests/src/python/test_qgsproviderconnection_postgres.py b/tests/src/python/test_qgsproviderconnection_postgres.py index 8be69f3e9992..d7f8f04f0e70 100644 --- a/tests/src/python/test_qgsproviderconnection_postgres.py +++ b/tests/src/python/test_qgsproviderconnection_postgres.py @@ -125,23 +125,31 @@ def test_postgis_connections(self): self.assertEqual(srids_and_types, [[0, 1], [0, 2], [0, 3], [0, 7], [3857, 1], [4326, 1]]) - # Check TopoGeometry layers are found in vector table names - + # Check TopoGeometry and Pointcloud layers are found in vector table names tables = conn.tables('qgis_test', QgsAbstractDatabaseProviderConnection.Vector) table_names = self._table_names(tables) self.assertTrue('TopoLayer1' in table_names) + self.assertTrue('PointCloudPointLayer' in table_names) + self.assertTrue('PointCloudPatchLayer' in table_names) + self.assertTrue('geometries_table' in table_names) # Revoke select permissions on topology.topology from qgis_test_user conn.executeSql('REVOKE SELECT ON topology.topology FROM qgis_test_user') + # Revoke select permissions on pointcloud_format from qgis_test_user + conn.executeSql('REVOKE SELECT ON pointcloud_formats FROM qgis_test_user') + # Re-connect as the qgis_test_role role newuri = self.uri + ' user=qgis_test_user password=qgis_test_user_password' - conn = md.createConnection(newuri, {}) + newconn = md.createConnection(newuri, {}) - tables = conn.tables('qgis_test', QgsAbstractDatabaseProviderConnection.Vector) + # Check TopoGeometry and Pointcloud layers are not found in vector table names + tables = newconn.tables('qgis_test', QgsAbstractDatabaseProviderConnection.Vector) table_names = self._table_names(tables) self.assertFalse('TopoLayer1' in table_names) + self.assertFalse('PointCloudPointLayer' in table_names) + self.assertFalse('PointCloudPatchLayer' in table_names) self.assertTrue('geometries_table' in table_names) # TODO: only revoke select permission on topology.layer, grant @@ -153,6 +161,13 @@ def test_postgis_connections(self): # TODO: only revoke select permission the actual topology # schema associated with TopoLayer1 + # TODO: only revoke select permission the pointcloud_columns + # table + + # Grant select permissions back on topology.topology to qgis_test_user + conn.executeSql('GRANT SELECT ON topology.topology TO qgis_test_user') + conn.executeSql('GRANT SELECT ON pointcloud_formats TO qgis_test_user') + # error: ERROR: relation "qgis_test.raster1" does not exist @unittest.skipIf(gdal.VersionInfo() < '2040000', 'This test requires GDAL >= 2.4.0') def test_postgis_raster_rename(self): diff --git a/tests/testdata/Dockerfile-postgis b/tests/testdata/Dockerfile-postgis index 5c6d36d9bfa8..c43a6265a8bf 100644 --- a/tests/testdata/Dockerfile-postgis +++ b/tests/testdata/Dockerfile-postgis @@ -5,3 +5,14 @@ ADD auth_system/certs_keys/postgres.key /etc/ssl/private/postgres_key.key ADD auth_system/certs_keys/issuer_ca_cert.pem /etc/ssl/certs/issuer_ca_cert.pem RUN chmod 400 /etc/ssl/private/postgres_key.key + +# Compile and install PointCloud. +# NOTE: release 1.2.0 would not build against PostgreSQL-11: +# https://github.com/pgpointcloud/pointcloud/issues/248 +RUN apt-get -y update; apt-get -y install build-essential autoconf postgresql-server-dev-11 libxml2-dev zlib1g-dev +RUN wget -O- \ + https://github.com/pgpointcloud/pointcloud/archive/dda0cb9c84365088737d9c157beeedf87ca59f01.tar.gz \ + | tar xz && \ + cd pointcloud-dda0cb9c84365088737d9c157beeedf87ca59f01 && \ + ./autogen.sh && ./configure && make && make install && \ + cd .. && rm -Rf pointcloud-dda0cb9c84365088737d9c157beeedf87ca59f01 diff --git a/tests/testdata/provider/testdata_pg.sh b/tests/testdata/provider/testdata_pg.sh index 9269d6efa1fd..b87eae1e8b2c 100755 --- a/tests/testdata/provider/testdata_pg.sh +++ b/tests/testdata/provider/testdata_pg.sh @@ -13,6 +13,7 @@ SCRIPTS=" tests/testdata/provider/testdata_pg_topology.sql tests/testdata/provider/testdata_pg_domain.sql tests/testdata/provider/testdata_pg_json.sql + tests/testdata/provider/testdata_pg_pointcloud.sql " dropdb --if-exists $DB diff --git a/tests/testdata/provider/testdata_pg_pointcloud.sql b/tests/testdata/provider/testdata_pg_pointcloud.sql new file mode 100644 index 000000000000..198f1c84200d --- /dev/null +++ b/tests/testdata/provider/testdata_pg_pointcloud.sql @@ -0,0 +1,70 @@ +DO $$ +DECLARE + layerid INTEGER; + +BEGIN + IF EXISTS ( SELECT * + FROM pg_catalog.pg_available_extensions e1, + pg_catalog.pg_available_extensions e2 + WHERE e1.name = 'pointcloud' + AND e2.name = 'pointcloud_postgis' ) + THEN + RAISE NOTICE 'Loading pointcloud'; + CREATE EXTENSION IF NOT EXISTS pointcloud; + CREATE EXTENSION IF NOT EXISTS pointcloud_postgis; + + TRUNCATE pointcloud_formats; + INSERT INTO pointcloud_formats (pcid, srid, schema) VALUES (1, 4326, $S$ + + + + 1 + 4 + X + int32_t + 0.01 + + + 2 + 4 + Y + int32_t + 0.01 + + + 3 + 4 + Z + int32_t + 0.01 + + + dimensional + 4326 + + + $S$); + + -- Pointcloud layer: qgis_test.PointCloudPointLayer + CREATE TABLE IF NOT EXISTS qgis_test."PointCloudPointLayer" ( + id serial primary key, + pt PCPOINT(1) + ); + INSERT INTO qgis_test."PointCloudPointLayer" (pt) + SELECT PC_MakePoint(1, ARRAY[-127, 45, 124.0]); + INSERT INTO qgis_test."PointCloudPointLayer" (pt) + SELECT PC_MakePoint(1, ARRAY[127, -45, 224.0]); + + -- Pointcloud layer: qgis_test.PointCloudPatchLayer + CREATE TABLE IF NOT EXISTS qgis_test."PointCloudPatchLayer" ( + id serial primary key, + pc PCPATCH(1) + ); + INSERT INTO qgis_test."PointCloudPatchLayer" (pc) + SELECT PC_Patch(pt) FROM qgis_test."PointCloudPointLayer"; + + END IF; +END; +$$; + From 2aef8afd7dc847464b03eaee2cf4d3e034e7c276 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Mon, 17 Feb 2020 11:44:32 +0100 Subject: [PATCH 2/5] Force use of PostgreSQL-11 headerfiles when building pointcloud --- tests/testdata/Dockerfile-postgis | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/testdata/Dockerfile-postgis b/tests/testdata/Dockerfile-postgis index c43a6265a8bf..6ce38dd8c71e 100644 --- a/tests/testdata/Dockerfile-postgis +++ b/tests/testdata/Dockerfile-postgis @@ -14,5 +14,7 @@ RUN wget -O- \ https://github.com/pgpointcloud/pointcloud/archive/dda0cb9c84365088737d9c157beeedf87ca59f01.tar.gz \ | tar xz && \ cd pointcloud-dda0cb9c84365088737d9c157beeedf87ca59f01 && \ - ./autogen.sh && ./configure && make && make install && \ + ./autogen.sh && \ + PG_CONFIG=/usr/lib/postgresql/11/bin/pg_config ./configure && \ + make && make install && \ cd .. && rm -Rf pointcloud-dda0cb9c84365088737d9c157beeedf87ca59f01 From c9c99f9672ad8195bc66d847bdd9a670456bdc7e Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 18 Feb 2020 08:29:09 +0100 Subject: [PATCH 3/5] Try installing pointcloud 1.2.1, and use PATH instead of PG_CONFIG because PG_CONFIG seems to be unsupported, at least at ./configure time (https://github.com/pgpointcloud/pointcloud/issues/258) --- tests/testdata/Dockerfile-postgis | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/testdata/Dockerfile-postgis b/tests/testdata/Dockerfile-postgis index 6ce38dd8c71e..0b509456534a 100644 --- a/tests/testdata/Dockerfile-postgis +++ b/tests/testdata/Dockerfile-postgis @@ -11,10 +11,11 @@ RUN chmod 400 /etc/ssl/private/postgres_key.key # https://github.com/pgpointcloud/pointcloud/issues/248 RUN apt-get -y update; apt-get -y install build-essential autoconf postgresql-server-dev-11 libxml2-dev zlib1g-dev RUN wget -O- \ - https://github.com/pgpointcloud/pointcloud/archive/dda0cb9c84365088737d9c157beeedf87ca59f01.tar.gz \ + https://github.com/pgpointcloud/pointcloud/archive/v1.2.1.tar.gz \ | tar xz && \ - cd pointcloud-dda0cb9c84365088737d9c157beeedf87ca59f01 && \ + cd pointcloud-1.2.1 && \ + PATH=/usr/lib/postgresql/11/bin/:$PATH && \ ./autogen.sh && \ - PG_CONFIG=/usr/lib/postgresql/11/bin/pg_config ./configure && \ + ./configure && \ make && make install && \ - cd .. && rm -Rf pointcloud-dda0cb9c84365088737d9c157beeedf87ca59f01 + cd .. && rm -Rf pointcloud-1.2.1 From ded072e9456376d12330e9cd43e5933f939fceda Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 18 Feb 2020 09:53:35 +0100 Subject: [PATCH 4/5] Use --with-pgconfig --- tests/testdata/Dockerfile-postgis | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testdata/Dockerfile-postgis b/tests/testdata/Dockerfile-postgis index 0b509456534a..a7bd900a2923 100644 --- a/tests/testdata/Dockerfile-postgis +++ b/tests/testdata/Dockerfile-postgis @@ -10,12 +10,12 @@ RUN chmod 400 /etc/ssl/private/postgres_key.key # NOTE: release 1.2.0 would not build against PostgreSQL-11: # https://github.com/pgpointcloud/pointcloud/issues/248 RUN apt-get -y update; apt-get -y install build-essential autoconf postgresql-server-dev-11 libxml2-dev zlib1g-dev +RUN dpkg -L postgresql-server-dev-11 RUN wget -O- \ https://github.com/pgpointcloud/pointcloud/archive/v1.2.1.tar.gz \ | tar xz && \ cd pointcloud-1.2.1 && \ - PATH=/usr/lib/postgresql/11/bin/:$PATH && \ ./autogen.sh && \ - ./configure && \ + ./configure --with-pgconfig=/usr/lib/postgresql/11/bin/pg_config && \ make && make install && \ cd .. && rm -Rf pointcloud-1.2.1 From f0f2bfc29ad16c7b643bd4736ceb21f20d54dc83 Mon Sep 17 00:00:00 2001 From: Sandro Santilli Date: Tue, 18 Feb 2020 11:20:38 +0100 Subject: [PATCH 5/5] Install postgresql-client-11 as well (works around a packaging bug in postgresql-server-dev-11) --- tests/testdata/Dockerfile-postgis | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/testdata/Dockerfile-postgis b/tests/testdata/Dockerfile-postgis index a7bd900a2923..bbb5ffce87fd 100644 --- a/tests/testdata/Dockerfile-postgis +++ b/tests/testdata/Dockerfile-postgis @@ -9,7 +9,13 @@ RUN chmod 400 /etc/ssl/private/postgres_key.key # Compile and install PointCloud. # NOTE: release 1.2.0 would not build against PostgreSQL-11: # https://github.com/pgpointcloud/pointcloud/issues/248 -RUN apt-get -y update; apt-get -y install build-essential autoconf postgresql-server-dev-11 libxml2-dev zlib1g-dev +RUN apt-get -y update; apt-get -y install \ + autoconf \ + build-essential \ + libxml2-dev \ + postgresql-client-11 \ + postgresql-server-dev-11 \ + zlib1g-dev RUN dpkg -L postgresql-server-dev-11 RUN wget -O- \ https://github.com/pgpointcloud/pointcloud/archive/v1.2.1.tar.gz \