Skip to content

Commit

Permalink
Add Zstandard sample dataset and unit test.
Browse files Browse the repository at this point in the history
  • Loading branch information
@connormanning
connormanning committed Jun 22, 2019
1 parent 1e00272 commit e1ef168
Show file tree
Hide file tree
Showing 7 changed files with 268 additions and 0 deletions.
9 changes: 9 additions & 0 deletions test/data/ept/ellipsoid-zstandard/ept-build.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"cacheSize": 64,
"maxNodeSize": 65536,
"minNodeSize": 16384,
"overflowDepth": 0,
"software": "Entwine",
"trustHeaders": true,
"version": "2.0.0"
}
Binary file added test/data/ept/ellipsoid-zstandard/ept-data/0-0-0-0.zst
View file
Binary file not shown.
3 changes: 3 additions & 0 deletions test/data/ept/ellipsoid-zstandard/ept-hierarchy/0-0-0-0.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"0-0-0-0": 100000
}
84 changes: 84 additions & 0 deletions test/data/ept/ellipsoid-zstandard/ept-sources/0.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
{
"ellipsoid.laz": {
"bounds": [
-8242746.0,
4966506.0,
-50.0,
-8242446.0,
4966706.0,
50.0
],
"metadata": {
"comp_spatialreference": "PROJCS[\"WGS 84 / Pseudo-Mercator\",GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]],PROJECTION[\"Mercator_1SP\"],PARAMETER[\"central_meridian\",0],PARAMETER[\"scale_factor\",1],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AXIS[\"X\",EAST],AXIS[\"Y\",NORTH],EXTENSION[\"PROJ4\",\"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs\"],AUTHORITY[\"EPSG\",\"3857\"]]",
"compressed": true,
"count": 100000,
"creation_doy": 178,
"creation_year": 2018,
"dataformat_id": 3,
"dataoffset": 538,
"filesource_id": 0,
"global_encoding": 0,
"global_encoding_base64": "AAA=",
"header_size": 227,
"major_version": 1,
"maxx": -8242446,
"maxy": 4966706,
"maxz": 50,
"minor_version": 2,
"minx": -8242746,
"miny": 4966506,
"minz": -50,
"offset_x": 0,
"offset_y": 0,
"offset_z": 0,
"point_length": 34,
"project_id": "00000000-0000-0000-0000-000000000000",
"scale_x": 0.01,
"scale_y": 0.01,
"scale_z": 0.01,
"software_id": "PDAL 1.7.2 (d016ad)",
"spatialreference": "PROJCS[\"WGS 84 / Pseudo-Mercator\",GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]],PROJECTION[\"Mercator_1SP\"],PARAMETER[\"central_meridian\",0],PARAMETER[\"scale_factor\",1],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AXIS[\"X\",EAST],AXIS[\"Y\",NORTH],EXTENSION[\"PROJ4\",\"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs\"],AUTHORITY[\"EPSG\",\"3857\"]]",
"srs": {
"compoundwkt": "PROJCS[\"WGS 84 / Pseudo-Mercator\",GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]],PROJECTION[\"Mercator_1SP\"],PARAMETER[\"central_meridian\",0],PARAMETER[\"scale_factor\",1],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AXIS[\"X\",EAST],AXIS[\"Y\",NORTH],EXTENSION[\"PROJ4\",\"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs\"],AUTHORITY[\"EPSG\",\"3857\"]]",
"horizontal": "PROJCS[\"WGS 84 / Pseudo-Mercator\",GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]],PROJECTION[\"Mercator_1SP\"],PARAMETER[\"central_meridian\",0],PARAMETER[\"scale_factor\",1],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AXIS[\"X\",EAST],AXIS[\"Y\",NORTH],EXTENSION[\"PROJ4\",\"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs\"],AUTHORITY[\"EPSG\",\"3857\"]]",
"isgeocentric": false,
"isgeographic": false,
"prettycompoundwkt": "PROJCS[\"WGS 84 / Pseudo-Mercator\",\n GEOGCS[\"WGS 84\",\n DATUM[\"WGS_1984\",\n SPHEROID[\"WGS 84\",6378137,298.257223563,\n AUTHORITY[\"EPSG\",\"7030\"]],\n AUTHORITY[\"EPSG\",\"6326\"]],\n PRIMEM[\"Greenwich\",0,\n AUTHORITY[\"EPSG\",\"8901\"]],\n UNIT[\"degree\",0.0174532925199433,\n AUTHORITY[\"EPSG\",\"9122\"]],\n AUTHORITY[\"EPSG\",\"4326\"]],\n PROJECTION[\"Mercator_1SP\"],\n PARAMETER[\"central_meridian\",0],\n PARAMETER[\"scale_factor\",1],\n PARAMETER[\"false_easting\",0],\n PARAMETER[\"false_northing\",0],\n UNIT[\"metre\",1,\n AUTHORITY[\"EPSG\",\"9001\"]],\n AXIS[\"X\",EAST],\n AXIS[\"Y\",NORTH],\n EXTENSION[\"PROJ4\",\"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs\"],\n AUTHORITY[\"EPSG\",\"3857\"]]",
"prettywkt": "PROJCS[\"WGS 84 / Pseudo-Mercator\",\n GEOGCS[\"WGS 84\",\n DATUM[\"WGS_1984\",\n SPHEROID[\"WGS 84\",6378137,298.257223563,\n AUTHORITY[\"EPSG\",\"7030\"]],\n AUTHORITY[\"EPSG\",\"6326\"]],\n PRIMEM[\"Greenwich\",0,\n AUTHORITY[\"EPSG\",\"8901\"]],\n UNIT[\"degree\",0.0174532925199433,\n AUTHORITY[\"EPSG\",\"9122\"]],\n AUTHORITY[\"EPSG\",\"4326\"]],\n PROJECTION[\"Mercator_1SP\"],\n PARAMETER[\"central_meridian\",0],\n PARAMETER[\"scale_factor\",1],\n PARAMETER[\"false_easting\",0],\n PARAMETER[\"false_northing\",0],\n UNIT[\"metre\",1,\n AUTHORITY[\"EPSG\",\"9001\"]],\n AXIS[\"X\",EAST],\n AXIS[\"Y\",NORTH],\n EXTENSION[\"PROJ4\",\"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs\"],\n AUTHORITY[\"EPSG\",\"3857\"]]",
"proj4": "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs",
"units": {
"horizontal": "metre",
"vertical": ""
},
"vertical": "",
"wkt": "PROJCS[\"WGS 84 / Pseudo-Mercator\",GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]],PROJECTION[\"Mercator_1SP\"],PARAMETER[\"central_meridian\",0],PARAMETER[\"scale_factor\",1],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AXIS[\"X\",EAST],AXIS[\"Y\",NORTH],EXTENSION[\"PROJ4\",\"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs\"],AUTHORITY[\"EPSG\",\"3857\"]]"
},
"system_id": "PDAL",
"vlr_0": {
"data": "AQABAAAABwAABAAAAQABAAEEAAABAAEAAgSxhxkAAAABCLGHBwAZAAYIAAABAI4jAAwAAAEAEQ8EDAAAAQApIw==",
"description": "GeoTiff GeoKeyDirectoryTag",
"record_id": 34735,
"user_id": "LASF_Projection"
},
"vlr_1": {
"data": "V0dTIDg0IC8gUHNldWRvLU1lcmNhdG9yfFdHUyA4NHwA",
"description": "GeoTiff GeoAsciiParamsTag",
"record_id": 34737,
"user_id": "LASF_Projection"
},
"vlr_2": {
"data": "AgAAAAMBAAAAAAAAUMMAAP////////////////////8DAAYAFAACAAcACAACAAgABgACAA==",
"description": "http://laszip.org",
"record_id": 22204,
"user_id": "laszip encoded"
}
},
"origin": 0,
"points": 100000,
"srs": {
"authority": "EPSG",
"horizontal": "3857",
"wkt": "PROJCS[\"WGS 84 / Pseudo-Mercator\",GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]],PROJECTION[\"Mercator_1SP\"],PARAMETER[\"central_meridian\",0],PARAMETER[\"scale_factor\",1],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AXIS[\"X\",EAST],AXIS[\"Y\",NORTH],EXTENSION[\"PROJ4\",\"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs\"],AUTHORITY[\"EPSG\",\"3857\"]]"
}
}
}
18 changes: 18 additions & 0 deletions test/data/ept/ellipsoid-zstandard/ept-sources/list.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
[
{
"bounds": [
-8242746.0,
4966506.0,
-50.0,
-8242446.0,
4966706.0,
50.0
],
"id": "ellipsoid.laz",
"inserts": 100000,
"path": "/Users/connor/code/entwine/test/data/ellipsoid.laz",
"points": 100000,
"status": "inserted",
"url": "0.json"
}
]
120 changes: 120 additions & 0 deletions test/data/ept/ellipsoid-zstandard/ept.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
{
"bounds": [
-8242748.0,
4966454.0,
-152.0,
-8242444.0,
4966758.0,
152.0
],
"boundsConforming": [
-8242747.0,
4966505.0,
-51.0,
-8242445.0,
4966707.0,
51.0
],
"dataType": "zstandard",
"hierarchyType": "json",
"points": 100000,
"schema": [
{
"name": "X",
"offset": -8242596.0,
"scale": 0.01,
"size": 4,
"type": "signed"
},
{
"name": "Y",
"offset": 4966606.0,
"scale": 0.01,
"size": 4,
"type": "signed"
},
{
"name": "Z",
"scale": 0.01,
"size": 4,
"type": "signed"
},
{
"name": "Intensity",
"size": 2,
"type": "unsigned"
},
{
"name": "ReturnNumber",
"size": 1,
"type": "unsigned"
},
{
"name": "NumberOfReturns",
"size": 1,
"type": "unsigned"
},
{
"name": "ScanDirectionFlag",
"size": 1,
"type": "unsigned"
},
{
"name": "EdgeOfFlightLine",
"size": 1,
"type": "unsigned"
},
{
"name": "Classification",
"size": 1,
"type": "unsigned"
},
{
"name": "ScanAngleRank",
"size": 4,
"type": "float"
},
{
"name": "UserData",
"size": 1,
"type": "unsigned"
},
{
"name": "PointSourceId",
"size": 2,
"type": "unsigned"
},
{
"name": "GpsTime",
"size": 8,
"type": "float"
},
{
"name": "Red",
"size": 2,
"type": "unsigned"
},
{
"name": "Green",
"size": 2,
"type": "unsigned"
},
{
"name": "Blue",
"size": 2,
"type": "unsigned"
},
{
"name": "OriginId",
"size": 4,
"type": "unsigned"
}
],
"span": 128,
"srs": {
"authority": "EPSG",
"horizontal": "3857",
"wkt": "PROJCS[\"WGS 84 / Pseudo-Mercator\",GEOGCS[\"WGS 84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]],AUTHORITY[\"EPSG\",\"4326\"]],PROJECTION[\"Mercator_1SP\"],PARAMETER[\"central_meridian\",0],PARAMETER[\"scale_factor\",1],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1,AUTHORITY[\"EPSG\",\"9001\"]],AXIS[\"X\",EAST],AXIS[\"Y\",NORTH],EXTENSION[\"PROJ4\",\"+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs\"],AUTHORITY[\"EPSG\",\"3857\"]]"
},
"version": "1.0.0"
}
34 changes: 34 additions & 0 deletions test/unit/io/EptReaderTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ namespace
// Also test a basic read of binary/zstandard versions of a smaller dataset.
const std::string ellipsoidEptBinaryPath(
"ept://" + Support::datapath("ept/ellipsoid-binary"));
const std::string ellipsoidEptZstandardPath(
"ept://" + Support::datapath("ept/ellipsoid-zstandard"));
const point_count_t ellipsoidNumPoints(100000);
const BOX3D ellipsoidBoundsConforming(-8242746, 4966506, -50,
-8242446, 4966706, 50);
Expand Down Expand Up @@ -166,6 +168,38 @@ TEST(EptReaderTest, fullReadBinary)
EXPECT_EQ(np, ellipsoidNumPoints);
}

TEST(EptReaderTest, fullReadZstandard)
{
Options options;
options.add("filename", ellipsoidEptZstandardPath);

PointTable table;

EptReader reader;
reader.setOptions(options);
reader.prepare(table);
const auto set(reader.execute(table));

double x, y, z;
uint64_t o;
uint64_t np(0);
for (const PointViewPtr& view : set)
{
for (point_count_t i(0); i < view->size(); ++i)
{
++np;

x = view->getFieldAs<double>(Dimension::Id::X, i);
y = view->getFieldAs<double>(Dimension::Id::Y, i);
z = view->getFieldAs<double>(Dimension::Id::Z, i);
o = view->getFieldAs<uint64_t>(Dimension::Id::OriginId, i);
ASSERT_TRUE(ellipsoidBoundsConforming.contains(x, y, z));
ASSERT_EQ(o, 0u);
}
}

EXPECT_EQ(np, ellipsoidNumPoints);
}

TEST(EptReaderTest, resolutionLimit)
{
Expand Down

0 comments on commit e1ef168

Please sign in to comment.