Skip to content

Commit ddd99d3

Browse files
blachablacha
authored
feat(cogify): retile imagery into COGS aligned to a tile matrix (#2759)
* feat(cogify): Cover imagery with tiles then turn tiles into COGS Simple CLI to automate the retiling of imagery into tiles that align to a tile matrix * docs: comment some interfaces * refactor: cleanup imports * refactor: fixup typing issue * refactor: fixup formatting * refactor: correct bundle path * refactor: add documentation about usage * refactor: include gdal version in output information * refactor: debug loading time * refactor: optimize the performance of tile covering * refactor: add unit tests for covering logic * fix: prevent target covering zoom going negative * fix: cog / cover descriptions were swapped * docs: expand more on why this exists * refactor: remove quadkey math
1 parent 3cd249d commit ddd99d3

File tree

35 files changed

+1654
-69
lines changed

35 files changed

+1654
-69
lines changed

‎packages/bathymetry/src/bathy.maker.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { GdalCommand } from '@basemaps/cli/build/gdal/gdal.command.js';
33
import { Bounds, Epsg, Tile, TileMatrixSet } from '@basemaps/geo';
44
import { fsa, LogType, s3ToVsis3 } from '@basemaps/shared';
55
import * as os from 'os';
6-
import type { Limit } from 'p-limit';
6+
import type { LimitFunction } from 'p-limit';
77
import PLimit from 'p-limit';
88
import * as path from 'path';
99
import { basename } from 'path';
@@ -50,7 +50,7 @@ export class BathyMaker {
5050
/** Current gdal version @see Gdal.version */
5151
gdalVersion: Promise<string>;
5252
/** Concurrent limiting queue, all work should be done inside the queue */
53-
q: Limit;
53+
q: LimitFunction;
5454

5555
constructor(ctx: BathyMakerContext) {
5656
this.config = { ...BathyMakerContextDefault, ...ctx };

‎packages/cli/src/cli/config/action.bundle.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { fsa, LogConfig } from '@basemaps/shared';
21
import { ConfigJson } from '@basemaps/config';
2+
import { fsa, LogConfig } from '@basemaps/shared';
33
import { CommandLineAction, CommandLineStringParameter } from '@rushstack/ts-command-line';
44

55
export const DefaultConfig = 'config/';

‎packages/cli/src/cog/cog.stac.job.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ import {
1717
fsa,
1818
titleizeImageryName,
1919
} from '@basemaps/shared';
20-
import { MultiPolygon, toFeatureCollection, toFeatureMultiPolygon } from '@linzjs/geojson';
2120
import { CliInfo } from '@basemaps/shared/build/cli/info.js';
21+
import { MultiPolygon, toFeatureCollection, toFeatureMultiPolygon } from '@linzjs/geojson';
2222
import { GdalCogBuilderDefaults, GdalCogBuilderResampling } from '../gdal/gdal.config.js';
2323
import { ProjectionLoader } from './projection.loader.js';
2424
import { CogStac, CogStacItem, CogStacItemExtensions, CogStacKeywords } from './stac.js';

‎packages/cli/src/cog/cutline.ts‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -246,11 +246,11 @@ export class Cutline {
246246
}
247247

248248
/**
249-
* Find the polygon covering of source imagery and a (optional) clip cutline. Truncates the
250-
* cutline to match.
251-
252-
* @param sourceMetadata
253-
*/
249+
* Find the polygon covering of source imagery and a (optional) clip cutline. Truncates the
250+
* cutline to match.
251+
*
252+
* @param sourceMetadata
253+
*/
254254
private findCovering(sourceMetadata: SourceMetadata): void {
255255
let srcPoly: MultiPolygon = [];
256256
const { resZoom } = sourceMetadata;
@@ -289,11 +289,11 @@ export class Cutline {
289289
}
290290

291291
/**
292-
* Pad the bounds to take in to consideration blending and 100 pixels of adjacent image data
293-
294-
* @param bounds
295-
* @param resZoom the imagery resolution target zoom level
296-
*/
292+
* Pad the bounds to take in to consideration blending and 100 pixels of adjacent image data
293+
*
294+
* @param bounds
295+
* @param resZoom the imagery resolution target zoom level
296+
*/
297297
private padBounds(bounds: Bounds, resZoom: number): Bounds {
298298
const px = this.tileMatrix.pixelScale(resZoom);
299299
// Ensure cutline blend does not interferre with non-costal edges

‎packages/cogify/README.md‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# @basemaps/cogify
2+
3+
CLI to retile imagery into a [Cloud Optimised Geotiffs (COG)](https://www.cogeo.org/) aligned to a [TileMatrix](https://www.ogc.org/standard/tms/)
4+
5+
## Why ?
6+
7+
LINZ gets most of it's imagery delivered as tiles in [NZTM2000](https://www.linz.govt.nz/guidance/geodetic-system/coordinate-systems-used-new-zealand/projections/new-zealand-transverse-mercator-2000-nztm2000) the imagery is tiled as rectangles against a tile grid such as the [1:5k](https://data.linz.govt.nz/layer/104691-nz-15k-tile-index/) or [1:1k](https://data.linz.govt.nz/layer/104692-nz-11k-tile-index/).
8+
9+
These grids are not designed for use in XYZ tiles services in [WebMercator/EPSG:3857](https://epsg.io/3857), If the tiles were converted directly to WebMercator there would be significant overlap between multiple source imagery tiffs and a output WebMercator tile. This would cause large overheads for basemaps to render tiles as it would need to fetch data from all of the tiffs inside of the output tile.
10+
11+
![NZTM Tile Index vs WebMercator](./static/nztm-tile-index.png)
12+
Above is an example of a web mercator zoom 11 tile (Red outline) and the number `1:5k` tiles required to render (Shaded black) it
13+
14+
This package contains the logic to process input imagery into a output Tile Matrix to create optimised COGs for web mapping purposes.
15+
16+
The output COGS perfectly align to a output tile, greatly increasing XYZ tile service performance.
17+
18+
### Process
19+
20+
To convert imagery to optimized COG, a output tile cover is created, this covers the source imagery in tiles from the output tile matrix.
21+
22+
```
23+
cogify cover --tile-matrix WebMercatorQuad s3://linz-imagery/.../porirua_2020_0.1m --target ./output
24+
```
25+
26+
The metadata for the optimized COGS is written into the output folder where the COG creation step can use [GDAL](https://github.com/gdal/gdal) to create the output tiff.
27+
28+
29+
```
30+
cogify create ./output/WebMercatorQuad/porirua_2020_0.1m/01GY8W69EJEMAKKXNHYMRF7DCY/14-16150-10245.json
31+
```
32+
33+
The output COG can be validated to ensure it matches the tile exactly.
34+
35+
```
36+
cogify validate --tile-matrix WebMercatorQuad ./output/WebMercatorQuad/porirua_2020_0.1m/01GY8W69EJEMAKKXNHYMRF7DCY/14-16150-10245.tiff
37+
```
38+
39+
## Usage
40+
41+
42+
Install `cogify` using `npm`
43+
```
44+
npm install -g @basemaps/cogify
45+
```
46+
47+
48+
```
49+
$ cogify --help
50+
51+
- cover - Create a covering configuration from a collection from source imagery
52+
- create - Create a COG from a covering configuration
53+
```
54+
55+
56+
### Covering
57+
58+
Create a tile covering for WebMeractorQuad from source imagery located in s3 and outputs the resulting configuration files into `./output/:projection/:imageryName/:id/collection.json`
59+
60+
```
61+
cogify cover --tile-matrix WebMercatorQuad s3://linz-imagery/new-zealand/north-island_2023_0.5m/rgb/2193/ --target ./output
62+
```
63+
64+
### Create
65+
66+
Create the first COG from the list
67+
```
68+
cogify create ./output/3857/north-island_2023_0.5m/:id/14-16150-10245.json
69+
```
70+

‎packages/cogify/package.json‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
{
2+
"name": "@basemaps/cogify",
3+
"version": "6.39.0",
4+
"private": false,
5+
"repository": {
6+
"type": "git",
7+
"url": "https://github.com/linz/basemaps.git",
8+
"directory": "packages/cogify"
9+
},
10+
"author": {
11+
"name": "Land Information New Zealand",
12+
"url": "https://linz.govt.nz",
13+
"organization": true
14+
},
15+
"license": "MIT",
16+
"main": "./build/index.js",
17+
"types": "./build/index.d.ts",
18+
"bin": {
19+
"cogify": "./dist/index.cjs"
20+
},
21+
"scripts": {
22+
"build": "tsc",
23+
"bundle": "../../scripts/bundle.js package.json",
24+
"test": "ospec --globs 'build/**/*.test.js'"
25+
},
26+
"bundle": [
27+
{
28+
"entry": "src/bin.ts",
29+
"minify": false,
30+
"outfile": "dist/index.cjs",
31+
"external": [
32+
"sharp",
33+
"pino-pretty"
34+
]
35+
}
36+
],
37+
"type": "module",
38+
"engines": {
39+
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
40+
},
41+
"dependencies": {},
42+
"devDependencies": {
43+
"@basemaps/geo": "^6.40.0",
44+
"@basemaps/cli": "^6.40.0",
45+
"@basemaps/shared": "^6.40.0",
46+
"@basemaps/config": "6.40.0",
47+
"cmd-ts": "^0.12.1",
48+
"p-limit": "^4.0.0",
49+
"stac-ts": "^1.0.0"
50+
},
51+
"publishConfig": {
52+
"access": "public"
53+
},
54+
"files": [
55+
"build/"
56+
]
57+
}

‎packages/cogify/src/bin.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
Error.stackTraceLimit = 100;
2+
3+
import { LogConfig } from '@basemaps/shared';
4+
import { run } from 'cmd-ts';
5+
import { CogifyCli } from './cogify/cli.js';
6+
7+
run(CogifyCli, process.argv.slice(2)).catch((err) => {
8+
const logger = LogConfig.get();
9+
logger.fatal({ err }, 'Command:Failed');
10+
11+
// Give the logger some time to flush before exiting
12+
setTimeout(() => process.exit(1), 25);
13+
});

‎packages/cogify/src/cogify/__test__/covering.test.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import { GoogleTms, QuadKey } from '@basemaps/geo';
2+
import o from 'ospec';
3+
import { addChildren, addSurrounding } from '../covering.js';
4+
5+
o.spec('getChildren', () => {
6+
o('should get children', () => {
7+
o(addChildren({ z: 0, x: 0, y: 0 })).deepEquals([
8+
{ z: 1, x: 0, y: 0 },
9+
{ z: 1, x: 1, y: 0 },
10+
{ z: 1, x: 0, y: 1 },
11+
{ z: 1, x: 1, y: 1 },
12+
]);
13+
});
14+
15+
['', '3', '310', '013', '3100123', '3103123231312301'].map((qk) => {
16+
o('should match QuadKey: ' + qk, () => {
17+
const tileChildren = addChildren(QuadKey.toTile(qk));
18+
const qkChildren = QuadKey.children(qk).map(QuadKey.toTile);
19+
o(tileChildren).deepEquals(qkChildren);
20+
});
21+
});
22+
});
23+
24+
o.spec('SurroundingTiles', () => {
25+
o('should not have surrounding tiles at z0', () => {
26+
const todo = addSurrounding({ z: 0, x: 0, y: 0 }, GoogleTms);
27+
o(todo).deepEquals([]);
28+
});
29+
30+
o('should add all surrounding tiles', () => {
31+
o(addSurrounding({ z: 2, x: 1, y: 1 }, GoogleTms)).deepEquals([
32+
{ z: 2, x: 1, y: 0 },
33+
{ z: 2, x: 2, y: 1 },
34+
{ z: 2, x: 1, y: 2 },
35+
{ z: 2, x: 0, y: 1 },
36+
]);
37+
});
38+
39+
o('should wrap at matrix extent', () => {
40+
// Top left tile
41+
o(addSurrounding({ z: 2, x: 0, y: 0 }, GoogleTms)).deepEquals([
42+
{ z: 2, x: 0, y: 3 }, // North - Wrapping North to South
43+
{ z: 2, x: 1, y: 0 }, // East
44+
{ z: 2, x: 0, y: 1 }, // South
45+
{ z: 2, x: 3, y: 0 }, // West - Wrapping West to East
46+
]);
47+
48+
// Bottom right tile
49+
o(addSurrounding({ z: 2, x: 3, y: 3 }, GoogleTms)).deepEquals([
50+
{ z: 2, x: 3, y: 2 }, // North
51+
{ z: 2, x: 0, y: 3 }, // East -- Wrapping East to West
52+
{ z: 2, x: 3, y: 0 }, // South -- Wrapping South to NOrth
53+
{ z: 2, x: 2, y: 3 }, // West
54+
]);
55+
});
56+
});

‎packages/cogify/src/cogify/cli.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import { subcommands } from 'cmd-ts';
2+
import { BasemapsCogifyCreateCommand } from './cli/cli.cog.js';
3+
import { BasemapsCogifyCoverCommand } from './cli/cli.cover.js';
4+
5+
export const CogifyCli = subcommands({
6+
name: 'cogify',
7+
cmds: {
8+
cover: BasemapsCogifyCoverCommand,
9+
create: BasemapsCogifyCreateCommand,
10+
},
11+
});

0 commit comments

Comments
 (0)