Skip to content

Commit c7e3605

Browse files
blachablacha
authored
feat(cogify): add --preset lerc_0.01 to create a 1cm error lerc cog (#2841)
1 parent 0ea552e commit c7e3605

File tree

8 files changed

+110
-25
lines changed

8 files changed

+110
-25
lines changed

‎packages/cogify/src/cogify/cli/cli.cog.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { ProjectionLoader, TileId, TileMatrixSets } from '@basemaps/geo';
22
import { LogType, fsa } from '@basemaps/shared';
33
import { CliId, CliInfo } from '@basemaps/shared/build/cli/info.js';
4-
import { CogTiff } from '@cogeotiff/core';
4+
import { CogTiff, TiffTag } from '@cogeotiff/core';
55
import { Metrics } from '@linzjs/metrics';
66
import { command, flag, restPositionals } from 'cmd-ts';
77
import { mkdir, rm } from 'fs/promises';
@@ -11,11 +11,14 @@ import { CutlineOptimizer } from '../../cutline.js';
1111
import { SourceDownloader, urlToString } from '../../download.js';
1212
import { HashTransform } from '../../hash.stream.js';
1313
import { getLogger, logArguments } from '../../log.js';
14-
import { gdalBuildCog, gdalBuildVrt, gdalBuildVrtWarp } from '../gdal.js';
14+
import { gdalBuildCog, gdalBuildVrt, gdalBuildVrtWarp } from '../gdal.command.js';
1515
import { GdalRunner } from '../gdal.runner.js';
1616
import { Url } from '../parsers.js';
1717
import { CogifyCreationOptions, CogifyStacItem, getCutline, getSources } from '../stac.js';
1818

19+
// FIXME: HACK @cogeotiff/core to include the Lerc tiff tag
20+
if (TiffTag[0xc5f2] == null) (TiffTag as any)[0xc5f2] = 'Lerc';
21+
1922
function extractSourceFiles(item: CogifyStacItem, baseUrl: URL): URL[] {
2023
return item.links.filter((link) => link.rel === 'linz_basemaps:source').map((link) => new URL(link.href, baseUrl));
2124
}

‎packages/cogify/src/cogify/cli/cli.cover.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,14 @@ import { GoogleTms, Nztm2000QuadTms, TileId } from '@basemaps/geo';
44
import { fsa } from '@basemaps/shared';
55
import { CliId, CliInfo } from '@basemaps/shared/build/cli/info.js';
66
import { Metrics } from '@linzjs/metrics';
7-
import { command, number, option, optional, restPositionals, string } from 'cmd-ts';
7+
import { command, number, oneOf, option, optional, restPositionals, string } from 'cmd-ts';
88
import { isArgo } from '../../argo.js';
99
import { CutlineOptimizer } from '../../cutline.js';
1010
import { getLogger, logArguments } from '../../log.js';
1111
import { TileCoverContext, createTileCover } from '../../tile.cover.js';
1212
import { createFileStats } from '../stac.js';
1313
import { Url } from '../parsers.js';
14+
import { Presets } from '../../preset.js';
1415

1516
const SupportedTileMatrix = [GoogleTms, Nztm2000QuadTms];
1617

@@ -29,6 +30,13 @@ export const BasemapsCogifyCoverCommand = command({
2930
defaultValue: () => 20,
3031
}),
3132
paths: restPositionals({ type: Url, displayName: 'path', description: 'Path to source imagery' }),
33+
preset: option({
34+
type: oneOf(Object.keys(Presets)),
35+
long: 'preset',
36+
description: 'GDAL compression preset',
37+
defaultValue: () => 'webp',
38+
defaultValueIsSerializable: true,
39+
}),
3240
tileMatrix: option({
3341
type: string,
3442
long: 'tile-matrix',
@@ -62,6 +70,7 @@ export const BasemapsCogifyCoverCommand = command({
6270
logger,
6371
metrics,
6472
cutline,
73+
preset: args.preset,
6574
};
6675

6776
const res = await createTileCover(ctx);

packages/cogify/src/cogify/gdal.ts renamed to packages/cogify/src/cogify/gdal.command.ts

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,7 @@
11
import { Epsg, EpsgCode, TileMatrixSets } from '@basemaps/geo';
22
import { GdalCommand } from './gdal.runner.js';
33
import { CogifyCreationOptions } from './stac.js';
4-
5-
export const CogifyDefaults = {
6-
compression: 'webp',
7-
blockSize: 512,
8-
quality: 90,
9-
warpResampling: 'bilinear',
10-
overviewResampling: 'lanczos',
11-
} as const;
4+
import { Presets } from '../preset.js';
125

136
export function gdalBuildVrt(id: string, source: string[]): GdalCommand {
147
if (source.length === 0) throw new Error('No source files given for :' + id);
@@ -33,7 +26,7 @@ export function gdalBuildVrtWarp(
3326
['-wo', 'NUM_THREADS=ALL_CPUS'], // Multithread the warp
3427
['-s_srs', Epsg.get(sourceProjection).toEpsgString()], // Source EPSG
3528
['-t_srs', tileMatrix.projection.toEpsgString()], // Target EPSG
36-
['-r', opt.warpResampling ?? CogifyDefaults.warpResampling],
29+
opt.warpResampling ? ['-r', opt.warpResampling] : undefined,
3730
cutline.path ? ['-cutline', cutline.path, '-cblend', cutline.blend] : undefined,
3831
sourceVrt,
3932
id + '.' + tileMatrix.identifier + '.vrt',
@@ -45,7 +38,7 @@ export function gdalBuildVrtWarp(
4538
}
4639

4740
export function gdalBuildCog(id: string, sourceVrt: string, opt: CogifyCreationOptions): GdalCommand {
48-
const cfg = { ...CogifyDefaults, ...opt };
41+
const cfg = { ...Presets[opt.preset], ...opt };
4942
const tileMatrix = TileMatrixSets.find(cfg.tileMatrix);
5043
if (tileMatrix == null) throw new Error('Unable to find tileMatrix: ' + cfg.tileMatrix);
5144

@@ -68,20 +61,22 @@ export function gdalBuildCog(id: string, sourceVrt: string, opt: CogifyCreationO
6861
['-of', 'COG'],
6962
['-co', 'NUM_THREADS=ALL_CPUS'], // Use all CPUS
7063
['--config', 'GDAL_NUM_THREADS', 'all_cpus'], // Also required to NUM_THREADS till gdal 3.7.x
71-
['-co', 'BIGTIFF=YES'], // Default to BIG_TIFF
64+
['-co', 'BIGTIFF=IF_NEEDED'], // BigTiff is somewhat slower and most (All?) of the COGS should be well below 4GB
7265
['-co', 'ADD_ALPHA=YES'],
7366
['-co', 'BLOCKSIZE=512'],
7467
['-co', `WARP_RESAMPLING=${cfg.warpResampling}`],
7568
['-co', `OVERVIEW_RESAMPLING=${cfg.overviewResampling}`],
7669
['-co', `COMPRESS=${cfg.compression}`],
77-
['-co', `QUALITY=${cfg.quality}`],
70+
cfg.quality ? ['-co', `QUALITY=${cfg.quality}`] : undefined,
71+
cfg.maxZError ? ['-co', `MAX_Z_ERROR=${cfg.maxZError}`] : undefined,
7872
['-co', 'SPARSE_OK=YES'],
7973
['-co', `TARGET_SRS=${tileMatrix.projection.toEpsgString()}`],
8074
['-co', `EXTENT=${tileExtent.join(',')},`],
8175
['-tr', targetResolution, targetResolution],
8276
sourceVrt,
8377
targetTiff,
8478
]
79+
.filter((f) => f != null)
8580
.flat()
8681
.map(String),
8782
};

‎packages/cogify/src/cogify/gdal.runner.ts‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { sha256base58 } from '@basemaps/config';
22
import { LogType } from '@basemaps/shared';
33
import { spawn } from 'child_process';
44
import { EventEmitter } from 'events';
5+
import { dirname } from 'path';
56

67
export interface GdalCommand {
78
/** Output file location */
@@ -12,6 +13,22 @@ export interface GdalCommand {
1213
args: string[];
1314
}
1415

16+
function getDockerContainer(): string {
17+
const containerPath = process.env['GDAL_DOCKER_CONTAINER'] ?? 'ghcr.io/osgeo/gdal';
18+
const tag = process.env['GDAL_DOCKER_CONTAINER_TAG'] ?? 'ubuntu-small-3.7.0';
19+
return `${containerPath}:${tag}`;
20+
}
21+
22+
/** Convert a GDAL command to run using docker */
23+
function toDockerArgs(cmd: GdalCommand): string[] {
24+
const dirName = dirname(cmd.output);
25+
26+
const args = ['run'];
27+
if (cmd.output) args.push(...['-v', `${dirName}:${dirName}`]);
28+
args.push(...[getDockerContainer(), cmd.command, ...cmd.args]);
29+
return args;
30+
}
31+
1532
export class GdalRunner {
1633
parser: GdalProgressParser = new GdalProgressParser();
1734
startTime: number;
@@ -52,7 +69,12 @@ export class GdalRunner {
5269
});
5370
this.startTime = performance.now();
5471

55-
const child = spawn(this.cmd.command, this.cmd.args);
72+
const useDocker = !!process.env['GDAL_DOCKER'];
73+
if (useDocker) {
74+
logger?.info({ command: this.cmd.command, commandHash, container: getDockerContainer() }, 'Gdal:Docker');
75+
}
76+
77+
const child = useDocker ? spawn('docker', toDockerArgs(this.cmd)) : spawn(this.cmd.command, this.cmd.args);
5678

5779
const outputBuff: Buffer[] = [];
5880
const errBuff: Buffer[] = [];

‎packages/cogify/src/cogify/stac.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ import { createHash } from 'node:crypto';
33
import { StacCollection, StacItem, StacLink } from 'stac-ts';
44

55
export interface CogifyCreationOptions {
6+
/** Preset GDAL config to use */
7+
preset: string;
8+
69
/** Tile to be created */
710
tile: Tile;
811

@@ -16,7 +19,7 @@ export interface CogifyCreationOptions {
1619
*
1720
* @default 'webp'
1821
*/
19-
compression?: 'webp' | 'jpeg';
22+
compression?: 'webp' | 'jpeg' | 'lerc';
2023

2124
/**
2225
* Output tile size
@@ -35,6 +38,9 @@ export interface CogifyCreationOptions {
3538
*/
3639
quality?: number;
3740

41+
/** Max Z Error only used when compression is `lerc` */
42+
maxZError?: number;
43+
3844
/**
3945
* Resampling for warping
4046
* @default 'bilinear'

‎packages/cogify/src/download.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ export class SourceDownloader {
5454
if (asset.asset == null) return false;
5555
// No more items need this asset, clean it up
5656
const targetFile = await asset.asset;
57-
logger.info({ source: asset.url, target: targetFile }, 'Cog:Source:Cleanup');
57+
logger.debug({ source: asset.url, target: targetFile }, 'Cog:Source:Cleanup');
5858
await fsa.delete(targetFile);
5959
return true;
6060
}
@@ -100,7 +100,7 @@ export class SourceDownloader {
100100
const targetFile = fsa.joinAll(this.cachePath, 'source', newFileName);
101101

102102
await this._checkHost(asset.url);
103-
logger.debug({ source: asset.url, target: targetFile }, 'Cog:Source:Download');
103+
logger.trace({ source: asset.url, target: targetFile }, 'Cog:Source:Download');
104104
const hashStream = fsa.stream(urlToString(asset.url)).pipe(new HashTransform('sha256'));
105105
const startTime = performance.now();
106106
await fsa.write(targetFile, hashStream);

‎packages/cogify/src/preset.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import { CogifyCreationOptions } from './cogify/stac';
2+
3+
export const CogifyDefaults = {
4+
compression: 'webp',
5+
blockSize: 512,
6+
quality: 90,
7+
warpResampling: 'bilinear',
8+
overviewResampling: 'lanczos',
9+
} as const;
10+
11+
export interface Preset {
12+
name: string;
13+
options: Partial<CogifyCreationOptions>;
14+
}
15+
16+
const webP: Preset = {
17+
name: 'webp',
18+
options: {
19+
blockSize: CogifyDefaults.blockSize,
20+
compression: CogifyDefaults.compression,
21+
quality: CogifyDefaults.quality,
22+
warpResampling: CogifyDefaults.warpResampling,
23+
overviewResampling: CogifyDefaults.overviewResampling,
24+
},
25+
};
26+
27+
const lerc10mm: Preset = {
28+
name: 'lerc_10mm',
29+
options: {
30+
blockSize: CogifyDefaults.blockSize,
31+
compression: 'lerc',
32+
maxZError: 0.01,
33+
// TODO should a different resampling be used for LERC?
34+
warpResampling: CogifyDefaults.warpResampling,
35+
overviewResampling: CogifyDefaults.overviewResampling,
36+
},
37+
};
38+
39+
const lerc1mm: Preset = {
40+
name: 'lerc_1mm',
41+
options: {
42+
blockSize: CogifyDefaults.blockSize,
43+
compression: 'lerc',
44+
maxZError: 0.001,
45+
// TODO should a different resampling be used for LERC?
46+
warpResampling: CogifyDefaults.warpResampling,
47+
overviewResampling: CogifyDefaults.overviewResampling,
48+
},
49+
};
50+
51+
export const Presets = { [webP.name]: webP, [lerc10mm.name]: lerc10mm, [lerc1mm.name]: lerc1mm };

‎packages/cogify/src/tile.cover.ts‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import { MultiPolygon, intersection, toFeatureCollection, union } from '@linzjs/
66
import { Metrics } from '@linzjs/metrics';
77
import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js';
88
import { createCovering } from './cogify/covering.js';
9-
import { CogifyDefaults } from './cogify/gdal.js';
109
import {
1110
CogifyLinkCutline,
1211
CogifyLinkSource,
@@ -15,6 +14,7 @@ import {
1514
createFileStats,
1615
} from './cogify/stac.js';
1716
import { CutlineOptimizer } from './cutline.js';
17+
import { Presets } from './preset.js';
1818

1919
export interface TileCoverContext {
2020
/** Unique id for the covering */
@@ -29,6 +29,8 @@ export interface TileCoverContext {
2929
metrics?: Metrics;
3030
/** Optional logger to trace covering creation */
3131
logger?: LogType;
32+
/** GDAL configuration preset */
33+
preset: string;
3234
}
3335
export interface TileCoverResult {
3436
/** Stac collection for the imagery */
@@ -136,15 +138,12 @@ export async function createTileCover(ctx: TileCoverContext): Promise<TileCoverR
136138
end_datetime: dateTime.end ?? undefined,
137139
'proj:epsg': ctx.tileMatrix.projection.code,
138140
'linz_basemaps:options': {
141+
preset: ctx.preset,
142+
...Presets[ctx.preset].options,
139143
tile,
140144
tileMatrix: ctx.tileMatrix.identifier,
141145
sourceEpsg: ctx.imagery.projection,
142-
blockSize: CogifyDefaults.blockSize,
143-
compression: CogifyDefaults.compression,
144-
quality: CogifyDefaults.quality,
145146
zoomLevel: targetBaseZoom,
146-
warpResampling: CogifyDefaults.warpResampling,
147-
overviewResampling: CogifyDefaults.overviewResampling,
148147
},
149148
'linz_basemaps:generated': {
150149
package: CliInfo.package,

0 commit comments

Comments
 (0)