fix(config): allow initializing config from URLs (#2830)

blacha · web-flow · commit 0ea552ec32ad · 2023-07-11T22:22:39.000Z
diff --git a/packages/cogify/src/cogify/cli/cli.config.ts b/packages/cogify/src/cogify/cli/cli.config.ts
@@ -1,5 +1,5 @@
 import { ConfigProviderMemory, base58 } from '@basemaps/config';
-import { ConfigImageryTiff, initConfigFromPaths } from '@basemaps/config/build/json/tiff.config.js';
+import { ConfigImageryTiff, initConfigFromUrls } from '@basemaps/config/build/json/tiff.config.js';
 import { Projection, TileMatrixSets } from '@basemaps/geo';
 import { fsa } from '@basemaps/shared';
 import { CliInfo } from '@basemaps/shared/build/cli/info.js';
@@ -30,7 +30,7 @@ export const BasemapsCogifyConfigCommand = command({
 
     const mem = new ConfigProviderMemory();
     metrics.start('imagery:load');
-    const cfg = await initConfigFromPaths(mem, [urlToString(args.path)]);
+    const cfg = await initConfigFromUrls(mem, [args.path]);
     metrics.end('imagery:load');
     logger.info({ imagery: cfg.imagery.length, titles: cfg.imagery.map((f) => f.title) }, 'ImageryConfig:Loaded');
 
diff --git a/packages/cogify/src/cogify/cli/cli.cover.ts b/packages/cogify/src/cogify/cli/cli.cover.ts
@@ -1,5 +1,5 @@
 import { ConfigProviderMemory } from '@basemaps/config';
-import { initConfigFromPaths } from '@basemaps/config/build/json/tiff.config.js';
+import { initConfigFromUrls } from '@basemaps/config/build/json/tiff.config.js';
 import { GoogleTms, Nztm2000QuadTms, TileId } from '@basemaps/geo';
 import { fsa } from '@basemaps/shared';
 import { CliId, CliInfo } from '@basemaps/shared/build/cli/info.js';
@@ -10,6 +10,7 @@ import { CutlineOptimizer } from '../../cutline.js';
 import { getLogger, logArguments } from '../../log.js';
 import { TileCoverContext, createTileCover } from '../../tile.cover.js';
 import { createFileStats } from '../stac.js';
+import { Url } from '../parsers.js';
 
 const SupportedTileMatrix = [GoogleTms, Nztm2000QuadTms];
 
@@ -27,7 +28,7 @@ export const BasemapsCogifyCoverCommand = command({
       description: 'Cutline blend amount see GDAL_TRANSLATE -cblend',
       defaultValue: () => 20,
     }),
-    paths: restPositionals({ type: string, displayName: 'path', description: 'Path to source imagery' }),
+    paths: restPositionals({ type: Url, displayName: 'path', description: 'Path to source imagery' }),
     tileMatrix: option({
       type: string,
       long: 'tile-matrix',
@@ -40,7 +41,7 @@ export const BasemapsCogifyCoverCommand = command({
 
     const mem = new ConfigProviderMemory();
     metrics.start('imagery:load');
-    const cfg = await initConfigFromPaths(mem, args.paths);
+    const cfg = await initConfigFromUrls(mem, args.paths);
     const imageryLoadTime = metrics.end('imagery:load');
     if (cfg.imagery.length === 0) throw new Error('No imagery found');
     const im = cfg.imagery[0];
diff --git a/packages/cogify/src/cogify/parsers.ts b/packages/cogify/src/cogify/parsers.ts
@@ -10,8 +10,6 @@ export const Url: Type<string, URL> = {
     try {
       return new URL(str);
     } catch (e) {
-      // Possibly already a URL
-      if (str.includes(':')) throw e;
       return pathToFileURL(str);
     }
   },
diff --git a/packages/config/src/json/__tests__/config.loader.test.ts b/packages/config/src/json/__tests__/config.loader.test.ts
@@ -0,0 +1,60 @@
+import { fsa } from '@chunkd/fs';
+import { FsMemory, SourceMemory } from '@chunkd/source-memory';
+import { fileURLToPath } from 'node:url';
+import o from 'ospec';
+import { ConfigProviderMemory } from '../../memory/memory.config.js';
+import { initConfigFromUrls } from '../tiff.config.js';
+
+const simpleTiff = new URL('../../../../__tests__/static/rgba8_tiled.tiff', import.meta.url);
+
+o.spec('config import', () => {
+  const fsMemory = new FsMemory();
+
+  // TODO SourceMemory adds `memory://` to every url even if it already has a `memory://` prefix
+  fsMemory.source = (filePath): SourceMemory => {
+    const bytes = fsMemory.files.get(filePath);
+    if (bytes == null) throw new Error('Failed to load file: ' + filePath);
+    return new SourceMemory(filePath.replace('memory://', ''), bytes);
+  };
+
+  o.before(() => fsa.register('memory://', fsMemory));
+  o.beforeEach(() => fsMemory.files.clear());
+
+  o('should load tiff from filesystem', async () => {
+    const buf = await fsa.read(fileURLToPath(simpleTiff));
+    await fsa.write('memory://tiffs/tile-tiff-name/tiff-a.tiff', buf);
+
+    const cfg = new ConfigProviderMemory();
+    const ret = await initConfigFromUrls(cfg, [new URL('memory://tiffs/tile-tiff-name')]);
+
+    o(ret.imagery.length).equals(1);
+    const imagery = ret.imagery[0];
+    o(imagery.name).equals('tile-tiff-name');
+    o(imagery.files).deepEquals([{ name: 'tiff-a.tiff', x: 0, y: -64, width: 64, height: 64 }]);
+  });
+
+  o('should create multiple imagery layers from multiple folders', async () => {
+    const buf = await fsa.read(fileURLToPath(simpleTiff));
+    await fsa.write('memory://tiffs/tile-tiff-a/tiff-a.tiff', buf);
+    await fsa.write('memory://tiffs/tile-tiff-b/tiff-b.tiff', buf);
+
+    const cfg = new ConfigProviderMemory();
+    const ret = await initConfigFromUrls(cfg, [
+      new URL('memory://tiffs/tile-tiff-a'),
+      new URL('memory://tiffs/tile-tiff-b/'),
+    ]);
+
+    o(ret.imagery.length).equals(2);
+    o(ret.imagery[0].name).equals('tile-tiff-a');
+    o(ret.imagery[0].files).deepEquals([{ name: 'tiff-a.tiff', x: 0, y: -64, width: 64, height: 64 }]);
+
+    o(ret.imagery[1].name).equals('tile-tiff-b');
+    o(ret.imagery[1].files).deepEquals([{ name: 'tiff-b.tiff', x: 0, y: -64, width: 64, height: 64 }]);
+
+    o(ret.tileSet.layers.length).equals(2);
+    o(ret.tileSet.layers[0][3857]).equals(ret.imagery[0].id);
+    o(ret.tileSet.layers[0].name).equals(ret.imagery[0].name);
+    o(ret.tileSet.layers[1][3857]).equals(ret.imagery[1].id);
+    o(ret.tileSet.layers[1].name).equals(ret.imagery[1].name);
+  });
+});
diff --git a/packages/config/src/json/tiff.config.ts b/packages/config/src/json/tiff.config.ts
@@ -10,8 +10,9 @@ import {
 import { fsa } from '@chunkd/fs';
 import { CogTiff } from '@cogeotiff/core';
 import pLimit, { LimitFunction } from 'p-limit';
-import { basename, resolve } from 'path';
+import { basename } from 'path';
 import { StacCollection } from 'stac-ts';
+import { fileURLToPath } from 'url';
 import { sha256base58 } from '../base58.node.js';
 import { ConfigImagery } from '../config/imagery.js';
 import { ConfigTileSetRaster, TileSetType } from '../config/tile.set.js';
@@ -47,9 +48,10 @@ export type ConfigImageryTiff = ConfigImagery & TiffSummary;
  *
  * @throws if any of the tiffs have differing EPSG or GSD
  **/
-function computeTiffSummary(target: string, tiffs: CogTiff[]): TiffSummary {
+function computeTiffSummary(target: URL, tiffs: CogTiff[]): TiffSummary {
   const res: Partial<TiffSummary> = { files: [] };
 
+  const targetPath = urlToString(target);
   let bounds: Bounds | undefined;
   for (const tiff of tiffs) {
     const firstImage = tiff.getImage(0);
@@ -78,7 +80,9 @@ function computeTiffSummary(target: string, tiffs: CogTiff[]): TiffSummary {
     else bounds = bounds.union(imgBounds);
 
     if (res.files == null) res.files = [];
-    res.files.push({ name: tiff.source.uri, ...imgBounds });
+
+    const relativePath = toRelative(targetPath, tiff.source.uri);
+    res.files.push({ name: relativePath, ...imgBounds });
   }
   res.bounds = bounds?.toJson();
   if (res.bounds == null) throw new Error('Failed to extract imagery bounds from:' + target);
@@ -87,11 +91,28 @@ function computeTiffSummary(target: string, tiffs: CogTiff[]): TiffSummary {
   return res as TiffSummary;
 }
 
+/** Convert a path to a relative path
+ * @param base the path to be relative to
+ * @param other the path to convert
+ */
+function toRelative(base: string, other: string): string {
+  if (!other.startsWith(base)) throw new Error('Paths are not relative');
+  const part = other.slice(base.length);
+  if (part.startsWith('/') || part.startsWith('\\')) return part.slice(1);
+  return part;
+}
+
+/** Convert a URL to a string using fileUrlToPath if the URL is a file:// */
+function urlToString(u: URL): string {
+  if (u.protocol === 'file:') return fileURLToPath(u);
+  return u.href;
+}
+
 /** Attempt to read a stac collection.json from the target path if it exists or return null if anything goes wrong. */
-async function loadStacFromPath(target: string): Promise<StacCollection | null> {
-  const collectionPath = fsa.join(target, 'collection.json');
+async function loadStacFromURL(target: URL): Promise<StacCollection | null> {
+  const collectionPath = new URL('collection.json', target);
   try {
-    return await fsa.readJson(collectionPath);
+    return await fsa.readJson(urlToString(collectionPath));
   } catch (e) {
     return null;
   }
@@ -104,30 +125,31 @@ async function loadStacFromPath(target: string): Promise<StacCollection | null>
  *
  * @returns Imagery configuration generated from the path
  */
-export async function imageryFromTiffPath(target: string, Q: LimitFunction, log?: LogType): Promise<ConfigImageryTiff> {
-  const sourceFiles = await fsa.toArray(fsa.list(target));
+export async function imageryFromTiffUrl(target: URL, Q: LimitFunction, log?: LogType): Promise<ConfigImageryTiff> {
+  const targetPath = urlToString(target);
+  const sourceFiles = await fsa.toArray(fsa.list(targetPath));
   const tiffs = await Promise.all(
     sourceFiles.filter(isTiff).map((c) => Q(() => new CogTiff(fsa.source(c)).init(true))),
   );
 
   try {
-    const stac = await loadStacFromPath(target);
+    const stac = await loadStacFromURL(target);
     const params = computeTiffSummary(target, tiffs);
 
-    const folderName = basename(target);
+    const folderName = basename(targetPath);
     const title = stac?.title ?? folderName;
     const tileMatrix =
       params.projection === EpsgCode.Nztm2000 ? Nztm2000QuadTms : TileMatrixSets.tryGet(params.projection);
 
     const imagery: ConfigImageryTiff = {
-      id: sha256base58(target),
+      id: sha256base58(target.href),
       name: folderName,
       title,
       updatedAt: Date.now(),
       projection: params.projection,
       tileMatrix: tileMatrix?.identifier ?? 'none',
       gsd: params.gsd,
-      uri: resolve(target),
+      uri: targetPath,
       bounds: params.bounds,
       files: params.files,
       collection: stac ?? undefined,
@@ -177,16 +199,16 @@ export async function imageryFromTiffPath(target: string, Q: LimitFunction, log?
  * @param concurrency number of tiff files to load at a time
  * @returns
  */
-export async function initConfigFromPaths(
+export async function initConfigFromUrls(
   provider: ConfigProviderMemory,
-  targets: string[],
+  targets: URL[],
   concurrency = 25,
   log?: LogType,
 ): Promise<{ tileSet: ConfigTileSetRaster; imagery: ConfigImageryTiff[] }> {
   const q = pLimit(concurrency);
 
   const imageryConfig: Promise<ConfigImageryTiff>[] = [];
-  for (const target of targets) imageryConfig.push(imageryFromTiffPath(target, q, log));
+  for (const target of targets) imageryConfig.push(imageryFromTiffUrl(target, q, log));
 
   const aerialTileSet: ConfigTileSetRaster = {
     id: 'ts_aerial',
diff --git a/packages/lambda-tiler/src/cli/render.tile.ts b/packages/lambda-tiler/src/cli/render.tile.ts
@@ -1,13 +1,14 @@
 import { ConfigProviderMemory } from '@basemaps/config';
-import { initConfigFromPaths } from '@basemaps/config/build/json/tiff.config.js';
+import { initConfigFromUrls } from '@basemaps/config/build/json/tiff.config.js';
 import { GoogleTms, ImageFormat } from '@basemaps/geo';
 import { LogConfig, setDefaultConfig } from '@basemaps/shared';
 import { fsa } from '@chunkd/fs';
 import { LambdaHttpRequest, LambdaUrlRequest, UrlEvent } from '@linzjs/lambda';
 import { Context } from 'aws-lambda';
 import { TileXyzRaster } from '../routes/tile.xyz.raster.js';
+import { pathToFileURL } from 'url';
 
-const target = `/home/blacha/tmp/basemaps/white-lines/nz-0.5m/`;
+const target = pathToFileURL(`/home/blacha/tmp/basemaps/white-lines/nz-0.5m/`);
 const tile = { z: 10, x: 1013, y: 633 };
 const tileMatrix = GoogleTms;
 const imageFormat = ImageFormat.Webp;
@@ -16,12 +17,12 @@ async function main(): Promise<void> {
   const log = LogConfig.get();
   const provider = new ConfigProviderMemory();
   setDefaultConfig(provider);
-  const { tileSet, imagery } = await initConfigFromPaths(provider, [target]);
+  const { tileSet, imagery } = await initConfigFromUrls(provider, [target]);
 
   if (tileSet.layers.length === 0) throw new Error('No imagery found in path: ' + target);
   log.info({ tileSet: tileSet.name, layers: tileSet.layers.length }, 'TileSet:Loaded');
   for (const im of imagery) {
-    log.info({ imagery: im.uri, title: im.title, tileMatrix: im.tileMatrix, files: im.files.length }, 'Imagery:Loaded');
+    log.info({ url: im.uri, title: im.title, tileMatrix: im.tileMatrix, files: im.files.length }, 'Imagery:Loaded');
   }
   const request = new LambdaUrlRequest({ headers: {} } as UrlEvent, {} as Context, log) as LambdaHttpRequest;
 
diff --git a/packages/server/src/cli.ts b/packages/server/src/cli.ts
@@ -1,11 +1,25 @@
 import { Env, LogConfig } from '@basemaps/shared';
 import { CliInfo } from '@basemaps/shared/build/cli/info.js';
-import { command, flag, number, option, optional, restPositionals, string } from 'cmd-ts';
+import { Type, command, flag, number, option, optional, restPositionals, string } from 'cmd-ts';
+import { pathToFileURL } from 'node:url';
 import { createServer } from './server.js';
 
 CliInfo.package = 'basemaps/server';
 
 const DefaultPort = 5000;
+/**
+ * Parse a input parameter as a URL,
+ * if it looks like a file path convert it using `pathToFileURL`
+ **/
+export const Url: Type<string, URL> = {
+  async from(str) {
+    try {
+      return new URL(str);
+    } catch (e) {
+      return pathToFileURL(str);
+    }
+  },
+};
 
 export const BasemapsServerCommand = command({
   name: 'basemaps-server',
@@ -26,7 +40,7 @@ export const BasemapsServerCommand = command({
       long: 'assets',
       description: 'Where the assets (sprites, fonts) are located',
     }),
-    paths: restPositionals({ type: string, displayName: 'path', description: 'Path to imagery' }),
+    paths: restPositionals({ type: Url, displayName: 'path', description: 'Path to imagery' }),
   },
   handler: async (args) => {
     const logger = LogConfig.get();
diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts
@@ -6,15 +6,15 @@ import {
   ConfigProviderDynamo,
   ConfigProviderMemory,
 } from '@basemaps/config';
-import { initConfigFromPaths } from '@basemaps/config/build/json/tiff.config.js';
+import { initConfigFromUrls } from '@basemaps/config/build/json/tiff.config.js';
 import { fsa, getDefaultConfig, LogType } from '@basemaps/shared';
 
 export type ServerOptions = ServerOptionsTiffs | ServerOptionsConfig;
 
 /** Load configuration from folders */
 export interface ServerOptionsTiffs {
   assets?: string;
-  paths: string[];
+  paths: URL[];
 }
 
 /** Load configuration from a config file/dynamodb */
@@ -34,7 +34,7 @@ export async function loadConfig(opts: ServerOptions, logger: LogType): Promise<
   // Load the config directly from the source tiff files
   if ('paths' in opts) {
     const mem = new ConfigProviderMemory();
-    const ret = await initConfigFromPaths(mem, opts.paths);
+    const ret = await initConfigFromUrls(mem, opts.paths);
     logger.info({ tileSet: ret.tileSet.name, layers: ret.tileSet.layers.length }, 'TileSet:Loaded');
     for (const im of ret.imagery) {
       logger.info(

Original file line number	Diff line number	Diff line change
`@@ -10,8 +10,6 @@ export const Url: Type<string, URL> = {`
`10`	`10`	`try {`
`11`	`11`	`return new URL(str);`
`12`	`12`	`} catch (e) {`
`13`		`- // Possibly already a URL`
`14`		`- if (str.includes(':')) throw e;`
`15`	`13`	`return pathToFileURL(str);`
`16`	`14`	`}`
`17`	`15`	`},`