feat(config): Insert a config bundle records in dynamodb to refference the config file in s3. (#2335)

* Insert a config bundle records in dynamodb to refference the config file in s3.

* Add configbunle to config provider memory

Author: Wentao-Kuang

packages/cli/src/cli/config/action.import.ts

@@ -1,5 +1,5 @@
 import { Env, fsa, LogConfig } from '@basemaps/shared';
-import { BaseConfig, ConfigBundled, ConfigProviderMemory } from '@basemaps/config';
+import { BaseConfig, Config, ConfigBundle, ConfigBundled, ConfigProviderMemory } from '@basemaps/config';
 import { CommandLineAction, CommandLineFlagParameter, CommandLineStringParameter } from '@rushstack/ts-command-line';
 import { Q, Updater } from './config.update.js';
 import { invalidateCache } from '../util.js';
@@ -27,7 +27,7 @@ export class CommandImport extends CommandLineAction {
     this.config = this.defineStringParameter({
       argumentName: 'CONFIG',
       parameterLongName: '--config',
-      description: 'Path of config json',
+      description: 'Path of config json, this can be both a local path or s3 location',
       required: true,
     });
     this.backup = this.defineStringParameter({
@@ -49,6 +49,8 @@ export class CommandImport extends CommandLineAction {
     const config = this.config.value;
     const backup = this.backup.value;
     if (config == null) throw new Error('Please provide a config json');
+    if (commit && !config.startsWith('s3://'))
+      throw new Error('To acturally import into dynamo has to use the config file from s3.');
 
     const HostPrefix = Env.isProduction() ? '' : 'dev.';
     const healthEndpoint = `https://${HostPrefix}basemaps.linz.govt.nz/v1/health`;
@@ -68,6 +70,17 @@ export class CommandImport extends CommandLineAction {
     for (const config of mem.objects.values()) this.update(config, commit);
     await Promise.all(this.promises);
 
+    if (commit) {
+      const configBundle: ConfigBundle = {
+        id: Config.ConfigBundle.id(configJson.id),
+        name: Config.ConfigBundle.id(`config-${configJson.hash}.json`),
+        path: config,
+        hash: configJson.hash,
+      };
+      logger.info({ config }, 'Import:ConfigBundle');
+      if (Config.ConfigBundle.isWriteable()) await Config.ConfigBundle.put(configBundle);
+    }
+
     if (commit && this.invalidations.length > 0) {
       // Lots of invalidations just invalidate everything
       if (this.invalidations.length > 10) {

packages/config/src/base.config.ts

@@ -1,5 +1,6 @@
 import { Epsg } from '@basemaps/geo';
 import { BaseConfig } from './config/base.js';
+import { ConfigBundle } from './config/config.bundle.js';
 import { ConfigPrefix, ConfigPrefixes } from './config/prefix.js';
 import { ConfigLayer, ConfigTileSet, TileSetType } from './config/tile.set.js';
 import {
@@ -34,6 +35,10 @@ export class ConfigInstance {
     return this.cfg.ProcessingJob;
   }
 
+  get ConfigBundle(): BasemapsConfigObject<ConfigBundle> {
+    return this.cfg.ConfigBundle;
+  }
+
   setConfigProvider(cfg: BasemapsConfigProvider): void {
     this.cfg = cfg;
   }
@@ -93,6 +98,7 @@ export abstract class BasemapsConfigProvider {
   abstract Style: BasemapsConfigObject<ConfigVectorStyle>;
   abstract Provider: BasemapsConfigObject<ConfigProvider>;
   abstract ProcessingJob: BasemapsConfigObject<ConfigProcessingJob>;
+  abstract ConfigBundle: BasemapsConfigObject<ConfigBundle>;
 }
 
 export abstract class BasemapsConfigObject<T extends BaseConfig> {

packages/config/src/config/config.bundle.ts

@@ -0,0 +1,9 @@
+import { BaseConfig } from './base.js';
+
+export interface ConfigBundle extends BaseConfig {
+  /** Path for the config bundle file */
+  path: string;
+
+  /** Hash of the config bundle file */
+  hash: string;
+}

packages/config/src/dynamo/dynamo.config.ts

@@ -9,6 +9,7 @@ import { ConfigTileSet } from '../config/tile.set.js';
 import { ConfigVectorStyle } from '../config/vector.style.js';
 import { ConfigDynamoBase } from './dynamo.config.base.js';
 import { ConfigDynamoCached } from './dynamo.config.cached.js';
+import { ConfigBundle } from '../config/config.bundle.js';
 
 export class ConfigProviderDynamo extends BasemapsConfigProvider {
   Prefix = ConfigPrefix;
@@ -22,6 +23,7 @@ export class ConfigProviderDynamo extends BasemapsConfigProvider {
   TileSet = new ConfigDynamoBase<ConfigTileSet>(this, ConfigPrefix.TileSet);
   Provider = new ConfigDynamoCached<ConfigProvider>(this, ConfigPrefix.Provider);
   ProcessingJob = new ConfigDynamoBase<ConfigProcessingJob>(this, ConfigPrefix.ProcessingJob);
+  ConfigBundle = new ConfigDynamoBase<ConfigBundle>(this, ConfigPrefix.ConfigBundle);
 
   constructor(tableName: string) {
     super();

packages/config/src/index.ts

@@ -3,6 +3,7 @@ export { BaseConfig } from './config/base.js';
 export { ConfigImagery } from './config/imagery.js';
 export { ConfigPrefix } from './config/prefix.js';
 export { ConfigProvider } from './config/provider.js';
+export { ConfigBundle } from './config/config.bundle.js';
 export {
   JobStatus,
   ProcessingJob,

packages/config/src/memory/memory.config.ts

@@ -9,6 +9,7 @@ import { ConfigProcessingJob } from '../config/processing.job.js';
 import { ConfigProvider } from '../config/provider.js';
 import { ConfigTileSet, TileSetType } from '../config/tile.set.js';
 import { ConfigVectorStyle } from '../config/vector.style.js';
+import { ConfigBundle } from '../config/config.bundle.js';
 import { standardizeLayerName } from '../json/name.convertor.js';
 
 /** bundle the configuration as a single JSON object */
@@ -51,6 +52,7 @@ export class ConfigProviderMemory extends BasemapsConfigProvider {
   TileSet = new MemoryConfigObject<ConfigTileSet>(this, ConfigPrefix.TileSet);
   Provider = new MemoryConfigObject<ConfigProvider>(this, ConfigPrefix.Provider);
   ProcessingJob = new MemoryConfigObject<ConfigProcessingJob>(this, ConfigPrefix.ProcessingJob);
+  ConfigBundle = new MemoryConfigObject<ConfigBundle>(this, ConfigPrefix.ConfigBundle);
 
   /** Memory cache of all objects */
   objects = new Map<string, BaseConfig>();