feat!: consolidate storage schemas and add Redis provider (MAPCO-11262)

.github/workflows/pull-request.yaml

@@ -14,12 +14,10 @@ jobs:
     steps:
       - name: Check out TS Project Git repository
         uses: actions/checkout@v4
-
       - name: Init nodejs
         uses: ./.github/actions/init-npm
         with:
           node-version: ${{ matrix.node }}
-
       - name: Run TS Project linters
         uses: wearerequired/lint-action@v2
         with:
@@ -40,14 +38,47 @@ jobs:
     steps:
       - name: Check out TS Project Git repository
         uses: actions/checkout@v4
-
       - name: Init nodejs
         uses: ./.github/actions/init-npm
-
       - name: Run tests
         run: npm run test
-
       - uses: actions/upload-artifact@v4
         with:
           name: Test Reporters ${{ matrix.node }}
           path: ./reports/**
+
+  upload_package:
+    needs: [eslint, tests]
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out TS Project Git repository
+        uses: actions/checkout@v6
+      - name: Init nodejs
+        uses: ./.github/actions/init-npm
+      - name: Pack the package
+        run: npm pack
+      - name: get properties
+        id: json_properties
+        uses: ActionsTools/read-json-action@main
+        with:
+          file_path: 'package.json'
+      - uses: azure/CLI@v2
+        with:
+          inlineScript: |
+            az storage blob upload --name raster-shared-${{ github.sha }}.tgz --account-name ghatmpstorage --container-name=npm-packages --file map-colonies-raster-shared-${{steps.json_properties.outputs.version}}.tgz --sas-token "${{ secrets.SAS_TOKEN }}"
+      - name: Comment on PR
+        uses: thollander/actions-comment-pull-request@v3
+        with:
+          message: |
+            Hi it is me, your friendly bot helper! :wave:
+            I've packed this commit for you :blush:
+            You can install it like this:
+            ```json
+            {
+              "dependencies": {
+                  "@map-colonies/raster-shared": "https://ghatmpstorage.blob.core.windows.net/npm-packages/raster-shared-${{ github.sha }}.tgz"
+              }
+            }
+            ```
+            The link will expire in a week :hourglass:

src/constants/core/constants.ts

@@ -67,6 +67,15 @@ export const SourceType = {
 
 export type SourceType = (typeof SourceType)[keyof typeof SourceType];
 
+/* eslint-disable @typescript-eslint/naming-convention */
+export const StorageProvider = {
+  ...pickEnum(SourceType, ['FS', 'S3']),
+  REDIS: 'REDIS',
+} as const;
+/* eslint-enable @typescript-eslint/naming-convention */
+
+export type StorageProvider = (typeof StorageProvider)[keyof typeof StorageProvider];
+
 /* eslint-disable @typescript-eslint/naming-convention */
 export const InstanceType = {
   INGESTION: 'ingestion',

src/constants/deletion/constants.ts

@@ -1,6 +1,7 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 export const DeletionJobTypes = {
   Delete_Layer: 'Delete_Layer',
+  Delete_Cache: 'Delete_Cache',
 } as const;
 
 export type DeletionJobTypes = (typeof DeletionJobTypes)[keyof typeof DeletionJobTypes];
@@ -9,7 +10,7 @@ export const DeletionTaskTypes = {
   Delete: 'delete',
   LayerTilesDeletion: 'tiles-deletion',
   ArtifactsDeletion: 'artifacts-deletion',
+  CacheDeletion: 'cache-deletion',
 } as const;
 
 export type DeletionTaskTypes = (typeof DeletionTaskTypes)[keyof typeof DeletionTaskTypes];
-/* eslint-enable @typescript-eslint/naming-convention */

src/constants/export/constants.ts

@@ -1,5 +1,5 @@
 import { ArtifactRasterType } from '@map-colonies/types';
-import { pickEnum } from '../../utils';
+import { pickEnum } from '../../utils/helpers.utils';
 
 /* eslint-disable @typescript-eslint/naming-convention */
 export const ExportArtifactType = pickEnum(ArtifactRasterType, ['GPKG', 'METADATA']);

src/schemas/core/storage.schema.ts

@@ -1,14 +1,34 @@
 import { z } from 'zod';
-import { SourceType } from '../../constants/core/constants';
+import { StorageProvider } from '../../constants/core/constants';
 
-export const fsStorageSchema = z.object({
-  storageProvider: z.literal(SourceType.FS),
-  subPath: z.string().min(1),
+export const s3BucketSchema = z.object({
+  bucket: z.string().min(1),
 });
 
-export const s3StorageSchema = z.object({
-  storageProvider: z.literal(SourceType.S3),
-  bucket: z.string().min(1),
+export const redisPrefixSchema = z.object({
+  prefix: z.string().min(1), // Full Redis key prefix, e.g. `myLayer-redis_WorldCRS84`
 });
 
-export const storageSchema = z.discriminatedUnion('storageProvider', [fsStorageSchema, s3StorageSchema]);
+export const fsSubPathSchema = z.object({
+  subPath: z.string().min(1),
+});
+
+export const fsStorageSchema = z
+  .object({
+    storageProvider: z.literal(StorageProvider.FS),
+  })
+  .merge(fsSubPathSchema);
+
+export const s3StorageSchema = z
+  .object({
+    storageProvider: z.literal(StorageProvider.S3),
+  })
+  .merge(s3BucketSchema);
+
+export const redisStorageSchema = z
+  .object({
+    storageProvider: z.literal(StorageProvider.REDIS),
+  })
+  .merge(redisPrefixSchema);
+
+export const storageSchema = z.discriminatedUnion('storageProvider', [fsStorageSchema, s3StorageSchema, redisStorageSchema]);

src/schemas/core/tile.schema.ts

@@ -1,5 +1,5 @@
 import { z } from 'zod';
-import { MAX_ZOOM_LEVEL, SourceType, TileOutputFormat } from '../../constants';
+import { MAX_ZOOM_LEVEL, TileOutputFormat } from '../../constants/core/constants';
 
 export const tileRangeSchema = z.object({
   zoom: z.number().int().min(0).max(MAX_ZOOM_LEVEL),
@@ -9,18 +9,14 @@ export const tileRangeSchema = z.object({
   maxY: z.number().int().min(0),
 });
 
-export const tilesDeletionParamsBaseSchema = z.object({
-  tilesPath: z.string().min(1), // Base path for the tiles to be deleted
+export const tileRangesSchema = z.object({
   ranges: z.array(tileRangeSchema).min(1), // Array of tile ranges to be deleted
-  fileExtension: z.literal(TileOutputFormat.PNG.toLowerCase()).or(z.literal(TileOutputFormat.JPEG.toLowerCase())), // File extension of the tiles (e.g., 'png', 'jpeg')
 });
 
-export const s3TilesDeletionParamsSchema = tilesDeletionParamsBaseSchema.extend({
-  sourceProvider: z.literal(SourceType.S3),
+/** Fields that only make sense for path-based tile stores. */
+export const tilesInfoSchema = z.object({
+  tilesRelativePath: z.string().min(1), // Base path for the tiles to be deleted
+  fileExtension: z.literal(TileOutputFormat.PNG.toLowerCase()).or(z.literal(TileOutputFormat.JPEG.toLowerCase())), // e.g. 'png', 'jpeg'
 });
 
-export const fsTilesDeletionParamsSchema = tilesDeletionParamsBaseSchema.extend({
-  sourceProvider: z.literal(SourceType.FS),
-});
-
-export const tilesDeletionParamsSchema = z.discriminatedUnion('sourceProvider', [s3TilesDeletionParamsSchema, fsTilesDeletionParamsSchema]);
+export const tilePyramidSchema = tileRangesSchema.merge(tilesInfoSchema).describe('tilePyramidSchema');

src/schemas/deletion/task.schema.ts

@@ -1,6 +1,16 @@
 import { z } from 'zod';
-import { storageSchema } from '../core';
+import { fsStorageSchema, redisStorageSchema, s3StorageSchema } from '../core/storage.schema';
+import { tilePyramidSchema, tileRangesSchema } from '../core/tile.schema';
 
+/**
+ * Shared by every Redis deletion shape: the key prefix that locates the entries, plus an
+ * optional wait for a mapproxy reload to settle before the keys are removed.
+ */
+const redisDeletionBaseSchema = redisStorageSchema.extend({
+  delaySeconds: z.number().int().nonnegative().optional(),
+});
+
+//#region DeleteTaskParams
 export const deleteTaskParamsSchema = z
   .object({
     deleteFromCatalog: z.boolean().default(false),
@@ -9,11 +19,37 @@ export const deleteTaskParamsSchema = z
     deletePolygonParts: z.boolean().default(false),
   })
   .describe('deleteTaskParamsSchema');
+//#endregion DeleteTaskParams
+
+//#region TilesDeletionParams
+export const s3TilesDeletionParamsSchema = s3StorageSchema.merge(tilePyramidSchema).strict();
+
+export const fsTilesDeletionParamsSchema = fsStorageSchema.merge(tilePyramidSchema).strict();
+
+export const redisTilesDeletionParamsSchema = redisDeletionBaseSchema.merge(tileRangesSchema).strict();
 
-export const deleteStoredResourcesParamsBaseSchema = z.object({
+export const tilesDeletionParamsSchema = z
+  .discriminatedUnion('storageProvider', [s3TilesDeletionParamsSchema, fsTilesDeletionParamsSchema, redisTilesDeletionParamsSchema])
+  .describe('tilesDeletionParamsSchema');
+//#endregion TilesDeletionParams
+
+//#region DeleteStoredResourcesParams
+export const resourcePathsSchema = z.object({
   paths: z.array(z.string().min(1)).min(1),
 });
 
-export const deleteStoredResourcesParamsSchema = deleteStoredResourcesParamsBaseSchema
-  .and(storageSchema)
+export const s3DeleteStoredResourcesParamsSchema = s3StorageSchema.merge(resourcePathsSchema);
+
+export const fsDeleteStoredResourcesParamsSchema = fsStorageSchema.merge(resourcePathsSchema);
+
+// The prefix IS the locator for a key-value store.
+export const redisDeleteStoredResourcesParamsSchema = redisDeletionBaseSchema.strict();
+
+export const deleteStoredResourcesParamsSchema = z
+  .discriminatedUnion('storageProvider', [
+    s3DeleteStoredResourcesParamsSchema,
+    fsDeleteStoredResourcesParamsSchema,
+    redisDeleteStoredResourcesParamsSchema,
+  ])
   .describe('deleteStoredResourcesParamsSchema');
+//#endregion DeleteStoredResourcesParams

src/types/core/storage.type.ts

@@ -1,6 +1,7 @@
 import type { z } from 'zod';
-import type { fsStorageSchema, s3StorageSchema, storageSchema } from '../../schemas';
+import type { fsStorageSchema, redisStorageSchema, s3StorageSchema, storageSchema } from '../../schemas';
 
 export type FsStorage = z.infer<typeof fsStorageSchema>;
 export type S3Storage = z.infer<typeof s3StorageSchema>;
+export type RedisStorage = z.infer<typeof redisStorageSchema>;
 export type Storage = z.infer<typeof storageSchema>;

src/types/core/tile.type.ts

@@ -1,5 +1,4 @@
 import z from 'zod';
-import { tileRangeSchema, tilesDeletionParamsSchema } from '../../schemas/core/tile.schema';
+import { tileRangeSchema } from '../../schemas/core/tile.schema';
 
-export type TilesDeletionParams = z.infer<typeof tilesDeletionParamsSchema>;
 export type TileRange = z.infer<typeof tileRangeSchema>;

src/types/deletion/task.type.ts

@@ -1,5 +1,30 @@
 import z from 'zod';
-import { deleteTaskParamsSchema, deleteStoredResourcesParamsSchema } from '../../schemas/deletion/task.schema';
+import {
+  deleteStoredResourcesParamsSchema,
+  deleteTaskParamsSchema,
+  fsDeleteStoredResourcesParamsSchema,
+  fsTilesDeletionParamsSchema,
+  redisDeleteStoredResourcesParamsSchema,
+  redisTilesDeletionParamsSchema,
+  s3DeleteStoredResourcesParamsSchema,
+  s3TilesDeletionParamsSchema,
+  tilesDeletionParamsSchema,
+} from '../../schemas/deletion/task.schema';
 
+//#region DeleteTaskParams
 export type DeleteTaskParams = z.infer<typeof deleteTaskParamsSchema>;
+//#endregion DeleteTaskParams
+
+//#region TilesDeletionParams
+export type TilesDeletionParams = z.infer<typeof tilesDeletionParamsSchema>;
+export type S3TilesDeletionParams = z.infer<typeof s3TilesDeletionParamsSchema>;
+export type FsTilesDeletionParams = z.infer<typeof fsTilesDeletionParamsSchema>;
+export type RedisTilesDeletionParams = z.infer<typeof redisTilesDeletionParamsSchema>;
+//#endregion TilesDeletionParams
+
+//#region DeleteStoredResourcesParams
 export type DeleteStoredResourcesParams = z.infer<typeof deleteStoredResourcesParamsSchema>;
+export type S3DeleteStoredResourcesParams = z.infer<typeof s3DeleteStoredResourcesParamsSchema>;
+export type FsDeleteStoredResourcesParams = z.infer<typeof fsDeleteStoredResourcesParamsSchema>;
+export type RedisDeleteStoredResourcesParams = z.infer<typeof redisDeleteStoredResourcesParamsSchema>;
+//#endregion DeleteStoredResourcesParams