feat: delete layer (MAPCO-1159)

.vscode/settings.json

@@ -1,5 +1,5 @@
 {
-  "typescript.tsdk": "node_modules/typescript/lib",
+  "js/ts.tsdk.path": "node_modules/typescript/lib",
   "files.eol": "\n",
   "editor.defaultFormatter": "esbenp.prettier-vscode",
   "files.insertFinalNewline": true

config/custom-environment-variables.json

@@ -73,35 +73,56 @@
       "__format": "boolean"
     }
   },
-  "s3": {
-    "endpoint": "S3_ENDPOINT",
-    "accessKeyId": "S3_ACCESS_KEY_ID",
-    "secretAccessKey": "S3_SECRET_ACCESS_KEY",
-    "sslEnabled": {
-      "__name": "S3_SSL_ENABLED",
-      "__format": "boolean"
+  "storage": {
+    "cleanupStorageProviders": {
+      "__name": "CLEANUP_STORAGE_PROVIDERS",
+      "__format": "json"
     },
-    "forcePathStyle": {
-      "__name": "S3_FORCE_PATH_STYLE",
-      "__format": "boolean"
+    "s3": {
+      "delete": {
+        "batchSize": {
+          "__name": "S3_DELETE_BATCH_SIZE",
+          "__format": "number"
+        }
+      },
+      "endpoint": "S3_ENDPOINT",
+      "accessKeyId": "S3_ACCESS_KEY_ID",
+      "secretAccessKey": "S3_SECRET_ACCESS_KEY",
+      "sslEnabled": {
+        "__name": "S3_SSL_ENABLED",
+        "__format": "boolean"
+      },
+      "forcePathStyle": {
+        "__name": "S3_FORCE_PATH_STYLE",
+        "__format": "boolean"
+      },
+      "region": "S3_REGION"
     },
-    "region": "S3_REGION"
+    "fs": {
+      "delete": {
+        "batchSize": {
+          "__name": "FS_DELETE_BATCH_SIZE",
+          "__format": "number"
+        }
+      },
+      "basePath": "FS_BASE_PATH",
+      "subPaths": {
+        "__name": "FS_SUB_PATHS",
+        "__format": "json"
+      }
+    }
   },
   "strategies": {
     "tilesDeletion": {
       "s3Bucket": "TILES_DELETION_S3_BUCKET",
-      "fsBasePath": "TILES_DELETION_FS_BASE_PATH",
+      "fsSubPath": "TILES_DELETION_FS_SUB_PATH",
       "batchSize": {
         "__name": "TILES_DELETION_BATCH_SIZE",
         "__format": "number"
       },
       "concurrency": {
         "__name": "TILES_DELETION_CONCURRENCY",
         "__format": "number"
-      },
-      "failureSampleSize": {
-        "__name": "TILES_DELETION_FAILURE_SAMPLE_SIZE",
-        "__format": "number"
       }
     }
   }

config/default.json

@@ -9,9 +9,7 @@
       "level": "info",
       "prettyPrint": false,
       "opentelemetryOptions": {
-        "enabled": false,
-        "url": "",
-        "resourceAttributes": {}
+        "enabled": false
       }
     }
   },
@@ -28,18 +26,26 @@
         {
           "job": "Ingestion_Swap_Update",
           "task": "tiles-deletion"
+        },
+        {
+          "job": "Delete_Layer",
+          "task": "tiles-deletion"
+        },
+        {
+          "job": "Delete_Layer",
+          "task": "artifacts-deletion"
         }
       ]
     }
   },
   "queue": {
-    "jobManagerBaseUrl": "http://localhost:8080",
-    "heartbeatBaseUrl": "http://localhost:8081",
+    "jobManagerBaseUrl": "http://localhost:8081",
+    "heartbeatBaseUrl": "http://localhost:8082",
     "heartbeatIntervalMs": 1000,
     "dequeueIntervalMs": 3000
   },
   "servicesUrl": {
-    "jobTracker": "http://localhost:8082"
+    "jobTracker": "http://localhost:8083"
   },
   "disableHttpClientLogs": false,
   "jobDefinitions": {
@@ -49,12 +55,23 @@
       },
       "swapUpdate": {
         "type": "Ingestion_Swap_Update"
+      },
+      "deleteLayer": {
+        "type": "Delete_Layer"
       }
     },
     "tasks": {
       "tilesDeletion": {
         "type": "tiles-deletion",
         "maxAttempts": 3
+      },
+      "layerDeletion": {
+        "type": "tiles-deletion",
+        "maxAttempts": 3
+      },
+      "artifactsDeletion": {
+        "type": "artifacts-deletion",
+        "maxAttempts": 3
       }
     }
   },
@@ -63,21 +80,35 @@
     "delay": "exponential",
     "shouldResetTimeout": true
   },
-  "s3": {
-    "endpoint": "http://localhost:9000",
-    "accessKeyId": "minioadmin",
-    "secretAccessKey": "minioadmin",
-    "sslEnabled": false,
-    "forcePathStyle": true,
-    "region": "us-east-1"
+  "storage": {
+    "cleanupStorageProviders": ["FS", "S3"],
+    "s3": {
+      "delete": {
+        "batchSize": 1000
+      },
+      "endpoint": "http://localhost:9000",
+      "accessKeyId": "minioadmin",
+      "secretAccessKey": "minioadmin",
+      "sslEnabled": false,
+      "forcePathStyle": true,
+      "region": "us-east-1"
+    },
+    "fs": {
+      "delete": {
+        "batchSize": 1000
+      },
+      "basePath": "/data",
+      "subPaths": {
+        "tilesSubPath": "tiles"
+      }
+    }
   },
   "strategies": {
     "tilesDeletion": {
       "s3Bucket": "",
-      "fsBasePath": "/tiles",
+      "fsSubPath": "tiles",
       "batchSize": 1000,
-      "concurrency": 10,
-      "failureSampleSize": 3
+      "concurrency": 10
     }
   }
 }

helm/templates/configmap.yaml

@@ -2,8 +2,10 @@
 {{- $serviceUrls := fromYaml (include "common.serviceUrls.merged" .) -}}
 {{- $storage := fromYaml (include "common.storage.merged" .) -}}
 {{- $s3 := ($storage.s3) | default dict -}}
-{{- $internalPvc := (($storage.fs).internalPvc) | default dict -}}
-{{- $tilesFSBasePath :=  clean (printf "/%s/%s" $internalPvc.mountPath ($internalPvc.tilesSubPath | default "tiles")) -}}
+{{- $fs := ($storage.fs) | default dict -}}
+{{- $internalPvc := (($fs).internalPvc) | default dict -}}
+{{- $fsBasePath := clean (printf "/%s" $internalPvc.mountPath) -}}
+{{- $fsTilesDeletionSubPath := clean (printf "%s" ($internalPvc.tilesSubPath | default "tiles")) -}}
 {{- if .Values.enabled -}}
 apiVersion: v1
 kind: ConfigMap
@@ -54,15 +56,29 @@ data:
   {{- with .Values.env.jobnik.worker }}
   JOBNIK_WORKER_CONCURRENCY: {{ .concurrency | default 1 | quote }}
   {{- end }}
+  CLEANUP_STORAGE_PROVIDERS: {{ $storage.cleanupStorageProviders | toJson | quote }}
+  {{- if has "S3" $storage.cleanupStorageProviders }}
+  S3_DELETE_BATCH_SIZE: {{ $s3.delete.batchSize | default 1000 | quote }}
   S3_ENDPOINT: {{ if $s3.endpointUrl }}{{ printf "%s://%s" ($s3.sslEnabled | ternary "https" "http") $s3.endpointUrl | quote }}{{ else }}{{ "" | quote }}{{ end }}
   S3_FORCE_PATH_STYLE: {{ $s3.forcePathStyle | default false | quote }}
   S3_SSL_ENABLED: {{ $s3.sslEnabled | default false | quote }}
   S3_REGION: {{ $s3.region | default "us-east-1" | quote }}
+  {{- end }}
+  {{- if has "FS" $storage.cleanupStorageProviders }}
+  FS_DELETE_BATCH_SIZE: {{ $fs.delete.batchSize | default 1000 | quote }}
+  FS_BASE_PATH: {{ $fsBasePath | quote }}
+  {{- $subPaths := dict -}}
+  {{- range $key, $val := $internalPvc -}}
+    {{- if and (hasSuffix "SubPath" $key) $val -}}
+      {{- $_ := set $subPaths (trimSuffix "SubPath" $key) (clean $val) -}}
+    {{- end -}}
+  {{- end }}
+  FS_SUB_PATHS: {{ $subPaths | toJson | quote }}
+  TILES_DELETION_FS_SUB_PATH: {{ $fsTilesDeletionSubPath | quote }}
+  {{- end }}
   TILES_DELETION_S3_BUCKET: {{ $s3.tilesBucket | default "" | quote }}
-  TILES_DELETION_FS_BASE_PATH: {{ $tilesFSBasePath | quote }}
   {{- with .Values.env.strategies.tilesDeletion }}
   TILES_DELETION_BATCH_SIZE: {{ .batchSize | default 1000 | quote }}
   TILES_DELETION_CONCURRENCY: {{ .concurrency | default 10 | quote }}
-  TILES_DELETION_FAILURE_SAMPLE_SIZE: {{ .failureSampleSize | default 3 | quote }}
   {{- end }}
 {{- end }}

helm/templates/deployment.yaml

@@ -54,10 +54,10 @@ spec:
           imagePullPolicy: {{ .pullPolicy | default "IfNotPresent" }}
           {{- end }}
           {{- if .Values.command }}
-          command: 
+          command:
           {{- toYaml .Values.command | nindent 12 }}
           {{- if .Values.args }}
-          args: 
+          args:
           {{- toYaml .Values.args | nindent 12 }}
           {{- end }}
           {{- end }}
@@ -122,7 +122,7 @@ spec:
             httpGet:
               path: {{ .Values.readinessProbe.path }}
               port: {{ .Values.env.targetPort }}
-          {{- end }}        
+          {{- end }}
           {{- if .Values.resources.enabled }}
           resources:
             {{- toYaml .Values.resources.value | nindent 12 }}

helm/values.yaml

@@ -13,20 +13,26 @@ serviceUrls:
   jobTracker: ""
 
 storage:
+  cleanupStorageProviders:
+    - FS
+    - S3
   s3:
+    delete:
+      batchSize: 1000
     endpointUrl: ""
     forcePathStyle: true
     secretName: ""
     sslEnabled: false
     region: ""
     tilesBucket: ""
   fs:
+    delete:
+      batchSize: 1000
     internalPvc:
       enabled: false
       name: ""
       mountPath: ""
-      tilesSubPath: ""
-
+      tilesSubPath: "" # e.g. folder/tiles
 
 mclabels:
   component: backend
@@ -127,7 +133,6 @@ env:
     tilesDeletion:
       batchSize: 1000
       concurrency: 10
-      failureSampleSize: 3
 
 resources:
   enabled: true

package.json

@@ -16,7 +16,7 @@
     "prebuild": "npm run clean",
     "build": "tsc --project tsconfig.build.json && tsc-alias -p tsconfig.build.json && npm run assets:copy",
     "start": "npm run build && cd dist && node --import ./instrumentation.mjs ./index.js",
-    "start:dev": "npm run build && cd dist && cross-env CONFIG_OFFLINE_MODE=true  node --enable-source-maps --import ./instrumentation.mjs ./index.js",
+    "start:dev": "npm run build && cd dist && cross-env CONFIG_OFFLINE_MODE=true node --enable-source-maps --import ./instrumentation.mjs ./index.js",
     "assets:copy": "copyfiles -f ./config/* ./dist/config && copyfiles ./package.json dist",
     "clean": "rimraf dist",
     "prepare": "node .husky/install.mjs"
@@ -37,7 +37,7 @@
     "@map-colonies/js-logger": "^5.0.0",
     "@map-colonies/mc-priority-queue": "^9.1.0",
     "@map-colonies/mc-utils": "^5.1.0",
-    "@map-colonies/raster-shared": "^8.1.0-alpha.3",
+    "@map-colonies/raster-shared": "^8.3.0-alpha.2",
     "@map-colonies/read-pkg": "^1.0.0",
     "@map-colonies/schemas": "^1.20.0",
     "@map-colonies/telemetry": "^10.0.1",