feat: cache-seeder 1'st implemenation (MAPCO-3850)

.eslintrc.json

@@ -1,5 +1,5 @@
 {
-  "ignorePatterns": ["**/*.js", "dist", "helm", "coverage", "reports", ".husky"],
+  "ignorePatterns": ["**/*.js", "dist", "tests/unit/mapproxyUtils", "tests/mockData", "helm", "coverage", "reports", ".husky"],
   "extends": ["@map-colonies/eslint-config/jest", "@map-colonies/eslint-config/ts-base"],
   "parserOptions": {
     "project": "./tsconfig.lint.json"

.gitignore

@@ -1,3 +1,6 @@
+#local helm configs
+helm/local
+
 # Logs
 logs
 *.log
@@ -110,3 +113,5 @@ dist
 # Jest
 jest_html_reporters.html
 reports
+local.yaml
+local.json

.prettierignore

@@ -13,3 +13,4 @@ node_modules
 coverage
 helm
 reports
+tests/mockData

README.md

@@ -1,83 +1,91 @@
-# Map Colonies typescript service template
+# Mapproxy Cache Seeder
 
 ----------------------------------
 
-This is a basic repo template for building new MapColonies web services in Typescript.
+![badge-alerts-lgtm](https://img.shields.io/lgtm/alerts/github/MapColonies/cache-seeder?style=for-the-badge)
 
-### Template Features:
+![grade-badge-lgtm](https://img.shields.io/lgtm/grade/javascript/github/MapColonies/cache-seeder?style=for-the-badge)
 
-- eslint configuration by [@map-colonies/eslint-config](https://github.com/MapColonies/eslint-config)
+![snyk](https://snyk.io/test/github/MapColonies/cache-seeder/badge.svg)
 
-- prettier configuration by [@map-colonies/prettier-config](https://github.com/MapColonies/prettier-config)
-
-- jest
-
-- .nvmrc
-
-- Multi stage production-ready Dockerfile
-
-- commitlint
-
-- git hooks
-
-- logging by [@map-colonies/js-logger](https://github.com/MapColonies/js-logger)
-
-- OpenAPI request validation
+----------------------------------
 
-- config load with [node-config](https://www.npmjs.com/package/node-config)
+This worker is responsible for tracking layer's update and seeding\cleaning cache on redis automatically by job-tasks polling
 
-- Tracing and metrics by [@map-colonies/telemetry](https://github.com/MapColonies/telemetry)
+Ingestion has an “Update” mode that updates tiles in storage (FS, S3).
+We use redis cache in order to serve tiles faster and reduce the load on our services.
 
-- github templates
+In order to refresh \ clean irrelevant cache data from redis the worker use [seed mechanism - mapproxy-utils cli](https://mapproxy.github.io/mapproxy/seed.html#id9)
 
-- bug report
+The worker polls each time seeding tasks from [job-manager service](https://github.com/MapColonies/job-manager).
+Each job (tileSeeding) include single task (tileSeeding) that include array of mapproxy caches (seed | cleanup) to execute:
 
-- feature request
+# Job-Task parameters API's according job manager
+<br />
 
-- pull request
+## Seed Task Parameters Structure
+| Field | What is it | Mandatory   | Type |
+| :---:   | :---: | :---: | :---: |
+| jobId | original jobId of ingestion\update job | + | UUID |
+| spanId | spanId of ingestion\update job | + | UUID |
+| cacheType | which cache type should be executed (current version only redis type) | + | String |
+| catalogId | Catalog ID of original layer | + | will be UUID |
+| seedTasks | seed option per single cache | + | array of seed objects |
 
-- github actions
+<br />
 
-- on pull_request
 
-- LGTM
+### seedTasks array's Object Structure
+| Field | What is it    | Mandatory   | Type |
+| :---:   | :---: | :---: | :---: |
+| grid | grid name of layer in mapproxy.yaml | + | string |
+| mode | define if it is seed or cleanup | + | enum - seed\clean |
+| layerId | layer name in mapproxy.yaml (current version only redis type) | + | String |
+| geometry | Coverage area for seeding | + | Geometry (GeoJson) |
+| fromZoomLevel | start zoom for seeding ranges | + | Integer |
+| toZoomLevel | end zoom for seeding ranges | + | Integer |
+| refreshBefore | ttl time - define if the cache is expired | + | Date ('ISO_8601' format: yyyy-MM-dd'T'HH:mm:ss) |
+| skipUncached | flag to skip seeding on uncached area and ranges | + | Boolean |
 
-- test
+------
 
-- lint
 
-- snyk
 
-## API
-Checkout the OpenAPI spec [here](/openapi3.yaml)
+## Run Locally
+### Prerequisites
+1. python >=3.8 
+2. install mapproxy (current v.1.16) or follow [here](https://mapproxy.org/docs/latest/install.html)
 
-## Installation
+```bash
+pip install MapProxy==1.16.0
+```
 
-Install deps with npm
 
+3. install external redis requirements for python
 ```bash
-npm install
+pip install redis==5.0.0 python-json-logger==2.0.4 prometheus-client==0.17.0 boto3==1.18.32 botocore==1.21.32 googleapis-common-protos==1.53.0 protobuf==3.20.3 shapely==2.0.2
 ```
-### Install Git Hooks
+4. Replace redis.py patch for current version (not included on current mapproxy redis module.)
+copy content from patch [redis.py](https://github.com/MapColonies/cache-seeder/blob/master/docker/patch/redis.py) and replace in python side-package installation.
 ```bash
-npx husky install
+cp redis.py <python or venv location>/site-packages/mapproxy/cache/redis.py
 ```
+<br />
 
-## Run Locally
-
+### Package installation
 Clone the project
 
 ```bash
 
-git clone https://link-to-project
+git clone https://github.com/MapColonies/cache-seeder.git
 
 ```
 
 Go to the project directory
 
 ```bash
 
-cd my-project
+cd cache-seeder
 
 ```
 
@@ -93,7 +101,7 @@ Start the server
 
 ```bash
 
-npm run start
+npm start
 
 ```
 
@@ -112,7 +120,5 @@ To only run unit tests:
 npm run test:unit
 ```
 
-To only run integration tests:
-```bash
-npm run test:integration
-```
+
+Note: Configure on configuration file the relevant data.

catalog-info.yaml

@@ -1,17 +1,19 @@
 apiVersion: backstage.io/v1alpha1
 kind: Component
 metadata:
-  name: ts-server-boilerplate
-  description: A boilerplate github repo for a REST API service in NodeJS for MapColonies
+  name: cache-seeder
+  description: This worker is responsible for execute seed&clean job for mapproxy redis caches wrapping mapproxy-utils package
   annotations:
-    github.com/project-slug: MapColonies/ts-server-boilerplate
+    github.com/project-slug: MapColonies/cache-seeder
   tags:
     - nodejs
     - typescript
     - expressjs
-    - boilerplate
+    - mapproxy-utils
+    - redis
+    - job-manager
 spec:
   type: service
   lifecycle: production
-  owner: DevInfra
-  system: boilerplate
+  owner: RasterTeam
+  system: Ingestion

config/custom-environment-variables.json

@@ -38,5 +38,35 @@
         }
       }
     }
+  },
+  "queue": {
+    "jobManagerBaseUrl": "QUEUE_JOB_MANAGER_BASE_URL",
+    "heartbeat": {
+      "heartbeatManagerBaseUrl": "QUEUE_HEART_BEAT_MANAGER_BASE_URL",
+      "heartbeatIntervalMs": {
+        "__name": "QUEUE_HEART_BEAT_INTERVAL_MS",
+        "__format": "number"
+      }
+    },
+    "dequeueIntervalMs": {
+      "__name": "QUEUE_DEQUEUE_INTERVAL_MS",
+      "__format": "number"
+    },
+    "jobType": "QUEUE_JOB_TYPE",
+    "tilesTaskType": "QUEUE_TASK_TYPE"
+  },
+  "mapproxy": {
+    "mapproxyApiUrl": "MAPPROXY_API_URL",
+    "mapproxyYamlDir": "MAPPROXY_YAML_DIR",
+    "seedYamlDir": "SEED_YAML_DIR",
+    "geometryTxtFile": "GEOMETRY_TXT_FILE"
+  },
+  "seedAttempts": {
+    "__name": "SEED_ATTEMPTS",
+    "__format": "number"
+  },
+  "seedConcurrency": {
+    "__name": "SEED_CONCURRENCY",
+    "__format": "number"
   }
 }

config/default.json

@@ -23,6 +23,31 @@
         "enabled": true,
         "options": null
       }
+    },
+    "httpRetry": {
+      "attempts": 5,
+      "delay": "exponential",
+      "shouldResetTimeout": true,
+      "disableHttpClientLogs": true
     }
-  }
+  },
+  "queue": {
+    "jobManagerBaseUrl": "http://localhost:8088",
+    "heartbeat": {
+      "heartbeatManagerBaseUrl": "http://localhost:8083",
+      "heartbeatIntervalMs": 3000
+    },
+    "dequeueIntervalMs": 1000,
+    "jobType": "TilesSeeding",
+    "tilesTaskType": "TilesSeeding"
+  },
+  "mapproxy": {
+    "mapproxyApiUrl": "http://localhost:8086",
+    "mapproxyYamlDir": "/mapproxy/mapproxy.yaml",
+    "seedYamlDir": "/mapproxy/seed.yaml",
+    "geometryTxtFile": "/mapproxy/coverage.json",
+    "seedProgressFileDir": "/mapproxy/mapproxy_seed_progress"
+  },
+  "seedAttempts": 5,
+  "seedConcurrency": 5
 }

docker/.dockerignore

@@ -0,0 +1,18 @@
+.nyc_output
+tests
+.circlei
+.github
+coverage
+node_modules
+.prettier*
+commitlint.config.json
+README.md
+.npmrc
+.redocly.yaml
+tsconfig.lint.json
+.nvmrc
+.git*
+commitlint.config.json
+dist
+build
+reports