feat(backend,frontend,client,common): implementation

.eslintrc.json

@@ -1,5 +1,5 @@
 {
-  "ignorePatterns": ["**/*.js", "dist", "helm", "coverage", "reports", ".husky"],
+  "ignorePatterns": ["**/*.js", "dist", "helm", "node_modules"],
   "extends": ["@map-colonies/eslint-config/jest", "@map-colonies/eslint-config/ts-base"],
   "parserOptions": {
     "project": "./tsconfig.lint.json"

.github/workflows/pull_request.yaml

@@ -4,22 +4,32 @@ on: [pull_request]
 
 jobs:
   eslint:
-    name: Run eslint
+    name: Run TS Project eslint
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        node: [18.x, 20.x]
+
     steps:
-      - name: Check out Git repository
-        uses: actions/checkout@v2
+      - name: Check out TS Project Git repository
+        uses: actions/checkout@v3
 
       - name: Set up Node.js
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
-          node-version: 20
+          node-version: ${{ matrix.node }}
 
-      - name: Install dependencies
+      - name: Install TS Project dependencies
         run: npm ci
 
-      - name: Run linters
+      - name: Build Lerna
+        run: npx lerna run build
+
+      - name: Run typecheck
+        run: npx lerna run typecheck
+
+      - name: Run TS Project linters
         uses: wearerequired/lint-action@v1
         with:
           github_token: ${{ secrets.github_token }}
@@ -28,11 +38,37 @@ jobs:
           prettier: true
           eslint_extensions: ts
 
+      - name: Run local linters
+        run: npx lerna run lint:fix
+
+  openapi-lint:
+    name: Run OpenAPI lint Check
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        package: [backend]
+
+    steps:
+      - name: Check out TS Project Git repository
+        uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20
+
+      - name: Install TS Project dependencies
+        run: npm ci
+
+      - name: Build Lerna
+        run: npx lerna run build
+
       - name: OpenAPI Lint Checks
         uses: nwestfall/openapi-action@v1.0.2
         with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
-         file: ./openapi3.yaml
+         file: ./packages/${{ matrix.package }}/openapi3.yaml
 
   security:
     runs-on: ubuntu-latest
@@ -56,28 +92,36 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Set up Node.js
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v3
         with:
           node-version: ${{ matrix.node }}
+          cache: npm
 
       - name: Install Node.js dependencies
         run: npm ci
 
+      - name: Build Lerna
+        run: npx lerna run build
+
       - name: Run tests
-        run: npm run test
+        run: npx lerna run test
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: Test Reporters
-          path: reports/**
+          path: packages/**/reports/**
 
   build_image:
     name: Build Image
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        package: [backend, frontend]
+
     steps:
       - name: Check out Git repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: build Docker image
-        run: docker build -t test-build:latest .
+        run: docker build -f ./packages/${{ matrix.package }}/Dockerfile -t ${{ matrix.package }}:latest .

.gitignore

@@ -1,3 +1,6 @@
+**/local.json
+**/myvalues.yaml
+
 # Logs
 logs
 *.log

.husky/commit-msg

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+npx --no -- commitlint --edit $1

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+npx --no -- lerna run lint

.licrc

@@ -5,7 +5,7 @@
 # This will indicate which licenses are not accepted.
 # The rest will be accepted, except for the unknown licenses or dependencies without licenses.
 unaccepted = ["LGPL"]
-# Note that only one of the previous options can be enabled at once. 
+# Note that only one of the previous options can be enabled at once.
 # If both of them are informed, only accepted will be considered.
 
 [dependencies]

.prettierignore

@@ -13,3 +13,5 @@ node_modules
 coverage
 helm
 reports
+jest.config*.js
+lerna.json

README.md

@@ -1,118 +1,101 @@
-# Map Colonies typescript service template
-
-----------------------------------
-
-This is a basic repo template for building new MapColonies web services in Typescript.
-
-### Template Features:
-
-- eslint configuration by [@map-colonies/eslint-config](https://github.com/MapColonies/eslint-config)
-
-- prettier configuration by [@map-colonies/prettier-config](https://github.com/MapColonies/prettier-config)
-
-- jest
-
-- .nvmrc
-
-- Multi stage production-ready Dockerfile
-
-- commitlint
-
-- git hooks
-
-- logging by [@map-colonies/js-logger](https://github.com/MapColonies/js-logger)
-
-- OpenAPI request validation
-
-- config load with [node-config](https://www.npmjs.com/package/node-config)
-
-- Tracing and metrics by [@map-colonies/telemetry](https://github.com/MapColonies/telemetry)
-
-- github templates
-
-- bug report
-
-- feature request
-
-- pull request
-
-- github actions
-
-- on pull_request
-
-- LGTM
-
-- test
-
-- lint
-
-- snyk
-
-## API
-Checkout the OpenAPI spec [here](/openapi3.yaml)
-
-## Installation
-
-Install deps with npm
-
-```bash
-npm install
-```
-### Install Git Hooks
-```bash
-npx husky install
+# detiler
+
+this monorepo includes detiler backend, frontend, client and their common library
+
+## detiler-backend
+deploys an api used to maintain each and every metatile's metadata and query it according to selected filters
+
+data is cached in redis and updated with each tile processing done by the [retiler](https://github.com/MapColonies/retiler) service
+
+- leverages from unnecessary tile processing skip
+- quick and easy kit comparison
+- data maintenance for immediate and future BI
+
+## detiler-frontend
+a react app containing `deck.gl` map components presenting selected kit's metatiles.
+
+data is colored in relation to some metric (state, update count, skip count, currentness, etc.) thus presenting for each tile it's metric in correlation with all other tiles.
+
+### tile metadata example:
+```json
+{
+  "z":17,
+  "x":19520,
+  "y":5321,
+  "kit":"my-default-kit",
+  "state":666,
+  "updatedAt":1711907506,
+  "renderedAt":1711907506,
+  "createdAt":1711302106,
+  "updateCount":13,
+  "renderCount":11,
+  "skipCount":2,
+  "geoshape":"POLYGON ((34.453125 31.541748046875, 34.453125 31.53076171875, 34.464111328125 31.53076171875, 34.464111328125 31.541748046875, 34.453125 31.541748046875))",
+  "coordinates":"34.458618, 31.536255"
+}
 ```
 
-## Run Locally
+### tile processing skip timeline:
+this process will occur in `retiler` by quering `detiler-backend`
 
-Clone the project
+1. a tile is being candidate for processing
+2. is the tile processing is attributed as forced? if so skip steps 3-5 and jump to step 6.
+3. query the tile's details from the backend --and `renderedAt` field
+4. fetch the tile's kit data timestamp (the timestamp is being maintained by osm2pgsql, with every append data timestamp is updated)
+5. compare the two, if tile's `renderedAt` time is later than kit data time - the tile has already been processed with the most current data, meaning its processing can be skipped. thus we have the following branch, either:
+    - `renderedAt` >= kit timestamp - processing should be skipped: update the tile's details - `state`, `updateCount`, `updatedAt` and `skipCount` accordingly
+    - `renderedAt` < kit timestamp - processing is needed
+6. process the tile and update the tile's details - `state`, `updateCount`, `updatedAt`, `renderCount` and `renderedAt` accordingly
 
-```bash
+## Configuration
+the frontend app including its environment variables are being processed in buildtime.
+to achieve runtime variables we inject for each variable it's value in runtime instead of a placeholder.
 
-git clone https://link-to-project
+see [.env.production](/packages/frontend/config/.env.production) and [env.sh](/packages/frontend/env.sh).
 
+## Redis
+### redis search index creation:
 ```
-
-Go to the project directory
-
-```bash
-
-cd my-project
-
+FT.CREATE tileDetailsIdx ON JSON PREFIX 1 tile:
+SCHEMA $.kit AS kit TEXT $.updatedAt AS updatedAt NUMERIC $.renderedAt AS renderedAt NUMERIC $.createdAt AS createdAt NUMERIC $.updateCount AS updateCount NUMERIC $.renderCount AS renderCount NUMERIC $.skipCount AS skipCount NUMERIC $.coordiantes AS coordinates GEO $.geoshape AS geoshape GEOSHAPE SPHERICAL $.state AS state NUMERIC $.z AS z NUMERIC $.x AS x NUMERIC $.y AS y NUMERIC
 ```
 
-Install dependencies
+### redis post processing:
+preferably we would use the `Redis Gears` module, but in the meantime the deprecated `Triggers and Functions` module is in use.
 
-```bash
-
-npm install
+the following post processing function is being used to maintain additional metadata for each kit - it's `maxState` and `maxUpdatedAt`
 
+execute with `redis-cli`
 ```
-
-Start the server
-
-```bash
-
-npm run start
-
+TFUNCTION LOAD REPLACE "#!js name=LibName api_version=1.0\n
+   function getMaximum(client, data) {
+       currentKit = client.call('json.get', data.key, 'kit');
+       currentState = client.call('json.get', data.key, 'state');
+       currentUpdatedAt = client.call('json.get', data.key, 'updatedAt');
+       key = 'kit:' + currentKit.slice(1, currentKit.length - 1);
+       maxState = client.call('hget', key, 'maxState');
+       if(maxState < currentState) {
+           client.call('hset', key, 'maxState', currentState);
+       }
+       maxUpdatedAt = client.call('hget', key, 'maxUpdatedAt');
+       if(maxUpdatedAt < currentUpdatedAt) {
+           client.call('hset', key, 'maxUpdatedAt', currentUpdatedAt);
+       }
+   }
+
+   redis.registerKeySpaceTrigger('KitLibrary', 'tile:', getMaximum);"
 ```
 
-## Running Tests
-
-To run tests, run the following command
-
-```bash
-
-npm run test
+## Development
+This repository is a monorepo managed by [`Lerna`](https://lerna.js.org/) and separated into multiple independent packages
 
+## Building
 ```
-
-To only run unit tests:
-```bash
-npm run test:unit
+npx lerna run build
 ```
 
-To only run integration tests:
-```bash
-npm run test:integration
+## Tests
+integration tests are missing due to `node-redis` library structure, [see open issue here](https://github.com/redis/node-redis/issues/2546)
+```
+npx lerna run test
 ```

commitlint.config.js

@@ -1,6 +1,8 @@
 module.exports = {
-    extends: ['@commitlint/config-conventional'],
-    rules: {
-        "scope-enum": [2, "always", ["deps", "configurations"]],
-    }
+   extends: ['@commitlint/config-conventional'],
+   rules: {
+      'scope-enum': async ctx =>
+        [2, 'always', ["deps", "configurations", "helm", "backend", "client", "common", "frontend"]],
+          "scope-empty": [2, "never"],
+   }
 };