Merge pull request #297 from kuzzleio/2.2.0-proposal

# [2.2.0](https://github.com/kuzzleio/kuzzle-device-manager/releases/tag/2.2.0) (2023-04-17)


#### Bug fixes

- [ [#290](#290) ] Historize asset for each same measurements   ([tdislay](https://github.com/tdislay))
- [ [#288](#288) ] Full path nested metadata when create/update assets   ([OlivierCavadenti](https://github.com/OlivierCavadenti))
- [ [#286](#286) ] ModelService upsert -> createOrReplace   ([fmauNeko](https://github.com/fmauNeko))
- [ [#282](#282) ] Fix engine creation   ([Aschen](https://github.com/Aschen))

#### New features

- [ [#296](#296) ] Export measures to CSV   ([Aschen](https://github.com/Aschen))
- [ [#291](#291) ] Add type of measure linked in each links of asset measures.   ([OlivierCavadenti](https://github.com/OlivierCavadenti))

#### Enhancements

- [ [#295](#295) ] Export default mappings   ([Aschen](https://github.com/Aschen))
- [ [#293](#293) ] Allow JSONObject in metadata   ([Aschen](https://github.com/Aschen))
- [ [#287](#287) ] Update metadata in existing assets when the model is modified   ([fmauNeko](https://github.com/fmauNeko))
- [ [#283](#283) ] Store reason of discarded payloads   ([OlivierCavadenti](https://github.com/OlivierCavadenti))

#### Others

- [ [#289](#289) ] Split tsconfig for tests   ([fmauNeko](https://github.com/fmauNeko))
---

.eslintrc.json

@@ -1,9 +1,11 @@
 {
-  "plugins": ["kuzzle"],
+  "plugins": ["kuzzle", "jest"],
   "extends": [
     "plugin:kuzzle/default",
     "plugin:kuzzle/node",
-    "plugin:kuzzle/typescript"
+    "plugin:kuzzle/typescript",
+    "plugin:jest/recommended",
+    "plugin:jest/style"
   ],
   "overrides": [
     {

.github/actions/deploy-typedoc/action.yml

@@ -0,0 +1,35 @@
+name: Typedoc
+description: Build and deploy Typedoc
+# Inputs should be env variables
+inputs:
+  SITE_BASE:
+    description: Destination path of the deployment
+    required: true
+  AWS_ACCESS_KEY_ID:
+    description: AWS Access key ID
+    required: true
+  AWS_SECRET_ACCESS_KEY:
+    description: AWS secret key
+    required: true
+  S3_BUCKET:
+    description: S3 bucket name
+    required: true
+  CLOUDFRONT_ID:
+    description: Cloudfront distribution ID
+    required: true
+  REGION:
+    description: AWS default region
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Build Typedoc and deploy
+      shell: bash
+      run: |
+        npm run docker npm ci
+        npx typedoc index.ts
+        echo "aws s3 sync ./docs s3://$S3_BUCKET$SITE_BASE --delete"
+        echo "aws cloudfront create-invalidation --distribution-id $CLOUDFRONT_ID --paths $SITE_BASE*"
+        aws s3 sync ./docs s3://$S3_BUCKET$SITE_BASE --delete
+        aws cloudfront create-invalidation --distribution-id $CLOUDFRONT_ID --paths $SITE_BASE*

.github/actions/functional-test/action.yml

@@ -10,4 +10,4 @@ runs:
         npm run build
         DOCKER_COMMAND="npm run prod" docker-compose up -d
         bash tests/wait-kuzzle.sh
-        npm run test:functional
+        npm run test

.github/actions/lint/action.yml

@@ -7,5 +7,5 @@ runs:
       shell: bash
       run: |
         npm run docker npm ci
-        npm run test:lint
+        npm run lint
         npm run build

.github/workflows/push_dev.workflow.yml

@@ -7,45 +7,45 @@ on:
       - 2-dev
 
 jobs:
-  # lint:
-  #   name: Lint
-  #   runs-on: ubuntu-22.04
-  #   steps:
-  #     - name: Install SSH key
-  #       uses: shimataro/ssh-key-action@v2
-  #       with:
-  #         key: ${{ secrets.SSH_KEY }}
-  #         name: id_rsa
-  #         known_hosts: unnecessary
-  #     - uses: actions/checkout@v2
-  #     - uses: ./.github/actions/remove-libs
-  #     - uses: actions/setup-node@v1.4.4
-  #       with:
-  #         node-version: "14"
-  #     - uses: ./.github/actions/lint
+  lint:
+    name: Lint
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Install SSH key
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.SSH_KEY }}
+          name: id_rsa
+          known_hosts: unnecessary
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/remove-libs
+      - uses: actions/setup-node@v1.4.4
+        with:
+          node-version: "14"
+      - uses: ./.github/actions/lint
 
-  # functional-tests:
-  #   name: Functional Tests
-  #   runs-on: ubuntu-22.04
-  #   needs: [lint]
-  #   steps:
-  #     - name: Install SSH key
-  #       uses: shimataro/ssh-key-action@v2
-  #       with:
-  #         key: ${{ secrets.SSH_KEY }}
-  #         name: id_rsa
-  #         known_hosts: unnecessary
-  #     - uses: actions/checkout@v2
-  #     - uses: ./.github/actions/remove-libs
-  #     - uses: actions/setup-node@v1.4.4
-  #       with:
-  #         node-version: "14"
-  #     - uses: ./.github/actions/functional-test
+  functional-tests:
+    name: Functional Tests
+    runs-on: ubuntu-22.04
+    needs: [lint]
+    steps:
+      - name: Install SSH key
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.SSH_KEY }}
+          name: id_rsa
+          known_hosts: unnecessary
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/remove-libs
+      - uses: actions/setup-node@v1.4.4
+        with:
+          node-version: "14"
+      - uses: ./.github/actions/functional-test
 
   doc-deploy:
     name: Deploy Doc to NEXT
     runs-on: ubuntu-latest
-    # needs: [functional-tests]
+    needs: [functional-tests]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -65,3 +65,27 @@ jobs:
           workflow_file_name: child_repo.workflow.yml
           ref: develop
           inputs: '{"repo_name": "${{ steps.extract-refs.outputs.repo }}", "branch": "${{ steps.extract-refs.outputs.branch }}", "version": "${{ steps.extract-refs.outputs.version }}"}'
+
+  deploy-typedoc:
+    name: Deploy Typedoc
+    runs-on: ubuntu-22.04
+    needs: [functional-tests]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1.4.4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - name: Install AWS CLI
+        shell: bash
+        run: pip3 --no-cache-dir install awscli==1.17.0
+      - uses: ./.github/actions/deploy-typedoc
+        env:
+          REGION: us-west-2
+          S3_BUCKET: docs-next.kuzzle.io
+          CLOUDFRONT_ID: E2ZCCEK9GRB49U
+          SITE_BASE: /typedoc/device-manager/2/
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

README.md

@@ -11,6 +11,7 @@ See the [documentation](https://next-docs.kuzzle.io/official-plugins/device-mana
 ## Usage
 
 In proper way to use this library in your program as an NPM module:
+
 ```
 npm install kuzzle-device-manager
 ```

doc/2/guides/digital-twins-devices-assets/index.md → doc/2/concepts/digital-twins-devices-assets/index.md

@@ -68,7 +68,7 @@ The feedback of device measurements can be done in two ways:
 - reception of data directly in the format accepted by the Kuzzle IoT Platform (see `device-manager/devices:receiveMeasures`)
 - reception of raw data and normalization by a `Decoder`
 
-The normalized measurements then systematically pass through the Ingestion Pipeline in order to be used by the Rules Engine.
+The normalized measurements then systematically pass through the Ingestion Pipeline in order to be used by the [Event System](/core/2/guides/develop-on-kuzzle/event-system/).
 
 ## Assets
 
@@ -80,7 +80,7 @@ This makes it possible in particular to manage the life cycle of the devices and
 
 ![Measure history](./measure-history.png)
 
-In the Foobar Kuzzle IoT, the assets carry the business specific to each use case. It is them that we will specialize in order to model the real world and meet the different needs for consultation, business rules, alerts, etc.
+In the Foobar Kuzzle IoT, the assets carry the business specific to each use case. It is them that we will specialize in order to model the real world and meet the different needs for consultation, workflows, alerts, etc.
 
 An asset is characterized by the following information:
 
@@ -108,6 +108,8 @@ When linking on the Kuzzle IoT Platform, it is necessary to associate one or mor
 
 Once linked, the measurements collected by the device will be logged for our asset with the name defined during the link.
 
+The API action [device-manager/devices:linkAsset](/official-plugins/device-manager/2/controllers/devices/link-asset) is used to link a device to an asset.
+
 ### Historization of assets state
 
 The successive states of the assets are systematically logged in the Kuzzle IoT Platform.

doc/2/concepts/index.md

@@ -0,0 +1,7 @@
+---
+code: false
+type: branch
+title: Concepts
+description: Kuzzle IoT Platform - Device Manager Concepts
+order: 100
+---

doc/2/guides/measures-ingestion/index.md → doc/2/concepts/measures-ingestion/index.md

@@ -9,7 +9,7 @@ description: Measures ingestion
 
 When it comes to receiving data, the Kuzzle IoT Platform is capable of receiving raw data or formatted data in the form of measures.
 
-These measures are then passed through the ingestion pipeline to allow the rules engine to perform processing at various stages.
+These measures are then passed through the ingestion pipeline to allow to perform processing at various stages.
 
 # Raw data
 
@@ -29,11 +29,11 @@ Each model of Decoder can be associated with a Decoder in order to normalize the
 
 A Decoder is responsible for
 
-- declare the measures it will process
-- register a corresponding API action
-- modify the mapping of the collection containing the raw data
-- validate the format of the received data
-- extract measures from the received data
+1. declare the measures it will process
+2. register a corresponding API action
+3. (optional) modify the mapping of the collection containing the raw data
+4. (optional) validate the format of the received data
+5. extract measures from the received data
 
 ### Declaration of measures
 
@@ -118,6 +118,11 @@ This method takes the raw data frame as a parameter and can indicate that:
 
 Depending on the result of the `validate` method, the API action will return either a `200` status (Case 1 and 2) or a `4**` status (case 3).
 
+For each case, a state and a reason is stored inside the payload document:
+1. the payload has a VALID state.
+2. the payload is discarded by user validation and has a SKIP state and a dedicated reason (which can be overridden by throwing a SkipError exception).
+3. the payload has an ERROR state and a reason equal to the error message.
+
 ```jsx
 class AbeewayDecoder extends Decoder {
   async validate(payload: JSONObject) {
@@ -239,6 +244,8 @@ The `payloads` collection of the `platform` index contains the following informa
 - `uuid`: unique identifier of the data received
 - `valid`: boolean indicating whether the data could be processed correctly
 - `apiAction`: API action that was used to send the data
+- `state`: the state of the payload : VALID if it's valid, SKIP if the payload is skipped by the user during validation or ERROR if an error is raised during payload reception.
+- `reason`: the reason of an error if the payload is in error (otherwise it will be undefined).
 
 For each measure contained in the Kuzzle IoT Platform, it is possible to go back to the raw data in order to analyze possible problems in the standardization stage.
 
@@ -248,7 +255,7 @@ The `payloadUuids` property contained in the measures allows you to search the `
 
 It is possible to execute additional processing within the ingestion pipeline by using one of the events provided for this purpose.
 
-Adding new business rules is done using the Kuzzle pipe mechanism.
+Adding new business rules is done using the Kuzzle pipe mechanism (See [Event System](/core/2/guides/develop-on-kuzzle/event-system/)).
 
 Depending on the treatments, it is better to choose one or the other and that is what we are going to see now.
 
@@ -268,6 +275,10 @@ The `device-manager:measures:process:before` event is triggered with an object c
 - `measures`: table of measures
 - `asset`: (optional) the last state of the asset linked to the device
 
+::: info
+An isolated version of the event is also available: `engine:<engine-id>:device-manager:measures:process:before`
+:::
+
 ### Enrich existing measures
 
 It is possible to modify the fields of existing measures directly by manipulating the table of measures.
@@ -382,6 +393,10 @@ The `device-manager:measures:process:after` event is triggered with an object co
 - `measures`: table of measures
 - `asset`: (optional) the new state of the asset linked to the device
 
+::: info
+An isolated version of the event is also available: `engine:<engine-id>:device-manager:measures:process:after`
+:::
+
 ## Ingestion Pipeline Concurrency
 
 In order to avoid race conditions in the pipeline, a Mutex ensures that the measures of a device are processed one after the other.

doc/2/controllers/assets/export-measures/index.md

@@ -0,0 +1,103 @@
+---
+code: true
+type: page
+title: exportMeasures
+description: Export measure history from an asset
+---
+
+# exportMeasures
+
+This action allow to export the measures history of an asset.
+
+The measures are exported as a CSV stream.
+
+The export process have two steps:
+
+1. execute the `exportMeasures` action with WebSocket or HTTP POST to prepare an export and retrieve an export link
+2. GET request to the generated export link through a tag `<a href="{export-link}">Download</a>`
+
+Those two steps are necessary to avoid the browser to crash when exporting a lot of data.
+
+Export link are valid for 2 minutes.
+
+::: info
+The generated export link does not have protocol, host and port. They should be added to the url.
+
+For example:
+
+```js
+// Query to `device-manager/assets:exportMeasures`
+const { result } = await sdk.query({ ... });
+
+const fullLink = `http${sdk.protocol.ssl ? 's' : ''}://${sdk.protocol.host}:${sdk.protocol.port}${result.link}`;
+```
+
+:::
+
+---
+
+## Query Syntax
+
+### HTTP
+
+```http
+POST: http://kuzzle:7512/_/device-manager/:engineId/assets/:_id/measures/_export
+GET: device-manager/:engineId/assets/:_id/measures/_export/:exportId
+```
+
+### Other protocols
+
+```js
+{
+  "controller": "device-manager/assets",
+  "action": "exportMeasures",
+  "engineId": "<engineId>",
+  "_id": "<assetId>",
+  "body": {
+    "query": {
+      // ...
+    },
+    "sort": [
+      // ...
+    ]
+  },
+
+  // optional:
+  "startAt": "<beginning of time range>",
+  "endAt": "<end of time range>",
+  "type": "<measure type>"
+}
+```
+
+---
+
+## Arguments
+
+- `engineId`: engine id
+- `_id`: asset id
+  ISO_8601
+- `startAt`: beginning of time range (ISO 8601)
+- `endAt`: end of time range (ISO 8601)
+- `type`: measure type
+
+## Body properties
+
+- `query`: the search query itself, using the [Koncorde Filters DSL](/core/2/api/koncorde-filters-syntax) syntax.
+- `sort`: contains a list of fields, used to [sort search results](https://www.elastic.co/guide/en/elasticsearch/reference/7.4/search-request-sort.html), in order of importance
+
+---
+
+## Response
+
+```js
+{
+  "status": 200,
+  "error": null,
+  "controller": "device-manager/assets",
+  "action": "search",
+  "requestId": "<unique request identifier>",
+  "result": {
+    "link": " /_/device-manager/<engine id>/devices/<asset id>/measures/_export/<export id>?jwt=<token>"
+  }
+}
+```