Skip to content

Commit

Permalink
feat(configuration): allow to specify Elasticsearch index settings (#337
Browse files Browse the repository at this point in the history
)

* Add missing settings fields in plugin configuration and use it during indices creation

* Use correct spelling and adapt collection creation arguments to match recent changes

* Update Docker Compose stack to use kuzzleio/kuzzle-runner:20

* Add documentation about the plugin configuration

* Remove plural on assets to avoid breaking change on the configuration format

* Remove plural on devices engine collection property to avoid breaking change
  • Loading branch information
alexandrebouthinon committed Jul 17, 2024
1 parent 865ae6d commit 6d22fa9
Show file tree
Hide file tree
Showing 5 changed files with 255 additions and 25 deletions.
198 changes: 198 additions & 0 deletions doc/2/concepts/configuration/index.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
---
code: false
type: page
title: Configuration
description: Plugin configuration
---

# Configuration

You can customize the behavior of the Device Manager plugin by providing a configuration object in the `plugins` section using the Kuzzle configuration file ([`kuzzlerc`](https://docs.kuzzle.io/core/2/guides/advanced/configuration/)).


```js
{
"plugins": {
"device-manager": {
// Configuration options
}
}
}
```

# Options

## Global

The following global options are available:

| Name | Type | Default | Description |
| --------------------- | ------- | ---------------- | -------------------------------------------------------------------------------------------------- |
| `ignoreStartupErrors` | boolean | `false` | If `true`, the plugin will not throw an error if the engine is not reachable at startup. |
| `engine.autoUpdate` | boolean | `true` | If `true`, the plugin will automatically update the engine collections when the plugin is started. |
| `adminIndex` | string | `device-manager` | The index name where the plugin stores its configuration and devices. |

## Collections

In order to customize the collections used by the plugin, you can provide a configuration object for each collection. All of these support the following properties:

- `name`: The name of the collection.
- `mappings`: The mappings of the collection (according to the [Elasticsearch mapping specification](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html)).
- `settings`: The settings of the collection (according to the [Elasticsearch settings specification](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html)). This property is optional but can be useful to define the number of shards and replicas of the Elasticsearch index or use custom analyzers.

### Admin collections

The following collections are used by the plugin. They're automatically created when the plugin is started and shared accross all the engines.

- `devices`: The collection that stores the devices.
- `config`: The collection that stores the plugin configuration.
- `payloads`: The collection that stores the raw payloads before they are processed and decoded by the plugin.

### Engine collections

The following collections are used by the plugin. They're automatically created when the plugin is started and are specific to each engine:

- `config`: The collection that stores the engine configuration.
- `device`: The collection that stores the devices.
- `measures`: The collection that stores the measures once extracted by the decoders.
- `asset`: The collection that stores the assets.
- `assetGroups`: The collection that stores the assets groups.
- `assetHistory`: The collection that stores the assets history.

# Example

```js
{
"plugins": {
"device-manager": {
"ignoreStartupErrors": false,
"engine": {
"autoUpdate": true
},
"adminIndex": "device-manager",
"adminCollections": {
"devices": {
"name": "devices",
"mappings": {
"properties": {
"name": { "type": "keyword" },
"description": { "type": "text" },
"type": { "type": "keyword" },
"status": { "type": "keyword" },
"lastSeen": { "type": "date" }
}
},
"settings": {
"index": {
"number_of_shards": 1,
"number_of_replicas": 1
}
}
},
"config": {
"name": "config",
"mappings": {
"properties": {
"name": { "type": "keyword" },
"value": { "type": "text" }
}
},
"settings": {
"index": {
"number_of_shards": 1,
"number_of_replicas": 1
}
}
},
"payloads": {
"name": "payloads",
"mappings": {
"properties": {
"payload": { "type": "text" }
}
}
}
},
"engineCollections": {
"config": {
"name": "config",
"mappings": {
"properties": {
"name": { "type": "keyword" },
"value": { "type": "text" }
}
}
},
"devices": {
"name": "devices",
"mappings": {
"properties": {
"name": { "type": "keyword" },
"description": { "type": "text" },
"type": { "type": "keyword" },
"status": { "type": "keyword" },
"lastSeen": { "type": "date" }
}
}
},
"measures": {
"name": "measures",
"mappings": {
"properties": {
"deviceId": { "type": "keyword" },
"timestamp": { "type": "date" },
"measures": { "type": "object" }
}
}
},
"asset": {
"name": "assets",
"mappings": {
"properties": {
"name": { "type": "keyword" },
"description": {
"type": "text",
"analyzer": "description"
},
"type": { "type": "keyword" },
"status": { "type": "keyword" }
}
},
"settings": {
// Custom analyzer definition
"analysis": {
"analyzer": {
"description": {
"type": "custom",
"tokenizer": "whitespace"
}
}
}
}
},
"assetGroups": {
"name": "assetGroups",
"mappings": {
"properties": {
"name": { "type": "keyword" },
"description": { "type": "text" },
"type": { "type": "keyword" },
"status": { "type": "keyword" }
}
}
},
"assetHistory": {
"name": "assetHistory",
"mappings": {
"properties": {
"assetId": { "type": "keyword" },
"timestamp": { "type": "date" },
"status": { "type": "keyword" }
}
}
}
}
}
}
}
```
2 changes: 1 addition & 1 deletion docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ services:
- esdata01:/usr/share/elasticsearch/data

kuzzle:
image: kuzzleio/kuzzle-runner:18
image: kuzzleio/kuzzle-runner:20
restart: unless-stopped
command: "bash /docker/start-backend.sh"
profiles: ["backend"]
Expand Down
38 changes: 23 additions & 15 deletions lib/modules/plugin/DeviceManagerEngine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -326,12 +326,12 @@ export class DeviceManagerEngine extends AbstractEngine<DeviceManagerPlugin> {
"asset",
engineGroup,
);
const settings = this.config.engineCollections.asset.settings;

await this.tryCreateCollection(
engineId,
InternalCollection.ASSETS,
await this.tryCreateCollection(engineId, InternalCollection.ASSETS, {
mappings,
);
settings,
});

return InternalCollection.ASSETS;
}
Expand All @@ -355,10 +355,15 @@ export class DeviceManagerEngine extends AbstractEngine<DeviceManagerPlugin> {

_.merge(mappings.properties.asset, assetsMappings);

const settings = this.config.engineCollections.assetHistory.settings;

await this.tryCreateCollection(
engineId,
InternalCollection.ASSETS_HISTORY,
mappings,
{
mappings,
settings,
},
);

return InternalCollection.ASSETS_HISTORY;
Expand All @@ -373,8 +378,11 @@ export class DeviceManagerEngine extends AbstractEngine<DeviceManagerPlugin> {
* @throws If it failed during the assets groups collection creation
*/
async createAssetsGroupsCollection(engineId: string) {
const settings = this.config.engineCollections.assetGroups.settings;

await this.tryCreateCollection(engineId, InternalCollection.ASSETS_GROUPS, {
mappings: assetGroupsMappings,
settings,
});

return InternalCollection.ASSETS_GROUPS;
Expand All @@ -391,12 +399,12 @@ export class DeviceManagerEngine extends AbstractEngine<DeviceManagerPlugin> {
async createDevicesCollection(engineId: string) {
const mappings =
await this.getDigitalTwinMappingsFromDB<DeviceModelContent>("device");
const settings = this.config.engineCollections.device.settings;

await this.tryCreateCollection(
engineId,
InternalCollection.DEVICES,
await this.tryCreateCollection(engineId, InternalCollection.DEVICES, {
mappings,
);
settings,
});

return InternalCollection.DEVICES;
}
Expand All @@ -411,12 +419,12 @@ export class DeviceManagerEngine extends AbstractEngine<DeviceManagerPlugin> {
*/
async createMeasuresCollection(engineId: string, engineGroup: string) {
const mappings = await this.getMeasuresMappingsFromDB(engineGroup);
const settings = this.config.engineCollections.measures.settings;

await this.tryCreateCollection(
engineId,
InternalCollection.MEASURES,
await this.tryCreateCollection(engineId, InternalCollection.MEASURES, {
mappings,
);
settings,
});

return InternalCollection.MEASURES;
}
Expand All @@ -433,15 +441,15 @@ export class DeviceManagerEngine extends AbstractEngine<DeviceManagerPlugin> {
private async tryCreateCollection(
engineIndex: string,
collection: string,
mappings:
options:
| CollectionMappings
| {
mappings?: CollectionMappings;
settings?: JSONObject;
},
) {
try {
await this.sdk.collection.create(engineIndex, collection, mappings);
await this.sdk.collection.create(engineIndex, collection, options);
} catch (error) {
throw new InternalError(
`Failed to create the collection [${collection}] for engine [${engineIndex}]: ${error.message}`,
Expand Down
18 changes: 13 additions & 5 deletions lib/modules/plugin/DeviceManagerPlugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -305,17 +305,14 @@ export class DeviceManagerPlugin extends Plugin {
dynamic: "strict",
properties: {},
},
settings: {},
},
devices: {
name: "devices",
mappings: devicesMappings,
settings: {},
},
payloads: {
name: "payloads",
mappings: payloadsMappings,
settings: {},
},
},
engineCollections: {
Expand All @@ -325,16 +322,24 @@ export class DeviceManagerPlugin extends Plugin {
dynamic: "strict",
properties: {},
},
settings: {},
},
asset: {
name: InternalCollection.ASSETS,
mappings: assetsMappings,
},
assetGroups: {
name: InternalCollection.ASSETS_GROUPS,
},
assetHistory: {
name: InternalCollection.ASSETS_HISTORY,
},
device: {
name: InternalCollection.DEVICES,
mappings: devicesMappings,
},
measures: {
name: InternalCollection.MEASURES,
},
},
};
/* eslint-enable sort-keys */
Expand Down Expand Up @@ -490,7 +495,10 @@ export class DeviceManagerPlugin extends Plugin {
});

await this.sdk.collection
.create(this.config.adminIndex, "payloads", this.getPayloadsMappings())
.create(this.config.adminIndex, "payloads", {
mappings: this.getPayloadsMappings(),
settings: this.config.adminCollections.payloads.settings,
})
.catch((error) => {
throw keepStack(
error,
Expand Down
Loading

0 comments on commit 6d22fa9

Please sign in to comment.