From 6172075effe21bb7c95caed08cc61b37b6fb8240 Mon Sep 17 00:00:00 2001 From: Haye de Wit <68965813+haye-webbio@users.noreply.github.com> Date: Thu, 2 Dec 2021 15:14:35 +0100 Subject: [PATCH 1/6] update index settings when updating a collection (#328) --- connectors/collection.js | 21 +++++++++++++++++++++ connectors/meilisearch/index.js | 14 ++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/connectors/collection.js b/connectors/collection.js index 79e96872..7204ff91 100644 --- a/connectors/collection.js +++ b/connectors/collection.js @@ -152,5 +152,26 @@ module.exports = ({ services, models, logger }) => { } return entries }, + + /** + * Returns MeiliSearch index settings from model definition. + * @param collection - Name of the Collection. + * @typedef Settings + * @type {import('meilisearch').Settings} + * @return {Settings} - MeiliSearch index settings + */ + getSettings: function (collection) { + const model = models[collection].meilisearch || {} + const settings = model.settings || {} + + if (typeof settings !== 'object') { + logger.warn( + `[MEILISEARCH]: "settings" provided in the model of the ${collection} must be an object.` + ) + return {} + } + + return settings + }, } } diff --git a/connectors/meilisearch/index.js b/connectors/meilisearch/index.js index 146f3599..08d29bb2 100644 --- a/connectors/meilisearch/index.js +++ b/connectors/meilisearch/index.js @@ -279,6 +279,20 @@ module.exports = async ({ storeConnector, collectionConnector }) => { entries: transformedEntries, }) + // Get MeiliSearch Index settings from model + const settings = collectionConnector.getSettings(collection) + + // Update MeiliSearch index settings if settings not empty + if (settings && Object.keys(settings).length !== 0) { + try { + await client.index(indexUid).updateSettings(settings) + } catch (error) { + console.error( + `[MEILISEARCH]: Failed updating MeiliSearch settings for collection: ${collection}. Please check your settings.` + ) + } + } + // Add documents in MeiliSearch const { updateId } = await client .index(indexUid) From 04030ae37eff404d6d63bbf5629facff90a494c0 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 2 Dec 2021 15:37:03 +0100 Subject: [PATCH 2/6] Add section in README about settings usage --- README.md | 21 +++++++++++++++++++ .../entries-transformers/dates-transformer.js | 2 +- .../filter-compatibility.js | 2 +- .../entries-transformers/remove-fields.js | 2 +- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7171f466..f17cf494 100644 --- a/README.md +++ b/README.md @@ -287,7 +287,28 @@ Resulting in `categories` being transformed like this in a `restaurant` entry. By transforming the `categories` into an array of names, it is now compatible with the [`filtering` feature](https://docs.meilisearch.com/reference/features/filtering_and_faceted_search.html#configuring-filters) in MeiliSearch. +#### 🏗 Add MeiliSearch Settings +Each index in MeiliSearch can be customized with specific settings. It is possible to add your [MeiliSearch settings](https://docs.meilisearch.com/reference/features/settings.html#settings) configuration to the indexes you create using `settings` field in your model's config. + +The settings are added when either adding a whole collection to MeiliSearch or when updating a collection in MeiliSearch. The settings are not updated when documents are added through the [`listeners`](-apply-hooks). + +**For example** +```js +module.exports = { + meilisearch: { + settings: { + filterableAttributes: ['genres'], + distinctAttribute: null, + searchableAttributes: ['title', 'description', 'genres'], + synonyms: { + wolverine: ['xmen', 'logan'], + logan: ['wolverine', 'xmen'] + } + } + }, +} +``` ### 🕵️‍♀️ Start Searching diff --git a/resources/entries-transformers/dates-transformer.js b/resources/entries-transformers/dates-transformer.js index 6b5ce634..9f5c1006 100644 --- a/resources/entries-transformers/dates-transformer.js +++ b/resources/entries-transformers/dates-transformer.js @@ -13,7 +13,7 @@ function dateToTimeStamp(date) { module.exports = { meilisearch: { - transformEntry(entry) { + transformEntry({ entry }) { const transformedEntry = { ...entry, // transform date format to timestamp diff --git a/resources/entries-transformers/filter-compatibility.js b/resources/entries-transformers/filter-compatibility.js index 103eb6dd..bc09d933 100644 --- a/resources/entries-transformers/filter-compatibility.js +++ b/resources/entries-transformers/filter-compatibility.js @@ -7,7 +7,7 @@ module.exports = { meilisearch: { - transformEntry(entry) { + transformEntry({ entry }) { const transformedEntry = { ...entry, categories: entry.categories.map(cat => cat.name), // map to only have categories name diff --git a/resources/entries-transformers/remove-fields.js b/resources/entries-transformers/remove-fields.js index c1693339..dc0e3ed8 100644 --- a/resources/entries-transformers/remove-fields.js +++ b/resources/entries-transformers/remove-fields.js @@ -7,7 +7,7 @@ module.exports = { meilisearch: { - transformEntry(entry) { + transformEntry({ entry }) { const transformedEntry = entry // remove created by and updated by fields delete transformedEntry.created_by From 4c93a8cc8049d2b4760640fc107b2d39ee1c4ad7 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 2 Dec 2021 15:38:28 +0100 Subject: [PATCH 3/6] Fix tests caused by strapi notifications --- cypress/integration/ui_spec.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cypress/integration/ui_spec.js b/cypress/integration/ui_spec.js index 04f43fa0..42a3b484 100644 --- a/cypress/integration/ui_spec.js +++ b/cypress/integration/ui_spec.js @@ -58,6 +58,7 @@ describe('Strapi Login flow', () => { }) it('Add credentials', () => { + cy.removeNotifications() cy.get('input[name="MSHost"]').clear().type(host) cy.get('input[name="MSApiKey"]').clear().type(apiKey) cy.get('.credentials_button').click() @@ -98,6 +99,7 @@ describe('Strapi Login flow', () => { }) cy.contains('Reload needed', { timeout: 10000 }) cy.reloadServer() + cy.removeNotifications() }) it('Check for successfull listened in develop mode', () => { @@ -205,6 +207,7 @@ describe('Strapi Login flow', () => { cy.contains('Reload needed', { timeout: 10000 }) } cy.reloadServer() + cy.removeNotifications() }) it('Check that collections are not in MeiliSearch anymore', () => { From 8b98055ef275d6eba71034fb815cdaa54e2b236d Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 2 Dec 2021 15:50:50 +0100 Subject: [PATCH 4/6] Add resources with settings examples --- README.md | 4 +- .../meilisearch-settings/basic-example.js | 17 +++++++++ .../make-relationship-filterable.js | 37 +++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 resources/meilisearch-settings/basic-example.js create mode 100644 resources/meilisearch-settings/make-relationship-filterable.js diff --git a/README.md b/README.md index f17cf494..ac2d9fbe 100644 --- a/README.md +++ b/README.md @@ -291,7 +291,7 @@ By transforming the `categories` into an array of names, it is now compatible wi Each index in MeiliSearch can be customized with specific settings. It is possible to add your [MeiliSearch settings](https://docs.meilisearch.com/reference/features/settings.html#settings) configuration to the indexes you create using `settings` field in your model's config. -The settings are added when either adding a whole collection to MeiliSearch or when updating a collection in MeiliSearch. The settings are not updated when documents are added through the [`listeners`](-apply-hooks). +The settings are added when either: adding a collection to MeiliSearch or when updating a collection in MeiliSearch. The settings are not updated when documents are added through the [`listeners`](-apply-hooks). **For example** ```js @@ -310,6 +310,8 @@ module.exports = { } ``` +[See resources](./resources/meilisearch-settings) for more settings examples. + ### 🕵️‍♀️ Start Searching Once you have a collection containing documents indexed in MeiliSearch, you can [start searching](https://docs.meilisearch.com/learn/getting_started/quick_start.html#search). diff --git a/resources/meilisearch-settings/basic-example.js b/resources/meilisearch-settings/basic-example.js new file mode 100644 index 00000000..0b44b935 --- /dev/null +++ b/resources/meilisearch-settings/basic-example.js @@ -0,0 +1,17 @@ +/** + * Adds the settings specific to this collection in MeiliSearch. + */ + +module.exports = { + meilisearch: { + settings: { + filterableAttributes: ['genres'], + distinctAttribute: null, + searchableAttributes: ['title', 'description', 'genres'], + synonyms: { + wolverine: ['xmen', 'logan'], + logan: ['wolverine', 'xmen'], + }, + }, + }, +} diff --git a/resources/meilisearch-settings/make-relationship-filterable.js b/resources/meilisearch-settings/make-relationship-filterable.js new file mode 100644 index 00000000..98461082 --- /dev/null +++ b/resources/meilisearch-settings/make-relationship-filterable.js @@ -0,0 +1,37 @@ +/** + * Imagine if you have a collection restaurant with a category field. + * Category is a collection of its own. They have a many-to-many relationship. + * + * In MeiliSearch you can use filters to for example in our case, only find italian restaurants. + * To be able to do that, you need to provide a list of values and add in the settings the field in `filterableAttributes`. + * See guide: https://docs.meilisearch.com/reference/features/filtering_and_faceted_search.html#configuring-filters + * + * In Strapi, when fetching an entry the many-to-many relationships are inside an object: + * + * { + * id: 1, + * restaurant_name: "The squared pizza", + * category: [ + * { id: 1, name: "italian" }, + * * { id: 2, name: "French" } + * ] + * } + * + * Since MeiliSearch is expecting `category: ["Italian", "french"]` and + * also `category` to be in `filterableAttributes` we can use the model configuration file to provide all these informations. + */ + +module.exports = { + meilisearch: { + settings: { + filterableAttributes: ['categories'], // add categories to filterable attributes. + }, + transformEntry({ entry }) { + const transformedEntry = { + ...entry, + categories: entry.categories.map(cat => cat.name), // map categories to only have categories name. + } + return transformedEntry + }, + }, +} From 2bdc99efd97cf9e4625b01e035243deb556ea2ff Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 2 Dec 2021 18:32:36 +0100 Subject: [PATCH 5/6] Add settings tests --- .../__tests__/custom-index-name.tests.js | 3 + .../__tests__/entry-transformer.tests.js | 4 +- connectors/__tests__/settings.tests.js | 159 ++++++++++++++++++ connectors/meilisearch/index.js | 18 +- .../api/restaurant/models/restaurant.js | 5 +- 5 files changed, 173 insertions(+), 16 deletions(-) create mode 100644 connectors/__tests__/settings.tests.js diff --git a/connectors/__tests__/custom-index-name.tests.js b/connectors/__tests__/custom-index-name.tests.js index ffe14f33..47a96511 100644 --- a/connectors/__tests__/custom-index-name.tests.js +++ b/connectors/__tests__/custom-index-name.tests.js @@ -6,8 +6,11 @@ const createCollectionConnector = require('../collection') jest.mock('meilisearch') const addDocumentsMock = jest.fn(() => 10) +const updateSettingsMock = jest.fn(() => 10) + const mockIndex = jest.fn(() => ({ addDocuments: addDocumentsMock, + updateSettings: updateSettingsMock, })) MeiliSearch.mockImplementation(() => { diff --git a/connectors/__tests__/entry-transformer.tests.js b/connectors/__tests__/entry-transformer.tests.js index e42d0c91..86637c36 100644 --- a/connectors/__tests__/entry-transformer.tests.js +++ b/connectors/__tests__/entry-transformer.tests.js @@ -6,8 +6,11 @@ const createCollectionConnector = require('../collection') jest.mock('meilisearch') const addDocumentsMock = jest.fn(() => 10) +const updateSettingsMock = jest.fn(() => 10) + const mockIndex = jest.fn(() => ({ addDocuments: addDocumentsMock, + updateSettings: updateSettingsMock, })) MeiliSearch.mockImplementation(() => { @@ -76,7 +79,6 @@ describe('Entry transformation', () => { }) afterEach(() => { - jest.resetAllMocks() jest.clearAllMocks() jest.restoreAllMocks() }) diff --git a/connectors/__tests__/settings.tests.js b/connectors/__tests__/settings.tests.js new file mode 100644 index 00000000..08e5b0d3 --- /dev/null +++ b/connectors/__tests__/settings.tests.js @@ -0,0 +1,159 @@ +const { MeiliSearch } = require('meilisearch') +const createMeiliSearchConnector = require('../meilisearch') +const createStoreConnector = require('../store') +const createCollectionConnector = require('../collection') + +jest.mock('meilisearch') + +const addDocumentsMock = jest.fn(() => 10) +const updateSettingsMock = jest.fn(() => 10) + +const mockIndex = jest.fn(() => ({ + addDocuments: addDocumentsMock, + updateSettings: updateSettingsMock, +})) + +MeiliSearch.mockImplementation(() => { + return { + getOrCreateIndex: () => { + return mockIndex + }, + index: mockIndex, + } +}) + +const storeClientMock = { + set: jest.fn(() => 'test'), + get: jest.fn(() => 'test'), +} + +const servicesMock = { + restaurant: { + count: jest.fn(() => { + return 11 + }), + find: jest.fn(() => { + return [{ id: '1', collection: [{ name: 'one' }, { name: 'two' }] }] + }), + }, +} + +const transformEntryMock = jest.fn(function ({ entry }) { + const transformedEntry = { + ...entry, + collection: entry.collection.map(cat => cat.name), + } + return transformedEntry +}) + +const loggerMock = { + warn: jest.fn(() => 'test'), +} + +describe('Test MeiliSearch settings', () => { + let storeConnector + beforeEach(async () => { + jest.clearAllMocks() + jest.restoreAllMocks() + storeConnector = createStoreConnector({ + storeClient: storeClientMock, + }) + }) + + test('Test not settings field in configuration', async () => { + const modelMock = { + restaurant: { + meilisearch: { + indexName: 'my_restaurant', + transformEntry: transformEntryMock, + }, + }, + } + const collectionConnector = createCollectionConnector({ + logger: loggerMock, + models: modelMock, + services: servicesMock, + }) + const meilisearchConnector = await createMeiliSearchConnector({ + collectionConnector, + storeConnector, + }) + const getIndexNameSpy = jest.spyOn(collectionConnector, 'getIndexName') + const getSettingsSpy = jest.spyOn(collectionConnector, 'getSettings') + + await meilisearchConnector.addCollectionInMeiliSearch('restaurant') + + expect(servicesMock.restaurant.count).toHaveBeenCalledTimes(1) + + expect(getIndexNameSpy).toHaveBeenCalledWith('restaurant') + expect(getIndexNameSpy).toHaveReturnedWith('my_restaurant') + expect(getSettingsSpy).toHaveBeenCalledWith('restaurant') + expect(getSettingsSpy).toHaveReturnedWith({}) + }) + + test('Test a empty setting object in configuration', async () => { + const modelMock = { + restaurant: { + meilisearch: { + indexName: 'my_restaurant', + transformEntry: transformEntryMock, + settings: {}, + }, + }, + } + const collectionConnector = createCollectionConnector({ + logger: loggerMock, + models: modelMock, + services: servicesMock, + }) + const meilisearchConnector = await createMeiliSearchConnector({ + collectionConnector, + storeConnector, + }) + const getIndexNameSpy = jest.spyOn(collectionConnector, 'getIndexName') + const getSettingsSpy = jest.spyOn(collectionConnector, 'getSettings') + + await meilisearchConnector.addCollectionInMeiliSearch('restaurant') + + expect(servicesMock.restaurant.count).toHaveBeenCalledTimes(1) + + expect(getIndexNameSpy).toHaveBeenCalledWith('restaurant') + expect(getIndexNameSpy).toHaveReturnedWith('my_restaurant') + expect(getSettingsSpy).toHaveBeenCalledWith('restaurant') + expect(getSettingsSpy).toHaveReturnedWith({}) + }) + + test('Test a setting object with one field in configuration', async () => { + const modelMock = { + restaurant: { + meilisearch: { + indexName: 'my_restaurant', + transformEntry: transformEntryMock, + settings: { + searchableAttributes: ['*'], + }, + }, + }, + } + const collectionConnector = createCollectionConnector({ + logger: loggerMock, + models: modelMock, + services: servicesMock, + }) + const meilisearchConnector = await createMeiliSearchConnector({ + collectionConnector, + storeConnector, + }) + const getIndexNameSpy = jest.spyOn(collectionConnector, 'getIndexName') + const getSettingsSpy = jest.spyOn(collectionConnector, 'getSettings') + + await meilisearchConnector.addCollectionInMeiliSearch('restaurant') + + expect(servicesMock.restaurant.count).toHaveBeenCalledTimes(1) + + expect(getIndexNameSpy).toHaveBeenCalledWith('restaurant') + expect(getIndexNameSpy).toHaveReturnedWith('my_restaurant') + expect(getSettingsSpy).toHaveBeenCalledWith('restaurant') + expect(getSettingsSpy).toHaveReturnedWith({ searchableAttributes: ['*'] }) + }) +}) diff --git a/connectors/meilisearch/index.js b/connectors/meilisearch/index.js index 08d29bb2..ea5657c6 100644 --- a/connectors/meilisearch/index.js +++ b/connectors/meilisearch/index.js @@ -264,6 +264,10 @@ module.exports = async ({ storeConnector, collectionConnector }) => { const client = MeiliSearch({ apiKey, host }) const indexUid = collectionConnector.getIndexName(collection) + // Get MeiliSearch Index settings from model + const settings = collectionConnector.getSettings(collection) + await client.index(indexUid).updateSettings(settings) + // Callback function for batching action const addDocuments = async (entries, collection) => { if (entries.length === 0) { @@ -279,20 +283,6 @@ module.exports = async ({ storeConnector, collectionConnector }) => { entries: transformedEntries, }) - // Get MeiliSearch Index settings from model - const settings = collectionConnector.getSettings(collection) - - // Update MeiliSearch index settings if settings not empty - if (settings && Object.keys(settings).length !== 0) { - try { - await client.index(indexUid).updateSettings(settings) - } catch (error) { - console.error( - `[MEILISEARCH]: Failed updating MeiliSearch settings for collection: ${collection}. Please check your settings.` - ) - } - } - // Add documents in MeiliSearch const { updateId } = await client .index(indexUid) diff --git a/playground/api/restaurant/models/restaurant.js b/playground/api/restaurant/models/restaurant.js index c5211dab..7c8858b7 100644 --- a/playground/api/restaurant/models/restaurant.js +++ b/playground/api/restaurant/models/restaurant.js @@ -13,6 +13,9 @@ module.exports = { }; return transformed; }, - indexName: "my_restaurant" + indexName: "my_restaurant", + settings: { + "searchableAttributes": ["*"] + } } } From 471123174dab3214550dd335edcd06106c6dae84 Mon Sep 17 00:00:00 2001 From: Charlotte Vermandel Date: Thu, 2 Dec 2021 18:34:46 +0100 Subject: [PATCH 6/6] Remove clear mocking --- connectors/__tests__/custom-index-name.tests.js | 1 - 1 file changed, 1 deletion(-) diff --git a/connectors/__tests__/custom-index-name.tests.js b/connectors/__tests__/custom-index-name.tests.js index 47a96511..b942a211 100644 --- a/connectors/__tests__/custom-index-name.tests.js +++ b/connectors/__tests__/custom-index-name.tests.js @@ -53,7 +53,6 @@ const loggerMock = { describe('Test custom index names', () => { let storeConnector beforeEach(async () => { - jest.resetAllMocks() jest.clearAllMocks() jest.restoreAllMocks() storeConnector = createStoreConnector({