From 71a3fa3f1dc97e23aeb1e5defaf58d216a1f7395 Mon Sep 17 00:00:00 2001 From: tkostuch Date: Tue, 28 Jan 2020 18:39:27 +0100 Subject: [PATCH 1/9] 3948 fetch attributes based on aggregates --- src/api/attribute/service.ts | 126 +++++++++++++++++++++++++++++++++++ src/api/catalog.js | 22 +++++- 2 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 src/api/attribute/service.ts diff --git a/src/api/attribute/service.ts b/src/api/attribute/service.ts new file mode 100644 index 00000000..e794c253 --- /dev/null +++ b/src/api/attribute/service.ts @@ -0,0 +1,126 @@ + +import request from 'request'; +import TagCache from 'redis-tag-cache' +import get from 'lodash/get'; +import cache from '../../lib/cache-instance' + +interface AttributeListParam { + [key: string]: number[] +} + +function getUri (config, indexName) { + return `${config.elasticsearch.protocol}://${config.elasticsearch.host}:${config.elasticsearch.port}/${indexName}/attribute/_search` +} + +async function getAttributeFromCache (attributeCode: string, config) { + if (config.server.useOutputCache && cache) { + try { + const res = await (cache as TagCache).get( + 'api:attribute-list' + attributeCode + ) + return res + } catch (err) { + console.error(err) + return null + } + } +} + +async function setAttributeInCache (attributeList, config) { + if (config.server.useOutputCache && cache) { + try { + await Promise.all( + attributeList.map(attribute => (cache as TagCache).set( + 'api:attribute-list' + attribute.attribute_code, + attribute + )) + ) + } catch (err) { + console.error(err) + } + } +} + +function clearAttributeOpitons (attribute, optionsIds: number[]) { + const stringOptionsIds = optionsIds.map(String) + return { + ...attribute, + options: (attribute.options || []).filter(option => stringOptionsIds.includes(option.value)) + } +} + +function list (attributesParam: AttributeListParam, config, indexName) { + return new Promise(async (resolve, reject) => { + let attributeCodes = Object.keys(attributesParam) + + const rawCachedAttributeList = await Promise.all( + attributeCodes.map(attributeCode => getAttributeFromCache(attributeCode, config)) + ) + + const cachedAttributeList = rawCachedAttributeList + .map((cachedAttribute, index) => { + if (cachedAttribute) { + const attributeOptionsIds = attributesParam[cachedAttribute.attribute_code] + attributeCodes.splice(index, 1) // side effect - reduce elements in needed attribute list + return clearAttributeOpitons(cachedAttribute, attributeOptionsIds) + } + }) + .filter(Boolean) + + if (!attributeCodes.length) { + return cachedAttributeList + } + + request({ + uri: getUri(config, indexName), + method: 'POST', + body: {'query': {'bool': {'filter': {'bool': {'must': [{'terms': {'attribute_code': attributeCodes}}]}}}}}, + json: true + }, async (err, res, body) => { + if (err) { + reject(err) + } + const fetchedAttributeList = get(body, 'hits.hits', []).map(hit => hit._source) + await setAttributeInCache(fetchedAttributeList, config) + resolve(cachedAttributeList.concat( + fetchedAttributeList.map(fetchedAttribute => { + const attributeOptionsIds = attributesParam[fetchedAttribute.attribute_code] + return clearAttributeOpitons(fetchedAttribute, attributeOptionsIds) + })) + ) + }) + }) +} + +function transformToMetadata ({ + is_visible_on_front, + is_visible, + default_frontend_label, + attribute_id, + entity_type_id, + id, + is_user_defined, + is_comparable, + attribute_code, + slug, + options +}) { + return { + is_visible_on_front, + is_visible, + default_frontend_label, + attribute_id, + entity_type_id, + id, + is_user_defined, + is_comparable, + attribute_code, + slug, + options + } +} + +module.exports = { + list, + transformToMetadata +} diff --git a/src/api/catalog.js b/src/api/catalog.js index 27f1215d..7120dc55 100755 --- a/src/api/catalog.js +++ b/src/api/catalog.js @@ -4,6 +4,7 @@ import ProcessorFactory from '../processor/factory'; import { adjustBackendProxyUrl } from '../lib/elastic' import cache from '../lib/cache-instance' import { sha3_224 } from 'js-sha3' +import AttributeService from './attribute/service' function _cacheStorageHandler (config, result, hash, tags) { if (config.server.useOutputCache && cache) { @@ -17,6 +18,20 @@ function _cacheStorageHandler (config, result, hash, tags) { } } +async function getProductsAttributesMetadata (body, config, indexName) { + const attributeListParam = Object.keys(body.aggregations) + .filter(key => body.aggregations[key].buckets.length) // leave only buckets with values + .reduce((acc, key) => { + const attributeCode = key.replace(/agg_terms_|agg_range_/, '') + return { + ...acc, + [attributeCode]: body.aggregations[key].buckets.map(bucket => bucket.key) + } + }, {}) + const attributeList = await AttributeService.list(attributeListParam, config, indexName) + return attributeList +} + export default ({config, db}) => function (req, res, body) { let groupId = null @@ -103,11 +118,14 @@ export default ({config, db}) => function (req, res, body) { let resultProcessor = factory.getAdapter(entityType, indexName, req, res) if (!resultProcessor) { resultProcessor = factory.getAdapter('default', indexName, req, res) } // get the default processor - if (entityType === 'product') { - resultProcessor.process(_resBody.hits.hits, groupId).then((result) => { + resultProcessor.process(_resBody.hits.hits, groupId).then(async (result) => { _resBody.hits.hits = result _cacheStorageHandler(config, _resBody, reqHash, tagsArray) + if (_resBody.aggregations) { + const attributesMetadata = await getProductsAttributesMetadata(_resBody, config, indexName) + _resBody.attribute_metadata = attributesMetadata.map(AttributeService.transformToMetadata) + } res.json(_resBody); }).catch((err) => { console.error(err) From 54373445361f0a1e94913cd406f97def63627294 Mon Sep 17 00:00:00 2001 From: tkostuch Date: Thu, 30 Jan 2020 08:33:47 +0100 Subject: [PATCH 2/9] 3948 add comments, add types, refactor, handle range and options --- src/api/attribute/service.ts | 62 ++++++++++++++++++++++++++++-------- src/api/catalog.ts | 44 ++++++++++++++++++++----- 2 files changed, 85 insertions(+), 21 deletions(-) diff --git a/src/api/attribute/service.ts b/src/api/attribute/service.ts index ec78bbc4..dcc389e0 100644 --- a/src/api/attribute/service.ts +++ b/src/api/attribute/service.ts @@ -4,14 +4,20 @@ import TagCache from 'redis-tag-cache' import get from 'lodash/get'; import cache from '../../lib/cache-instance' -interface AttributeListParam { +export interface AttributeListParam { [key: string]: number[] } -function getUri (config, indexName) { +/** + * Build ES uri fro attributes + */ +function getUri (config, indexName: string): string { return `${config.elasticsearch.protocol}://${config.elasticsearch.host}:${config.elasticsearch.port}/${indexName}/attribute/_search` } +/** + * Returns attributes from cache + */ async function getAttributeFromCache (attributeCode: string, config) { if (config.server.useOutputCache && cache) { try { @@ -26,6 +32,9 @@ async function getAttributeFromCache (attributeCode: string, config) { } } +/** + * Save attributes in cache + */ async function setAttributeInCache (attributeList, config) { if (config.server.useOutputCache && cache) { try { @@ -41,36 +50,49 @@ async function setAttributeInCache (attributeList, config) { } } +/** + * Returns attribute with only needed options + * @param attribute - attribute object + * @param optionsIds - list of only needed options ids + */ function clearAttributeOpitons (attribute, optionsIds: number[]) { const stringOptionsIds = optionsIds.map(String) return { ...attribute, - options: (attribute.options || []).filter(option => stringOptionsIds.includes(option.value)) + options: (attribute.options || []).filter(option => stringOptionsIds.includes(String(option.value))) } } -function list (attributesParam: AttributeListParam, config, indexName) { +function list (attributesParam: AttributeListParam, config, indexName: string): Promise { return new Promise(async (resolve, reject) => { + // we start with all attributeCodes that are requested let attributeCodes = Object.keys(attributesParam) + // here we check if some of attribute are in cache const rawCachedAttributeList = await Promise.all( attributeCodes.map(attributeCode => getAttributeFromCache(attributeCode, config)) ) const cachedAttributeList = rawCachedAttributeList + .filter(Boolean) // remove empty results from cache.get .map((cachedAttribute, index) => { if (cachedAttribute) { const attributeOptionsIds = attributesParam[cachedAttribute.attribute_code] - attributeCodes.splice(index, 1) // side effect - reduce elements in needed attribute list + + // side effect - we want to reduce starting 'attributeCodes' because some of them are in cache + attributeCodes.splice(index, 1) + + // clear unused options return clearAttributeOpitons(cachedAttribute, attributeOptionsIds) } }) - .filter(Boolean) + // if all requested attributes are in cache then we can return here if (!attributeCodes.length) { - return cachedAttributeList + return resolve(cachedAttributeList) } + // fetch attributes for rest attributeCodes request({ uri: getUri(config, indexName), method: 'POST', @@ -81,17 +103,29 @@ function list (attributesParam: AttributeListParam, config, indexName) { reject(err) } const fetchedAttributeList = get(body, 'hits.hits', []).map(hit => hit._source) + + // save atrributes in cache await setAttributeInCache(fetchedAttributeList, config) - resolve(cachedAttributeList.concat( - fetchedAttributeList.map(fetchedAttribute => { + + // cached and fetched attributes + const allAttributes = [ + ...cachedAttributeList, + ...fetchedAttributeList.map(fetchedAttribute => { const attributeOptionsIds = attributesParam[fetchedAttribute.attribute_code] + + // clear unused options return clearAttributeOpitons(fetchedAttribute, attributeOptionsIds) - })) - ) + }) + ] + + return resolve(allAttributes) }) }) } +/** + * Returns only needed data for filters in vsf + */ function transformToMetadata ({ is_visible_on_front, is_visible, @@ -103,7 +137,8 @@ function transformToMetadata ({ is_comparable, attribute_code, slug, - options + options = [], + buckets = [] }) { return { is_visible_on_front, @@ -116,7 +151,8 @@ function transformToMetadata ({ is_comparable, attribute_code, slug, - options + options, + buckets } } diff --git a/src/api/catalog.ts b/src/api/catalog.ts index edbee811..e098c438 100755 --- a/src/api/catalog.ts +++ b/src/api/catalog.ts @@ -4,7 +4,7 @@ import ProcessorFactory from '../processor/factory'; import { adjustBackendProxyUrl } from '../lib/elastic' import cache from '../lib/cache-instance' import { sha3_224 } from 'js-sha3' -import AttributeService from './attribute/service' +import AttributeService, { AttributeListParam } from './attribute/service' import bodybuilder from 'bodybuilder' import { elasticsearch, SearchQuery } from 'storefront-query-builder' @@ -20,17 +20,45 @@ function _cacheStorageHandler (config, result, hash, tags) { } } -async function getProductsAttributesMetadata (body, config, indexName): Promise { - const attributeListParam = Object.keys(body.aggregations) +/** + * Transforms ES aggregates into valid format for AttributeService - {[attribute_code]: [bucketId1, bucketId2]} + * @param body - products response body + * @param config - global config + * @param indexName - current indexName + */ +async function getProductsAttributesMetadata (body, config, indexName: string): Promise { + const attributeListParam: AttributeListParam = Object.keys(body.aggregations) .filter(key => body.aggregations[key].buckets.length) // leave only buckets with values .reduce((acc, key) => { - const attributeCode = key.replace(/agg_terms_|agg_range_/, '') - return { - ...acc, - [attributeCode]: body.aggregations[key].buckets.map(bucket => bucket.key) + const attributeCode = key.replace(/^(agg_terms_|agg_range_)|(_options)$/g, '') + const bucketsIds = body.aggregations[key].buckets.map(bucket => bucket.key) + + if (!acc[attributeCode]) { + acc[attributeCode] = [] } + + // there can be more then one attributes for example 'agg_terms_color' and 'agg_terms_color_options' + // we need to get buckets from both + acc[attributeCode] = [...new Set([...acc[attributeCode], ...bucketsIds])] + + return acc }, {}) - const attributeList = await AttributeService.list(attributeListParam, config, indexName) + + // find attribute list + const attributeList: any[] = await AttributeService.list(attributeListParam, config, indexName) + + // add buckets for range attributes + Object.keys(body.aggregations) + .filter(key => key.match(/^(agg_range_)/g)) + .forEach(key => { + const attributeCode = key.replace(/^(agg_range_)/g, '') + const buckets = body.aggregations[key].buckets + const attribute = attributeList.find(attr => attr.attribute_code === attributeCode) + if (attribute) { + attribute.buckets = buckets + } + }) + return attributeList } From 0be0f70d2b668f8e33524c241dbb5490507f0a25 Mon Sep 17 00:00:00 2001 From: tkostuch Date: Thu, 30 Jan 2020 08:46:06 +0100 Subject: [PATCH 3/9] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index af9028d0..a7213e61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add url module - @gibkigonzo (#3942) - The `response_format` query parameter to the `/api/catalog` endpoint. Currently there is just one additional format supported: `response_format=compact`. When used, the response format got optimized by: a) remapping the results, removing the `_source` from the `hits.hits`; b) compressing the JSON fields names according to the `config.products.fieldsToCompact`; c) removing the JSON fields from the `product.configurable_children` when their values === parent product values; overall response size reduced over -70% - @pkarw - The support for `SearchQuery` instead of the ElasticSearch DSL as for the input to `/api/catalog` - using `storefront-query-builder` package - @pkarw - https://github.com/DivanteLtd/vue-storefront/issues/2167 +- Create attribute service that allows to fetch attributes with specific options - used for products aggregates - @gibkigonzo (https://github.com/DivanteLtd/vue-storefront/pull/4001, https://github.com/DivanteLtd/mage2vuestorefront/pull/99) ## [1.11.0] - 2019.12.20 From 9e500170e3a662e9f4e90263b5f4bb24f6e0eeca Mon Sep 17 00:00:00 2001 From: tkostuch Date: Thu, 30 Jan 2020 09:27:45 +0100 Subject: [PATCH 4/9] 3948 remove buckets, because we still have aggregations --- src/api/attribute/service.ts | 6 ++---- src/api/catalog.ts | 12 ------------ 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/src/api/attribute/service.ts b/src/api/attribute/service.ts index dcc389e0..c0eca194 100644 --- a/src/api/attribute/service.ts +++ b/src/api/attribute/service.ts @@ -137,8 +137,7 @@ function transformToMetadata ({ is_comparable, attribute_code, slug, - options = [], - buckets = [] + options = [] }) { return { is_visible_on_front, @@ -151,8 +150,7 @@ function transformToMetadata ({ is_comparable, attribute_code, slug, - options, - buckets + options } } diff --git a/src/api/catalog.ts b/src/api/catalog.ts index e098c438..c0eac750 100755 --- a/src/api/catalog.ts +++ b/src/api/catalog.ts @@ -47,18 +47,6 @@ async function getProductsAttributesMetadata (body, config, indexName: string): // find attribute list const attributeList: any[] = await AttributeService.list(attributeListParam, config, indexName) - // add buckets for range attributes - Object.keys(body.aggregations) - .filter(key => key.match(/^(agg_range_)/g)) - .forEach(key => { - const attributeCode = key.replace(/^(agg_range_)/g, '') - const buckets = body.aggregations[key].buckets - const attribute = attributeList.find(attr => attr.attribute_code === attributeCode) - if (attribute) { - attribute.buckets = buckets - } - }) - return attributeList } From 5ff1f971666a22335edb2a4e64a159e94b03d5d1 Mon Sep 17 00:00:00 2001 From: tkostuch Date: Thu, 30 Jan 2020 11:00:25 +0100 Subject: [PATCH 5/9] 3948 use elasticsearch-js instead of request --- src/api/attribute/service.ts | 102 +++++++++++----------- src/graphql/elasticsearch/queryBuilder.ts | 1 - 2 files changed, 51 insertions(+), 52 deletions(-) diff --git a/src/api/attribute/service.ts b/src/api/attribute/service.ts index c0eca194..88ef56e6 100644 --- a/src/api/attribute/service.ts +++ b/src/api/attribute/service.ts @@ -1,8 +1,9 @@ -import request from 'request'; import TagCache from 'redis-tag-cache' import get from 'lodash/get'; import cache from '../../lib/cache-instance' +import { adjustQuery, getClient as getElasticClient } from './../../lib/elastic' +import bodybuilder from 'bodybuilder' export interface AttributeListParam { [key: string]: number[] @@ -63,64 +64,63 @@ function clearAttributeOpitons (attribute, optionsIds: number[]) { } } -function list (attributesParam: AttributeListParam, config, indexName: string): Promise { - return new Promise(async (resolve, reject) => { - // we start with all attributeCodes that are requested - let attributeCodes = Object.keys(attributesParam) +async function list (attributesParam: AttributeListParam, config, indexName: string): Promise { + // we start with all attributeCodes that are requested + let attributeCodes = Object.keys(attributesParam) - // here we check if some of attribute are in cache - const rawCachedAttributeList = await Promise.all( - attributeCodes.map(attributeCode => getAttributeFromCache(attributeCode, config)) - ) + // here we check if some of attribute are in cache + const rawCachedAttributeList = await Promise.all( + attributeCodes.map(attributeCode => getAttributeFromCache(attributeCode, config)) + ) - const cachedAttributeList = rawCachedAttributeList - .filter(Boolean) // remove empty results from cache.get - .map((cachedAttribute, index) => { - if (cachedAttribute) { - const attributeOptionsIds = attributesParam[cachedAttribute.attribute_code] + const cachedAttributeList = rawCachedAttributeList + .filter(Boolean) // remove empty results from cache.get + .map((cachedAttribute, index) => { + if (cachedAttribute) { + const attributeOptionsIds = attributesParam[cachedAttribute.attribute_code] - // side effect - we want to reduce starting 'attributeCodes' because some of them are in cache - attributeCodes.splice(index, 1) + // side effect - we want to reduce starting 'attributeCodes' because some of them are in cache + attributeCodes.splice(index, 1) - // clear unused options - return clearAttributeOpitons(cachedAttribute, attributeOptionsIds) - } - }) - - // if all requested attributes are in cache then we can return here - if (!attributeCodes.length) { - return resolve(cachedAttributeList) - } - - // fetch attributes for rest attributeCodes - request({ - uri: getUri(config, indexName), - method: 'POST', - body: {'query': {'bool': {'filter': {'bool': {'must': [{'terms': {'attribute_code': attributeCodes}}]}}}}}, - json: true - }, async (err, res, body) => { - if (err) { - reject(err) + // clear unused options + return clearAttributeOpitons(cachedAttribute, attributeOptionsIds) } - const fetchedAttributeList = get(body, 'hits.hits', []).map(hit => hit._source) - - // save atrributes in cache - await setAttributeInCache(fetchedAttributeList, config) + }) - // cached and fetched attributes - const allAttributes = [ - ...cachedAttributeList, - ...fetchedAttributeList.map(fetchedAttribute => { - const attributeOptionsIds = attributesParam[fetchedAttribute.attribute_code] + // if all requested attributes are in cache then we can return here + if (!attributeCodes.length) { + return cachedAttributeList + } - // clear unused options - return clearAttributeOpitons(fetchedAttribute, attributeOptionsIds) - }) - ] + // fetch attributes for rest attributeCodes + try { + const query = adjustQuery({ + index: indexName, + type: 'attribute', + body: bodybuilder().filter('terms', 'attribute_code', attributeCodes).build() + }, 'attribute', config) + const response = await getElasticClient(config).search(query) + const fetchedAttributeList = get(response.body, 'hits.hits', []).map(hit => hit._source) + + // save atrributes in cache + await setAttributeInCache(fetchedAttributeList, config) + + // cached and fetched attributes + const allAttributes = [ + ...cachedAttributeList, + ...fetchedAttributeList.map(fetchedAttribute => { + const attributeOptionsIds = attributesParam[fetchedAttribute.attribute_code] + + // clear unused options + return clearAttributeOpitons(fetchedAttribute, attributeOptionsIds) + }) + ] - return resolve(allAttributes) - }) - }) + return allAttributes + } catch (err) { + console.error(err) + return [] + } } /** diff --git a/src/graphql/elasticsearch/queryBuilder.ts b/src/graphql/elasticsearch/queryBuilder.ts index 29e46f7d..c9c2f3f7 100644 --- a/src/graphql/elasticsearch/queryBuilder.ts +++ b/src/graphql/elasticsearch/queryBuilder.ts @@ -19,6 +19,5 @@ export function buildQuery ({ if (search !== '') { builtQuery['min_score'] = config.get('elasticsearch.min_score') } - return builtQuery; } From fc57251921c1163992f3dc7621747bb0de2c32d2 Mon Sep 17 00:00:00 2001 From: tkostuch Date: Thu, 30 Jan 2020 11:01:17 +0100 Subject: [PATCH 6/9] 3948 remove getUri --- src/api/attribute/service.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/api/attribute/service.ts b/src/api/attribute/service.ts index 88ef56e6..249c1c6d 100644 --- a/src/api/attribute/service.ts +++ b/src/api/attribute/service.ts @@ -9,13 +9,6 @@ export interface AttributeListParam { [key: string]: number[] } -/** - * Build ES uri fro attributes - */ -function getUri (config, indexName: string): string { - return `${config.elasticsearch.protocol}://${config.elasticsearch.host}:${config.elasticsearch.port}/${indexName}/attribute/_search` -} - /** * Returns attributes from cache */ From f6c61f9adba75b26800651a25641081e6e220cc6 Mon Sep 17 00:00:00 2001 From: tkostuch Date: Thu, 30 Jan 2020 11:28:09 +0100 Subject: [PATCH 7/9] 3948 add transformAggsToAttributeListParam, and support graphql --- src/api/attribute/service.ts | 30 ++++++++++++++- src/api/catalog.ts | 38 +++---------------- src/graphql/elasticsearch/catalog/resolver.js | 8 ++++ .../elasticsearch/catalog/schema.graphqls | 2 + 4 files changed, 44 insertions(+), 34 deletions(-) diff --git a/src/api/attribute/service.ts b/src/api/attribute/service.ts index 249c1c6d..f7ff0be7 100644 --- a/src/api/attribute/service.ts +++ b/src/api/attribute/service.ts @@ -9,6 +9,33 @@ export interface AttributeListParam { [key: string]: number[] } +/** + * Transforms ES aggregates into valid format for AttributeService - {[attribute_code]: [bucketId1, bucketId2]} + * @param body - products response body + * @param config - global config + * @param indexName - current indexName + */ +function transformAggsToAttributeListParam (aggregations): AttributeListParam { + const attributeListParam: AttributeListParam = Object.keys(aggregations) + .filter(key => aggregations[key].buckets.length) // leave only buckets with values + .reduce((acc, key) => { + const attributeCode = key.replace(/^(agg_terms_|agg_range_)|(_options)$/g, '') + const bucketsIds = aggregations[key].buckets.map(bucket => bucket.key) + + if (!acc[attributeCode]) { + acc[attributeCode] = [] + } + + // there can be more then one attributes for example 'agg_terms_color' and 'agg_terms_color_options' + // we need to get buckets from both + acc[attributeCode] = [...new Set([...acc[attributeCode], ...bucketsIds])] + + return acc + }, {}) + + return attributeListParam +} + /** * Returns attributes from cache */ @@ -149,5 +176,6 @@ function transformToMetadata ({ export default { list, - transformToMetadata + transformToMetadata, + transformAggsToAttributeListParam } diff --git a/src/api/catalog.ts b/src/api/catalog.ts index c0eac750..25620adc 100755 --- a/src/api/catalog.ts +++ b/src/api/catalog.ts @@ -4,7 +4,7 @@ import ProcessorFactory from '../processor/factory'; import { adjustBackendProxyUrl } from '../lib/elastic' import cache from '../lib/cache-instance' import { sha3_224 } from 'js-sha3' -import AttributeService, { AttributeListParam } from './attribute/service' +import AttributeService from './attribute/service' import bodybuilder from 'bodybuilder' import { elasticsearch, SearchQuery } from 'storefront-query-builder' @@ -20,36 +20,6 @@ function _cacheStorageHandler (config, result, hash, tags) { } } -/** - * Transforms ES aggregates into valid format for AttributeService - {[attribute_code]: [bucketId1, bucketId2]} - * @param body - products response body - * @param config - global config - * @param indexName - current indexName - */ -async function getProductsAttributesMetadata (body, config, indexName: string): Promise { - const attributeListParam: AttributeListParam = Object.keys(body.aggregations) - .filter(key => body.aggregations[key].buckets.length) // leave only buckets with values - .reduce((acc, key) => { - const attributeCode = key.replace(/^(agg_terms_|agg_range_)|(_options)$/g, '') - const bucketsIds = body.aggregations[key].buckets.map(bucket => bucket.key) - - if (!acc[attributeCode]) { - acc[attributeCode] = [] - } - - // there can be more then one attributes for example 'agg_terms_color' and 'agg_terms_color_options' - // we need to get buckets from both - acc[attributeCode] = [...new Set([...acc[attributeCode], ...bucketsIds])] - - return acc - }, {}) - - // find attribute list - const attributeList: any[] = await AttributeService.list(attributeListParam, config, indexName) - - return attributeList -} - function _outputFormatter (responseBody, format = 'standard') { if (format === 'compact') { // simple formatter delete responseBody.took @@ -161,8 +131,10 @@ export default ({config, db}) => async function (req, res, body) { _resBody.hits.hits = result _cacheStorageHandler(config, _resBody, reqHash, tagsArray) if (_resBody.aggregations) { - const attributesMetadata = await getProductsAttributesMetadata(_resBody, config, indexName) - _resBody.attribute_metadata = attributesMetadata.map(AttributeService.transformToMetadata) + const attributeListParam = AttributeService.transformAggsToAttributeListParam(_resBody.aggregations) + // find attribute list + const attributeList = await AttributeService.list(attributeListParam, config, indexName) + _resBody.attribute_metadata = attributeList.map(AttributeService.transformToMetadata) } res.json(_outputFormatter(_resBody, responseFormat)); }).catch((err) => { diff --git a/src/graphql/elasticsearch/catalog/resolver.js b/src/graphql/elasticsearch/catalog/resolver.js index 3d4a80e1..98ea5995 100644 --- a/src/graphql/elasticsearch/catalog/resolver.js +++ b/src/graphql/elasticsearch/catalog/resolver.js @@ -4,6 +4,7 @@ import { buildQuery } from '../queryBuilder'; import esResultsProcessor from './processor' import { getIndexName } from '../mapping' import { adjustQuery } from './../../../lib/elastic' +import AttributeService from './../../../api/attribute/service' const resolver = { Query: { @@ -67,6 +68,13 @@ async function list (filter, sort, currentPage, pageSize, search, context, rootV } response.aggregations = esResponse.aggregations + + if (response.aggregations) { + const attributeListParam = AttributeService.transformAggsToAttributeListParam(response.aggregations) + const attributeList = await AttributeService.list(attributeListParam, config, esIndex) + response.attribute_metadata = attributeList.map(AttributeService.transformToMetadata) + } + response.sort_fields = {} if (sortOptions.length > 0) { response.sort_fields.options = sortOptions diff --git a/src/graphql/elasticsearch/catalog/schema.graphqls b/src/graphql/elasticsearch/catalog/schema.graphqls index 86fdcffe..348c9aaa 100644 --- a/src/graphql/elasticsearch/catalog/schema.graphqls +++ b/src/graphql/elasticsearch/catalog/schema.graphqls @@ -7,6 +7,7 @@ type Products @doc(description: "The Products object is the top-level object ret total_count: Int @doc(description: "The number of products returned") # filters: [LayerFilter] @doc(description: "Layered navigation filters array") // @TODO: add filters to response instead of aggregations aggregations: JSON @doc(description: "Layered navigation filters array as aggregations") + attribute_metadata: JSON @doc(description: "Transformed aggregations into attributes") sort_fields: SortFields @doc(description: "An object that includes the default sort field and all available sort fields") } @@ -14,6 +15,7 @@ type ESResponse { hits: JSON suggest: JSON aggregations: JSON + attribute_metadata: JSON } type Query { From 09b0d1496ae5a807bdf527c2ab8a645155aa0255 Mon Sep 17 00:00:00 2001 From: tkostuch Date: Thu, 30 Jan 2020 17:19:29 +0100 Subject: [PATCH 8/9] 3948 move filter after map --- src/api/attribute/service.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/api/attribute/service.ts b/src/api/attribute/service.ts index f7ff0be7..94841192 100644 --- a/src/api/attribute/service.ts +++ b/src/api/attribute/service.ts @@ -94,18 +94,20 @@ async function list (attributesParam: AttributeListParam, config, indexName: str ) const cachedAttributeList = rawCachedAttributeList - .filter(Boolean) // remove empty results from cache.get .map((cachedAttribute, index) => { if (cachedAttribute) { const attributeOptionsIds = attributesParam[cachedAttribute.attribute_code] - // side effect - we want to reduce starting 'attributeCodes' because some of them are in cache + // side effect - we want to reduce starting 'attributeCodes' if some of them are in cache attributeCodes.splice(index, 1) // clear unused options return clearAttributeOpitons(cachedAttribute, attributeOptionsIds) } }) + // remove empty results from cache.get + // this needs to be after .map because we want to have same indexes as are in attributeCodes + .filter(Boolean) // if all requested attributes are in cache then we can return here if (!attributeCodes.length) { From 738b1ac608f382b2b8aa36abe63995c9bfedb823 Mon Sep 17 00:00:00 2001 From: tkostuch Date: Fri, 7 Feb 2020 07:42:35 +0100 Subject: [PATCH 9/9] 3948 disable loadByAttributeMetadata by default --- config/default.json | 3 ++- src/api/catalog.ts | 2 +- src/graphql/elasticsearch/catalog/resolver.js | 2 +- src/graphql/elasticsearch/catalog/schema.graphqls | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/default.json b/config/default.json index 02e2cb7e..7fe8c0de 100644 --- a/config/default.json +++ b/config/default.json @@ -332,7 +332,8 @@ "includeFields": [ "children_data", "id", "children_count", "sku", "name", "is_active", "parent_id", "level", "url_key" ] }, "attribute": { - "includeFields": [ "attribute_code", "id", "entity_type_id", "options", "default_value", "is_user_defined", "frontend_label", "attribute_id", "default_frontend_label", "is_visible_on_front", "is_visible", "is_comparable" ] + "includeFields": [ "attribute_code", "id", "entity_type_id", "options", "default_value", "is_user_defined", "frontend_label", "attribute_id", "default_frontend_label", "is_visible_on_front", "is_visible", "is_comparable" ], + "loadByAttributeMetadata": false }, "productList": { "sort": "", diff --git a/src/api/catalog.ts b/src/api/catalog.ts index 25620adc..355e835c 100755 --- a/src/api/catalog.ts +++ b/src/api/catalog.ts @@ -130,7 +130,7 @@ export default ({config, db}) => async function (req, res, body) { resultProcessor.process(_resBody.hits.hits, groupId).then(async (result) => { _resBody.hits.hits = result _cacheStorageHandler(config, _resBody, reqHash, tagsArray) - if (_resBody.aggregations) { + if (_resBody.aggregations && config.entities.attribute.loadByAttributeMetadata) { const attributeListParam = AttributeService.transformAggsToAttributeListParam(_resBody.aggregations) // find attribute list const attributeList = await AttributeService.list(attributeListParam, config, indexName) diff --git a/src/graphql/elasticsearch/catalog/resolver.js b/src/graphql/elasticsearch/catalog/resolver.js index 98ea5995..bd39332c 100644 --- a/src/graphql/elasticsearch/catalog/resolver.js +++ b/src/graphql/elasticsearch/catalog/resolver.js @@ -69,7 +69,7 @@ async function list (filter, sort, currentPage, pageSize, search, context, rootV response.aggregations = esResponse.aggregations - if (response.aggregations) { + if (response.aggregations && config.entities.attribute.loadByAttributeMetadata) { const attributeListParam = AttributeService.transformAggsToAttributeListParam(response.aggregations) const attributeList = await AttributeService.list(attributeListParam, config, esIndex) response.attribute_metadata = attributeList.map(AttributeService.transformToMetadata) diff --git a/src/graphql/elasticsearch/catalog/schema.graphqls b/src/graphql/elasticsearch/catalog/schema.graphqls index 348c9aaa..f8c624bb 100644 --- a/src/graphql/elasticsearch/catalog/schema.graphqls +++ b/src/graphql/elasticsearch/catalog/schema.graphqls @@ -7,7 +7,7 @@ type Products @doc(description: "The Products object is the top-level object ret total_count: Int @doc(description: "The number of products returned") # filters: [LayerFilter] @doc(description: "Layered navigation filters array") // @TODO: add filters to response instead of aggregations aggregations: JSON @doc(description: "Layered navigation filters array as aggregations") - attribute_metadata: JSON @doc(description: "Transformed aggregations into attributes") + attribute_metadata: JSON @doc(description: "Transformed aggregations into attributes - you need to allow 'config.entities.attribute.loadByAttributeMetadata'") sort_fields: SortFields @doc(description: "An object that includes the default sort field and all available sort fields") }