diff --git a/CHANGELOG.md b/CHANGELOG.md index a8d1b5d0..4d35a6e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add url module - @gibkigonzo (#3942) - The `response_format` query parameter to the `/api/catalog` endpoint. Currently there is just one additional format supported: `response_format=compact`. When used, the response format got optimized by: a) remapping the results, removing the `_source` from the `hits.hits`; b) compressing the JSON fields names according to the `config.products.fieldsToCompact`; c) removing the JSON fields from the `product.configurable_children` when their values === parent product values; overall response size reduced over -70% - @pkarw - The support for `SearchQuery` instead of the ElasticSearch DSL as for the input to `/api/catalog` - using `storefront-query-builder` package - @pkarw - https://github.com/DivanteLtd/vue-storefront/issues/2167 +- Create attribute service that allows to fetch attributes with specific options - used for products aggregates - @gibkigonzo (https://github.com/DivanteLtd/vue-storefront/pull/4001, https://github.com/DivanteLtd/mage2vuestorefront/pull/99) - Add ElasticSearch client support for HTTP authentication - @cewald (#397) ### Fixed diff --git a/config/default.json b/config/default.json index 04b6ad95..32f934f6 100644 --- a/config/default.json +++ b/config/default.json @@ -333,7 +333,8 @@ "includeFields": [ "children_data", "id", "children_count", "sku", "name", "is_active", "parent_id", "level", "url_key" ] }, "attribute": { - "includeFields": [ "attribute_code", "id", "entity_type_id", "options", "default_value", "is_user_defined", "frontend_label", "attribute_id", "default_frontend_label", "is_visible_on_front", "is_visible", "is_comparable" ] + "includeFields": [ "attribute_code", "id", "entity_type_id", "options", "default_value", "is_user_defined", "frontend_label", "attribute_id", "default_frontend_label", "is_visible_on_front", "is_visible", "is_comparable" ], + "loadByAttributeMetadata": false }, "productList": { "sort": "", diff --git a/src/api/attribute/service.ts b/src/api/attribute/service.ts new file mode 100644 index 00000000..94841192 --- /dev/null +++ b/src/api/attribute/service.ts @@ -0,0 +1,183 @@ + +import TagCache from 'redis-tag-cache' +import get from 'lodash/get'; +import cache from '../../lib/cache-instance' +import { adjustQuery, getClient as getElasticClient } from './../../lib/elastic' +import bodybuilder from 'bodybuilder' + +export interface AttributeListParam { + [key: string]: number[] +} + +/** + * Transforms ES aggregates into valid format for AttributeService - {[attribute_code]: [bucketId1, bucketId2]} + * @param body - products response body + * @param config - global config + * @param indexName - current indexName + */ +function transformAggsToAttributeListParam (aggregations): AttributeListParam { + const attributeListParam: AttributeListParam = Object.keys(aggregations) + .filter(key => aggregations[key].buckets.length) // leave only buckets with values + .reduce((acc, key) => { + const attributeCode = key.replace(/^(agg_terms_|agg_range_)|(_options)$/g, '') + const bucketsIds = aggregations[key].buckets.map(bucket => bucket.key) + + if (!acc[attributeCode]) { + acc[attributeCode] = [] + } + + // there can be more then one attributes for example 'agg_terms_color' and 'agg_terms_color_options' + // we need to get buckets from both + acc[attributeCode] = [...new Set([...acc[attributeCode], ...bucketsIds])] + + return acc + }, {}) + + return attributeListParam +} + +/** + * Returns attributes from cache + */ +async function getAttributeFromCache (attributeCode: string, config) { + if (config.server.useOutputCache && cache) { + try { + const res = await (cache as TagCache).get( + 'api:attribute-list' + attributeCode + ) + return res + } catch (err) { + console.error(err) + return null + } + } +} + +/** + * Save attributes in cache + */ +async function setAttributeInCache (attributeList, config) { + if (config.server.useOutputCache && cache) { + try { + await Promise.all( + attributeList.map(attribute => (cache as TagCache).set( + 'api:attribute-list' + attribute.attribute_code, + attribute + )) + ) + } catch (err) { + console.error(err) + } + } +} + +/** + * Returns attribute with only needed options + * @param attribute - attribute object + * @param optionsIds - list of only needed options ids + */ +function clearAttributeOpitons (attribute, optionsIds: number[]) { + const stringOptionsIds = optionsIds.map(String) + return { + ...attribute, + options: (attribute.options || []).filter(option => stringOptionsIds.includes(String(option.value))) + } +} + +async function list (attributesParam: AttributeListParam, config, indexName: string): Promise { + // we start with all attributeCodes that are requested + let attributeCodes = Object.keys(attributesParam) + + // here we check if some of attribute are in cache + const rawCachedAttributeList = await Promise.all( + attributeCodes.map(attributeCode => getAttributeFromCache(attributeCode, config)) + ) + + const cachedAttributeList = rawCachedAttributeList + .map((cachedAttribute, index) => { + if (cachedAttribute) { + const attributeOptionsIds = attributesParam[cachedAttribute.attribute_code] + + // side effect - we want to reduce starting 'attributeCodes' if some of them are in cache + attributeCodes.splice(index, 1) + + // clear unused options + return clearAttributeOpitons(cachedAttribute, attributeOptionsIds) + } + }) + // remove empty results from cache.get + // this needs to be after .map because we want to have same indexes as are in attributeCodes + .filter(Boolean) + + // if all requested attributes are in cache then we can return here + if (!attributeCodes.length) { + return cachedAttributeList + } + + // fetch attributes for rest attributeCodes + try { + const query = adjustQuery({ + index: indexName, + type: 'attribute', + body: bodybuilder().filter('terms', 'attribute_code', attributeCodes).build() + }, 'attribute', config) + const response = await getElasticClient(config).search(query) + const fetchedAttributeList = get(response.body, 'hits.hits', []).map(hit => hit._source) + + // save atrributes in cache + await setAttributeInCache(fetchedAttributeList, config) + + // cached and fetched attributes + const allAttributes = [ + ...cachedAttributeList, + ...fetchedAttributeList.map(fetchedAttribute => { + const attributeOptionsIds = attributesParam[fetchedAttribute.attribute_code] + + // clear unused options + return clearAttributeOpitons(fetchedAttribute, attributeOptionsIds) + }) + ] + + return allAttributes + } catch (err) { + console.error(err) + return [] + } +} + +/** + * Returns only needed data for filters in vsf + */ +function transformToMetadata ({ + is_visible_on_front, + is_visible, + default_frontend_label, + attribute_id, + entity_type_id, + id, + is_user_defined, + is_comparable, + attribute_code, + slug, + options = [] +}) { + return { + is_visible_on_front, + is_visible, + default_frontend_label, + attribute_id, + entity_type_id, + id, + is_user_defined, + is_comparable, + attribute_code, + slug, + options + } +} + +export default { + list, + transformToMetadata, + transformAggsToAttributeListParam +} diff --git a/src/api/catalog.ts b/src/api/catalog.ts index 16b6f246..355e835c 100755 --- a/src/api/catalog.ts +++ b/src/api/catalog.ts @@ -4,6 +4,7 @@ import ProcessorFactory from '../processor/factory'; import { adjustBackendProxyUrl } from '../lib/elastic' import cache from '../lib/cache-instance' import { sha3_224 } from 'js-sha3' +import AttributeService from './attribute/service' import bodybuilder from 'bodybuilder' import { elasticsearch, SearchQuery } from 'storefront-query-builder' @@ -125,11 +126,16 @@ export default ({config, db}) => async function (req, res, body) { let resultProcessor = factory.getAdapter(entityType, indexName, req, res) if (!resultProcessor) { resultProcessor = factory.getAdapter('default', indexName, req, res) } // get the default processor - if (entityType === 'product') { - resultProcessor.process(_resBody.hits.hits, groupId).then((result) => { + resultProcessor.process(_resBody.hits.hits, groupId).then(async (result) => { _resBody.hits.hits = result _cacheStorageHandler(config, _resBody, reqHash, tagsArray) + if (_resBody.aggregations && config.entities.attribute.loadByAttributeMetadata) { + const attributeListParam = AttributeService.transformAggsToAttributeListParam(_resBody.aggregations) + // find attribute list + const attributeList = await AttributeService.list(attributeListParam, config, indexName) + _resBody.attribute_metadata = attributeList.map(AttributeService.transformToMetadata) + } res.json(_outputFormatter(_resBody, responseFormat)); }).catch((err) => { console.error(err) diff --git a/src/graphql/elasticsearch/catalog/resolver.js b/src/graphql/elasticsearch/catalog/resolver.js index 3d4a80e1..bd39332c 100644 --- a/src/graphql/elasticsearch/catalog/resolver.js +++ b/src/graphql/elasticsearch/catalog/resolver.js @@ -4,6 +4,7 @@ import { buildQuery } from '../queryBuilder'; import esResultsProcessor from './processor' import { getIndexName } from '../mapping' import { adjustQuery } from './../../../lib/elastic' +import AttributeService from './../../../api/attribute/service' const resolver = { Query: { @@ -67,6 +68,13 @@ async function list (filter, sort, currentPage, pageSize, search, context, rootV } response.aggregations = esResponse.aggregations + + if (response.aggregations && config.entities.attribute.loadByAttributeMetadata) { + const attributeListParam = AttributeService.transformAggsToAttributeListParam(response.aggregations) + const attributeList = await AttributeService.list(attributeListParam, config, esIndex) + response.attribute_metadata = attributeList.map(AttributeService.transformToMetadata) + } + response.sort_fields = {} if (sortOptions.length > 0) { response.sort_fields.options = sortOptions diff --git a/src/graphql/elasticsearch/catalog/schema.graphqls b/src/graphql/elasticsearch/catalog/schema.graphqls index 86fdcffe..f8c624bb 100644 --- a/src/graphql/elasticsearch/catalog/schema.graphqls +++ b/src/graphql/elasticsearch/catalog/schema.graphqls @@ -7,6 +7,7 @@ type Products @doc(description: "The Products object is the top-level object ret total_count: Int @doc(description: "The number of products returned") # filters: [LayerFilter] @doc(description: "Layered navigation filters array") // @TODO: add filters to response instead of aggregations aggregations: JSON @doc(description: "Layered navigation filters array as aggregations") + attribute_metadata: JSON @doc(description: "Transformed aggregations into attributes - you need to allow 'config.entities.attribute.loadByAttributeMetadata'") sort_fields: SortFields @doc(description: "An object that includes the default sort field and all available sort fields") } @@ -14,6 +15,7 @@ type ESResponse { hits: JSON suggest: JSON aggregations: JSON + attribute_metadata: JSON } type Query { diff --git a/src/graphql/elasticsearch/queryBuilder.ts b/src/graphql/elasticsearch/queryBuilder.ts index 29e46f7d..c9c2f3f7 100644 --- a/src/graphql/elasticsearch/queryBuilder.ts +++ b/src/graphql/elasticsearch/queryBuilder.ts @@ -19,6 +19,5 @@ export function buildQuery ({ if (search !== '') { builtQuery['min_score'] = config.get('elasticsearch.min_score') } - return builtQuery; }