fetch attributes based on aggregates

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add url module - @gibkigonzo (#3942)
 - The `response_format` query parameter to the `/api/catalog` endpoint. Currently there is just one additional format supported: `response_format=compact`. When used, the response format got optimized by: a) remapping the results, removing the `_source` from the `hits.hits`; b) compressing the JSON fields names according to the `config.products.fieldsToCompact`; c) removing the JSON fields from the `product.configurable_children` when their values === parent product values; overall response size reduced over -70% - @pkarw
 - The support for `SearchQuery` instead of the ElasticSearch DSL as for the input to `/api/catalog` - using `storefront-query-builder` package - @pkarw - https://github.com/DivanteLtd/vue-storefront/issues/2167
+- Create attribute service that allows to fetch attributes with specific options - used for products aggregates - @gibkigonzo (https://github.com/DivanteLtd/vue-storefront/pull/4001, https://github.com/DivanteLtd/mage2vuestorefront/pull/99)
 - Add ElasticSearch client support for HTTP authentication - @cewald (#397)
 
 ### Fixed

config/default.json

@@ -333,7 +333,8 @@
         "includeFields": [ "children_data", "id", "children_count", "sku", "name", "is_active", "parent_id", "level", "url_key" ]
       },
       "attribute": {
-        "includeFields": [ "attribute_code", "id", "entity_type_id", "options", "default_value", "is_user_defined", "frontend_label", "attribute_id", "default_frontend_label", "is_visible_on_front", "is_visible", "is_comparable" ]
+        "includeFields": [ "attribute_code", "id", "entity_type_id", "options", "default_value", "is_user_defined", "frontend_label", "attribute_id", "default_frontend_label", "is_visible_on_front", "is_visible", "is_comparable" ],
+        "loadByAttributeMetadata": false
       },
       "productList": {
         "sort": "",

src/api/attribute/service.ts

@@ -0,0 +1,183 @@
+
+import TagCache from 'redis-tag-cache'
+import get from 'lodash/get';
+import cache from '../../lib/cache-instance'
+import { adjustQuery, getClient as getElasticClient } from './../../lib/elastic'
+import bodybuilder from 'bodybuilder'
+
+export interface AttributeListParam {
+  [key: string]: number[]
+}
+
+/**
+ * Transforms ES aggregates into valid format for AttributeService - {[attribute_code]: [bucketId1, bucketId2]}
+ * @param body - products response body
+ * @param config - global config
+ * @param indexName - current indexName
+ */
+function transformAggsToAttributeListParam (aggregations): AttributeListParam {
+  const attributeListParam: AttributeListParam = Object.keys(aggregations)
+    .filter(key => aggregations[key].buckets.length) // leave only buckets with values
+    .reduce((acc, key) => {
+      const attributeCode = key.replace(/^(agg_terms_|agg_range_)|(_options)$/g, '')
+      const bucketsIds = aggregations[key].buckets.map(bucket => bucket.key)
+
+      if (!acc[attributeCode]) {
+        acc[attributeCode] = []
+      }
+
+      // there can be more then one attributes for example 'agg_terms_color' and 'agg_terms_color_options'
+      // we need to get buckets from both
+      acc[attributeCode] = [...new Set([...acc[attributeCode], ...bucketsIds])]
+
+      return acc
+    }, {})
+
+  return attributeListParam
+}
+
+/**
+ * Returns attributes from cache
+ */
+async function getAttributeFromCache (attributeCode: string, config) {
+  if (config.server.useOutputCache && cache) {
+    try {
+      const res = await (cache as TagCache).get(
+        'api:attribute-list' + attributeCode
+      )
+      return res
+    } catch (err) {
+      console.error(err)
+      return null
+    }
+  }
+}
+
+/**
+ * Save attributes in cache
+ */
+async function setAttributeInCache (attributeList, config) {
+  if (config.server.useOutputCache && cache) {
+    try {
+      await Promise.all(
+        attributeList.map(attribute => (cache as TagCache).set(
+          'api:attribute-list' + attribute.attribute_code,
+          attribute
+        ))
+      )
+    } catch (err) {
+      console.error(err)
+    }
+  }
+}
+
+/**
+ * Returns attribute with only needed options
+ * @param attribute - attribute object
+ * @param optionsIds - list of only needed options ids
+ */
+function clearAttributeOpitons (attribute, optionsIds: number[]) {
+  const stringOptionsIds = optionsIds.map(String)
+  return {
+    ...attribute,
+    options: (attribute.options || []).filter(option => stringOptionsIds.includes(String(option.value)))
+  }
+}
+
+async function list (attributesParam: AttributeListParam, config, indexName: string): Promise<any[]> {
+  // we start with all attributeCodes that are requested
+  let attributeCodes = Object.keys(attributesParam)
+
+  // here we check if some of attribute are in cache
+  const rawCachedAttributeList = await Promise.all(
+    attributeCodes.map(attributeCode => getAttributeFromCache(attributeCode, config))
+  )
+
+  const cachedAttributeList = rawCachedAttributeList
+    .map((cachedAttribute, index) => {
+      if (cachedAttribute) {
+        const attributeOptionsIds = attributesParam[cachedAttribute.attribute_code]
+
+        // side effect - we want to reduce starting 'attributeCodes' if some of them are in cache
+        attributeCodes.splice(index, 1)
+
+        // clear unused options
+        return clearAttributeOpitons(cachedAttribute, attributeOptionsIds)
+      }
+    })
+    // remove empty results from cache.get
+    // this needs to be after .map because we want to have same indexes as are in attributeCodes
+    .filter(Boolean)
+
+  // if all requested attributes are in cache then we can return here
+  if (!attributeCodes.length) {
+    return cachedAttributeList
+  }
+
+  // fetch attributes for rest attributeCodes
+  try {
+    const query = adjustQuery({
+      index: indexName,
+      type: 'attribute',
+      body: bodybuilder().filter('terms', 'attribute_code', attributeCodes).build()
+    }, 'attribute', config)
+    const response = await getElasticClient(config).search(query)
+    const fetchedAttributeList = get(response.body, 'hits.hits', []).map(hit => hit._source)
+
+    // save atrributes in cache
+    await setAttributeInCache(fetchedAttributeList, config)
+
+    // cached and fetched attributes
+    const allAttributes = [
+      ...cachedAttributeList,
+      ...fetchedAttributeList.map(fetchedAttribute => {
+        const attributeOptionsIds = attributesParam[fetchedAttribute.attribute_code]
+
+        // clear unused options
+        return clearAttributeOpitons(fetchedAttribute, attributeOptionsIds)
+      })
+    ]
+
+    return allAttributes
+  } catch (err) {
+    console.error(err)
+    return []
+  }
+}
+
+/**
+ * Returns only needed data for filters in vsf
+ */
+function transformToMetadata ({
+  is_visible_on_front,
+  is_visible,
+  default_frontend_label,
+  attribute_id,
+  entity_type_id,
+  id,
+  is_user_defined,
+  is_comparable,
+  attribute_code,
+  slug,
+  options = []
+}) {
+  return {
+    is_visible_on_front,
+    is_visible,
+    default_frontend_label,
+    attribute_id,
+    entity_type_id,
+    id,
+    is_user_defined,
+    is_comparable,
+    attribute_code,
+    slug,
+    options
+  }
+}
+
+export default {
+  list,
+  transformToMetadata,
+  transformAggsToAttributeListParam
+}

src/api/catalog.ts

@@ -4,6 +4,7 @@ import ProcessorFactory from '../processor/factory';
 import { adjustBackendProxyUrl } from '../lib/elastic'
 import cache from '../lib/cache-instance'
 import { sha3_224 } from 'js-sha3'
+import AttributeService from './attribute/service'
 import bodybuilder from 'bodybuilder'
 import { elasticsearch, SearchQuery } from 'storefront-query-builder'
 
@@ -125,11 +126,16 @@ export default ({config, db}) => async function (req, res, body) {
         let resultProcessor = factory.getAdapter(entityType, indexName, req, res)
 
         if (!resultProcessor) { resultProcessor = factory.getAdapter('default', indexName, req, res) } // get the default processor
-
         if (entityType === 'product') {
-          resultProcessor.process(_resBody.hits.hits, groupId).then((result) => {
+          resultProcessor.process(_resBody.hits.hits, groupId).then(async (result) => {
             _resBody.hits.hits = result
             _cacheStorageHandler(config, _resBody, reqHash, tagsArray)
+            if (_resBody.aggregations && config.entities.attribute.loadByAttributeMetadata) {
+              const attributeListParam = AttributeService.transformAggsToAttributeListParam(_resBody.aggregations)
+              // find attribute list
+              const attributeList = await AttributeService.list(attributeListParam, config, indexName)
+              _resBody.attribute_metadata = attributeList.map(AttributeService.transformToMetadata)
+            }
             res.json(_outputFormatter(_resBody, responseFormat));
           }).catch((err) => {
             console.error(err)

src/graphql/elasticsearch/catalog/resolver.js

@@ -4,6 +4,7 @@ import { buildQuery } from '../queryBuilder';
 import esResultsProcessor from './processor'
 import { getIndexName } from '../mapping'
 import { adjustQuery } from './../../../lib/elastic'
+import AttributeService from './../../../api/attribute/service'
 
 const resolver = {
   Query: {
@@ -67,6 +68,13 @@ async function list (filter, sort, currentPage, pageSize, search, context, rootV
   }
 
   response.aggregations = esResponse.aggregations
+
+  if (response.aggregations && config.entities.attribute.loadByAttributeMetadata) {
+    const attributeListParam = AttributeService.transformAggsToAttributeListParam(response.aggregations)
+    const attributeList = await AttributeService.list(attributeListParam, config, esIndex)
+    response.attribute_metadata = attributeList.map(AttributeService.transformToMetadata)
+  }
+
   response.sort_fields = {}
   if (sortOptions.length > 0) {
     response.sort_fields.options = sortOptions

src/graphql/elasticsearch/catalog/schema.graphqls

@@ -7,13 +7,15 @@ type Products @doc(description: "The Products object is the top-level object ret
     total_count: Int @doc(description: "The number of products returned")
     # filters: [LayerFilter] @doc(description: "Layered navigation filters array") // @TODO: add filters to response instead of aggregations
     aggregations: JSON @doc(description: "Layered navigation filters array as aggregations")
+    attribute_metadata: JSON @doc(description: "Transformed aggregations into attributes - you need to allow 'config.entities.attribute.loadByAttributeMetadata'")
     sort_fields: SortFields @doc(description: "An object that includes the default sort field and all available sort fields")
 }
 
 type ESResponse {
   hits: JSON
   suggest: JSON
   aggregations: JSON
+  attribute_metadata: JSON
 }
 
 type Query {

src/graphql/elasticsearch/queryBuilder.ts

@@ -19,6 +19,5 @@ export function buildQuery ({
   if (search !== '') {
     builtQuery['min_score'] = config.get('elasticsearch.min_score')
   }
-
   return builtQuery;
 }