diff --git a/CHANGELOG.md b/CHANGELOG.md index e00eeb8f..af9028d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Add url module - @gibkigonzo (#3942) - -### Fixed - -### Changed / Improved - +- The `response_format` query parameter to the `/api/catalog` endpoint. Currently there is just one additional format supported: `response_format=compact`. When used, the response format got optimized by: a) remapping the results, removing the `_source` from the `hits.hits`; b) compressing the JSON fields names according to the `config.products.fieldsToCompact`; c) removing the JSON fields from the `product.configurable_children` when their values === parent product values; overall response size reduced over -70% - @pkarw +- The support for `SearchQuery` instead of the ElasticSearch DSL as for the input to `/api/catalog` - using `storefront-query-builder` package - @pkarw - https://github.com/DivanteLtd/vue-storefront/issues/2167 ## [1.11.0] - 2019.12.20 diff --git a/config/default.json b/config/default.json index 63f8fe49..e3419a62 100644 --- a/config/default.json +++ b/config/default.json @@ -36,7 +36,83 @@ "taxrule", "review" ], - "apiVersion": "5.6" + "apiVersion": "5.6", + + "searchScoring": { + "attributes": { + "attribute_code": { + "scoreValues": { "attribute_value": { "weight": 1 } } + } + }, + "fuzziness": 2, + "cutoff_frequency": 0.01, + "max_expansions": 3, + "minimum_should_match": "75%", + "prefix_length": 2, + "boost_mode": "multiply", + "score_mode": "multiply", + "max_boost": 100, + "function_min_score": 1 + }, + "searchableAttributes": { + "name": { + "boost": 4 + }, + "sku": { + "boost": 2 + }, + "category.name": { + "boost": 1 + } + } + }, + "products": { + "fieldsToCompress": ["max_regular_price", "max_price", "max_regular_price", "minimal_regular_price", "final_price", "price", "special_price", "original_final_price", "original_price", "original_special_price", "final_price_incl_tax", "price_incl_tax", "special_price_incl_tax", "final_price_tax", "price_tax", "special_price_tax", "image", "small_image", "thumbnail"], + "fieldsToCompact": { + "minimal_price": "mp", + "has_options": "ho", + "url_key": "u", + "status": "s", + "required_options": "ro", + "name": "nm", + "tax_class_id": "tci", + "description": "desc", + "minimal_regular_price": "mrp", + "final_price": "fp", + "price": "p", + "special_price": "sp", + "original_final_price": "ofp", + "original_price": "op", + "original_special_price": "osp", + "final_price_incl_tax": "fpit", + "original_price_incl_tax": "opit", + "price_incl_tax": "pit", + "special_price_incl_tax": "spit", + "final_price_tax": "fpt", + "price_tax": "pt", + "special_price_tax": "spt", + "original_price_tax": "opt", + "image": "i", + "small_image": "si", + "thumbnail": "t" + }, + "filterFieldMapping": { + "category.name": "category.name.keyword" + }, + "filterAggregationSize": { + "default": 10, + "size": 10, + "color": 10 + }, + "priceFilterKey": "final_price", + "priceFilters": { + "ranges": [ + { "from": 0, "to": 50 }, + { "from": 50, "to": 100 }, + { "from": 100, "to": 150 }, + { "from": 150 } + ] + } }, "redis": { "host": "localhost", @@ -76,7 +152,7 @@ "tax": { "defaultCountry": "DE", "defaultRegion": "", - "deprecatedPriceFieldsSupport": true, + "deprecatedPriceFieldsSupport": false, "calculateServerSide": true, "sourcePriceIncludesTax": false, "finalPriceIncludesTax": true, @@ -113,7 +189,7 @@ "defaultRegion": "", "calculateServerSide": true, "sourcePriceIncludesTax": false, - "deprecatedPriceFieldsSupport": true, + "deprecatedPriceFieldsSupport": false, "finalPriceIncludesTax": true, "userGroupId": null, "useOnlyDefaultUserGroupId": false @@ -143,7 +219,7 @@ "usePlatformTotals": true, "setConfigurableProductOptions": true, "sourcePriceIncludesTax": false, - "deprecatedPriceFieldsSupport": true, + "deprecatedPriceFieldsSupport": false, "finalPriceIncludesTax": false, "userGroupId": null, "useOnlyDefaultUserGroupId": false diff --git a/nodemon.json b/nodemon.json index 10e396db..cd6759e7 100644 --- a/nodemon.json +++ b/nodemon.json @@ -1,7 +1,7 @@ { "verbose": true, "debug": false, - "exec": "ts-node src", + "exec": "node -r ts-node/register src/", "watch": ["./src"], "ext": "ts, js", "inspect": true diff --git a/package.json b/package.json index a323281f..8505e43a 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "resource-router-middleware": "^0.6.0", "sharp": "^0.23.4", "soap": "^0.25.0", + "storefront-query-builder": "^0.0.9", "syswide-cas": "latest", "winston": "^2.4.2" }, @@ -122,7 +123,7 @@ "ts-jest": "^24.0.2", "ts-node": "^8.1.0", "tslib": "^1.9.3", - "typescript": "3.3.*" + "typescript": "3.7.*" }, "bugs": { "url": "https://github.com/DivanteLtd/vue-storefront-api/issues" diff --git a/src/api/cart.js b/src/api/cart.ts similarity index 100% rename from src/api/cart.js rename to src/api/cart.ts diff --git a/src/api/catalog.js b/src/api/catalog.ts similarity index 80% rename from src/api/catalog.js rename to src/api/catalog.ts index 27f1215d..16b6f246 100755 --- a/src/api/catalog.js +++ b/src/api/catalog.ts @@ -4,6 +4,8 @@ import ProcessorFactory from '../processor/factory'; import { adjustBackendProxyUrl } from '../lib/elastic' import cache from '../lib/cache-instance' import { sha3_224 } from 'js-sha3' +import bodybuilder from 'bodybuilder' +import { elasticsearch, SearchQuery } from 'storefront-query-builder' function _cacheStorageHandler (config, result, hash, tags) { if (config.server.useOutputCache && cache) { @@ -17,7 +19,23 @@ function _cacheStorageHandler (config, result, hash, tags) { } } -export default ({config, db}) => function (req, res, body) { +function _outputFormatter (responseBody, format = 'standard') { + if (format === 'compact') { // simple formatter + delete responseBody.took + delete responseBody.timed_out + delete responseBody._shards + if (responseBody.hits) { + delete responseBody.hits.max_score + responseBody.total = responseBody.hits.total + responseBody.hits = responseBody.hits.hits.map(hit => { + return Object.assign(hit._source, { _score: hit._score }) + }) + } + } + return responseBody +} + +export default ({config, db}) => async function (req, res, body) { let groupId = null // Request method handling: exit if not GET or POST @@ -26,15 +44,19 @@ export default ({config, db}) => function (req, res, body) { throw new Error('ERROR: ' + req.method + ' request method is not supported.') } - let requestBody = {} + let responseFormat = 'standard' + let requestBody = req.body if (req.method === 'GET') { if (req.query.request) { // this is in fact optional requestBody = JSON.parse(decodeURIComponent(req.query.request)) } - } else { - requestBody = req.body } + if (req.query.request_format === 'search-query') { // search query and not Elastic DSL - we need to translate it + requestBody = await elasticsearch.buildQueryBodyFromSearchQuery({ config, queryChain: bodybuilder(), searchQuery: new SearchQuery(requestBody) }) + } + if (req.query.response_format) responseFormat = req.query.response_format + const urlSegments = req.url.split('/'); let indexName = '' @@ -108,7 +130,7 @@ export default ({config, db}) => function (req, res, body) { resultProcessor.process(_resBody.hits.hits, groupId).then((result) => { _resBody.hits.hits = result _cacheStorageHandler(config, _resBody, reqHash, tagsArray) - res.json(_resBody); + res.json(_outputFormatter(_resBody, responseFormat)); }).catch((err) => { console.error(err) }) @@ -116,7 +138,7 @@ export default ({config, db}) => function (req, res, body) { resultProcessor.process(_resBody.hits.hits).then((result) => { _resBody.hits.hits = result _cacheStorageHandler(config, _resBody, reqHash, tagsArray) - res.json(_resBody); + res.json(_outputFormatter(_resBody, responseFormat)); }).catch((err) => { console.error(err) }) diff --git a/src/api/img.js b/src/api/img.ts similarity index 100% rename from src/api/img.js rename to src/api/img.ts diff --git a/src/api/index.js b/src/api/index.ts similarity index 97% rename from src/api/index.js rename to src/api/index.ts index 603ee434..a9da3444 100755 --- a/src/api/index.js +++ b/src/api/index.ts @@ -38,7 +38,7 @@ export default ({ config, db }) => { api.use('/sync', sync({ config, db })) // mount the url resource - api.use('/url', url({ config, db })) + api.use('/url', url({ config })) // perhaps expose some API metadata at the root api.get('/', (req, res) => { diff --git a/src/api/invalidate.js b/src/api/invalidate.ts similarity index 71% rename from src/api/invalidate.js rename to src/api/invalidate.ts index bb3ea19a..1d7bf5aa 100644 --- a/src/api/invalidate.js +++ b/src/api/invalidate.ts @@ -4,9 +4,9 @@ import cache from '../lib/cache-instance' import request from 'request' function invalidateCache (req, res) { - if (config.server.useOutputCache) { + if (config.get('server.useOutputCache')) { if (req.query.tag && req.query.key) { // clear cache pages for specific query tag - if (req.query.key !== config.server.invalidateCacheKey) { + if (req.query.key !== config.get('server.invalidateCacheKey')) { console.error('Invalid cache invalidation key') apiStatus(res, 'Invalid cache invalidation key', 500) return @@ -14,13 +14,13 @@ function invalidateCache (req, res) { console.log(`Clear cache request for [${req.query.tag}]`) let tags = [] if (req.query.tag === '*') { - tags = config.server.availableCacheTags + tags = config.get('server.availableCacheTags') } else { tags = req.query.tag.split(',') } const subPromises = [] tags.forEach(tag => { - if (config.server.availableCacheTags.indexOf(tag) >= 0 || config.server.availableCacheTags.find(t => { + if ((config.get('server.availableCacheTags') as [string]).indexOf(tag) >= 0 || (config.get('server.availableCacheTags') as [string]).find(t => { return tag.indexOf(t) === 0 })) { subPromises.push(cache.invalidate(tag).then(() => { @@ -36,9 +36,9 @@ function invalidateCache (req, res) { apiStatus(res, error, 500) console.error(error) }) - if (config.server.invalidateCacheForwarding) { // forward invalidate request to the next server in the chain - if (!req.query.forwardedFrom && config.server.invalidateCacheForwardUrl) { // don't forward forwarded requests - request(config.server.invalidateCacheForwardUrl + req.query.tag + '&forwardedFrom=vs', {}, (err, res, body) => { + if (config.get('server.invalidateCacheForwarding')) { // forward invalidate request to the next server in the chain + if (!req.query.forwardedFrom && config.get('server.invalidateCacheForwardUrl')) { // don't forward forwarded requests + request(config.get('server.invalidateCacheForwardUrl') + req.query.tag + '&forwardedFrom=vs', {}, (err, res, body) => { if (err) { console.error(err); } try { if (body && JSON.parse(body).code !== 200) console.log(body); diff --git a/src/api/order.js b/src/api/order.ts similarity index 93% rename from src/api/order.js rename to src/api/order.ts index 4df374cd..ee995baf 100755 --- a/src/api/order.js +++ b/src/api/order.ts @@ -43,7 +43,7 @@ export default ({ config, db }) => resource({ console.log(JSON.stringify(incomingOrder)) for (let product of req.body.products) { - let key = config.tax.calculateServerSide ? { priceInclTax: product.priceInclTax } : { price: product.price } + let key: { id?: string, sku?: string, priceInclTax?: number, price?: number } = config.tax.calculateServerSide ? { priceInclTax: product.priceInclTax } : { price: product.price } if (config.tax.alwaysSyncPlatformPricesOver) { key.id = product.id } else { diff --git a/src/api/product.js b/src/api/product.ts similarity index 100% rename from src/api/product.js rename to src/api/product.ts diff --git a/src/api/review.js b/src/api/review.ts similarity index 100% rename from src/api/review.js rename to src/api/review.ts diff --git a/src/api/stock.js b/src/api/stock.ts similarity index 100% rename from src/api/stock.js rename to src/api/stock.ts diff --git a/src/api/sync.js b/src/api/sync.ts similarity index 100% rename from src/api/sync.js rename to src/api/sync.ts diff --git a/src/api/url/index.js b/src/api/url/index.ts similarity index 76% rename from src/api/url/index.js rename to src/api/url/index.ts index 15562615..e433b633 100644 --- a/src/api/url/index.js +++ b/src/api/url/index.ts @@ -1,10 +1,12 @@ import { Router } from 'express'; import createMapRoute from './map'; -module.exports = ({ config }) => { +const url = ({ config }) => { const router = Router() router.use('/map', createMapRoute({ config })) return router } + +export default url diff --git a/src/api/url/map.js b/src/api/url/map.ts similarity index 98% rename from src/api/url/map.js rename to src/api/url/map.ts index cf9abfb4..3894b2e1 100644 --- a/src/api/url/map.js +++ b/src/api/url/map.ts @@ -42,7 +42,7 @@ const checkFieldValueEquality = ({ config, response, value }) => { return Boolean(isEqualValue) } -module.exports = ({ config }) => { +const map = ({ config }) => { const router = Router() router.post('/:index', (req, res) => { const { url, excludeFields, includeFields } = req.body @@ -88,3 +88,5 @@ module.exports = ({ config }) => { return router } + +export default map diff --git a/src/api/user.js b/src/api/user.ts similarity index 100% rename from src/api/user.js rename to src/api/user.ts diff --git a/src/graphql/elasticsearch/queryBuilder.js b/src/graphql/elasticsearch/queryBuilder.js deleted file mode 100644 index 8eb3927b..00000000 --- a/src/graphql/elasticsearch/queryBuilder.js +++ /dev/null @@ -1,185 +0,0 @@ -import bodybuilder from 'bodybuilder'; -import getBoosts from '../../lib/boost' -import map from 'lodash/map'; -import getMapping from './mapping' -import config from 'config' - -function processNestedFieldFilter (attribute, value) { - let processedFilter = { - 'attribute': attribute, - 'value': value - }; - let filterAttributeKeys = Object.keys(value); - for (let filterAttributeKey of filterAttributeKeys) { - if (value[filterAttributeKey] && !Array.isArray(value[filterAttributeKey]) && typeof value[filterAttributeKey] === 'object') { - processedFilter = processNestedFieldFilter(attribute + '.' + filterAttributeKey, value[filterAttributeKey]); - } - } - return processedFilter; -} - -/** - * - * @param {Object} object - * @param {String} scope - * @returns {boolean} - */ -function checkIfObjectHasScope ({ object, scope }) { - return object.scope === scope || (Array.isArray(object.scope) && object.scope.find(scrope => scrope === scope)); -} - -function applyFilters (filter, query, type) { - if (filter.length === 0) { - return query - } - const rangeOperators = ['gt', 'lt', 'gte', 'lte', 'moreq', 'from', 'to'] - const optionsPrefix = '_options' - - const appliedFilters = []; - if (filter) { - for (var attribute in filter) { - let processedFilter = processNestedFieldFilter(attribute, filter[attribute]) - let appliedAttributeValue = processedFilter['value'] - const scope = appliedAttributeValue.scope || 'default'; - delete appliedAttributeValue.scope; - appliedFilters.push({ - attribute: processedFilter['attribute'], - value: appliedAttributeValue, - scope: scope - }); - } - } - - // process applied filters - if (appliedFilters.length > 0) { - let hasCatalogFilters = false; - - // apply default filters - appliedFilters.forEach((filter) => { - if (checkIfObjectHasScope({ object: filter, scope: 'default' }) && Object.keys(filter.value).length) { - if (rangeOperators.every(rangeOperator => Object.prototype.hasOwnProperty.call(filter.value, rangeOperator))) { - // process range filters - query = query.filter('range', filter.attribute, filter.value); - } else { - // process terms filters - filter.value = filter.value[Object.keys(filter.value)[0]]; - if (!Array.isArray(filter.value)) { - filter.value = [filter.value]; - } - query = query.filter('terms', getMapping(filter.attribute), filter.value) - } - } else if (filter.scope === 'catalog') { - hasCatalogFilters = true; - } - }) - - // apply catalog scope filters - let attrFilterBuilder = (filterQr, attrPostfix = '') => { - appliedFilters.forEach((catalogfilter) => { - const valueKeys = Object.keys(catalogfilter.value); - if (checkIfObjectHasScope({ object: catalogfilter, scope: 'catalog' }) && valueKeys.length) { - const isRange = valueKeys.filter(value => rangeOperators.indexOf(value) !== -1) - if (isRange.length) { - let rangeAttribute = catalogfilter.attribute - if (rangeAttribute === 'price') { - rangeAttribute = 'final_price' - } - // process range filters - filterQr = filterQr.andFilter('range', rangeAttribute, catalogfilter.value); - } else { - // process terms filters - let newValue = catalogfilter.value[Object.keys(catalogfilter.value)[0]] - if (!Array.isArray(newValue)) { - newValue = [newValue]; - } - if (attrPostfix === '') { - filterQr = filterQr.andFilter('terms', getMapping(catalogfilter.attribute), newValue) - } else { - filterQr = filterQr.andFilter('terms', catalogfilter.attribute + attrPostfix, newValue) - } - } - } - }) - return filterQr - } - - if (hasCatalogFilters) { - query = query.filterMinimumShouldMatch(1).orFilter('bool', (b) => attrFilterBuilder(b)) - .orFilter('bool', (b) => attrFilterBuilder(b, optionsPrefix).filter('match', 'type_id', 'configurable')); // the queries can vary based on the product type - } - - // Add aggregations for filters - if (appliedFilters.length > 0 && type === 'product') { - for (let attrToFilter of appliedFilters) { - if (attrToFilter.scope === 'catalog') { - if (attrToFilter.attribute !== 'price') { - query = query.aggregation('terms', getMapping(attrToFilter.attribute)) - query = query.aggregation('terms', attrToFilter.attribute + optionsPrefix) - } else { - query = query.aggregation('terms', attrToFilter.attribute) - query.aggregation('range', 'price', { - ranges: [ - { from: 0, to: 50 }, - { from: 50, to: 100 }, - { from: 100, to: 150 }, - { from: 150 } - ] - }) - } - } - } - } - } - - return query; -} - -function applySearchQuery (search, query) { - if (search !== '') { - query = query.andQuery('bool', b => b.orQuery('match_phrase_prefix', 'name', { query: search, boost: getBoosts('name'), slop: 2 }) - .orQuery('match_phrase', 'category.name', { query: search, boost: getBoosts('category.name') }) - .orQuery('match_phrase', 'short_description', { query: search, boost: getBoosts('short_description') }) - .orQuery('match_phrase', 'description', { query: search, boost: getBoosts('description') }) - .orQuery('bool', b => b.orQuery('terms', 'sku', search.split('-')) - .orQuery('terms', 'configurable_children.sku', search.split('-')) - .orQuery('match_phrase', 'sku', { query: search, boost: getBoosts('sku') }) - .orQuery('match_phrase', 'configurable_children.sku', { query: search, boost: getBoosts('configurable_children.sku') })) - ); - } - - return query; -} - -function applySort (sort, query) { - if (sort) { - map(sort, (value, key) => { - query.sort(key, value); - }); - } - - return query; -} - -export function buildQuery ({ - filter = [], - sort = '', - currentPage = 1, - pageSize = 10, - search = '', - type = 'product' -}) { - let query = bodybuilder(); - - query = applySearchQuery(search, query); - query = applyFilters(filter, query, type); - query = applySort(sort, query); - - query = query.from((currentPage - 1) * pageSize).size(pageSize); - - let builtQuery = query.build() - if (search !== '') { - builtQuery['min_score'] = config.elasticsearch.min_score - } - - return builtQuery; -} diff --git a/src/graphql/elasticsearch/queryBuilder.ts b/src/graphql/elasticsearch/queryBuilder.ts new file mode 100644 index 00000000..29e46f7d --- /dev/null +++ b/src/graphql/elasticsearch/queryBuilder.ts @@ -0,0 +1,24 @@ +import bodybuilder from 'bodybuilder'; +import { elasticsearch, ElasticsearchQueryConfig } from 'storefront-query-builder' +import config from 'config' + +export function buildQuery ({ + filter = [], + sort = '', + currentPage = 1, + pageSize = 10, + search = '', + type = 'product' +}) { + let queryChain = bodybuilder(); + elasticsearch.buildQueryBodyFromFilterObject({ config: (config as ElasticsearchQueryConfig), queryChain, filter, search }) + queryChain = elasticsearch.applySort({ sort, queryChain }); + queryChain = queryChain.from((currentPage - 1) * pageSize).size(pageSize); + + let builtQuery = queryChain.build() + if (search !== '') { + builtQuery['min_score'] = config.get('elasticsearch.min_score') + } + + return builtQuery; +} diff --git a/src/lib/cache-instance.js b/src/lib/cache-instance.js index d6fe0a25..b126b25c 100644 --- a/src/lib/cache-instance.js +++ b/src/lib/cache-instance.js @@ -1,15 +1,10 @@ -const fs = require('fs') -const path = require('path') const TagCache = require('redis-tag-cache').default const config = require('config') -let cache = false if (config.server.useOutputCache) { const redisConfig = Object.assign(config.redis) - cache = new TagCache({ + module.exports = new TagCache({ redis: redisConfig, defaultTimeout: config.server.outputCacheDefaultTtl }) } - -module.exports = cache diff --git a/src/lib/elastic.js b/src/lib/elastic.js index ca296219..62fba7f5 100644 --- a/src/lib/elastic.js +++ b/src/lib/elastic.js @@ -33,16 +33,22 @@ function adjustIndexName (indexName, entityType, config) { function adjustBackendProxyUrl (req, indexName, entityType, config) { let url + const queryString = require('query-string'); + const parsedQuery = queryString.parseUrl(req.url).query + if (parseInt(config.elasticsearch.apiVersion) < 6) { // legacy for ES 5 - url = config.elasticsearch.host + ':' + config.elasticsearch.port + (req.query.request ? _updateQueryStringParameter(req.url, 'request', null) : req.url) + delete parsedQuery.request + delete parsedQuery.request_format + delete parsedQuery.response_format + url = config.elasticsearch.host + ':' + config.elasticsearch.port + '/' + indexName + '/' + entityType + '/_search?' + queryString.stringify(parsedQuery) } else { - const queryString = require('query-string'); - const parsedQuery = queryString.parseUrl(req.url).query parsedQuery._source_includes = parsedQuery._source_include parsedQuery._source_excludes = parsedQuery._source_exclude delete parsedQuery._source_exclude delete parsedQuery._source_include delete parsedQuery.request + delete parsedQuery.request_format + delete parsedQuery.response_format url = config.elasticsearch.host + ':' + config.elasticsearch.port + '/' + adjustIndexName(indexName, entityType, config) + '/_search?' + queryString.stringify(parsedQuery) } if (!url.startsWith('http')) { diff --git a/src/platform/factory.ts b/src/platform/factory.ts index 30267e1d..2b859b74 100644 --- a/src/platform/factory.ts +++ b/src/platform/factory.ts @@ -24,4 +24,4 @@ class PlatformFactory { } } -module.exports = PlatformFactory; +export default PlatformFactory; diff --git a/src/processor/product.js b/src/processor/product.js index f83f0ced..6fa84add 100644 --- a/src/processor/product.js +++ b/src/processor/product.js @@ -3,6 +3,16 @@ import { sgnSrc } from '../lib/util' const jwa = require('jwa'); const hmac = jwa('HS256'); +function compactItem (item, fieldsToCompact) { + for (let [key, value] of Object.entries(fieldsToCompact)) { + if (typeof item[key] !== 'undefined') { + item[value] = item[key] + delete item[key] + } + } + return item +} + class ProductProcessor { constructor (config, entityType, indexName, req, res) { this._config = config @@ -43,6 +53,29 @@ class ProductProcessor { throw Error('error with resultset for processor chaining') } + // compact price fields + if (this._req.query.response_format === 'compact') { + resultSet[0] = resultSet[0].map((item) => { + const fieldsToCompress = this._config.products.fieldsToCompress + const fieldsToCompact = this._config.products.fieldsToCompact + if (!item._source) { return item } + if (item._source.configurable_children) { + item._source.configurable_children = item._source.configurable_children.map((subItem) => { + if (subItem) { + fieldsToCompress.forEach(field => { + if (item._source[field] === subItem[field]) { + delete subItem[field] // remove fields that are non distinct + } + }) + } + return compactItem(subItem, fieldsToCompact) + }) + } + item._source = compactItem(item._source, fieldsToCompact) + return compactItem(item, fieldsToCompact) + }) + } + if (this._req.query._source_exclude && this._req.query._source_exclude.indexOf('sgn') < 0) { const rs = resultSet[0].map((item) => { if (!item._source) { return item } diff --git a/tsconfig.json b/tsconfig.json index d6b3ed88..5c178c49 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,17 +3,18 @@ "target": "es6", "strict": false, "allowJs": true, - "importHelpers": true, + "importHelpers": false, "module": "commonjs", "moduleResolution": "node", "experimentalDecorators": true, "emitDecoratorMetadata": true, "esModuleInterop": true, - "resolveJsonModule": false, + "resolveJsonModule": true, "outDir": "dist", "sourceMap": true, "baseUrl": ".", - "lib": ["es7"] + "lib": ["es7"], + "preserveSymlinks": true }, "include": [ "src/**/*" diff --git a/yarn.lock b/yarn.lock index 305ce47b..d261f5ab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -972,10 +972,7 @@ binary-extensions@^1.0.0: bl@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/bl/-/bl-3.0.0.tgz#3611ec00579fd18561754360b21e9f784500ff88" -<<<<<<< HEAD -======= integrity sha512-EUAyP5UHU5hxF8BPT0LKW8gjYLhq1DQIcneOX/pL/m2Alo+OYDQAJlHq+yseMP50Os2nHXOSic6Ss3vSQeyf4A== ->>>>>>> upstream/master dependencies: readable-stream "^3.0.1" @@ -1076,11 +1073,7 @@ buffer-equal-constant-time@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819" -<<<<<<< HEAD buffer-from@1.x, buffer-from@^1.0.0: -======= -buffer-from@^1.0.0: ->>>>>>> upstream/master version "1.1.1" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" @@ -1232,10 +1225,6 @@ chownr@^1.1.1: version "1.1.2" resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.2.tgz#a18f1e0b269c8a6a5d3c86eb298beb14c3dd7bf6" -chownr@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.3.tgz#42d837d5239688d55f303003a508230fa6727142" - chownr@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.3.tgz#42d837d5239688d55f303003a508230fa6727142" @@ -1308,6 +1297,15 @@ cliui@^4.0.0: strip-ansi "^4.0.0" wrap-ansi "^2.0.0" +clone-deep@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-4.0.1.tgz#c19fd9bdbbf85942b4fd979c84dcf7d5f07c2387" + integrity sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ== + dependencies: + is-plain-object "^2.0.4" + kind-of "^6.0.2" + shallow-clone "^3.0.0" + cluster-key-slot@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" @@ -1654,10 +1652,7 @@ decompress-response@^3.3.0: decompress-response@^4.2.0: version "4.2.1" resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-4.2.1.tgz#414023cc7a302da25ce2ec82d0d5238ccafd8986" -<<<<<<< HEAD -======= integrity sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw== ->>>>>>> upstream/master dependencies: mimic-response "^2.0.0" @@ -4292,10 +4287,7 @@ minipass@^2.2.1, minipass@^2.3.5: minipass@^3.0.0: version "3.1.1" resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.1.1.tgz#7607ce778472a185ad6d89082aa2070f79cedcd5" -<<<<<<< HEAD -======= integrity sha512-UFqVihv6PQgwj8/yTGvl9kPz7xIAY+R5z6XYjRInD3Gk3qx6QGSD6zEcpeG4Dy/lQnv1J6zv8ejV90hyYIKf3w== ->>>>>>> upstream/master dependencies: yallist "^4.0.0" @@ -5414,15 +5406,6 @@ readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable string_decoder "~1.1.1" util-deprecate "~1.0.1" -readable-stream@^3.0.1, readable-stream@^3.1.1: - version "3.4.0" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.4.0.tgz#a51c26754658e0a3c21dbf59163bd45ba6f447fc" - integrity sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ== - dependencies: - inherits "^2.0.3" - string_decoder "^1.1.1" - util-deprecate "^1.0.1" - readdirp@^2.0.0, readdirp@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-2.2.1.tgz#0e87622a3325aa33e892285caf8b4e846529a525" @@ -5763,11 +5746,6 @@ semver@^6.0.0, semver@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" -semver@^6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" - integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== - send@0.17.1: version "0.17.1" resolved "https://registry.yarnpkg.com/send/-/send-0.17.1.tgz#c1d8b059f7900f7466dd4938bdc44e11ddb376c8" @@ -5819,6 +5797,13 @@ sha1@^1.1.1: charenc ">= 0.0.1" crypt ">= 0.0.1" +shallow-clone@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3" + integrity sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA== + dependencies: + kind-of "^6.0.2" + sharp@^0.23.4: version "0.23.4" resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.23.4.tgz#ca36067cb6ff7067fa6c77b01651cb9a890f8eb3" @@ -6120,6 +6105,12 @@ stealthy-require@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b" +"storefront-query-builder@https://github.com/DivanteLtd/storefront-query-builder.git": + version "0.0.8" + resolved "https://github.com/DivanteLtd/storefront-query-builder.git#26d17c9ae043b54725032687f116b21e83e02a77" + dependencies: + clone-deep "^4.0.1" + stream-events@^1.0.1, stream-events@^1.0.4: version "1.0.5" resolved "https://registry.yarnpkg.com/stream-events/-/stream-events-1.0.5.tgz#bbc898ec4df33a4902d892333d47da9bf1c406d5" @@ -6520,9 +6511,10 @@ typedarray@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" -typescript@3.3.*: - version "3.3.4000" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.3.4000.tgz#76b0f89cfdbf97827e1112d64f283f1151d6adf0" +typescript@3.7.*: + version "3.7.4" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.4.tgz#1743a5ec5fef6a1fa9f3e4708e33c81c73876c19" + integrity sha512-A25xv5XCtarLwXpcDNZzCGvW2D1S3/bACratYBx2sax8PefsFhlYmkQicKHvpYflFS8if4zne5zT5kpJ7pzuvw== uglify-js@^2.6.1: version "2.8.29"