From 551cd35396f449d4a37d0e27e8b8ae36ed4e59fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Tue, 28 Jan 2020 18:08:40 -0300 Subject: [PATCH 01/15] development inited --- index.js | 5 --- lib/config-validator.js | 33 +++++++++++++++ lib/index.js | 9 ----- lib/solr-error.js | 2 +- lib/solr-filters.js | 10 +++++ lib/solr.js | 7 +++- package-lock.json | 89 +++++++++++++++++++++++++++++++++++++++++ package.json | 7 +++- 8 files changed, 144 insertions(+), 18 deletions(-) delete mode 100644 index.js create mode 100644 lib/config-validator.js delete mode 100644 lib/index.js create mode 100644 lib/solr-filters.js diff --git a/index.js b/index.js deleted file mode 100644 index 4819e25..0000000 --- a/index.js +++ /dev/null @@ -1,5 +0,0 @@ -'use strict'; - -const { Solr } = require('./lib'); - -module.exports = Solr; diff --git a/lib/config-validator.js b/lib/config-validator.js new file mode 100644 index 0000000..12d42c9 --- /dev/null +++ b/lib/config-validator.js @@ -0,0 +1,33 @@ +'use strict'; + +const { struct } = require('superstuct'); + +const SolrError = require('./solr-error'); + +const configStruct = struct.partial({ + url: 'string', + core: 'string' +}); + +class ConfigValidator { + + /** + * Validates the received config struct + * @param {Object} config the config to validate + * @returns {Object} received config object + * @throws if the struct is invalid + */ + static validate(config) { + + try { + + return configStruct(config); + + } catch(err) { + err.message = `Error validating connection config: ${err.message}`; + throw new SolrError(err, SolrError.codes.INVALID_CONFIG); + } + } +} + +module.exports = ConfigValidator; diff --git a/lib/index.js b/lib/index.js deleted file mode 100644 index f8caa24..0000000 --- a/lib/index.js +++ /dev/null @@ -1,9 +0,0 @@ -'use strict'; - -const Solr = require('./solr'); -const SolrError = require('./solr-error'); - -module.exports = { - Solr, - SolrError -}; diff --git a/lib/solr-error.js b/lib/solr-error.js index ba5d907..20a847e 100644 --- a/lib/solr-error.js +++ b/lib/solr-error.js @@ -5,7 +5,7 @@ class SolrError extends Error { static get codes() { return { - // your errors here... + INVALID_CONFIG: 1 }; } diff --git a/lib/solr-filters.js b/lib/solr-filters.js new file mode 100644 index 0000000..c5503f7 --- /dev/null +++ b/lib/solr-filters.js @@ -0,0 +1,10 @@ +'use strict'; + +const SolrError = require('./solr-error'); + +class SolrFilters { + + +} + +module.exports = SolrFilters; diff --git a/lib/solr.js b/lib/solr.js index 718ca7f..8e446fc 100644 --- a/lib/solr.js +++ b/lib/solr.js @@ -2,9 +2,14 @@ const SolrError = require('./solr-error'); +const ConfigValidator = require('./config-validator'); + class Solr { - // package code... + constructor(config) { + this._config = ConfigValidator.validate(config); + } + } diff --git a/package-lock.json b/package-lock.json index 30ff4bf..a7f575c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -331,6 +331,17 @@ "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=", "dev": true }, + "clone-deep": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-2.0.2.tgz", + "integrity": "sha512-SZegPTKjCgpQH63E+eN6mVEEPdQBOUzjyJm5Pora4lrwWRFS8I0QAxV/KD6vV/i0WuijHZWQC1fMsPEdxfdVCQ==", + "requires": { + "for-own": "^1.0.0", + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.0", + "shallow-clone": "^1.0.0" + } + }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -838,6 +849,19 @@ "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==", "dev": true }, + "for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=" + }, + "for-own": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/for-own/-/for-own-1.0.0.tgz", + "integrity": "sha1-xjMy9BXO3EsE2/5wz4NklMU8tEs=", + "requires": { + "for-in": "^1.0.1" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1186,12 +1210,25 @@ "integrity": "sha1-YTObbyR1/Hcv2cnYP1yFddwVSuE=", "dev": true }, + "is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=" + }, "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", "dev": true }, + "is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "requires": { + "isobject": "^3.0.1" + } + }, "is-promise": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", @@ -1240,6 +1277,11 @@ "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, + "isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" + }, "istanbul-lib-coverage": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.5.tgz", @@ -1315,6 +1357,11 @@ "integrity": "sha512-FrLwOgm+iXrPV+5zDU6Jqu4gCRXbWEQg2O3SKONsWE4w7AXFRkryS53bpWdaL9cNol+AmR3AEYz6kn+o0fCPnw==", "dev": true }, + "kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + }, "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", @@ -1386,6 +1433,22 @@ "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", "dev": true }, + "mixin-object": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz", + "integrity": "sha1-T7lJRB2rGCVA8f4DW6YOGUel5X4=", + "requires": { + "for-in": "^0.1.3", + "is-extendable": "^0.1.1" + }, + "dependencies": { + "for-in": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz", + "integrity": "sha1-2Hc5COMSVhCZUrH9ubP6hn0ndeE=" + } + } + }, "mkdirp": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", @@ -2902,6 +2965,23 @@ "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w=", "dev": true }, + "shallow-clone": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-1.0.0.tgz", + "integrity": "sha512-oeXreoKR/SyNJtRJMAKPDSvd28OqEwG4eR/xc856cRGBII7gX9lvAqDxusPm0846z/w/hWYjI1NpKwJ00NHzRA==", + "requires": { + "is-extendable": "^0.1.1", + "kind-of": "^5.0.0", + "mixin-object": "^2.0.1" + }, + "dependencies": { + "kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==" + } + } + }, "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", @@ -3056,6 +3136,15 @@ "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", "dev": true }, + "superstruct": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/superstruct/-/superstruct-0.6.1.tgz", + "integrity": "sha512-LDbOKL5sNbOJ00Q36iYRhSexKIptZje0/mhNznnz04wT9CmsPDZg/K/UV1dgYuCwNMuOBHTbVROZsGB9EhhK4w==", + "requires": { + "clone-deep": "^2.0.1", + "kind-of": "^6.0.1" + } + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", diff --git a/package.json b/package.json index 8fb0934..f9ac876 100644 --- a/package.json +++ b/package.json @@ -2,13 +2,13 @@ "name": "@janiscommerce/solr", "version": "1.0.0", "description": "Apache Solr Driver", - "main": "index.js", + "main": "./lib/solr.js", "scripts": { "test": "export TEST_ENV=true; mocha --exit -R nyan --recursive tests/", "test-ci": "nyc --reporter=html --reporter=text mocha --recursive tests/", "watch-test": "export TEST_ENV=true; mocha --exit -R nyan -w --recursive tests/", "coverage": "nyc npm test", - "lint": "eslint index.js lib/ tests/" + "lint": "eslint lib/ tests/" }, "repository": { "type": "git", @@ -31,5 +31,8 @@ ], "directories": { "test": "tests" + }, + "dependencies": { + "superstruct": "0.6.1" } } From 13ecda739751cdbc1205ef3b088a789c57ca4295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Wed, 29 Jan 2020 18:01:27 -0300 Subject: [PATCH 02/15] Added get and multiInsert basic methods --- dev.js | 27 ++++ lib/config-validator.js | 6 +- lib/helpers/endpoint.js | 21 +++ lib/helpers/request.js | 67 +++++++++ lib/solr-error.js | 3 +- lib/solr.js | 38 +++++ package-lock.json | 311 ++++++++++++++++++++++++++++++++++++++-- package.json | 1 + 8 files changed, 458 insertions(+), 16 deletions(-) create mode 100644 dev.js create mode 100644 lib/helpers/endpoint.js create mode 100644 lib/helpers/request.js diff --git a/dev.js b/dev.js new file mode 100644 index 0000000..e8e1ddf --- /dev/null +++ b/dev.js @@ -0,0 +1,27 @@ +'use strict'; + +const Solr = require('./lib/solr'); + +const solr = new Solr({ + url: 'http://localhost:8983' +}); + +class Model { + static get table() { + return 'dev'; + } +} + +const model = new Model(); + +(async () => { + + console.log(await solr.multiInsert(model, [ + { + some: 'thing' + } + ])); + + console.log(await solr.get(model)); + +})(); diff --git a/lib/config-validator.js b/lib/config-validator.js index 12d42c9..8e4ad3a 100644 --- a/lib/config-validator.js +++ b/lib/config-validator.js @@ -1,12 +1,12 @@ 'use strict'; -const { struct } = require('superstuct'); +const { struct } = require('superstruct'); const SolrError = require('./solr-error'); const configStruct = struct.partial({ - url: 'string', - core: 'string' + url: 'string' /* , + core: 'string' */ }); class ConfigValidator { diff --git a/lib/helpers/endpoint.js b/lib/helpers/endpoint.js new file mode 100644 index 0000000..cd64378 --- /dev/null +++ b/lib/helpers/endpoint.js @@ -0,0 +1,21 @@ +'use strict'; + +const GET_ENDPOINT_BASE = '{{host}}/solr/{{core}}/query'; +const UPDATE_ENDPOINT_BASE = '{{host}}/solr/{{core}}/update?commit=true'; + +class Endpoint { + + static get(host, core) { + return this._buildEndpoint(GET_ENDPOINT_BASE, { host, core }); + } + + static update(host, core) { + return this._buildEndpoint(UPDATE_ENDPOINT_BASE, { host, core }); + } + + static _buildEndpoint(url, replacements) { + return Object.entries(replacements).reduce((endpoint, [key, value]) => endpoint.replace(`{{${key}}}`, value), url); + } +} + +module.exports = Endpoint; diff --git a/lib/helpers/request.js b/lib/helpers/request.js new file mode 100644 index 0000000..567b8c3 --- /dev/null +++ b/lib/helpers/request.js @@ -0,0 +1,67 @@ +'use strict'; + +const { promisify } = require('util'); + +const request = promisify(require('request')); + +class Request { + + /** + * Make an http request with method GET to the specified endpoint with the received request body and headers + * @param {String} endpoint the endpoint to make the request + * @param {Object|Array} requestBody The request body + * @param {Object} headers The request headers + * @returns {Object|Array|null} with the JSON response or null if the request ended with 4XX code + * @throws when the response code is 5XX + */ + static async get(endpoint, requestBody, headers) { + const httpRequest = this._buildHttpRequest(endpoint, requestBody, headers); + return this._makeRequest(httpRequest, 'GET'); + } + + /** + * Make an http request with method POST to the specified endpoint with the received request body and headers + * @param {String} endpoint the endpoint to make the request + * @param {Object|Array} requestBody The request body + * @param {Object} headers The request headers + * @returns {Object|Array|null} with the JSON response or null if the request ended with 4XX code + * @throws when the response code is 5XX + */ + static async post(endpoint, requestBody, headers) { + const httpRequest = this._buildHttpRequest(endpoint, requestBody, headers); + return this._makeRequest(httpRequest, 'POST'); + } + + static _buildHttpRequest(endpoint, body, headers) { + + const httpRequest = { + url: endpoint, + headers: { + ...headers, + 'Content-Type': 'application/json' + } + }; + + if(body) + httpRequest.body = JSON.stringify(body); + + return httpRequest; + } + + static async _makeRequest(httpRequest, method) { + + const { statusCode, statusMessage, body } = await request({ ...httpRequest, method }); + + if(statusCode >= 500) + throw new Error(statusMessage); + + if(statusCode >= 400) { + console.error(statusMessage, body); + return null; + } + + return body ? JSON.parse(body) : {}; + } +} + +module.exports = Request; diff --git a/lib/solr-error.js b/lib/solr-error.js index 20a847e..3df255a 100644 --- a/lib/solr-error.js +++ b/lib/solr-error.js @@ -5,7 +5,8 @@ class SolrError extends Error { static get codes() { return { - INVALID_CONFIG: 1 + INVALID_CONFIG: 1, + INVALID_MODEL: 2 }; } diff --git a/lib/solr.js b/lib/solr.js index 8e446fc..a5d2410 100644 --- a/lib/solr.js +++ b/lib/solr.js @@ -3,6 +3,8 @@ const SolrError = require('./solr-error'); const ConfigValidator = require('./config-validator'); +const Endpoint = require('./helpers/endpoint'); +const Request = require('./helpers/request'); class Solr { @@ -10,7 +12,43 @@ class Solr { this._config = ConfigValidator.validate(config); } + async multiInsert(model, items) { + this._validateModel(model); + + const endpoint = Endpoint.update(this._url, model.constructor.table); + + return !!await Request.post(endpoint, items); + } + + async get(model, params) { + + this._validateModel(model); + + const endpoint = Endpoint.get(this._url, model.constructor.table); + + // TODO: filters + + const res = await Request.get(endpoint, { + query: '*:*' + // filters + }); + + return res.response.docs; + } + + _validateModel(model) { + + if(!model) + throw new SolrError('Invalid or empty model.', SolrError.codes.INVALID_MODEL); + + if(!model.constructor.table) + throw new SolrError('Invalid model: Should have an static getter for table name.', SolrError.codes.INVALID_MODEL); + } + + get _url() { + return this._config.url; + } } module.exports = Solr; diff --git a/package-lock.json b/package-lock.json index a7f575c..291b601 100644 --- a/package-lock.json +++ b/package-lock.json @@ -168,7 +168,6 @@ "version": "6.11.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.11.0.tgz", "integrity": "sha512-nCprB/0syFYy9fVYU1ox1l2KN8S9I+tziH8D4zdZuLT3N6RMlGSGt5FSTpAiHB/Whv8Qs1cWHma1aMKZyaHRKA==", - "dev": true, "requires": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -233,18 +232,54 @@ "es-abstract": "^1.17.0-next.1" } }, + "asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "requires": { + "safer-buffer": "~2.1.0" + } + }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, "astral-regex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" + }, + "aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" + }, + "aws4": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz", + "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==" + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "requires": { + "tweetnacl": "^0.14.3" + } + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -293,6 +328,11 @@ "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true }, + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -357,6 +397,14 @@ "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", "dev": true }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, "commander": { "version": "2.15.1", "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", @@ -381,6 +429,11 @@ "integrity": "sha1-/ozxhP9mcLa67wGp1IYaXL7EEgo=", "dev": true }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, "cosmiconfig": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-5.2.1.tgz", @@ -434,6 +487,14 @@ "which": "^1.2.9" } }, + "dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "^1.0.0" + } + }, "debug": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", @@ -458,6 +519,11 @@ "object-keys": "^1.0.12" } }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, "diff": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", @@ -473,6 +539,15 @@ "esutils": "^2.0.2" } }, + "ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, "emoji-regex": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", @@ -776,6 +851,11 @@ "strip-eof": "^1.0.0" } }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, "external-editor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", @@ -787,17 +867,20 @@ "tmp": "^0.0.33" } }, + "extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" + }, "fast-deep-equal": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz", - "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==", - "dev": true + "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==" }, "fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" }, "fast-levenshtein": { "version": "2.0.6", @@ -862,6 +945,21 @@ "for-in": "^1.0.1" } }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -895,6 +993,14 @@ "pump": "^3.0.0" } }, + "getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "^1.0.0" + } + }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", @@ -927,6 +1033,20 @@ "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", "dev": true }, + "har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, "has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -960,6 +1080,16 @@ "integrity": "sha512-kssjab8CvdXfcXMXVcvsXum4Hwdq9XGtRD3TteMEvEbq0LXyiNQr6AprqKqfeaDXze7SxWvRxdpwE6ku7ikLkg==", "dev": true }, + "http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, "husky": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/husky/-/husky-2.7.0.tgz", @@ -1265,6 +1395,11 @@ "has-symbols": "^1.0.1" } }, + "is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" + }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -1282,6 +1417,11 @@ "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, "istanbul-lib-coverage": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.5.tgz", @@ -1327,6 +1467,11 @@ "esprima": "^4.0.0" } }, + "jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, "jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -1339,11 +1484,15 @@ "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", "dev": true }, + "json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" + }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -1351,6 +1500,22 @@ "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", "dev": true }, + "json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" + }, + "jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, "just-extend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.0.2.tgz", @@ -1412,6 +1577,19 @@ "integrity": "sha512-gKO5uExCXvSm6zbF562EvM+rd1kQDnB9AZBbiQVzf1ZmdDpxUSvpnAaVOP83N/31mRK8Ml8/VE8DMvsAZQ+7wg==", "dev": true }, + "mime-db": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", + "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==" + }, + "mime-types": { + "version": "2.1.26", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.26.tgz", + "integrity": "sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==", + "requires": { + "mime-db": "1.43.0" + } + }, "mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", @@ -2620,6 +2798,11 @@ } } }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, "object-inspect": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", @@ -2810,6 +2993,11 @@ "pify": "^2.0.0" } }, + "performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" + }, "pify": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", @@ -2846,6 +3034,11 @@ "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", "dev": true }, + "psl": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.7.0.tgz", + "integrity": "sha512-5NsSEDv8zY70ScRnOTn7bK7eanl2MvFrOrS/R6x+dBt5g1ghnj9Zv90kO8GwT8gxcu2ANyFprnFYB85IogIJOQ==" + }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -2859,8 +3052,12 @@ "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "dev": true + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + }, + "qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, "read-pkg": { "version": "2.0.0", @@ -2889,6 +3086,33 @@ "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==", "dev": true }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, "resolve": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.15.0.tgz", @@ -2947,11 +3171,15 @@ "tslib": "^1.9.0" } }, + "safe-buffer": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", + "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" + }, "safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "semver": { "version": "5.7.1", @@ -3079,6 +3307,22 @@ "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "dev": true }, + "sshpk": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", @@ -3221,12 +3465,41 @@ "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=", "dev": true }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + } + } + }, "tslib": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz", "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==", "dev": true }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, "type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", @@ -3252,11 +3525,15 @@ "version": "4.2.2", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", - "dev": true, "requires": { "punycode": "^2.1.0" } }, + "uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + }, "validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", @@ -3267,6 +3544,16 @@ "spdx-expression-parse": "^3.0.0" } }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", diff --git a/package.json b/package.json index f9ac876..57c4e39 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "test": "tests" }, "dependencies": { + "request": "^2.88.0", "superstruct": "0.6.1" } } From a82f0a0f8e63bc465c0662776100e5d55e2a4166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Thu, 30 Jan 2020 18:05:10 -0300 Subject: [PATCH 03/15] Added basic filters support --- dev.js | 27 ++++++++++++++-- lib/helpers/endpoint.js | 18 +++++++---- lib/helpers/filters.js | 72 +++++++++++++++++++++++++++++++++++++++++ lib/helpers/query.js | 25 ++++++++++++++ lib/helpers/request.js | 12 +++---- lib/helpers/schema.js | 49 ++++++++++++++++++++++++++++ lib/solr-error.js | 4 ++- lib/solr-filters.js | 10 ------ lib/solr.js | 42 +++++++++++++++++++++--- 9 files changed, 228 insertions(+), 31 deletions(-) create mode 100644 lib/helpers/filters.js create mode 100644 lib/helpers/query.js create mode 100644 lib/helpers/schema.js delete mode 100644 lib/solr-filters.js diff --git a/dev.js b/dev.js index e8e1ddf..50ed668 100644 --- a/dev.js +++ b/dev.js @@ -10,18 +10,41 @@ class Model { static get table() { return 'dev'; } + + static get fields() { + return { + some: true, + other: { type: 'string' }, + another: { type: 'number' } + }; + } } const model = new Model(); +const Filters = require('./lib/helpers/filters'); + (async () => { - console.log(await solr.multiInsert(model, [ + console.log(Filters.build({ + field: { type: 'equal', value: 'sarasa' }, + otherField: { type: 'notEqual', value: 'sarasa' }, + anotherField: { type: 'greater', value: 5200 }, + loquesea: { type: 'greaterOrEqual', value: 54 }, + miau: { type: 'lesser', value: 65 }, + guau: { type: 'lesserOrEqual', value: 32 } + })); + + // console.log(await solr.get(model)); + + // .log(await solr.updateSchemas(model)); + + /* console.log(await solr.multiInsert(model, [ { some: 'thing' } ])); - console.log(await solr.get(model)); + console.log(await solr.get(model)); */ })(); diff --git a/lib/helpers/endpoint.js b/lib/helpers/endpoint.js index cd64378..bad0a4e 100644 --- a/lib/helpers/endpoint.js +++ b/lib/helpers/endpoint.js @@ -1,16 +1,22 @@ 'use strict'; -const GET_ENDPOINT_BASE = '{{host}}/solr/{{core}}/query'; -const UPDATE_ENDPOINT_BASE = '{{host}}/solr/{{core}}/update?commit=true'; +const path = require('path'); + +const ENDPOINT_BASE = '{{url}}/solr/{{core}}'; class Endpoint { - static get(host, core) { - return this._buildEndpoint(GET_ENDPOINT_BASE, { host, core }); + static get presets() { + + return { + get: 'query', + update: 'update?commit=true', + schemas: 'schema' + }; } - static update(host, core) { - return this._buildEndpoint(UPDATE_ENDPOINT_BASE, { host, core }); + static create(endpoint, url, core, replacements) { + return this._buildEndpoint(path.join(ENDPOINT_BASE, endpoint), { url, core, ...replacements }); } static _buildEndpoint(url, replacements) { diff --git a/lib/helpers/filters.js b/lib/helpers/filters.js new file mode 100644 index 0000000..ea1e105 --- /dev/null +++ b/lib/helpers/filters.js @@ -0,0 +1,72 @@ +'use strict'; + +const SolrError = require('../solr-error'); + +class Filters { + + static build(filters) { + + const builtFilters = { + query: '*:*', + ...this._formatByType(filters) + }; + + return builtFilters; + } + + static _formatByType(filters) { + + const types = { + equal: this._formatEq, + notEqual: this._formatNe, + greater: this._formatGt, + greaterOrEqual: this._formatGte, + lesser: this._formatLt, + lesserOrEqual: this._formatLte + }; + + return Object.entries(filters).reduce((filtersByType, [field, filter]) => { + + const type = filter.type || 'equal'; + + const formatter = types[type]; + + if(!formatter) + throw new SolrError(`'${type}' is not a valid or supported filter type.`, SolrError.codes.INVALID_FILTER_TYPE); + + if(!filtersByType[field]) + filtersByType[field] = {}; + + filtersByType[field] = formatter(field, filter); + + return filtersByType; + + }, {}); + } + + static _formatEq(field, { value }) { + return `${field}:${value}`; + } + + static _formatNe(field, { value }) { + return `-${field}:${value}`; + } + + static _formatGt(field, { value }) { + return `${field}:{${value} TO *}`; + } + + static _formatGte(field, { value }) { + return `${field}:[${value} TO *]`; + } + + static _formatLt(field, { value }) { + return `${field}:{* TO ${value}}`; + } + + static _formatLte(field, { value }) { + return `${field}:[* TO ${value}]`; + } +} + +module.exports = Filters; diff --git a/lib/helpers/query.js b/lib/helpers/query.js new file mode 100644 index 0000000..9266e43 --- /dev/null +++ b/lib/helpers/query.js @@ -0,0 +1,25 @@ +'use strict'; + +class Query { + + build({ limit, page, order, filters }) { + + const query = {}; + + filters = this._buildFilters(filters); + + return query; + } + + _buildFilters(filters) { + + const builtFilters = { + query: '*:*' + }; + + + return builtFilters; + } +} + +module.exports = Query; diff --git a/lib/helpers/request.js b/lib/helpers/request.js index 567b8c3..e9969ae 100644 --- a/lib/helpers/request.js +++ b/lib/helpers/request.js @@ -4,6 +4,8 @@ const { promisify } = require('util'); const request = promisify(require('request')); +const SolrError = require('../solr-error'); + class Request { /** @@ -16,6 +18,7 @@ class Request { */ static async get(endpoint, requestBody, headers) { const httpRequest = this._buildHttpRequest(endpoint, requestBody, headers); + console.log(httpRequest); return this._makeRequest(httpRequest, 'GET'); } @@ -52,13 +55,8 @@ class Request { const { statusCode, statusMessage, body } = await request({ ...httpRequest, method }); - if(statusCode >= 500) - throw new Error(statusMessage); - - if(statusCode >= 400) { - console.error(statusMessage, body); - return null; - } + if(statusCode >= 500 || statusCode >= 400) + throw new SolrError(`[${statusCode}] (${statusMessage}): ${body}`, SolrError.codes.REQUEST_FAILED); return body ? JSON.parse(body) : {}; } diff --git a/lib/helpers/schema.js b/lib/helpers/schema.js new file mode 100644 index 0000000..a158f85 --- /dev/null +++ b/lib/helpers/schema.js @@ -0,0 +1,49 @@ +'use strict'; + +const FIELD_TYPES = { + string: 'string', + boolean: 'boolean', + date: 'pdate', + number: 'pint', + float: 'pfloat', + double: 'pdouble', + long: 'plong' +}; + +const DEFAULT_FIELDS = { + dateCreated: { type: 'date' }, + dateModified: { type: 'date' }, + userCreated: { type: 'string' }, + userModified: { type: 'string' }, + status: { type: 'string' } +}; + +class Schema { + + static buildQuery(method, model) { + + const { fields } = model.constructor; + + if(!fields) + return; + + const query = Object.entries(fields).reduce((schemas, [field, params]) => { + + const fieldType = params.type || 'string'; + + schemas[field] = { + name: field, + type: FIELD_TYPES[fieldType] + }; + + return schemas; + + }, DEFAULT_FIELDS); + + return { + [`${method}-field`]: Object.values(query) + }; + } +} + +module.exports = Schema; diff --git a/lib/solr-error.js b/lib/solr-error.js index 3df255a..6a25d2a 100644 --- a/lib/solr-error.js +++ b/lib/solr-error.js @@ -6,7 +6,9 @@ class SolrError extends Error { return { INVALID_CONFIG: 1, - INVALID_MODEL: 2 + INVALID_MODEL: 2, + INVALID_FILTER_TYPE: 3, + REQUEST_FAILED: 4 }; } diff --git a/lib/solr-filters.js b/lib/solr-filters.js deleted file mode 100644 index c5503f7..0000000 --- a/lib/solr-filters.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; - -const SolrError = require('./solr-error'); - -class SolrFilters { - - -} - -module.exports = SolrFilters; diff --git a/lib/solr.js b/lib/solr.js index a5d2410..77de89f 100644 --- a/lib/solr.js +++ b/lib/solr.js @@ -5,6 +5,7 @@ const SolrError = require('./solr-error'); const ConfigValidator = require('./config-validator'); const Endpoint = require('./helpers/endpoint'); const Request = require('./helpers/request'); +const Schema = require('./helpers/schema'); class Solr { @@ -12,11 +13,16 @@ class Solr { this._config = ConfigValidator.validate(config); } + async insert(model, item) { + return this.multiInsert(model, [item]); + } + async multiInsert(model, items) { this._validateModel(model); - const endpoint = Endpoint.update(this._url, model.constructor.table); + const { table } = model.constructor; + const endpoint = Endpoint.create(Endpoint.presets.update, this._url, table); return !!await Request.post(endpoint, items); } @@ -25,9 +31,8 @@ class Solr { this._validateModel(model); - const endpoint = Endpoint.get(this._url, model.constructor.table); - - // TODO: filters + const { table } = model.constructor; + const endpoint = Endpoint.create(Endpoint.presets.get, this._url, table); const res = await Request.get(endpoint, { query: '*:*' @@ -37,13 +42,40 @@ class Solr { return res.response.docs; } + async createSchemas(model) { + + this._validateModel(model); + + const { table } = model.constructor; + const query = Schema.buildQuery('add', model); + const endpoint = Endpoint.create(Endpoint.presets.schemas, this._url, table); + + return Request.post(endpoint, query); + } + + async updateSchemas(model) { + + this._validateModel(model); + + const { table } = model.constructor; + const query = Schema.buildQueryByModel('replace', model); + const endpoint = Endpoint.create(Endpoint.presets.schemas, this._url, table); + + return Request.post(endpoint, query); + } + _validateModel(model) { if(!model) throw new SolrError('Invalid or empty model.', SolrError.codes.INVALID_MODEL); - if(!model.constructor.table) + const { table, fields } = model.constructor; + + if(!table) throw new SolrError('Invalid model: Should have an static getter for table name.', SolrError.codes.INVALID_MODEL); + + if((fields && typeof fields !== 'object') || Array.isArray(fields)) + throw new SolrError('Invalid model: Fields should be an object', SolrError.codes.INVALID_MODEL); } get _url() { From ae9c7a4fae1f7b6444731c955ec83150f02aa698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Fri, 31 Jan 2020 18:04:22 -0300 Subject: [PATCH 04/15] Added schemas build for json nested object and response formatters for json objects in solr responses --- dev.js | 55 +++++++++++++++++++------- develop.js | 75 ++++++++++++++++++++++++++++++++++++ lib/helpers/res-formatter.js | 49 +++++++++++++++++++++++ lib/helpers/schema.js | 59 +++++++++++++++++++++++----- package-lock.json | 5 +++ package.json | 1 + 6 files changed, 220 insertions(+), 24 deletions(-) create mode 100644 develop.js create mode 100644 lib/helpers/res-formatter.js diff --git a/dev.js b/dev.js index 50ed668..b5f5e19 100644 --- a/dev.js +++ b/dev.js @@ -15,7 +15,26 @@ class Model { return { some: true, other: { type: 'string' }, - another: { type: 'number' } + another: { type: 'number' }, + somearr: { type: ['string'] }, + otherarr: { type: ['number'] }, + someobj: { + type: { + prop: 'string', + otherprop: 'number', + anotherprop: ['date'], + superprop: { + magicprop: ['string'] + }, + damnprop: { + damneditem: 'string', + otherdamneditem: ['number'], + superdamndeditem: { + ultradamneditem: 'date' + } + } + } + } }; } } @@ -24,7 +43,26 @@ const model = new Model(); const Filters = require('./lib/helpers/filters'); -(async () => { +const Schemas = require('./lib/helpers/schema'); + +const ResponseFormatter = require('./lib/helpers/res-formatter'); + +console.log( + JSON.stringify( + ResponseFormatter.format([ + { + some: 'field', + 'jsonfield.prop': 'value', + 'jsonfield.subprop.prop': 'othervalue', + 'jsonfield.subprop.otherprop': 14, + id: '51d1ba30-8596-46e8-b4ae-b83190dd6ac5', + _version_: 1657249623905927168 + } + ]), null, 2 + ) +); + +/* (async () => { console.log(Filters.build({ field: { type: 'equal', value: 'sarasa' }, @@ -35,16 +73,5 @@ const Filters = require('./lib/helpers/filters'); guau: { type: 'lesserOrEqual', value: 32 } })); - // console.log(await solr.get(model)); - - // .log(await solr.updateSchemas(model)); - - /* console.log(await solr.multiInsert(model, [ - { - some: 'thing' - } - ])); - - console.log(await solr.get(model)); */ -})(); +})(); */ diff --git a/develop.js b/develop.js new file mode 100644 index 0000000..d4e8221 --- /dev/null +++ b/develop.js @@ -0,0 +1,75 @@ +'use strict'; + +const assert = require('assert'); + +class Model { + static get table() { + return 'dev'; + } + + static get fields() { + return { + some: true, + other: { type: 'string' }, + another: { type: 'number' }, + somearr: { type: ['string'] }, + otherarr: { type: ['number'] }, + someobj: { + type: { + prop: 'string', + otherprop: 'number', + anotherprop: ['date'], + superprop: { + magicprop: ['string'] + }, + damnprop: { + damneditem: 'string', + otherdamneditem: ['number'], + superdamndeditem: { + ultradamneditem: 'date' + } + } + } + } + }; + } +} + +const model = new Model(); + +const Schemas = require('./lib/helpers/schema'); + +const expectedSchema = { + + 'add-field': [ + + { name: 'some', type: 'string' }, + { name: 'other', type: 'string' }, + { name: 'another', type: 'pint' }, + { name: 'somearr', type: 'string', multiValues: true }, + { name: 'otherarr', type: 'pint', multiValues: true }, + { name: 'someobj.prop', type: 'string' }, + { name: 'someobj.otherprop', type: 'pint' }, + { name: 'someobj.anotherprop', type: 'pdate', multiValues: true }, + { + name: 'someobj.superprop.magicprop', + type: 'string', + multiValues: true + }, + { name: 'someobj.damnprop.damneditem', type: 'string' }, + { + name: 'someobj.damnprop.otherdamneditem', + type: 'pint', + multiValues: true + }, + { + name: 'someobj.damnprop.superdamndeditem.ultradamneditem', + type: 'pdate' + } + ] +}; + +/* assert.deepStrictEqual( + Schemas.buildQuery('add', model), + expectedSchema +); */ diff --git a/lib/helpers/res-formatter.js b/lib/helpers/res-formatter.js new file mode 100644 index 0000000..7f081c9 --- /dev/null +++ b/lib/helpers/res-formatter.js @@ -0,0 +1,49 @@ +'use strict'; + +const merge = require('lodash.merge'); + +const IGNORED_FIELDS = ['_version_']; + +class ResponseFormatter { + + static format(response) { + + return response.map(item => { + + // Remove solr ignored fields + IGNORED_FIELDS.forEach(ignoredField => delete item[ignoredField]); + + return Object.entries(item).reduce((formattedItem, [field, value]) => { + + if(!field.includes('.')) + return { ...formattedItem, [field]: value }; + + return merge(formattedItem, this._buildJsonField(field, value)); + + }, {}); + }); + } + + static _buildJsonField(field, value) { + + const fields = field.split('.'); + + const [rootField] = fields; + + // Need to reverse the properties array to iterate first by the property that will have the item value + const properties = fields.splice(1).reverse(); + + return properties.reduce((item, property, i) => { + + if(i === 0) + item[rootField][property] = value; + else + item[rootField] = { [property]: item[rootField] }; + + return item; + + }, { [rootField]: {} }); + } +} + +module.exports = ResponseFormatter; diff --git a/lib/helpers/schema.js b/lib/helpers/schema.js index a158f85..9f66c75 100644 --- a/lib/helpers/schema.js +++ b/lib/helpers/schema.js @@ -27,22 +27,61 @@ class Schema { if(!fields) return; - const query = Object.entries(fields).reduce((schemas, [field, params]) => { + const builtSchemas = Object.entries(fields).reduce((schemas, [field, schema]) => { - const fieldType = params.type || 'string'; + schemas.push(...this._buildSchema(field, schema)); - schemas[field] = { - name: field, - type: FIELD_TYPES[fieldType] - }; + return schemas; + + }, []); + + return { [`${method}-field`]: builtSchemas }; + } + + static _buildSchema(field, schema) { + + const { type } = schema; + + if(Array.isArray(type)) + return this._buildArraySchema(field, type); + + if(this._isObject(type)) + return this._buildObjectSchema(field, type); + + return [{ + name: field, + type: this._getType(type) + }]; + } + + static _buildArraySchema(field, fieldType) { + + const [type] = fieldType; + + return [{ + name: field, + type: this._getType(type), + multiValues: true + }]; + } + + static _buildObjectSchema(field, type) { + + return Object.entries(type).reduce((schemas, [property, schema]) => { + + schemas.push(...this._buildSchema(`${field}.${property}`, { type: schema })); return schemas; - }, DEFAULT_FIELDS); + }, []); + } + + static _getType(type) { + return FIELD_TYPES[type] || FIELD_TYPES.string; + } - return { - [`${method}-field`]: Object.values(query) - }; + static _isObject(object) { + return object !== null && typeof object === 'object' && !Array.isArray(object); } } diff --git a/package-lock.json b/package-lock.json index 291b601..03c9894 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1571,6 +1571,11 @@ "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", "dev": true }, + "lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" + }, "lolex": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lolex/-/lolex-4.2.0.tgz", diff --git a/package.json b/package.json index 57c4e39..4ba7782 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "test": "tests" }, "dependencies": { + "lodash.merge": "^4.6.2", "request": "^2.88.0", "superstruct": "0.6.1" } From 024f734d34ef63ba901d00f9af9e1138243b9fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Mon, 3 Feb 2020 17:56:16 -0300 Subject: [PATCH 05/15] Added get params support: sorting, paging, limits and filters --- dev.js | 74 +++++++------------ lib/helpers/filters.js | 21 +++--- lib/helpers/query.js | 37 +++++++--- lib/helpers/request.js | 1 - lib/helpers/{res-formatter.js => response.js} | 4 +- lib/helpers/schema.js | 7 +- lib/solr-error.js | 3 +- lib/solr.js | 19 +++-- 8 files changed, 84 insertions(+), 82 deletions(-) rename lib/helpers/{res-formatter.js => response.js} (94%) diff --git a/dev.js b/dev.js index b5f5e19..f75f767 100644 --- a/dev.js +++ b/dev.js @@ -13,25 +13,16 @@ class Model { static get fields() { return { - some: true, - other: { type: 'string' }, - another: { type: 'number' }, - somearr: { type: ['string'] }, - otherarr: { type: ['number'] }, - someobj: { + text: true, + numeric: { type: 'number' }, + array: { type: ['string'] }, + arrayOfNumbers: { type: ['number'] }, + date: { type: 'date' }, + object: { type: { - prop: 'string', - otherprop: 'number', - anotherprop: ['date'], - superprop: { - magicprop: ['string'] - }, - damnprop: { - damneditem: 'string', - otherdamneditem: ['number'], - superdamndeditem: { - ultradamneditem: 'date' - } + property: 'string', + otherProperty: { + subproperty: 'number' } } } @@ -41,37 +32,28 @@ class Model { const model = new Model(); -const Filters = require('./lib/helpers/filters'); +(async () => { -const Schemas = require('./lib/helpers/schema'); + // console.log(await solr.updateSchemas(model)); -const ResponseFormatter = require('./lib/helpers/res-formatter'); - -console.log( - JSON.stringify( - ResponseFormatter.format([ - { - some: 'field', - 'jsonfield.prop': 'value', - 'jsonfield.subprop.prop': 'othervalue', - 'jsonfield.subprop.otherprop': 14, - id: '51d1ba30-8596-46e8-b4ae-b83190dd6ac5', - _version_: 1657249623905927168 + console.log(await solr.insert(model, { + text: 'sarasa', + numeric: 22, + array: ['sarasa'], + arrayOfNumbers: [1, 2, 3], + date: new Date() + /* object: { + property: 'foobar', + otherProperty: { + subproperty: 17 } - ]), null, 2 - ) -); - -/* (async () => { - - console.log(Filters.build({ - field: { type: 'equal', value: 'sarasa' }, - otherField: { type: 'notEqual', value: 'sarasa' }, - anotherField: { type: 'greater', value: 5200 }, - loquesea: { type: 'greaterOrEqual', value: 54 }, - miau: { type: 'lesser', value: 65 }, - guau: { type: 'lesserOrEqual', value: 32 } + } */ })); + console.log( + await solr.get(model, { + + }) + ); -})(); */ +})(); diff --git a/lib/helpers/filters.js b/lib/helpers/filters.js index ea1e105..9a7cde0 100644 --- a/lib/helpers/filters.js +++ b/lib/helpers/filters.js @@ -6,12 +6,9 @@ class Filters { static build(filters) { - const builtFilters = { - query: '*:*', - ...this._formatByType(filters) - }; + const filtersByType = this._formatByType(filters); - return builtFilters; + return Object.values(filtersByType).reduce((formattedFilters, filter) => (formattedFilters ? `${formattedFilters} AND ${filter}` : filter), ''); } static _formatByType(filters) { @@ -37,34 +34,34 @@ class Filters { if(!filtersByType[field]) filtersByType[field] = {}; - filtersByType[field] = formatter(field, filter); + filtersByType[field] = formatter(field, filter.value || filter); return filtersByType; }, {}); } - static _formatEq(field, { value }) { + static _formatEq(field, value) { return `${field}:${value}`; } - static _formatNe(field, { value }) { + static _formatNe(field, value) { return `-${field}:${value}`; } - static _formatGt(field, { value }) { + static _formatGt(field, value) { return `${field}:{${value} TO *}`; } - static _formatGte(field, { value }) { + static _formatGte(field, value) { return `${field}:[${value} TO *]`; } - static _formatLt(field, { value }) { + static _formatLt(field, value) { return `${field}:{* TO ${value}}`; } - static _formatLte(field, { value }) { + static _formatLte(field, value) { return `${field}:[* TO ${value}]`; } } diff --git a/lib/helpers/query.js b/lib/helpers/query.js index 9266e43..b64dfab 100644 --- a/lib/helpers/query.js +++ b/lib/helpers/query.js @@ -1,24 +1,43 @@ 'use strict'; +const Filters = require('./filters'); + +const SolrError = require('../solr-error'); + +const DEFAULT_LIMIT = 500; + class Query { - build({ limit, page, order, filters }) { + static build(params = {}) { - const query = {}; + const limit = params.limit || DEFAULT_LIMIT; - filters = this._buildFilters(filters); + const filters = params.filters ? { filter: Filters.build(params.filters) } : {}; - return query; - } + const order = params.order ? this._getSorting(params.order) : {}; - _buildFilters(filters) { + const page = params.page ? { offset: (params.page * limit) - limit } : {}; - const builtFilters = { - query: '*:*' + return { + query: '*:*', + limit, + ...filters, + ...order, + ...page }; + } + static _getSorting(order) { - return builtFilters; + return { + sort: Object.entries(order).reduce((sortings, [field, term]) => { + + const sort = `${field} ${term}`; + + return sortings ? `${sortings} AND ${sort}` : sort; + + }, '') + }; } } diff --git a/lib/helpers/request.js b/lib/helpers/request.js index e9969ae..d07f37c 100644 --- a/lib/helpers/request.js +++ b/lib/helpers/request.js @@ -18,7 +18,6 @@ class Request { */ static async get(endpoint, requestBody, headers) { const httpRequest = this._buildHttpRequest(endpoint, requestBody, headers); - console.log(httpRequest); return this._makeRequest(httpRequest, 'GET'); } diff --git a/lib/helpers/res-formatter.js b/lib/helpers/response.js similarity index 94% rename from lib/helpers/res-formatter.js rename to lib/helpers/response.js index 7f081c9..e56bd2a 100644 --- a/lib/helpers/res-formatter.js +++ b/lib/helpers/response.js @@ -4,7 +4,7 @@ const merge = require('lodash.merge'); const IGNORED_FIELDS = ['_version_']; -class ResponseFormatter { +class Response { static format(response) { @@ -46,4 +46,4 @@ class ResponseFormatter { } } -module.exports = ResponseFormatter; +module.exports = Response; diff --git a/lib/helpers/schema.js b/lib/helpers/schema.js index 9f66c75..3d9e9a0 100644 --- a/lib/helpers/schema.js +++ b/lib/helpers/schema.js @@ -22,10 +22,7 @@ class Schema { static buildQuery(method, model) { - const { fields } = model.constructor; - - if(!fields) - return; + const fields = { ...DEFAULT_FIELDS, ...model.constructor.fields }; const builtSchemas = Object.entries(fields).reduce((schemas, [field, schema]) => { @@ -61,7 +58,7 @@ class Schema { return [{ name: field, type: this._getType(type), - multiValues: true + multiValued: true }]; } diff --git a/lib/solr-error.js b/lib/solr-error.js index 6a25d2a..30d5551 100644 --- a/lib/solr-error.js +++ b/lib/solr-error.js @@ -8,7 +8,8 @@ class SolrError extends Error { INVALID_CONFIG: 1, INVALID_MODEL: 2, INVALID_FILTER_TYPE: 3, - REQUEST_FAILED: 4 + REQUEST_FAILED: 4, + INTERNAL_SOLR_ERROR: 5 }; } diff --git a/lib/solr.js b/lib/solr.js index 77de89f..873a380 100644 --- a/lib/solr.js +++ b/lib/solr.js @@ -3,9 +3,11 @@ const SolrError = require('./solr-error'); const ConfigValidator = require('./config-validator'); +const Response = require('./helpers/response'); const Endpoint = require('./helpers/endpoint'); const Request = require('./helpers/request'); const Schema = require('./helpers/schema'); +const Query = require('./helpers/query'); class Solr { @@ -34,12 +36,17 @@ class Solr { const { table } = model.constructor; const endpoint = Endpoint.create(Endpoint.presets.get, this._url, table); - const res = await Request.get(endpoint, { - query: '*:*' - // filters - }); + const res = await Request.get(endpoint, Query.build(params)); - return res.response.docs; + try { + + const { docs } = res.response; + + return Response.format(docs); + + } catch(err) { + throw new SolrError(`Invalid Solr response: ${err.message}`, SolrError.codes.INTERNAL_SOLR_ERROR); + } } async createSchemas(model) { @@ -58,7 +65,7 @@ class Solr { this._validateModel(model); const { table } = model.constructor; - const query = Schema.buildQueryByModel('replace', model); + const query = Schema.buildQuery('replace', model); const endpoint = Endpoint.create(Endpoint.presets.schemas, this._url, table); return Request.post(endpoint, query); From 770562901570b12ecb9ca204e69977858e78613e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Tue, 4 Feb 2020 18:05:27 -0300 Subject: [PATCH 06/15] Added getTotals method also fixed query and filters builder --- dev.js | 104 ++++++++++++++++++------ develop.js | 75 ----------------- lib/helpers/endpoint.js | 2 +- lib/helpers/filters.js | 40 ++++++--- lib/helpers/query.js | 26 +++--- lib/solr-error.js | 6 +- lib/solr.js | 174 +++++++++++++++++++++++++++++++++++++--- package.json | 3 +- 8 files changed, 287 insertions(+), 143 deletions(-) delete mode 100644 develop.js diff --git a/dev.js b/dev.js index f75f767..9226a96 100644 --- a/dev.js +++ b/dev.js @@ -7,22 +7,27 @@ const solr = new Solr({ }); class Model { + static get table() { return 'dev'; } static get fields() { return { - text: true, - numeric: { type: 'number' }, - array: { type: ['string'] }, - arrayOfNumbers: { type: ['number'] }, - date: { type: 'date' }, + string: true, + number: { type: 'number' }, + bool: { type: 'boolean' }, + float: { type: 'float' }, + stringArray: { type: ['string'] }, + numberArray: { type: ['number'] }, object: { type: { - property: 'string', - otherProperty: { - subproperty: 'number' + string: 'string', + number: 'number', + bool: 'boolean', + float: 'float', + object: { + boolArray: ['boolean'] } } } @@ -34,26 +39,77 @@ const model = new Model(); (async () => { + // createSchemas + // console.log(await solr.createSchemas(model)); + + // updateSchemas // console.log(await solr.updateSchemas(model)); - console.log(await solr.insert(model, { - text: 'sarasa', - numeric: 22, - array: ['sarasa'], - arrayOfNumbers: [1, 2, 3], - date: new Date() - /* object: { - property: 'foobar', - otherProperty: { - subproperty: 17 + // insert + /* console.log(await solr.insert(model, { + string: 'some string', + number: 32, + bool: true, + float: 1.32, + stringArray: ['a', 'b', 'c'], + numberArray: [1, 2, 3], + object: { + string: 'other string', + number: 77, + bool: false, + float: 2.32, + object: { + boolArray: [true, false, false, true] } - } */ - })); + } + })); */ - console.log( - await solr.get(model, { + // multiInsert + /* console.log(await solr.multiInsert(model, [ + { + string: 'other string', + number: 64, + bool: false, + float: 2.64, + stringArray: ['e', 'f', 'g'], + numberArray: [2, 4, 6], + object: { + string: 'another string', + number: 32, + bool: true, + float: 4.64, + object: { + boolArray: [false, true, true, false] + } + } + }, + { + string: 'another string', + number: 16, + bool: false, + float: 0.16, + stringArray: ['h', 'i', 'j'], + numberArray: [3, 6, 9], + object: { + string: 'super string', + number: 55, + bool: true, + float: 1.16, + object: { + boolArray: [true, true, false, false] + } + } + } + ])); */ - }) - ); + // get + console.log(await solr.get(model, { + filters: { + stringArray: { type: 'equal', value: ['a', 'b', 'c'] } + // string: { type: 'equal', value: 'another string' } + } + })); + // getTotals + console.log(await solr.getTotals(model)); })(); diff --git a/develop.js b/develop.js deleted file mode 100644 index d4e8221..0000000 --- a/develop.js +++ /dev/null @@ -1,75 +0,0 @@ -'use strict'; - -const assert = require('assert'); - -class Model { - static get table() { - return 'dev'; - } - - static get fields() { - return { - some: true, - other: { type: 'string' }, - another: { type: 'number' }, - somearr: { type: ['string'] }, - otherarr: { type: ['number'] }, - someobj: { - type: { - prop: 'string', - otherprop: 'number', - anotherprop: ['date'], - superprop: { - magicprop: ['string'] - }, - damnprop: { - damneditem: 'string', - otherdamneditem: ['number'], - superdamndeditem: { - ultradamneditem: 'date' - } - } - } - } - }; - } -} - -const model = new Model(); - -const Schemas = require('./lib/helpers/schema'); - -const expectedSchema = { - - 'add-field': [ - - { name: 'some', type: 'string' }, - { name: 'other', type: 'string' }, - { name: 'another', type: 'pint' }, - { name: 'somearr', type: 'string', multiValues: true }, - { name: 'otherarr', type: 'pint', multiValues: true }, - { name: 'someobj.prop', type: 'string' }, - { name: 'someobj.otherprop', type: 'pint' }, - { name: 'someobj.anotherprop', type: 'pdate', multiValues: true }, - { - name: 'someobj.superprop.magicprop', - type: 'string', - multiValues: true - }, - { name: 'someobj.damnprop.damneditem', type: 'string' }, - { - name: 'someobj.damnprop.otherdamneditem', - type: 'pint', - multiValues: true - }, - { - name: 'someobj.damnprop.superdamndeditem.ultradamneditem', - type: 'pdate' - } - ] -}; - -/* assert.deepStrictEqual( - Schemas.buildQuery('add', model), - expectedSchema -); */ diff --git a/lib/helpers/endpoint.js b/lib/helpers/endpoint.js index bad0a4e..537e389 100644 --- a/lib/helpers/endpoint.js +++ b/lib/helpers/endpoint.js @@ -10,7 +10,7 @@ class Endpoint { return { get: 'query', - update: 'update?commit=true', + update: 'update/json/docs?commit=true', schemas: 'schema' }; } diff --git a/lib/helpers/filters.js b/lib/helpers/filters.js index 9a7cde0..06d3ead 100644 --- a/lib/helpers/filters.js +++ b/lib/helpers/filters.js @@ -5,10 +5,8 @@ const SolrError = require('../solr-error'); class Filters { static build(filters) { - const filtersByType = this._formatByType(filters); - - return Object.values(filtersByType).reduce((formattedFilters, filter) => (formattedFilters ? `${formattedFilters} AND ${filter}` : filter), ''); + return Object.values(filtersByType); } static _formatByType(filters) { @@ -22,19 +20,39 @@ class Filters { lesserOrEqual: this._formatLte }; - return Object.entries(filters).reduce((filtersByType, [field, filter]) => { + return Object.entries(filters).reduce((filtersByType, [field, fieldFilters]) => { + + if(!Array.isArray(fieldFilters)) + fieldFilters = [fieldFilters]; + + const builtFilter = fieldFilters.reduce((filter, terms) => { + + let { type, value } = terms; + + if(!type && !value) + value = terms; + + if(!type) + type = 'equal'; + + if(!value) + throw new SolrError(`Invalid filters for field '${field}': Missing value for filter type ${type}.`, SolrError.codes.INVALID_FILTER_VALUE); + + const formatter = types[type]; + + if(!formatter) + throw new SolrError(`'${type}' is not a valid or supported filter type.`, SolrError.codes.INVALID_FILTER_TYPE); - const type = filter.type || 'equal'; + const formattedFilter = formatter(field, value); - const formatter = types[type]; + return filter ? `${filter} OR ${formattedFilter}` : formattedFilter; - if(!formatter) - throw new SolrError(`'${type}' is not a valid or supported filter type.`, SolrError.codes.INVALID_FILTER_TYPE); + }, ''); if(!filtersByType[field]) filtersByType[field] = {}; - filtersByType[field] = formatter(field, filter.value || filter); + filtersByType[field] = builtFilter; return filtersByType; @@ -42,11 +60,11 @@ class Filters { } static _formatEq(field, value) { - return `${field}:${value}`; + return `${field}:"${value}"`; } static _formatNe(field, value) { - return `-${field}:${value}`; + return `-${field}:"${value}"`; } static _formatGt(field, value) { diff --git a/lib/helpers/query.js b/lib/helpers/query.js index b64dfab..31d6eed 100644 --- a/lib/helpers/query.js +++ b/lib/helpers/query.js @@ -2,42 +2,34 @@ const Filters = require('./filters'); -const SolrError = require('../solr-error'); - -const DEFAULT_LIMIT = 500; class Query { - static build(params = {}) { + static build(params) { - const limit = params.limit || DEFAULT_LIMIT; + const { limit, page } = params; const filters = params.filters ? { filter: Filters.build(params.filters) } : {}; - - const order = params.order ? this._getSorting(params.order) : {}; - - const page = params.page ? { offset: (params.page * limit) - limit } : {}; + const order = params.order ? { sort: this._getSorting(params.order) } : {}; return { query: '*:*', + offset: (page * limit) - limit, limit, ...filters, - ...order, - ...page + ...order }; } static _getSorting(order) { - return { - sort: Object.entries(order).reduce((sortings, [field, term]) => { + return Object.entries(order).reduce((sortings, [field, term]) => { - const sort = `${field} ${term}`; + const sort = `${field} ${term}`; - return sortings ? `${sortings} AND ${sort}` : sort; + return sortings ? `${sortings}, ${sort}` : sort; - }, '') - }; + }, ''); } } diff --git a/lib/solr-error.js b/lib/solr-error.js index 30d5551..842cbd6 100644 --- a/lib/solr-error.js +++ b/lib/solr-error.js @@ -8,8 +8,10 @@ class SolrError extends Error { INVALID_CONFIG: 1, INVALID_MODEL: 2, INVALID_FILTER_TYPE: 3, - REQUEST_FAILED: 4, - INTERNAL_SOLR_ERROR: 5 + INVALID_FILTER_VALUE: 4, + REQUEST_FAILED: 5, + INTERNAL_SOLR_ERROR: 6, + INVALID_PARAMETERS: 7 }; } diff --git a/lib/solr.js b/lib/solr.js index 873a380..abb5e37 100644 --- a/lib/solr.js +++ b/lib/solr.js @@ -1,54 +1,180 @@ 'use strict'; +const UUID = require('uuid/v4'); + const SolrError = require('./solr-error'); const ConfigValidator = require('./config-validator'); + const Response = require('./helpers/response'); const Endpoint = require('./helpers/endpoint'); const Request = require('./helpers/request'); const Schema = require('./helpers/schema'); const Query = require('./helpers/query'); +const DEFAULT_LIMIT = 500; + class Solr { constructor(config) { this._config = ConfigValidator.validate(config); } + /** + * Inserts an item into Solr + * @param {Model} model Model instance + * @param {Object} item The item to insert + * @returns {String} The ID of the inserted item + * @throws When something goes wrong + * @example + * await insert(model, { field: 'value' }); + * // Expected result + * 'f429592c-8318-4507-a2ea-1b7fc388162a' + */ async insert(model, item) { - return this.multiInsert(model, [item]); + + this._validateModel(model); + + if(Array.isArray(item) || typeof item !== 'object') + throw new SolrError('Invalid item: Should be an object, also not an array.', SolrError.codes.INVALID_PARAMETERS); + + this._prepareItem(item); + + const { table } = model.constructor; + const endpoint = Endpoint.create(Endpoint.presets.update, this._url, table); + + const res = await Request.post(endpoint, item); + + this._validateResponse(res); + + return item.id; } + /** + * Inserts multiple items into Solr + * @param {Model} model Model instance + * @param {Array.} items The items to insert + * @returns {Array.} The inserted items + * @throws When something goes wrong + * @example + * await multiInsert(model, [ + * { field: 'value' }, + * { field: 'other value' } + * ]); + * // Expected result + * [ + * { id: '699ff4c9-ec23-44ab-adb6-36f19b1712f6', field: 'value' }, + * { id: 'f429592c-8318-4507-a2ea-1b7fc388162a', field: 'other value' } + * ] + */ async multiInsert(model, items) { this._validateModel(model); + if(!Array.isArray(items)) + throw new SolrError('Invalid items: Should be an array.', SolrError.codes.INVALID_PARAMETERS); + + items.map(this._prepareItem); + const { table } = model.constructor; const endpoint = Endpoint.create(Endpoint.presets.update, this._url, table); - return !!await Request.post(endpoint, items); + const res = await Request.post(endpoint, items); + + this._validateResponse(res); + + return items; } - async get(model, params) { + /** + * Get data from Solr database + * @param {Model} model Model instance + * @param {Object} params Get parameters (limit, filters, order, page) + * @returns {Array.} Solr get result + * @throws When something goes wrong + * @example + * await get(model, { + * limit: 10, + * page: 2, + * order: { field: 'asc' }, + * filters: { + * field: { type: 'greater', value: 32 } + * } + * }); + * // Expected result + * [ + * { + * id: '699ff4c9-ec23-44ab-adb6-36f19b1712f6' + * field: 34 + * }, + * { + * id: 'f429592c-8318-4507-a2ea-1b7fc388162a', + * field: 33 + * } + * ] + */ + async get(model, params = {}) { this._validateModel(model); const { table } = model.constructor; const endpoint = Endpoint.create(Endpoint.presets.get, this._url, table); - const res = await Request.get(endpoint, Query.build(params)); + const page = params.page || 1; + const limit = params.limit || DEFAULT_LIMIT; + + const query = Query.build({ ...params, page, limit }); + + console.log(query); + + const res = await Request.get(endpoint, query); + + this._validateResponse(res); - try { + const { docs } = res.response; - const { docs } = res.response; + model.lastQueryHasResults = !!docs.length; + model.totalsParams = { page, limit, query }; - return Response.format(docs); + return Response.format(docs); + } + + /** + * Get the paginated totals from the latest get query + * @param {Model} model Model instance + * @returns {Object} total, page size, pages and page from the results + */ + async getTotals(model) { + + this._validateModel(model); + + if(!model.lastQueryHasResults) + return { total: 0, pages: 0 }; + + const { page, limit, query } = model.totalsParams; + + const { table } = model.constructor; + const endpoint = Endpoint.create(Endpoint.presets.get, this._url, table); + + const res = await Request.get(endpoint, query); + + this._validateResponse(res); - } catch(err) { - throw new SolrError(`Invalid Solr response: ${err.message}`, SolrError.codes.INTERNAL_SOLR_ERROR); - } + const { numFound } = res.response; + + return { + total: numFound, + pageSize: limit, + pages: Math.ceil(numFound / limit), + page + }; } + /** + * Create the field schemas with the specified field types + * @param {Model} model Model instance + * @throws When something goes wrong + */ async createSchemas(model) { this._validateModel(model); @@ -57,9 +183,16 @@ class Solr { const query = Schema.buildQuery('add', model); const endpoint = Endpoint.create(Endpoint.presets.schemas, this._url, table); - return Request.post(endpoint, query); + const res = await Request.post(endpoint, query); + + this._validateResponse(res); } + /** + * Update the existing field schemas with the specified field types + * @param {Model} model Model instance + * @throws When something goes wrong + */ async updateSchemas(model) { this._validateModel(model); @@ -68,7 +201,24 @@ class Solr { const query = Schema.buildQuery('replace', model); const endpoint = Endpoint.create(Endpoint.presets.schemas, this._url, table); - return Request.post(endpoint, query); + const res = await Request.post(endpoint, query); + + this._validateResponse(res); + } + + _validateResponse(res) { + + const { responseHeader, response } = res; + + if(!responseHeader || responseHeader.status !== 0) + throw new SolrError(`Invalid Solr response: ${res}`, SolrError.codes.INTERNAL_SOLR_ERROR); + + if(response && !response.docs) + throw new SolrError(`Invalid Solr response: ${response}`, SolrError.codes.INTERNAL_SOLR_ERROR); + } + + _prepareItem(item) { + item.id = UUID(); } _validateModel(model) { diff --git a/package.json b/package.json index 4ba7782..3d3ba84 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "dependencies": { "lodash.merge": "^4.6.2", "request": "^2.88.0", - "superstruct": "0.6.1" + "superstruct": "0.6.1", + "uuid": "^3.4.0" } } From 2c13c3c26fb1f2b100f92e52e75d47b10d7e4513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Wed, 5 Feb 2020 18:16:16 -0300 Subject: [PATCH 07/15] added more test cases and fixed minor bugs --- lib/helpers/filters.js | 20 +- lib/helpers/request.js | 2 +- lib/helpers/schema.js | 18 +- lib/helpers/utils.js | 9 + lib/solr.js | 25 ++- package-lock.json | 19 ++ package.json | 3 +- tests/solr-test.js | 422 ++++++++++++++++++++++++++++++++++++++++- 8 files changed, 483 insertions(+), 35 deletions(-) create mode 100644 lib/helpers/utils.js diff --git a/lib/helpers/filters.js b/lib/helpers/filters.js index 06d3ead..053b2ab 100644 --- a/lib/helpers/filters.js +++ b/lib/helpers/filters.js @@ -12,8 +12,8 @@ class Filters { static _formatByType(filters) { const types = { - equal: this._formatEq, - notEqual: this._formatNe, + equal: this._formatEq.bind(this), + notEqual: this._formatNe.bind(this), greater: this._formatGt, greaterOrEqual: this._formatGte, lesser: this._formatLt, @@ -59,12 +59,24 @@ class Filters { }, {}); } + static _prepareValue(value) { + + if(Array.isArray(value)) + return `(${value.reduce((formattedValue, item) => (formattedValue ? `${formattedValue} AND "${item}"` : `"${item}"`), '')})`; + + return `"${value}"`; + } + static _formatEq(field, value) { - return `${field}:"${value}"`; + + value = this._prepareValue(value); + return `${field}:${value}`; } static _formatNe(field, value) { - return `-${field}:"${value}"`; + + value = this._prepareValue(value); + return `-${field}:${value}`; } static _formatGt(field, value) { diff --git a/lib/helpers/request.js b/lib/helpers/request.js index d07f37c..9b85ad6 100644 --- a/lib/helpers/request.js +++ b/lib/helpers/request.js @@ -54,7 +54,7 @@ class Request { const { statusCode, statusMessage, body } = await request({ ...httpRequest, method }); - if(statusCode >= 500 || statusCode >= 400) + if(statusCode >= 400) throw new SolrError(`[${statusCode}] (${statusMessage}): ${body}`, SolrError.codes.REQUEST_FAILED); return body ? JSON.parse(body) : {}; diff --git a/lib/helpers/schema.js b/lib/helpers/schema.js index 3d9e9a0..0b95b69 100644 --- a/lib/helpers/schema.js +++ b/lib/helpers/schema.js @@ -1,5 +1,7 @@ 'use strict'; +const { isObject } = require('./utils'); + const FIELD_TYPES = { string: 'string', boolean: 'boolean', @@ -10,19 +12,11 @@ const FIELD_TYPES = { long: 'plong' }; -const DEFAULT_FIELDS = { - dateCreated: { type: 'date' }, - dateModified: { type: 'date' }, - userCreated: { type: 'string' }, - userModified: { type: 'string' }, - status: { type: 'string' } -}; - class Schema { static buildQuery(method, model) { - const fields = { ...DEFAULT_FIELDS, ...model.constructor.fields }; + const { fields } = model.constructor; const builtSchemas = Object.entries(fields).reduce((schemas, [field, schema]) => { @@ -42,7 +36,7 @@ class Schema { if(Array.isArray(type)) return this._buildArraySchema(field, type); - if(this._isObject(type)) + if(isObject(type)) return this._buildObjectSchema(field, type); return [{ @@ -76,10 +70,6 @@ class Schema { static _getType(type) { return FIELD_TYPES[type] || FIELD_TYPES.string; } - - static _isObject(object) { - return object !== null && typeof object === 'object' && !Array.isArray(object); - } } module.exports = Schema; diff --git a/lib/helpers/utils.js b/lib/helpers/utils.js new file mode 100644 index 0000000..075a67d --- /dev/null +++ b/lib/helpers/utils.js @@ -0,0 +1,9 @@ +'use strict'; + +function isObject(object) { + return object !== null && typeof object === 'object' && !Array.isArray(object); +} + +module.exports = { + isObject +}; diff --git a/lib/solr.js b/lib/solr.js index abb5e37..397f7f3 100644 --- a/lib/solr.js +++ b/lib/solr.js @@ -2,6 +2,8 @@ const UUID = require('uuid/v4'); +const { isObject } = require('./helpers/utils'); + const SolrError = require('./solr-error'); const ConfigValidator = require('./config-validator'); @@ -35,10 +37,10 @@ class Solr { this._validateModel(model); - if(Array.isArray(item) || typeof item !== 'object') + if(!isObject(item)) throw new SolrError('Invalid item: Should be an object, also not an array.', SolrError.codes.INVALID_PARAMETERS); - this._prepareItem(item); + item = this._prepareItem(item); const { table } = model.constructor; const endpoint = Endpoint.create(Endpoint.presets.update, this._url, table); @@ -74,7 +76,7 @@ class Solr { if(!Array.isArray(items)) throw new SolrError('Invalid items: Should be an array.', SolrError.codes.INVALID_PARAMETERS); - items.map(this._prepareItem); + items = items.map(this._prepareItem); const { table } = model.constructor; const endpoint = Endpoint.create(Endpoint.presets.update, this._url, table); @@ -125,8 +127,6 @@ class Solr { const query = Query.build({ ...params, page, limit }); - console.log(query); - const res = await Request.get(endpoint, query); this._validateResponse(res); @@ -181,6 +181,7 @@ class Solr { const { table } = model.constructor; const query = Schema.buildQuery('add', model); + const endpoint = Endpoint.create(Endpoint.presets.schemas, this._url, table); const res = await Request.post(endpoint, query); @@ -211,14 +212,18 @@ class Solr { const { responseHeader, response } = res; if(!responseHeader || responseHeader.status !== 0) - throw new SolrError(`Invalid Solr response: ${res}`, SolrError.codes.INTERNAL_SOLR_ERROR); + throw new SolrError('Invalid Solr response: No responseHeader received.', SolrError.codes.INTERNAL_SOLR_ERROR); if(response && !response.docs) - throw new SolrError(`Invalid Solr response: ${response}`, SolrError.codes.INTERNAL_SOLR_ERROR); + throw new SolrError(`Invalid Solr response: ${JSON.stringify(response)}`, SolrError.codes.INTERNAL_SOLR_ERROR); } _prepareItem(item) { - item.id = UUID(); + + return { + id: UUID(), + ...item + }; } _validateModel(model) { @@ -231,8 +236,8 @@ class Solr { if(!table) throw new SolrError('Invalid model: Should have an static getter for table name.', SolrError.codes.INVALID_MODEL); - if((fields && typeof fields !== 'object') || Array.isArray(fields)) - throw new SolrError('Invalid model: Fields should be an object', SolrError.codes.INVALID_MODEL); + if(fields && !isObject(fields)) + throw new SolrError('Invalid model: Fields should be an object, also not an array.', SolrError.codes.INVALID_MODEL); } get _url() { diff --git a/package-lock.json b/package-lock.json index 03c9894..9704f62 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1748,6 +1748,19 @@ } } }, + "nock": { + "version": "11.7.2", + "resolved": "https://registry.npmjs.org/nock/-/nock-11.7.2.tgz", + "integrity": "sha512-7swr5bL1xBZ5FctyubjxEVySXOSebyqcL7Vy1bx1nS9IUqQWj81cmKjVKJLr8fHhtzI1MV8nyCdENA/cGcY1+Q==", + "dev": true, + "requires": { + "debug": "^4.1.0", + "json-stringify-safe": "^5.0.1", + "lodash": "^4.17.13", + "mkdirp": "^0.5.0", + "propagate": "^2.0.0" + } + }, "normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", @@ -3039,6 +3052,12 @@ "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", "dev": true }, + "propagate": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/propagate/-/propagate-2.0.1.tgz", + "integrity": "sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==", + "dev": true + }, "psl": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.7.0.tgz", diff --git a/package.json b/package.json index 3d3ba84..76f39a5 100644 --- a/package.json +++ b/package.json @@ -18,11 +18,12 @@ "license": "ISC", "homepage": "https://github.com/janis-commerce/solr.git#readme", "devDependencies": { - "husky": "^2.4.1", "eslint": "^5.16.0", "eslint-config-airbnb-base": "^13.1.0", "eslint-plugin-import": "^2.17.3", + "husky": "^2.4.1", "mocha": "^5.2.0", + "nock": "^11.7.2", "nyc": "^13.1.0", "sinon": "^7.3.2" }, diff --git a/tests/solr-test.js b/tests/solr-test.js index 28636fb..c0f340b 100644 --- a/tests/solr-test.js +++ b/tests/solr-test.js @@ -1,15 +1,427 @@ 'use strict'; -const assert = require('assert'); - +const assert = require('assert'); + +const nock = require('nock'); + const sandbox = require('sinon').createSandbox(); -const Solr = require('./../index'); +const Solr = require('../lib/solr'); -const SolrError = require('./../lib/solr-error') +const SolrError = require('../lib/solr-error'); describe('Solr', () => { - // your tests here... + afterEach(() => { + sandbox.restore(); + nock.cleanAll(); + }); + + class FakeModel { + + static get table() { + return 'some-core'; + } + + static get fields() { + return { + string: true, + number: { type: 'number' }, + float: { type: 'float' }, + double: { type: 'double' }, + long: { type: 'long' }, + boolean: { type: 'boolean' }, + date: { type: 'date' }, + array: { type: ['string'] }, + object: { + type: { + property: 'string', + subproperty: { + property: ['number'] + } + } + } + }; + } + } + + const builtSchemas = [ + { + name: 'string', + type: 'string' + }, + { + name: 'number', + type: 'pint' + }, + { + name: 'float', + type: 'pfloat' + }, + { + name: 'double', + type: 'pdouble' + }, + { + name: 'long', + type: 'plong' + }, + { + name: 'boolean', + type: 'boolean' + }, + { + name: 'date', + type: 'pdate' + }, + { + name: 'array', + type: 'string', + multiValued: true + }, + { + name: 'object.property', + type: 'string' + }, + { + name: 'object.subproperty.property', + type: 'pint', + multiValued: true + } + ]; + + const host = 'http://some-host.com'; + + const endpoints = { + update: '/solr/some-core/update/json/docs?commit=true', + get: '/solr/some-core/query', + schema: '/solr/some-core/schema' + }; + + const model = new FakeModel(); + + const solr = new Solr({ + url: host + }); + + describe('insert()', () => { + + const item = { + id: 'some-id', + some: 'data' + }; + + it('Should call Solr POST api to insert the received item', async () => { + + const request = nock(host) + .post(endpoints.update, item) + .reply(200, { + responseHeader: { + status: 0 + } + }); + + const result = await solr.insert(model, item); + + assert.deepEqual(result, item.id); + request.done(); + }); + + it('Should throw when the received model is invalid', async () => { + + await assert.rejects(solr.insert(null, item), { + name: 'SolrError', + code: SolrError.codes.INVALID_MODEL + }); + }); + + [ + + null, + undefined, + 'string', + 1, + ['array'] + + ].forEach(invalidItem => { + + it('Should throw when the received item is not an object', async () => { + + await assert.rejects(solr.insert(model, invalidItem), { + name: 'SolrError', + code: SolrError.codes.INVALID_PARAMETERS + }); + }); + }); + + it('Should throw when the Solr response code is bigger or equal than 400', async () => { + + const request = nock(host) + .post(endpoints.update, item) + .reply(400, { + responseHeader: { + status: 400 + } + }); + + await assert.rejects(solr.insert(model, item), { + name: 'SolrError', + code: SolrError.codes.REQUEST_FAILED + }); + + request.done(); + }); + + it('Should throw when the Solr response is invalid', async () => { + + const request = nock(host) + .post(endpoints.update) + .reply(200, { + responseHeader: { + status: 1 + } + }); + + await assert.rejects(solr.insert(model, item), { + name: 'SolrError', + code: SolrError.codes.INTERNAL_SOLR_ERROR + }); + + request.done(); + }); + }); + + describe('multiInsert()', () => { + + const items = [ + { + id: 'some-id', + some: 'data' + }, + { + id: 'other-id', + other: 'data' + } + ]; + + it('Should call Solr POST api to insert the received items', async () => { + + const request = nock(host) + .post(endpoints.update, items) + .reply(200, { + responseHeader: { + status: 0 + } + }); + + const result = await solr.multiInsert(model, items); + + assert.deepStrictEqual(result, items); + request.done(); + }); + + it('Should throw when the received model is invalid', async () => { + + sandbox.stub(FakeModel, 'fields') + .get(() => []); + + await assert.rejects(solr.multiInsert(model, items), { + name: 'SolrError', + code: SolrError.codes.INVALID_MODEL + }); + }); + + [ + + null, + undefined, + 'string', + 1, + { not: 'an array' } + + ].forEach(invalidItem => { + + it('Should throw when the received items are not an array', async () => { + + await assert.rejects(solr.multiInsert(model, invalidItem), { + name: 'SolrError', + code: SolrError.codes.INVALID_PARAMETERS + }); + }); + }); + + it('Should throw when the Solr response code is bigger or equal than 400', async () => { + + const request = nock(host) + .post(endpoints.update, items) + .reply(400, { + responseHeader: { + status: 400 + } + }); + + await assert.rejects(solr.multiInsert(model, items), { + name: 'SolrError', + code: SolrError.codes.REQUEST_FAILED + }); + + request.done(); + }); + + it('Should throw when the Solr response is invalid', async () => { + + const request = nock(host) + .post(endpoints.update) + .reply(200, { + responseHeader: { + status: 1 + } + }); + + await assert.rejects(solr.multiInsert(model, items), { + name: 'SolrError', + code: SolrError.codes.INTERNAL_SOLR_ERROR + }); + + request.done(); + }); + }); + + describe('get()', () => { + + it('Should call Solr GET api to get the items and format it correctly', async () => { + + const request = nock(host) + .get(endpoints.get, { + query: '*:*', + offset: 0, + limit: 500 + }) + .reply(200, { + responseHeader: { + status: 0 + }, + response: { + docs: [ + { + id: 'some-id', + some: 'data', + 'object.property': 'some-property', + 'object.subproperty.property': [1, 2, 3], + _version_: 1122111221 + } + ] + } + }); + + const result = await solr.get(model); + + assert.deepStrictEqual(result, [ + { + id: 'some-id', + some: 'data', + object: { + property: 'some-property', + subproperty: { + property: [1, 2, 3] + } + } + } + ]); + + request.done(); + }); + + it('Should call Solr GET api to get the items with the specified params', async () => { + + const request = nock(host) + .get(endpoints.get, { + query: '*:*', + offset: 5, + limit: 5, + filter: ['-id:"other-id"', 'some:"data"'], + sort: 'id asc, some desc' + }) + .reply(200, { + responseHeader: { + status: 0 + }, + response: { + docs: [ + { + id: 'some-id', + some: 'data' + } + ] + } + }); + + const result = await solr.get(model, { + limit: 5, + page: 2, + order: { id: 'asc', some: 'desc' }, + filters: { + id: { type: 'notEqual', value: 'other-id' }, + some: 'data' + } + }); + + assert.deepStrictEqual(result, [ + { + id: 'some-id', + some: 'data' + } + ]); + + request.done(); + }); + + it('Should throw when the received model is invalid', async () => { + + await assert.rejects(solr.get(null), { + name: 'SolrError', + code: SolrError.codes.INVALID_MODEL + }); + }); + }); + + describe('createSchemas()', () => { + + it('Should call Solr POST api to create the schemas', async () => { + + const request = nock(host) + .post(endpoints.schema, { + 'add-field': builtSchemas + }) + .reply(200, { + responseHeader: { + status: 0 + } + }); + + await assert.doesNotReject(solr.createSchemas(model)); + + request.done(); + }); + }); + + describe('updateSchemas()', () => { + + it('Should call Solr POST api to create the schemas', async () => { + + const request = nock(host) + .post(endpoints.schema, { + 'replace-field': builtSchemas + }) + .reply(200, { + responseHeader: { + status: 0 + } + }); + + await assert.doesNotReject(solr.updateSchemas(model)); + request.done(); + }); + }); }); From c0c5409cf928ea4dbe7c903e909d0cc982f7abe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Thu, 6 Feb 2020 18:12:50 -0300 Subject: [PATCH 08/15] Added externalFields and fieldOverride with model fields static getter support also added readme content and unit tests --- CHANGELOG.md | 2 + README.md | 165 +++++++++++++++++++++++++++++- dev.js | 115 --------------------- lib/config-validator.js | 4 +- lib/helpers/filters.js | 101 ++++++++++++------- lib/helpers/query.js | 4 +- lib/helpers/request.js | 8 +- lib/helpers/schema.js | 10 +- lib/solr-error.js | 7 +- lib/solr.js | 26 +++-- tests/helpers/filters.js | 88 ++++++++++++++++ tests/solr-test.js | 212 ++++++++++++++++++++++++++++++++++++++- 12 files changed, 558 insertions(+), 184 deletions(-) delete mode 100644 dev.js create mode 100644 tests/helpers/filters.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 4386bc4..5e9a481 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,3 +6,5 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Solr DB Driver Package \ No newline at end of file diff --git a/README.md b/README.md index d433ede..bb981f6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# solr +# Solr [![Build Status](https://travis-ci.org/janis-commerce/solr.svg?branch=master)](https://travis-ci.org/janis-commerce/solr) [![Coverage Status](https://coveralls.io/repos/github/janis-commerce/solr/badge.svg?branch=master)](https://coveralls.io/github/janis-commerce/solr?branch=master) @@ -10,8 +10,171 @@ Apache Solr Driver npm install @janiscommerce/solr ``` +## Models +Whenever the `Model` type is mentioned in this document, it refers to an instance of [@janiscommerce/model](https://www.npmjs.com/package/@janiscommerce/model). + +This is used to configure which collection should be used, which unique indexes it has, among other stuff. + ## API +### `new Solr(config)` +Constructs the Solr driver instance, connected with the `config` object. + +**Config properties** + +- url `String` (required): Solr URL + +**Config usage** +```js +{ + url: 'http://localhost:8983' +} +``` + +### ***async*** `insert(model, item)` +Inserts one document in a solr core + +- model: `Model`: A model instance +- item: `Object`: The item to save in the solr core + +- Resolves `String`: The *ID* of the inserted item +- Rejects: `SolrError` When something bad occurs + +### ***async*** `multiInsert(model, items)` +Inserts multiple documents in a solr core + +- model: `Model`: A model instance +- item: `Array`: The items to save in the solr core + +- Resolves: `Array`: Items inserted +- Rejects: `SolrError` When something bad occurs + +### ***async*** `get(model, [parameters])` +Searches documents in a solr core + +- model `Model`: A model instance +- parameters: `Object` (optional): The query parameters. Default: `{}` + +- Resolves: `Array`: An array of documents +- Rejects: `SolrError` When something bad occurs + +**Available parameters (all of them are optional)** + +- order `Object`: Sets the sorting criteria of the matched documents, for example: `{ myField: 'asc', myOtherField: 'desc' }` +- limit `Number`: Sets the page size when fetching documents. Default: `500` +- page `Number`: Sets the current page to retrieve. +- filters `Object`: Sets the criteria to match documents. + +Parameters example: +```js +{ + limit: 1000, // Default 500 + page: 2, + order: { + itemField: 'asc' + }, + filters: { + itemField: 'foobar', + otherItemField: { + type: 'notEqual', + value: 'foobar' + }, + anotherItemField: ['foo', 'bar'] + } +} +``` + +#### Filters + +The filters have a simpler structure what raw solr filters, in order to simplify it's usage. + +**Single valued filters** + +These filters sets a criteria using the specified `type` and `value`: +```js +{ + filters: { + myField: {type: 'notEqual', value: 'foobar'} + } +} +``` + +**Multivalued filters** + +These filters are an array of multiple filters that sets the criterias using an *OR* operation with the specified `type` and `value` +```js +{ + filters: { + myField: ['foobar', { type: 'notEqual', value: 'barfoo' }] + } +} +``` + +**Filter types** + +The filter types can be defined in the model static getter `fields` like this: +```js +class MyModel extends Model { + + static get fields(){ + + return { + myField: { + type: 'greaterOrEqual' + } + } + } +} +``` + +It can also be overriden in each query like this: +```js +solr.get(myModel, { + filters: { + myField: { + type: 'lesserOrEqual', + value: 10 + } + } +}) +``` + +The following table shows all the supported filter types: + +| Type | Solr equivalence | +| -------------- | ------------------ | +| equal | field:"value" | +| notEqual | -field:"value" | +| greater | field:{value TO *} | +| greaterOrEqual | field:[value TO *] | +| lesser | field:{* TO value} | +| lesserOrEqual | field:[* TO value] | + +If the type isn't defined in the model or in the query, it defaults to `equal`. + +**Internal field names** + +The name of a filter and the field that it will match can differ. To archieve that, you must declare it in the model static getter `fields`: + +```js +class MyModel extends Model { + + static get fields() { + + return { + externalFieldName: { + field: 'internalFieldName' + } + } + } +} +``` + +#### Nested filters +If you want to filter by fields inside objects, you can use nested filters. For example: +```js + +``` ## Usage ```js diff --git a/dev.js b/dev.js deleted file mode 100644 index 9226a96..0000000 --- a/dev.js +++ /dev/null @@ -1,115 +0,0 @@ -'use strict'; - -const Solr = require('./lib/solr'); - -const solr = new Solr({ - url: 'http://localhost:8983' -}); - -class Model { - - static get table() { - return 'dev'; - } - - static get fields() { - return { - string: true, - number: { type: 'number' }, - bool: { type: 'boolean' }, - float: { type: 'float' }, - stringArray: { type: ['string'] }, - numberArray: { type: ['number'] }, - object: { - type: { - string: 'string', - number: 'number', - bool: 'boolean', - float: 'float', - object: { - boolArray: ['boolean'] - } - } - } - }; - } -} - -const model = new Model(); - -(async () => { - - // createSchemas - // console.log(await solr.createSchemas(model)); - - // updateSchemas - // console.log(await solr.updateSchemas(model)); - - // insert - /* console.log(await solr.insert(model, { - string: 'some string', - number: 32, - bool: true, - float: 1.32, - stringArray: ['a', 'b', 'c'], - numberArray: [1, 2, 3], - object: { - string: 'other string', - number: 77, - bool: false, - float: 2.32, - object: { - boolArray: [true, false, false, true] - } - } - })); */ - - // multiInsert - /* console.log(await solr.multiInsert(model, [ - { - string: 'other string', - number: 64, - bool: false, - float: 2.64, - stringArray: ['e', 'f', 'g'], - numberArray: [2, 4, 6], - object: { - string: 'another string', - number: 32, - bool: true, - float: 4.64, - object: { - boolArray: [false, true, true, false] - } - } - }, - { - string: 'another string', - number: 16, - bool: false, - float: 0.16, - stringArray: ['h', 'i', 'j'], - numberArray: [3, 6, 9], - object: { - string: 'super string', - number: 55, - bool: true, - float: 1.16, - object: { - boolArray: [true, true, false, false] - } - } - } - ])); */ - - // get - console.log(await solr.get(model, { - filters: { - stringArray: { type: 'equal', value: ['a', 'b', 'c'] } - // string: { type: 'equal', value: 'another string' } - } - })); - - // getTotals - console.log(await solr.getTotals(model)); -})(); diff --git a/lib/config-validator.js b/lib/config-validator.js index 8e4ad3a..8a5382d 100644 --- a/lib/config-validator.js +++ b/lib/config-validator.js @@ -5,8 +5,8 @@ const { struct } = require('superstruct'); const SolrError = require('./solr-error'); const configStruct = struct.partial({ - url: 'string' /* , - core: 'string' */ + url: 'string' + // core: 'string' Los cores ya no estan separados de las collections, reemplaza a el getter table del modelo }); class ConfigValidator { diff --git a/lib/helpers/filters.js b/lib/helpers/filters.js index 053b2ab..4d18431 100644 --- a/lib/helpers/filters.js +++ b/lib/helpers/filters.js @@ -1,17 +1,66 @@ 'use strict'; +const { isObject } = require('./utils'); + const SolrError = require('../solr-error'); +const DEFAULT_FILTER_TYPE = 'equal'; + class Filters { - static build(filters) { - const filtersByType = this._formatByType(filters); + static build(filters, modelFields) { + + const filtersGroup = this._parseFilterGroup(filters, modelFields); + + const filtersByType = this._formatByType(filtersGroup); + return Object.values(filtersByType); } - static _formatByType(filters) { + static _parseFilterGroup(filterGroup, modelFields) { + + return Object.entries(filterGroup).reduce((parsedFilterGroup, [filterName, filterData]) => { + + const modelField = modelFields && filterName in modelFields ? modelFields[filterName] : {}; + + const filterKey = modelField.field || filterName; + + const filterValues = this._getFilterObjects(filterData, modelField); + + parsedFilterGroup[filterKey] = filterKey in parsedFilterGroup ? [...parsedFilterGroup[filterKey], ...filterValues] : filterValues; + + return parsedFilterGroup; + + }, {}); + } + + static _getFilterObjects(filterData, modelField) { + + if(!Array.isArray(filterData)) + filterData = [filterData]; + + return filterData.map(filterObject => { + + if(!isObject(filterObject)) + filterObject = { value: filterObject }; + + const { value } = filterObject; - const types = { + if(isObject(value) || Array.isArray(value)) + throw new SolrError('Invalid filters: JSON/Array filters are not supported by Solr.', SolrError.codes.UNSUPPORTED_FILTER); + + const type = filterObject.type || modelField.type || DEFAULT_FILTER_TYPE; + + if(!value) + throw new SolrError(`Invalid filters: Missing value for filter type ${type}.`, SolrError.codes.INVALID_FILTER_VALUE); + + return { type, value }; + }); + } + + static get _formatters() { + + return { equal: this._formatEq.bind(this), notEqual: this._formatNe.bind(this), greater: this._formatGt, @@ -19,64 +68,38 @@ class Filters { lesser: this._formatLt, lesserOrEqual: this._formatLte }; + } - return Object.entries(filters).reduce((filtersByType, [field, fieldFilters]) => { - - if(!Array.isArray(fieldFilters)) - fieldFilters = [fieldFilters]; - - const builtFilter = fieldFilters.reduce((filter, terms) => { - - let { type, value } = terms; - - if(!type && !value) - value = terms; + static _formatByType(filtersGroup) { - if(!type) - type = 'equal'; + return Object.entries(filtersGroup).reduce((filtersByType, [field, filterData]) => { - if(!value) - throw new SolrError(`Invalid filters for field '${field}': Missing value for filter type ${type}.`, SolrError.codes.INVALID_FILTER_VALUE); + const filterByType = filterData.reduce((filters, { type, value }) => { - const formatter = types[type]; + const formatter = this._formatters[type]; if(!formatter) throw new SolrError(`'${type}' is not a valid or supported filter type.`, SolrError.codes.INVALID_FILTER_TYPE); const formattedFilter = formatter(field, value); - return filter ? `${filter} OR ${formattedFilter}` : formattedFilter; + return filters ? `${filters} OR ${formattedFilter}` : formattedFilter; }, ''); - if(!filtersByType[field]) - filtersByType[field] = {}; - - filtersByType[field] = builtFilter; + filtersByType[field] = filterByType; return filtersByType; }, {}); } - static _prepareValue(value) { - - if(Array.isArray(value)) - return `(${value.reduce((formattedValue, item) => (formattedValue ? `${formattedValue} AND "${item}"` : `"${item}"`), '')})`; - - return `"${value}"`; - } - static _formatEq(field, value) { - - value = this._prepareValue(value); - return `${field}:${value}`; + return `${field}:"${value}"`; } static _formatNe(field, value) { - - value = this._prepareValue(value); - return `-${field}:${value}`; + return `-${field}:"${value}"`; } static _formatGt(field, value) { diff --git a/lib/helpers/query.js b/lib/helpers/query.js index 31d6eed..948df05 100644 --- a/lib/helpers/query.js +++ b/lib/helpers/query.js @@ -7,9 +7,9 @@ class Query { static build(params) { - const { limit, page } = params; + const { limit, page, fields } = params; - const filters = params.filters ? { filter: Filters.build(params.filters) } : {}; + const filters = params.filters ? { filter: Filters.build(params.filters, fields) } : {}; const order = params.order ? { sort: this._getSorting(params.order) } : {}; return { diff --git a/lib/helpers/request.js b/lib/helpers/request.js index 9b85ad6..84de788 100644 --- a/lib/helpers/request.js +++ b/lib/helpers/request.js @@ -41,12 +41,10 @@ class Request { headers: { ...headers, 'Content-Type': 'application/json' - } + }, + body: JSON.stringify(body) }; - if(body) - httpRequest.body = JSON.stringify(body); - return httpRequest; } @@ -57,7 +55,7 @@ class Request { if(statusCode >= 400) throw new SolrError(`[${statusCode}] (${statusMessage}): ${body}`, SolrError.codes.REQUEST_FAILED); - return body ? JSON.parse(body) : {}; + return JSON.parse(body); } } diff --git a/lib/helpers/schema.js b/lib/helpers/schema.js index 0b95b69..7f06665 100644 --- a/lib/helpers/schema.js +++ b/lib/helpers/schema.js @@ -14,15 +14,13 @@ const FIELD_TYPES = { class Schema { - static buildQuery(method, model) { + static buildQuery(method, schemas) { - const { fields } = model.constructor; + const builtSchemas = Object.entries(schemas).reduce((fields, [field, schema]) => { - const builtSchemas = Object.entries(fields).reduce((schemas, [field, schema]) => { + fields.push(...this._buildSchema(field, schema)); - schemas.push(...this._buildSchema(field, schema)); - - return schemas; + return fields; }, []); diff --git a/lib/solr-error.js b/lib/solr-error.js index 842cbd6..dcd3e14 100644 --- a/lib/solr-error.js +++ b/lib/solr-error.js @@ -9,9 +9,10 @@ class SolrError extends Error { INVALID_MODEL: 2, INVALID_FILTER_TYPE: 3, INVALID_FILTER_VALUE: 4, - REQUEST_FAILED: 5, - INTERNAL_SOLR_ERROR: 6, - INVALID_PARAMETERS: 7 + UNSUPPORTED_FILTER: 5, + REQUEST_FAILED: 6, + INTERNAL_SOLR_ERROR: 7, + INVALID_PARAMETERS: 8 }; } diff --git a/lib/solr.js b/lib/solr.js index 397f7f3..4908363 100644 --- a/lib/solr.js +++ b/lib/solr.js @@ -119,13 +119,14 @@ class Solr { this._validateModel(model); - const { table } = model.constructor; + const { table, fields } = model.constructor; + const endpoint = Endpoint.create(Endpoint.presets.get, this._url, table); const page = params.page || 1; const limit = params.limit || DEFAULT_LIMIT; - const query = Query.build({ ...params, page, limit }); + const query = Query.build({ ...params, page, limit, fields }); const res = await Request.get(endpoint, query); @@ -179,8 +180,12 @@ class Solr { this._validateModel(model); - const { table } = model.constructor; - const query = Schema.buildQuery('add', model); + const { table, schemas } = model.constructor; + + if(!schemas) + return; + + const query = Schema.buildQuery('add', schemas); const endpoint = Endpoint.create(Endpoint.presets.schemas, this._url, table); @@ -198,8 +203,12 @@ class Solr { this._validateModel(model); - const { table } = model.constructor; - const query = Schema.buildQuery('replace', model); + const { table, schemas } = model.constructor; + + if(!schemas) + return; + + const query = Schema.buildQuery('replace', schemas); const endpoint = Endpoint.create(Endpoint.presets.schemas, this._url, table); const res = await Request.post(endpoint, query); @@ -231,13 +240,16 @@ class Solr { if(!model) throw new SolrError('Invalid or empty model.', SolrError.codes.INVALID_MODEL); - const { table, fields } = model.constructor; + const { table, fields, schemas } = model.constructor; if(!table) throw new SolrError('Invalid model: Should have an static getter for table name.', SolrError.codes.INVALID_MODEL); if(fields && !isObject(fields)) throw new SolrError('Invalid model: Fields should be an object, also not an array.', SolrError.codes.INVALID_MODEL); + + if(schemas && !isObject(schemas)) + throw new SolrError('Invalid model: Schemas should be an object, also not an array.', SolrError.codes.INVALID_MODEL); } get _url() { diff --git a/tests/helpers/filters.js b/tests/helpers/filters.js new file mode 100644 index 0000000..e6821e0 --- /dev/null +++ b/tests/helpers/filters.js @@ -0,0 +1,88 @@ +'use strict'; + +const assert = require('assert'); + +const SolrError = require('../../lib/solr-error'); + +const Filters = require('../../lib/helpers/filters'); + +describe('Helpers', () => { + + describe('Filters', () => { + + describe('build()', () => { + + const modelFields = { + customField: { + field: 'equal', + type: 'notEqual' + }, + customDefault: { + type: 'greaterOrEqual' + } + }; + + it('Should build the filters for the received fields and model fields', () => { + + const filters = Filters.build({ + + equalDefault: 'something', + customDefault: 32, + equal: { type: 'equal', value: 'something' }, + notEqual: { type: 'notEqual', value: 'something' }, + greater: { type: 'greater', value: 10 }, + greaterOrEqual: { type: 'greaterOrEqual', value: 10 }, + lesser: { type: 'lesser', value: 10 }, + lesserOrEqual: { type: 'lesserOrEqual', value: 10 }, + multiFilters: ['some', 'other', { type: 'greater', value: 5 }], + customField: { value: 'foobar' } + + }, modelFields); + + assert.deepStrictEqual(filters, [ + 'equalDefault:"something"', + 'customDefault:[32 TO *]', + 'equal:"something" OR -equal:"foobar"', + '-notEqual:"something"', + 'greater:{10 TO *}', + 'greaterOrEqual:[10 TO *]', + 'lesser:{* TO 10}', + 'lesserOrEqual:[* TO 10]', + 'multiFilters:"some" OR multiFilters:"other" OR multiFilters:{5 TO *}' + ]); + }); + + [ + + { field: { type: 'equal', value: ['array'] } }, + { field: { type: 'equal', value: { an: 'object' } } } + + ].forEach(filters => { + + it('Should throw when the received filters is not supported', () => { + + assert.throws(() => Filters.build(filters), { + name: 'SolrError', + code: SolrError.codes.UNSUPPORTED_FILTER + }); + }); + }); + + it('Should throw when the received filter doesn\'t have a value', () => { + + assert.throws(() => Filters.build({ field: { type: 'equal' } }), { + name: 'SolrError', + code: SolrError.codes.INVALID_FILTER_VALUE + }); + }); + + it('Should throw when the received filter have an unknown or unsupported type', () => { + + assert.throws(() => Filters.build({ field: { type: 'unknown', value: 'some' } }), { + name: 'SolrError', + code: SolrError.codes.INVALID_FILTER_TYPE + }); + }); + }); + }); +}); diff --git a/tests/solr-test.js b/tests/solr-test.js index c0f340b..add11ce 100644 --- a/tests/solr-test.js +++ b/tests/solr-test.js @@ -24,6 +24,18 @@ describe('Solr', () => { } static get fields() { + return { + some: { + type: 'notEqual' + }, + myCustomField: { + field: 'date', + type: 'greaterOrEqual' + } + }; + } + + static get schemas() { return { string: true, number: { type: 'number' }, @@ -104,6 +116,30 @@ describe('Solr', () => { url: host }); + describe('constructor', () => { + + [ + + null, + undefined, + 1, + 'string', + ['array'], + { invalid: 'config' }, + { host: ['not a string'] } + + ].forEach(config => { + + it('Should throw when the received config is invalid', async () => { + assert.throws(() => new Solr(config), { + name: 'SolrError', + code: SolrError.codes.INVALID_CONFIG + }); + }); + }); + + }); + describe('insert()', () => { const item = { @@ -339,7 +375,11 @@ describe('Solr', () => { query: '*:*', offset: 5, limit: 5, - filter: ['-id:"other-id"', 'some:"data"'], + filter: [ + 'id:"some-id"', + '-some:"data"', + 'date:[10 TO *] OR date:[11 TO *] OR date:[12 TO *]' + ], sort: 'id asc, some desc' }) .reply(200, { @@ -361,8 +401,9 @@ describe('Solr', () => { page: 2, order: { id: 'asc', some: 'desc' }, filters: { - id: { type: 'notEqual', value: 'other-id' }, - some: 'data' + id: 'some-id', + some: 'data', + myCustomField: [10, 11, 12] } }); @@ -376,15 +417,128 @@ describe('Solr', () => { request.done(); }); + it('Should throw when the Solr response code is bigger or equal than 400', async () => { + + const request = nock(host) + .get(endpoints.get) + .reply(400, { + responseHeader: { + status: 400 + } + }); + + await assert.rejects(solr.get(model), { + name: 'SolrError', + code: SolrError.codes.REQUEST_FAILED + }); + + request.done(); + }); + + it('Should throw when the Solr response is invalid', async () => { + + const request = nock(host) + .get(endpoints.get) + .reply(200, { + responseHeader: { + status: 0 + }, + response: {} + }); + + await assert.rejects(solr.get(model), { + name: 'SolrError', + code: SolrError.codes.INTERNAL_SOLR_ERROR + }); + + request.done(); + }); + it('Should throw when the received model is invalid', async () => { - await assert.rejects(solr.get(null), { + sandbox.stub(FakeModel, 'table') + .get(() => undefined); + + await assert.rejects(solr.get(model), { name: 'SolrError', code: SolrError.codes.INVALID_MODEL }); }); }); + describe('getTotals', () => { + + afterEach(() => { + delete model.lastQueryHasResults; + delete model.totalsParams; + }); + + it('Should call Solr GET api to get the items count', async () => { + + const request = nock(host) + .get(endpoints.get, { + query: '*:*', + limit: 500, + offset: 0 + }) + .reply(200, { + responseHeader: { + status: 0 + }, + response: { + numFound: 10, + docs: Array(10).fill({ item: 'some-item' }) + } + }) + .persist(); + + await solr.get(model); + + const result = await solr.getTotals(model); + + assert.deepStrictEqual(result, { + total: 10, + pageSize: 500, + pages: 1, + page: 1 + }); + + request.done(); + }); + + it('Should return the default empty results when get can \'t find any item', async () => { + + const request = nock(host) + .get(endpoints.get, { + query: '*:*', + limit: 500, + offset: 0 + }) + .reply(200, { + responseHeader: { + status: 0 + }, + response: { + numFound: 0, + docs: [] + } + }); + + await solr.get(model); + + const result = await solr.getTotals(model); + + assert.deepStrictEqual(result, { total: 0, pages: 0 }); + + request.done(); + }); + + it('Should return the default empty results when get was not called', async () => { + const result = await solr.getTotals(model); + assert.deepStrictEqual(result, { total: 0, pages: 0 }); + }); + }); + describe('createSchemas()', () => { it('Should call Solr POST api to create the schemas', async () => { @@ -403,6 +557,31 @@ describe('Solr', () => { request.done(); }); + + it('Should return an empty object when there are no schemas in the model', async () => { + + sandbox.stub(FakeModel, 'schemas') + .get(() => undefined); + + const request = nock(host) + .post(endpoints.schema) + .reply(200); + + await assert.doesNotReject(solr.createSchemas(model)); + + assert.deepEqual(request.isDone(), false); + }); + + it('Should throw when the model is invalid', async () => { + + sandbox.stub(FakeModel, 'schemas') + .get(() => ['not an object']); + + await assert.rejects(solr.createSchemas(model), { + name: 'SolrError', + code: SolrError.codes.INVALID_MODEL + }); + }); }); describe('updateSchemas()', () => { @@ -423,5 +602,30 @@ describe('Solr', () => { request.done(); }); + + it('Should return an empty object when there are no schemas in the model', async () => { + + sandbox.stub(FakeModel, 'schemas') + .get(() => undefined); + + const request = nock(host) + .post(endpoints.schema) + .reply(200); + + await assert.doesNotReject(solr.updateSchemas(model)); + + assert.deepEqual(request.isDone(), false); + }); + + it('Should throw when the model is invalid', async () => { + + sandbox.stub(FakeModel, 'schemas') + .get(() => ['not an object']); + + await assert.rejects(solr.updateSchemas(model), { + name: 'SolrError', + code: SolrError.codes.INVALID_MODEL + }); + }); }); }); From f7611fc1ff2a738239cf30a25e9a751ac02c1d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Fri, 7 Feb 2020 14:03:53 -0300 Subject: [PATCH 09/15] Added readme --- README.md | 251 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 248 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bb981f6..7e0b618 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Inserts one document in a solr core - model: `Model`: A model instance - item: `Object`: The item to save in the solr core -- Resolves `String`: The *ID* of the inserted item +- Resolves: `String`: The *ID* of the inserted item - Rejects: `SolrError` When something bad occurs ### ***async*** `multiInsert(model, items)` @@ -110,6 +110,20 @@ These filters are an array of multiple filters that sets the criterias using an } ``` +**Unsupported filters** + +Due Solr limitations, some filters are unsupported, like Array/Object filters: +```js +{ + filters: { + myField: { type: 'equal', value: [1,2,3] }, + myOtherField: { type: 'notEqual', value: { some: 'thing' } } // Also you can use nested filters instead + } +} +``` + +See [nested filters](#nested%20filters) to see how to filter by object properties and sub properties. + **Filter types** The filter types can be defined in the model static getter `fields` like this: @@ -173,13 +187,244 @@ class MyModel extends Model { #### Nested filters If you want to filter by fields inside objects, you can use nested filters. For example: ```js +{ + /* Sample document to match + { + id: 'some-id', + someField: { + foo: 'bar' + } + } + */ + solr.get(myModel, { + filters: { + 'someField.foo': 'bar' + } + }); +} ``` +### ***async*** `getTotals(model)` +Gets information about the quantity of documents matched by the last call to the `get()` method. + +- model: `Model`: A model instance used for the query. **IMPORTANT**: This must be the same instance. + +- Resolves: `Object`: An object containing the totalizers +- Rejects: `SolrError`: When something bad occurs + +Return example: +```js +{ + total: 140, + pageSize: 60, + pages: 3, + page: 1 +} +``` + +If no query was executed before, it will just return the `total` and `pages` properties with a value of zero. + +### ***async*** `createSchemas(model)` +Build the schemas using the schemas defined in the model static getter `schemas`. + +- model `Model`: A model instance + +- Rejects: `SolrError`: When something bad occurs + +- **IMPORTANT**: + - This method must be executed before any operation, otherwise Solr will set all new fields as an `array` of `strings`. + - This method can't replace or delete already existing schemas in Solr. + +If you need details about how to define the schemas in the model, see the schemas [below](#schemas) + +### ***async*** `updateSchemas(model)` +Update the schemas by replacing the current schemas in Solr with the schemas defined in the model static getter `schemas`. +**IMPORTANT**: This method can't create or delete already existing schemas in Solr. + +- model `Model`: A model instance + +- Rejects: `SolrError`: When something bad occurs + +If you need details about how to define the schemas in the model, see the schemas [below](#schemas) + +#### Schemas + +The schemas are required by Solr to use the correct data types on every field, by default, Solr sets new fields as an `array` of `strings`, even in objects and his properties. + +**Field types** + +The field types can be defined in the model static getter `schemas` like this: +```js +class MyModel extends Model { + + static get schemas(){ + + return { + myStringField: true, // Default type string + myNumberString: { type: 'number' }, + myArrayOfStrings: { type: ['string'] }, + myObject: { + property: 'string' + } + } + } +} +``` + +The following table shows all the supported field types: + +| Type | Solr equivalence | +| -------------- | ---------------- | +| string | string | +| boolean | boolean | +| date | pdate | +| number | pint | +| float | pfloat | +| double | pdouble | +| long | plong | + +In case of an array field, the Solr equivalence will be the same for each field, only will set the `multiValued` property to `true`. + +If the type isn't defined, it defaults to `string`. + +**Single valued field** + +These are the most common fields, only stores a single value: +```js +{ + myField: { type: 'number' }, + myOtherField: { type: 'date' } +} +``` + +**Multi valued fields (Array)** + +These fields stores an array with multiple values **of the same type**: +```js +{ + myField: { type: ['string'] }, // Array of strings + myOtherField: { type: ['number'] } // Array of numbers +} +``` + +**Objects (JSON)** + +These fields stores an object with multiple properties that **can be of different types**: +```js +{ + myField: { + type: { + property: 'string', + subProperty: { + property: ['string'] + }, + otherProperty: 'date' + } + } +} +``` + +**IMPORTANT**: Due Solr compatibility issues, the object fields will be created **internally** as single fields like this: +```js +// Schema +{ + myField: { + type: { + property: 'string', + otherProperty: 'number', + subProperty: { + property: ['float'] + } + } + } +} + +// Formatted for Solr +{ + 'myField.property': { type: 'string' }, + 'myField.otherProperty': { type: 'number' }, + 'myField.subProperty.property': { type: ['float'] } +} +``` + +It will show as a **full object** on `get` operations. + +## Errors + +The errors are informed with a `SolrError`. +This object has a code that can be useful for debugging or error handling. +The codes are the following: + +| Code | Description | +|------|-------------------------- | +| 1 | Invalid connection config | +| 2 | Invalid model | +| 3 | Invalid filter type | +| 4 | Invalid filter value | +| 5 | Unsupported filter | +| 6 | Request failed | +| 7 | Internal Solr error | +| 8 | Invalid parameters | + ## Usage ```js const Solr = require('@janiscommerce/solr'); -``` +const Model = require('./myModel'); + +const solr = new Solr({ + url: 'localhost:8983' +}); + +const model = new Model(); + +(async () => { + + //Insert + await solr.insert(model, { + id: 'some-id', + name: 'test' + }); + // > 'some-id' + + // multiInsert + await solr.multiInsert(model, [ + { id: 'some-id', name: 'test 1' }, + { id: 'other-id', name: 'test 2' }, + { id: 'another-id', name: 'test 3' } + ]); + // > + /* + [ + { id: 'some-id', name: 'test 1' }, + { id: 'other-id', name: 'test 2' }, + { id: 'another-id', name: 'test 3' } + ] + */ + + // get + await solr.get(model, {}); + // > [...] // Every document in the solr core, up to 500 documents (by default) + + await solr.get(model, { filters: { id: 'some-id' } }); + // > [{ id: 'some-id', name: 'test' }] + + await solr.get(model, { limit: 10, page: 2, filters: { name: 'foo' } }); + // > The second page of 10 documents matching name equals to 'foo'. + + await solr.get(model, { order: { id: 'desc' } }); + // > [...] Every document in the solr core, ordered by descending id, up to 500 documents (by default) + + // getTotals + await solr.getTotals(model); + // > { page: 1, limit: 500, pages: 1, total: 4 } + + // createSchemas + await solr.createSchemas(model); + + // updateSchemas + await solr.updateSchemas(model); -## Examples +})(); +``` \ No newline at end of file From 46bf2b5ff4d7b5819a4e10886a3699c64594ab01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Mon, 10 Feb 2020 16:55:38 -0300 Subject: [PATCH 10/15] Added createCore method, moved core from model to config --- README.md | 43 +++++++++----- lib/config-validator.js | 4 +- lib/helpers/endpoint.js | 8 +-- lib/solr.js | 82 ++++++++++++++++---------- tests/solr-test.js | 127 +++++++++++++++++++++++++--------------- 5 files changed, 165 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 7e0b618..c8ba553 100644 --- a/README.md +++ b/README.md @@ -225,40 +225,41 @@ Return example: If no query was executed before, it will just return the `total` and `pages` properties with a value of zero. -### ***async*** `createSchemas(model)` -Build the schemas using the schemas defined in the model static getter `schemas`. +### ***async*** `createSchema(model, core)` +Build the fields schema using the schema defined in the model static getter `schema`. - model `Model`: A model instance +- core `String`: The core where the field schema will created. Default: `core` value from instance config. - Rejects: `SolrError`: When something bad occurs - **IMPORTANT**: - This method must be executed before any operation, otherwise Solr will set all new fields as an `array` of `strings`. - - This method can't replace or delete already existing schemas in Solr. + - This method can't replace or delete already existing fields schema in Solr. -If you need details about how to define the schemas in the model, see the schemas [below](#schemas) +If you need details about how to define the fields schema in the model, see the schema apart [below](#schema) -### ***async*** `updateSchemas(model)` -Update the schemas by replacing the current schemas in Solr with the schemas defined in the model static getter `schemas`. -**IMPORTANT**: This method can't create or delete already existing schemas in Solr. +### ***async*** `updateSchema(model)` +Update the fields schema by replacing the current fields schema in Solr with the defined in the model static getter `schema`. +**IMPORTANT**: This method can't create or delete already existing fields schema in Solr. - model `Model`: A model instance - Rejects: `SolrError`: When something bad occurs -If you need details about how to define the schemas in the model, see the schemas [below](#schemas) +If you need details about how to define the fields schema in the model, see the schema apart [below](#schema) -#### Schemas +#### Fields schema -The schemas are required by Solr to use the correct data types on every field, by default, Solr sets new fields as an `array` of `strings`, even in objects and his properties. +The fields schema are required by Solr to use the correct data types on every field, by default, Solr sets new fields as an `array` of `strings`, even in objects and his properties. **Field types** -The field types can be defined in the model static getter `schemas` like this: +The field types can be defined in the model static getter `schema` like this: ```js class MyModel extends Model { - static get schemas(){ + static get schema(){ return { myStringField: true, // Default type string @@ -350,6 +351,14 @@ These fields stores an object with multiple properties that **can be of differen It will show as a **full object** on `get` operations. +### ***async*** `createCore(model, name)` +Creates a new core in the Solr URL, then build the fields schema for that core. + +- model `Model`: A model instance +- name `String`: The name for the core to create + +- Rejects: `SolrError`: When something bad occurs + ## Errors The errors are informed with a `SolrError`. @@ -420,11 +429,13 @@ const model = new Model(); await solr.getTotals(model); // > { page: 1, limit: 500, pages: 1, total: 4 } - // createSchemas - await solr.createSchemas(model); + // createSchema + await solr.createSchema(model); - // updateSchemas - await solr.updateSchemas(model); + // updateSchema + await solr.updateSchema(model); + + // createCore })(); ``` \ No newline at end of file diff --git a/lib/config-validator.js b/lib/config-validator.js index 8a5382d..f7e3d40 100644 --- a/lib/config-validator.js +++ b/lib/config-validator.js @@ -5,8 +5,8 @@ const { struct } = require('superstruct'); const SolrError = require('./solr-error'); const configStruct = struct.partial({ - url: 'string' - // core: 'string' Los cores ya no estan separados de las collections, reemplaza a el getter table del modelo + url: 'string', + core: 'string' }); class ConfigValidator { diff --git a/lib/helpers/endpoint.js b/lib/helpers/endpoint.js index 537e389..3b7323e 100644 --- a/lib/helpers/endpoint.js +++ b/lib/helpers/endpoint.js @@ -2,16 +2,16 @@ const path = require('path'); -const ENDPOINT_BASE = '{{url}}/solr/{{core}}'; +const ENDPOINT_BASE = '{{url}}/solr'; class Endpoint { static get presets() { return { - get: 'query', - update: 'update/json/docs?commit=true', - schemas: 'schema' + get: '{{core}}/query', + update: '{{core}}/update/json/docs?commit=true', + schema: '{{core}}/schema' }; } diff --git a/lib/solr.js b/lib/solr.js index 4908363..51797d4 100644 --- a/lib/solr.js +++ b/lib/solr.js @@ -24,7 +24,7 @@ class Solr { /** * Inserts an item into Solr - * @param {Model} model Model instance + * @param {Model} model Model instance (not used but required for @janiscommerce/model compatibilty) * @param {Object} item The item to insert * @returns {String} The ID of the inserted item * @throws When something goes wrong @@ -35,15 +35,12 @@ class Solr { */ async insert(model, item) { - this._validateModel(model); - if(!isObject(item)) throw new SolrError('Invalid item: Should be an object, also not an array.', SolrError.codes.INVALID_PARAMETERS); item = this._prepareItem(item); - const { table } = model.constructor; - const endpoint = Endpoint.create(Endpoint.presets.update, this._url, table); + const endpoint = Endpoint.create(Endpoint.presets.update, this._url, this._core); const res = await Request.post(endpoint, item); @@ -54,7 +51,7 @@ class Solr { /** * Inserts multiple items into Solr - * @param {Model} model Model instance + * @param {Model} model Model instance (not used but required for @janiscommerce/model compatibilty) * @param {Array.} items The items to insert * @returns {Array.} The inserted items * @throws When something goes wrong @@ -71,15 +68,12 @@ class Solr { */ async multiInsert(model, items) { - this._validateModel(model); - if(!Array.isArray(items)) throw new SolrError('Invalid items: Should be an array.', SolrError.codes.INVALID_PARAMETERS); items = items.map(this._prepareItem); - const { table } = model.constructor; - const endpoint = Endpoint.create(Endpoint.presets.update, this._url, table); + const endpoint = Endpoint.create(Endpoint.presets.update, this._url, this._core); const res = await Request.post(endpoint, items); @@ -119,9 +113,9 @@ class Solr { this._validateModel(model); - const { table, fields } = model.constructor; + const { fields } = model.constructor; - const endpoint = Endpoint.create(Endpoint.presets.get, this._url, table); + const endpoint = Endpoint.create(Endpoint.presets.get, this._url, this._core); const page = params.page || 1; const limit = params.limit || DEFAULT_LIMIT; @@ -154,8 +148,7 @@ class Solr { const { page, limit, query } = model.totalsParams; - const { table } = model.constructor; - const endpoint = Endpoint.create(Endpoint.presets.get, this._url, table); + const endpoint = Endpoint.create(Endpoint.presets.get, this._url, this._core); const res = await Request.get(endpoint, query); @@ -172,22 +165,23 @@ class Solr { } /** - * Create the field schemas with the specified field types + * Create the fields schema with the specified field types * @param {Model} model Model instance + * @param {string} core The core name where create the schemas (Default: core from config) * @throws When something goes wrong */ - async createSchemas(model) { + async createSchema(model, core = this._core) { this._validateModel(model); - const { table, schemas } = model.constructor; + const { schema } = model.constructor; - if(!schemas) + if(!schema) return; - const query = Schema.buildQuery('add', schemas); + const query = Schema.buildQuery('add', schema); - const endpoint = Endpoint.create(Endpoint.presets.schemas, this._url, table); + const endpoint = Endpoint.create(Endpoint.presets.schema, this._url, core); const res = await Request.post(endpoint, query); @@ -195,27 +189,52 @@ class Solr { } /** - * Update the existing field schemas with the specified field types + * Update the existing fields schema with the specified field types * @param {Model} model Model instance * @throws When something goes wrong */ - async updateSchemas(model) { + async updateSchema(model) { this._validateModel(model); - const { table, schemas } = model.constructor; + const { schema } = model.constructor; - if(!schemas) + if(!schema) return; - const query = Schema.buildQuery('replace', schemas); - const endpoint = Endpoint.create(Endpoint.presets.schemas, this._url, table); + const query = Schema.buildQuery('replace', schema); + const endpoint = Endpoint.create(Endpoint.presets.schema, this._url, this._core); const res = await Request.post(endpoint, query); this._validateResponse(res); } + /** + * Create a new core into Solr URL then create the fields schema. + * @param {Model} model Model instance + * @param {string} name The name for the new core to create + */ + async createCore(model, name) { + + try { + + this._validateModel(model); + + } catch(err) { + // Should not explode when the model is invalid, also must not create any core. + return; + } + + const endpoint = Endpoint.create('admin/cores?action=CREATE&name={{name}}&configSet=_default', this._url, null, { name }); + + const res = await Request.post(endpoint); + + this._validateResponse(res); + + return this.createSchema(model, name); + } + _validateResponse(res) { const { responseHeader, response } = res; @@ -240,21 +259,22 @@ class Solr { if(!model) throw new SolrError('Invalid or empty model.', SolrError.codes.INVALID_MODEL); - const { table, fields, schemas } = model.constructor; - - if(!table) - throw new SolrError('Invalid model: Should have an static getter for table name.', SolrError.codes.INVALID_MODEL); + const { fields, schema } = model.constructor; if(fields && !isObject(fields)) throw new SolrError('Invalid model: Fields should be an object, also not an array.', SolrError.codes.INVALID_MODEL); - if(schemas && !isObject(schemas)) + if(schema && !isObject(schema)) throw new SolrError('Invalid model: Schemas should be an object, also not an array.', SolrError.codes.INVALID_MODEL); } get _url() { return this._config.url; } + + get _core() { + return this._config.core; + } } module.exports = Solr; diff --git a/tests/solr-test.js b/tests/solr-test.js index add11ce..133ba48 100644 --- a/tests/solr-test.js +++ b/tests/solr-test.js @@ -19,10 +19,6 @@ describe('Solr', () => { class FakeModel { - static get table() { - return 'some-core'; - } - static get fields() { return { some: { @@ -35,7 +31,7 @@ describe('Solr', () => { }; } - static get schemas() { + static get schema() { return { string: true, number: { type: 'number' }, @@ -113,7 +109,8 @@ describe('Solr', () => { const model = new FakeModel(); const solr = new Solr({ - url: host + url: host, + core: 'some-core' }); describe('constructor', () => { @@ -126,7 +123,8 @@ describe('Solr', () => { 'string', ['array'], { invalid: 'config' }, - { host: ['not a string'] } + { url: ['not a string'], core: 'valid' }, + { url: 'valid', core: ['not a string'] } ].forEach(config => { @@ -163,14 +161,6 @@ describe('Solr', () => { request.done(); }); - it('Should throw when the received model is invalid', async () => { - - await assert.rejects(solr.insert(null, item), { - name: 'SolrError', - code: SolrError.codes.INVALID_MODEL - }); - }); - [ null, @@ -256,17 +246,6 @@ describe('Solr', () => { request.done(); }); - it('Should throw when the received model is invalid', async () => { - - sandbox.stub(FakeModel, 'fields') - .get(() => []); - - await assert.rejects(solr.multiInsert(model, items), { - name: 'SolrError', - code: SolrError.codes.INVALID_MODEL - }); - }); - [ null, @@ -456,8 +435,8 @@ describe('Solr', () => { it('Should throw when the received model is invalid', async () => { - sandbox.stub(FakeModel, 'table') - .get(() => undefined); + sandbox.stub(FakeModel, 'fields') + .get(() => ['not an object']); await assert.rejects(solr.get(model), { name: 'SolrError', @@ -539,9 +518,9 @@ describe('Solr', () => { }); }); - describe('createSchemas()', () => { + describe('createSchema()', () => { - it('Should call Solr POST api to create the schemas', async () => { + it('Should call Solr POST api to create the schema using core name from config', async () => { const request = nock(host) .post(endpoints.schema, { @@ -553,40 +532,57 @@ describe('Solr', () => { } }); - await assert.doesNotReject(solr.createSchemas(model)); + await assert.doesNotReject(solr.createSchema(model)); + + request.done(); + }); + + it('Should call Solr POST api to create the schema using the specified core name', async () => { + + const request = nock(host) + .post('/solr/foobar-core/schema', { + 'add-field': builtSchemas + }) + .reply(200, { + responseHeader: { + status: 0 + } + }); + + await assert.doesNotReject(solr.createSchema(model, 'foobar-core')); request.done(); }); - it('Should return an empty object when there are no schemas in the model', async () => { + it('Should return an empty object when there are no schema in the model', async () => { - sandbox.stub(FakeModel, 'schemas') + sandbox.stub(FakeModel, 'schema') .get(() => undefined); const request = nock(host) .post(endpoints.schema) .reply(200); - await assert.doesNotReject(solr.createSchemas(model)); + await assert.doesNotReject(solr.createSchema(model)); assert.deepEqual(request.isDone(), false); }); it('Should throw when the model is invalid', async () => { - sandbox.stub(FakeModel, 'schemas') + sandbox.stub(FakeModel, 'schema') .get(() => ['not an object']); - await assert.rejects(solr.createSchemas(model), { + await assert.rejects(solr.createSchema(model), { name: 'SolrError', code: SolrError.codes.INVALID_MODEL }); }); }); - describe('updateSchemas()', () => { + describe('updateSchema()', () => { - it('Should call Solr POST api to create the schemas', async () => { + it('Should call Solr POST api to create the schema', async () => { const request = nock(host) .post(endpoints.schema, { @@ -598,34 +594,73 @@ describe('Solr', () => { } }); - await assert.doesNotReject(solr.updateSchemas(model)); + await assert.doesNotReject(solr.updateSchema(model)); request.done(); }); - it('Should return an empty object when there are no schemas in the model', async () => { + it('Should return an empty object when there are no schema in the model', async () => { - sandbox.stub(FakeModel, 'schemas') + sandbox.stub(FakeModel, 'schema') .get(() => undefined); const request = nock(host) .post(endpoints.schema) .reply(200); - await assert.doesNotReject(solr.updateSchemas(model)); + await assert.doesNotReject(solr.updateSchema(model)); assert.deepEqual(request.isDone(), false); }); it('Should throw when the model is invalid', async () => { - sandbox.stub(FakeModel, 'schemas') - .get(() => ['not an object']); - - await assert.rejects(solr.updateSchemas(model), { + await assert.rejects(solr.updateSchema(), { name: 'SolrError', code: SolrError.codes.INVALID_MODEL }); }); }); + + describe('createCore()', () => { + + it('Should create a core into the Solr URL', async () => { + + sandbox.stub(Solr.prototype, 'createSchema') + .returns(); + + const request = nock(host) + .post('/solr/admin/cores?action=CREATE&name=new-core&configSet=_default') + .reply(200, { + responseHeader: { + status: 0 + } + }); + + await assert.doesNotReject(solr.createCore(model, 'new-core')); + + request.done(); + sandbox.assert.calledOnce(Solr.prototype.createSchema); + sandbox.assert.calledWithExactly(Solr.prototype.createSchema, model, 'new-core'); + }); + + it('Should not reject when the received model is invalid', async () => { + + sandbox.stub(Solr.prototype, 'createSchema') + .returns(); + + const request = nock(host) + .post('/solr/admin/cores?action=CREATE&name=new-core&configSet=_default') + .reply(200, { + responseHeader: { + status: 0 + } + }); + + await assert.doesNotReject(solr.createCore(null, 'new-core')); + + assert.deepEqual(request.isDone(), false); + sandbox.assert.notCalled(Solr.prototype.createSchema); + }); + }); }); From 694d4d0cd8ba0d1bf4834a044167bc418d5259e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Wed, 12 Feb 2020 18:04:58 -0300 Subject: [PATCH 11/15] Added more methods and configs --- lib/config-validator.js | 11 +++- lib/helpers/endpoint.js | 1 + lib/helpers/query.js | 38 +++++++++++- lib/solr.js | 129 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 174 insertions(+), 5 deletions(-) diff --git a/lib/config-validator.js b/lib/config-validator.js index f7e3d40..e3403a7 100644 --- a/lib/config-validator.js +++ b/lib/config-validator.js @@ -6,7 +6,9 @@ const SolrError = require('./solr-error'); const configStruct = struct.partial({ url: 'string', - core: 'string' + core: 'string', + user: 'string?', + password: 'string?' }); class ConfigValidator { @@ -21,7 +23,12 @@ class ConfigValidator { try { - return configStruct(config); + const validConfig = configStruct(config); + + if(validConfig.user && !validConfig.password) + throw new Error(`Password required for user '${validConfig.user}'`); + + return validConfig; } catch(err) { err.message = `Error validating connection config: ${err.message}`; diff --git a/lib/helpers/endpoint.js b/lib/helpers/endpoint.js index 3b7323e..998ec17 100644 --- a/lib/helpers/endpoint.js +++ b/lib/helpers/endpoint.js @@ -11,6 +11,7 @@ class Endpoint { return { get: '{{core}}/query', update: '{{core}}/update/json/docs?commit=true', + updateCommands: '{{core}}/update?commit=true', schema: '{{core}}/schema' }; } diff --git a/lib/helpers/query.js b/lib/helpers/query.js index 948df05..77de7dc 100644 --- a/lib/helpers/query.js +++ b/lib/helpers/query.js @@ -1,11 +1,12 @@ 'use strict'; -const Filters = require('./filters'); +const SolrError = require('../solr-error'); +const Filters = require('./filters'); class Query { - static build(params) { + static get(params) { const { limit, page, fields } = params; @@ -21,6 +22,39 @@ class Query { }; } + static delete(params) { + + const { fields } = params; + + const filters = params.filters ? Filters.build(params.filters, fields) : []; + + return { + delete: { + query: filters.reduce((stringQuery, terms) => (stringQuery ? ` AND ${terms}` : terms), '') + } + }; + } + + static distinct(params) { + + const { key, fields } = params; + + if(typeof key !== 'string') + throw new SolrError(`Distinct key must be a string, received: ${typeof key}.`); + + const filters = params.filters ? Filters.build(params.filters, fields) : {}; + + return { + query: '*:*', + fields: key, + params: { + group: true, + 'group.field': key + }, + ...filters + }; + } + static _getSorting(order) { return Object.entries(order).reduce((sortings, [field, term]) => { diff --git a/lib/solr.js b/lib/solr.js index 51797d4..3b64880 100644 --- a/lib/solr.js +++ b/lib/solr.js @@ -120,7 +120,7 @@ class Solr { const page = params.page || 1; const limit = params.limit || DEFAULT_LIMIT; - const query = Query.build({ ...params, page, limit, fields }); + const query = Query.get({ ...params, page, limit, fields }); const res = await Request.get(endpoint, query); @@ -164,6 +164,133 @@ class Solr { }; } + /** + * Removes an item from the database + * @param {Model} model Model instance + * @param {Object} item The item to delete + * @throws When something goes wrong + * @example + * await remove(model, { id: 'some-id', value: 'some-value' }); + */ + async remove(model, item) { + + if(!isObject(item)) + throw new SolrError('Invalid item: Should be an object, also not an array.', SolrError.codes.INVALID_PARAMETERS); + + if(!item.id) + throw new SolrError('Invalid item: Should have an ID.', SolrError.codes.INVALID_PARAMETERS); + + const endpoint = Endpoint.create(Endpoint.presets.updateCommands, this._url, this._core); + + const res = await Request.post(endpoint, { delete: { id: item.id } }); + + this._validateResponse(res); + } + + /** + * Multi remove items from the database + * @param {Model} model Model instance + * @param {Object} filters solr filters + * @throws When something goes wrong + * @example + * await multiRemove(model, { myField: { type: 'greater', value: 10 } }); + */ + async mutliRemove(model, filters) { + + this._validateModel(model); + + const { fields } = model.constructor; + + const endpoint = Endpoint.create(Endpoint.presets.updateCommands, this._url, this._core); + + const query = Query.delete({ filters, fields }); + + const res = await Request.post(endpoint, query); + + this._validateResponse(res); + } + + async distinct(model, params = {}) { + + this._validateModel(model); + + const endpoint = Endpoint.create(Endpoint.presets.get, this._url, this._core); + + const query = Query.distinct(params); + + const res = await Request.get(endpoint, query); + + this._validateResponse(res); + + // Formatear respuesta + /* +{ + "responseHeader": { + "status": 0, + "QTime": 1, + "params": { + "json": "{\n\t\"query\": \"*:*\",\n\t\"fields\": \"text\",\n\t\"params\": {\n\t\t\"group\": true,\n\t\t\"group.field\": \"text\"\n\t}\n}" + } + }, + "grouped": { + "text": { + "matches": 5, + "groups": [ + { + "groupValue": "sarasa", + "doclist": { + "numFound": 2, + "start": 0, + "docs": [ + { + "text": "sarasa" + } + ] + } + }, + { + "groupValue": "foo", + "doclist": { + "numFound": 1, + "start": 0, + "docs": [ + { + "text": "foo" + } + ] + } + }, + { + "groupValue": "bar", + "doclist": { + "numFound": 1, + "start": 0, + "docs": [ + { + "text": "bar" + } + ] + } + }, + { + "groupValue": "foobar", + "doclist": { + "numFound": 1, + "start": 0, + "docs": [ + { + "text": "foobar" + } + ] + } + } + ] + } + } +} + */ + } + /** * Create the fields schema with the specified field types * @param {Model} model Model instance From 27f91585b79db20b90f3930cce5e9ce726351678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Thu, 13 Feb 2020 18:43:15 -0300 Subject: [PATCH 12/15] Added remove, multiRemove, distinct methods and authentication support --- lib/config-validator.js | 2 +- lib/helpers/query.js | 12 +- lib/helpers/request.js | 20 ++- lib/helpers/response.js | 45 ++++++ lib/helpers/utils.js | 7 +- lib/solr.js | 164 ++++++++------------ tests/solr-test.js | 320 +++++++++++++++++++++++++++++++++++++++- 7 files changed, 452 insertions(+), 118 deletions(-) diff --git a/lib/config-validator.js b/lib/config-validator.js index e3403a7..388f204 100644 --- a/lib/config-validator.js +++ b/lib/config-validator.js @@ -25,7 +25,7 @@ class ConfigValidator { const validConfig = configStruct(config); - if(validConfig.user && !validConfig.password) + if((validConfig.user && !validConfig.password) || (validConfig.password && !validConfig.user)) throw new Error(`Password required for user '${validConfig.user}'`); return validConfig; diff --git a/lib/helpers/query.js b/lib/helpers/query.js index 77de7dc..ff1208e 100644 --- a/lib/helpers/query.js +++ b/lib/helpers/query.js @@ -28,11 +28,9 @@ class Query { const filters = params.filters ? Filters.build(params.filters, fields) : []; - return { - delete: { - query: filters.reduce((stringQuery, terms) => (stringQuery ? ` AND ${terms}` : terms), '') - } - }; + const query = filters.reduce((stringQuery, terms) => (stringQuery ? `${stringQuery} AND ${terms}` : terms), ''); + + return query ? { delete: { query } } : {}; } static distinct(params) { @@ -40,9 +38,9 @@ class Query { const { key, fields } = params; if(typeof key !== 'string') - throw new SolrError(`Distinct key must be a string, received: ${typeof key}.`); + throw new SolrError(`Distinct key must be a string, received: ${typeof key}.`, SolrError.codes.INVALID_PARAMETERS); - const filters = params.filters ? Filters.build(params.filters, fields) : {}; + const filters = params.filters ? { filter: Filters.build(params.filters, fields) } : {}; return { query: '*:*', diff --git a/lib/helpers/request.js b/lib/helpers/request.js index 84de788..7025f0e 100644 --- a/lib/helpers/request.js +++ b/lib/helpers/request.js @@ -4,6 +4,8 @@ const { promisify } = require('util'); const request = promisify(require('request')); +const { base64 } = require('./utils'); + const SolrError = require('../solr-error'); class Request { @@ -16,8 +18,8 @@ class Request { * @returns {Object|Array|null} with the JSON response or null if the request ended with 4XX code * @throws when the response code is 5XX */ - static async get(endpoint, requestBody, headers) { - const httpRequest = this._buildHttpRequest(endpoint, requestBody, headers); + static async get(endpoint, requestBody, auth, headers) { + const httpRequest = this._buildHttpRequest(endpoint, requestBody, auth, headers); return this._makeRequest(httpRequest, 'GET'); } @@ -29,12 +31,17 @@ class Request { * @returns {Object|Array|null} with the JSON response or null if the request ended with 4XX code * @throws when the response code is 5XX */ - static async post(endpoint, requestBody, headers) { - const httpRequest = this._buildHttpRequest(endpoint, requestBody, headers); + static async post(endpoint, requestBody, auth, headers) { + const httpRequest = this._buildHttpRequest(endpoint, requestBody, auth, headers); return this._makeRequest(httpRequest, 'POST'); } - static _buildHttpRequest(endpoint, body, headers) { + static _buildHttpRequest(endpoint, body, auth, headers) { + + let credentials; + + if(auth.user && auth.password) + credentials = `Basic ${base64(`${auth.user}:${auth.password}`)}`; const httpRequest = { url: endpoint, @@ -45,6 +52,9 @@ class Request { body: JSON.stringify(body) }; + if(credentials) + httpRequest.headers.Authorization = credentials; + return httpRequest; } diff --git a/lib/helpers/response.js b/lib/helpers/response.js index e56bd2a..23b81c8 100644 --- a/lib/helpers/response.js +++ b/lib/helpers/response.js @@ -2,8 +2,22 @@ const merge = require('lodash.merge'); +const { inspect } = require('util'); + +const { superstruct } = require('superstruct'); + +const { isObject } = require('./utils'); + +const SolrError = require('../solr-error'); + const IGNORED_FIELDS = ['_version_']; +const struct = superstruct({ + types: { + equalToZero: value => value === 0 + } +}); + class Response { static format(response) { @@ -24,6 +38,37 @@ class Response { }); } + static validate(res, terms = {}) { + + const responseStruct = struct.partial( + + this._buildNestedStruct({ + responseHeader: { status: 'equalToZero' }, + ...terms + }) + ); + + try { + + responseStruct(res); + + } catch(err) { + throw new SolrError(`Invalid Solr response: ${err.message} ${inspect(res)}`, SolrError.codes.INTERNAL_SOLR_ERROR); + } + } + + static _buildNestedStruct(terms) { + + return Object.entries(terms).reduce((structs, [field, term]) => { + + if(!isObject(term)) + return { ...structs, [field]: term }; + + return { ...structs, [field]: struct.partial(this._buildNestedStruct(term)) }; + + }, {}); + } + static _buildJsonField(field, value) { const fields = field.split('.'); diff --git a/lib/helpers/utils.js b/lib/helpers/utils.js index 075a67d..cd6939b 100644 --- a/lib/helpers/utils.js +++ b/lib/helpers/utils.js @@ -4,6 +4,11 @@ function isObject(object) { return object !== null && typeof object === 'object' && !Array.isArray(object); } +function base64(value) { + return Buffer.from(value).toString('base64'); +} + module.exports = { - isObject + isObject, + base64 }; diff --git a/lib/solr.js b/lib/solr.js index 3b64880..7c629fe 100644 --- a/lib/solr.js +++ b/lib/solr.js @@ -42,9 +42,9 @@ class Solr { const endpoint = Endpoint.create(Endpoint.presets.update, this._url, this._core); - const res = await Request.post(endpoint, item); + const res = await Request.post(endpoint, item, this._auth); - this._validateResponse(res); + Response.validate(res); return item.id; } @@ -75,9 +75,9 @@ class Solr { const endpoint = Endpoint.create(Endpoint.presets.update, this._url, this._core); - const res = await Request.post(endpoint, items); + const res = await Request.post(endpoint, items, this._auth); - this._validateResponse(res); + Response.validate(res); return items; } @@ -122,9 +122,11 @@ class Solr { const query = Query.get({ ...params, page, limit, fields }); - const res = await Request.get(endpoint, query); + const res = await Request.get(endpoint, query, this._auth); - this._validateResponse(res); + Response.validate(res, { + response: { docs: ['object'] } + }); const { docs } = res.response; @@ -150,9 +152,11 @@ class Solr { const endpoint = Endpoint.create(Endpoint.presets.get, this._url, this._core); - const res = await Request.get(endpoint, query); + const res = await Request.get(endpoint, query, this._auth); - this._validateResponse(res); + Response.validate(res, { + response: { numFound: 'number' } + }); const { numFound } = res.response; @@ -182,9 +186,9 @@ class Solr { const endpoint = Endpoint.create(Endpoint.presets.updateCommands, this._url, this._core); - const res = await Request.post(endpoint, { delete: { id: item.id } }); + const res = await Request.post(endpoint, { delete: { id: item.id } }, this._auth); - this._validateResponse(res); + Response.validate(res); } /** @@ -205,90 +209,46 @@ class Solr { const query = Query.delete({ filters, fields }); - const res = await Request.post(endpoint, query); + const res = await Request.post(endpoint, query, this._auth); - this._validateResponse(res); + Response.validate(res); } + /** + * Get distinct values of a field + * @param {Model} model Model instance + * @param {Object} params parameters (key and filters) + * @return {Array} results + * @example + * await distinct(model, { key: 'myField', { filters: { field: { type: 'lesser', value: 32 } } } }) + * // Expected result + * ['some data', 'other data'] + */ async distinct(model, params = {}) { this._validateModel(model); + const { fields } = model.constructor; + const endpoint = Endpoint.create(Endpoint.presets.get, this._url, this._core); - const query = Query.distinct(params); - - const res = await Request.get(endpoint, query); - - this._validateResponse(res); - - // Formatear respuesta - /* -{ - "responseHeader": { - "status": 0, - "QTime": 1, - "params": { - "json": "{\n\t\"query\": \"*:*\",\n\t\"fields\": \"text\",\n\t\"params\": {\n\t\t\"group\": true,\n\t\t\"group.field\": \"text\"\n\t}\n}" - } - }, - "grouped": { - "text": { - "matches": 5, - "groups": [ - { - "groupValue": "sarasa", - "doclist": { - "numFound": 2, - "start": 0, - "docs": [ - { - "text": "sarasa" - } - ] - } - }, - { - "groupValue": "foo", - "doclist": { - "numFound": 1, - "start": 0, - "docs": [ - { - "text": "foo" - } - ] - } - }, - { - "groupValue": "bar", - "doclist": { - "numFound": 1, - "start": 0, - "docs": [ - { - "text": "bar" - } - ] - } - }, - { - "groupValue": "foobar", - "doclist": { - "numFound": 1, - "start": 0, - "docs": [ - { - "text": "foobar" - } - ] - } - } - ] - } - } -} - */ + const query = Query.distinct({ ...params, fields }); + + const res = await Request.get(endpoint, query, this._auth); + + Response.validate(res, { + grouped: { [params.key]: { groups: ['object'] } } + }); + + const { groups } = res.grouped[params.key]; + + return groups.map(({ doclist }) => { + + // When grouping by specific field, the doclist.docs array will be always with one item + const [value] = Object.values(doclist.docs[0]); + + return value; + }); } /** @@ -310,9 +270,9 @@ class Solr { const endpoint = Endpoint.create(Endpoint.presets.schema, this._url, core); - const res = await Request.post(endpoint, query); + const res = await Request.post(endpoint, query, this._auth); - this._validateResponse(res); + Response.validate(res); } /** @@ -332,9 +292,9 @@ class Solr { const query = Schema.buildQuery('replace', schema); const endpoint = Endpoint.create(Endpoint.presets.schema, this._url, this._core); - const res = await Request.post(endpoint, query); + const res = await Request.post(endpoint, query, this._auth); - this._validateResponse(res); + Response.validate(res); } /** @@ -355,24 +315,13 @@ class Solr { const endpoint = Endpoint.create('admin/cores?action=CREATE&name={{name}}&configSet=_default', this._url, null, { name }); - const res = await Request.post(endpoint); + const res = await Request.post(endpoint, null, this._auth); - this._validateResponse(res); + Response.validate(res); return this.createSchema(model, name); } - _validateResponse(res) { - - const { responseHeader, response } = res; - - if(!responseHeader || responseHeader.status !== 0) - throw new SolrError('Invalid Solr response: No responseHeader received.', SolrError.codes.INTERNAL_SOLR_ERROR); - - if(response && !response.docs) - throw new SolrError(`Invalid Solr response: ${JSON.stringify(response)}`, SolrError.codes.INTERNAL_SOLR_ERROR); - } - _prepareItem(item) { return { @@ -402,6 +351,17 @@ class Solr { get _core() { return this._config.core; } + + get _auth() { + + if(!this._config.user) + return {}; + + return { + user: this._config.user, + password: this._config.password + }; + } } module.exports = Solr; diff --git a/tests/solr-test.js b/tests/solr-test.js index 133ba48..ca426c3 100644 --- a/tests/solr-test.js +++ b/tests/solr-test.js @@ -6,6 +6,8 @@ const nock = require('nock'); const sandbox = require('sinon').createSandbox(); +const { base64 } = require('../lib/helpers/utils'); + const Solr = require('../lib/solr'); const SolrError = require('../lib/solr-error'); @@ -102,6 +104,7 @@ describe('Solr', () => { const endpoints = { update: '/solr/some-core/update/json/docs?commit=true', + updateCommands: '/solr/some-core/update?commit=true', get: '/solr/some-core/query', schema: '/solr/some-core/schema' }; @@ -124,7 +127,11 @@ describe('Solr', () => { ['array'], { invalid: 'config' }, { url: ['not a string'], core: 'valid' }, - { url: 'valid', core: ['not a string'] } + { url: 'valid', core: ['not a string'] }, + { url: 'valid', core: 'valid', user: ['not a string'], password: 'valid' }, + { url: 'valid', core: 'valid', user: 'valid', password: ['not a string'] }, + { url: 'valid', core: 'valid', user: 'valid' }, + { url: 'valid', core: 'valid', password: 'valid' } ].forEach(config => { @@ -396,6 +403,58 @@ describe('Solr', () => { request.done(); }); + it('Should call Solr GET api to get the items using auth credentials', async () => { + + const authorizedSolr = new Solr({ + url: host, + core: 'some-core', + user: 'some-user', + password: 'some-password' + }); + + const expectedCredentials = `Basic ${base64('some-user:some-password')}`; + + const request = nock(host, { reqheaders: { Authorization: expectedCredentials } }) + .get(endpoints.get, { + query: '*:*', + offset: 0, + limit: 500 + }) + .reply(200, { + responseHeader: { + status: 0 + }, + response: { + docs: [ + { + id: 'some-id', + some: 'data', + 'object.property': 'some-property', + 'object.subproperty.property': [1, 2, 3], + _version_: 1122111221 + } + ] + } + }); + + const result = await authorizedSolr.get(model); + + assert.deepStrictEqual(result, [ + { + id: 'some-id', + some: 'data', + object: { + property: 'some-property', + subproperty: { + property: [1, 2, 3] + } + } + } + ]); + + request.done(); + }); + it('Should throw when the Solr response code is bigger or equal than 400', async () => { const request = nock(host) @@ -445,7 +504,7 @@ describe('Solr', () => { }); }); - describe('getTotals', () => { + describe('getTotals()', () => { afterEach(() => { delete model.lastQueryHasResults; @@ -518,6 +577,263 @@ describe('Solr', () => { }); }); + describe('remove()', () => { + + const item = { + id: 'some-id', + some: 'data' + }; + + it('Should call Solr POST api to delete the received item', async () => { + + const request = nock(host) + .post(endpoints.updateCommands, { + delete: { id: item.id } + }) + .reply(200, { + responseHeader: { + status: 0 + } + }); + + await assert.doesNotReject(solr.remove(model, item)); + + request.done(); + }); + + [ + + null, + undefined, + 'string', + 1, + ['array'], + { item: 'without id' } + + ].forEach(invalidItem => { + + it('Should throw when the received item is not an object or not have ID', async () => { + + await assert.rejects(solr.remove(model, invalidItem), { + name: 'SolrError', + code: SolrError.codes.INVALID_PARAMETERS + }); + }); + }); + + it('Should throw when the Solr response code is bigger or equal than 400', async () => { + + const request = nock(host) + .post(endpoints.updateCommands, { + delete: { id: item.id } + }) + .reply(400, { + responseHeader: { + status: 400 + } + }); + + await assert.rejects(solr.remove(model, item), { + name: 'SolrError', + code: SolrError.codes.REQUEST_FAILED + }); + + request.done(); + }); + + it('Should throw when the Solr response is invalid', async () => { + + const request = nock(host) + .post(endpoints.updateCommands, { + delete: { id: item.id } + }) + .reply(200, { + responseHeader: { + status: 1 + } + }); + + await assert.rejects(solr.remove(model, item), { + name: 'SolrError', + code: SolrError.codes.INTERNAL_SOLR_ERROR + }); + + request.done(); + }); + }); + + describe('multiRemove()', () => { + + it('Should call Solr POST api to delete by the received filters', async () => { + + const request = nock(host) + .post(endpoints.updateCommands, { + delete: { query: 'field:"value" AND otherField:[* TO 10]' } + }) + .reply(200, { + responseHeader: { + status: 0 + } + }); + + await assert.doesNotReject(solr.mutliRemove(model, { field: 'value', otherField: { type: 'lesserOrEqual', value: 10 } })); + + request.done(); + }); + + it('Should throw when the Solr response code is bigger or equal than 400', async () => { + + const request = nock(host) + .post(endpoints.updateCommands, {}) + .reply(400, { + responseHeader: { + status: 400 + } + }); + + await assert.rejects(solr.mutliRemove(model), { + name: 'SolrError', + code: SolrError.codes.REQUEST_FAILED + }); + + request.done(); + }); + + it('Should throw when the Solr response is invalid', async () => { + + const request = nock(host) + .post(endpoints.updateCommands, { + delete: { query: 'field:"value"' } + }) + .reply(200, { + responseHeader: { + status: 1 + } + }); + + await assert.rejects(solr.mutliRemove(model, { field: 'value' }), { + name: 'SolrError', + code: SolrError.codes.INTERNAL_SOLR_ERROR + }); + + request.done(); + }); + + it('Should throw when the received model is invalid', async () => { + + await assert.rejects(solr.mutliRemove(), { + name: 'SolrError', + code: SolrError.codes.INVALID_MODEL + }); + }); + }); + + describe('distinct()', () => { + + it('Should call Solr GET api to get the items distinct and format it correctly', async () => { + + const request = nock(host) + .get(endpoints.get, { + query: '*:*', + fields: 'someField', + params: { + group: true, + 'group.field': 'someField' + }, + filter: ['otherField:"true"'] + }) + .reply(200, { + responseHeader: { + status: 0 + }, + grouped: { + someField: { + groups: [ + { + doclist: { docs: [{ someField: 'some' }] } + }, + { + doclist: { docs: [{ someField: 'other' }] } + } + ] + } + } + }); + + const result = await solr.distinct(model, { key: 'someField', filters: { otherField: true } }); + + assert.deepStrictEqual(result, ['some', 'other']); + + request.done(); + }); + + it('Should throw when the received key is not a string or not exists', async () => { + + await assert.rejects(solr.distinct(model, { key: ['not a string'] }), { + name: 'SolrError', + code: SolrError.codes.INVALID_PARAMETERS + }); + }); + + it('Should throw when the Solr response code is bigger or equal than 400', async () => { + + const request = nock(host) + .get(endpoints.get, { + query: '*:*', + fields: 'someField', + params: { + group: true, + 'group.field': 'someField' + } + }) + .reply(400, { + responseHeader: { + status: 400 + } + }); + + await assert.rejects(solr.distinct(model, { key: 'someField' }), { + name: 'SolrError', + code: SolrError.codes.REQUEST_FAILED + }); + + request.done(); + }); + + it('Should throw when the Solr response is invalid', async () => { + + const request = nock(host) + .get(endpoints.get, { + query: '*:*', + fields: 'someField', + params: { + group: true, + 'group.field': 'someField' + } + }) + .reply(200, { + responseHeader: { + status: 1 + } + }); + + await assert.rejects(solr.distinct(model, { key: 'someField' }), { + name: 'SolrError', + code: SolrError.codes.INTERNAL_SOLR_ERROR + }); + + request.done(); + }); + + it('Should throw when the received model is invalid', async () => { + + await assert.rejects(solr.distinct(), { + name: 'SolrError', + code: SolrError.codes.INVALID_MODEL + }); + }); + }); + describe('createSchema()', () => { it('Should call Solr POST api to create the schema using core name from config', async () => { From fc46784b18109c523a66ec9caec87c81a386615c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Thu, 13 Feb 2020 18:44:20 -0300 Subject: [PATCH 13/15] Updated unit tests --- tests/solr-test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/solr-test.js b/tests/solr-test.js index ca426c3..5f97956 100644 --- a/tests/solr-test.js +++ b/tests/solr-test.js @@ -129,6 +129,7 @@ describe('Solr', () => { { url: ['not a string'], core: 'valid' }, { url: 'valid', core: ['not a string'] }, { url: 'valid', core: 'valid', user: ['not a string'], password: 'valid' }, + { url: 'valid', core: 'valid', user: ['not a string'], password: ['not a string'] }, { url: 'valid', core: 'valid', user: 'valid', password: ['not a string'] }, { url: 'valid', core: 'valid', user: 'valid' }, { url: 'valid', core: 'valid', password: 'valid' } From c992073f970c4b6c48164bb3cf86ecd30a72b985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Thu, 13 Feb 2020 19:06:39 -0300 Subject: [PATCH 14/15] Updated docs --- README.md | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c8ba553..64c0051 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,17 @@ Constructs the Solr driver instance, connected with the `config` object. **Config properties** - url `String` (required): Solr URL +- core `String` (required): Solr Core +- user `String` (optional): Auth user +- password `String` (optional but required if an user is specified): Auth password **Config usage** ```js { - url: 'http://localhost:8983' + url: 'http://localhost:8983', + core: 'some-core', + user: 'some-user', + password: 'some-password' } ``` @@ -49,6 +55,15 @@ Inserts multiple documents in a solr core - Resolves: `Array`: Items inserted - Rejects: `SolrError` When something bad occurs +### ***async*** `distict(model, [parameters])` +Searches distinct values of a property in a solr core + +- model: `Model`: A model instance +- parameters: `Object` (optional): The query parameters. Default: {}. It only accepts `key` (the field name to get distinct values from), and `filters` -- described below in `get()` method. + +- Resolves `Array`: An array of documents +- Rejects `SolrError`: When something bad occurs + ### ***async*** `get(model, [parameters])` Searches documents in a solr core @@ -225,6 +240,22 @@ Return example: If no query was executed before, it will just return the `total` and `pages` properties with a value of zero. +### ***async*** `remove(model, item)` +Removes a document in a solr core + +- model: `Model`: A model instance +- item: `Object`: The item to be removed + +- Rejects `SolrError`: When something bad occurs + +### ***async*** `multiRemove(model, filters)` +Removes one or more documents in a solr core + +- model: `Model`: A model instance +- filters: `Object`: Filters criteria to match documents + +- Rejects `SolrError`: When something bad occurs + ### ***async*** `createSchema(model, core)` Build the fields schema using the schema defined in the model static getter `schema`. @@ -429,6 +460,16 @@ const model = new Model(); await solr.getTotals(model); // > { page: 1, limit: 500, pages: 1, total: 4 } + // distinct + await solr.distinct(model, { key: 'fieldName', filters: { someField: true } }); + // > ['some-value', 'other-value'] + + // remove + await solr.remove(model, { id: 'some-id', field: 'value' }); + + // multiRemove + await solr.multiRemove(model, { field: { type: 'greater', value: 10 } }); + // createSchema await solr.createSchema(model); @@ -436,6 +477,6 @@ const model = new Model(); await solr.updateSchema(model); // createCore - + await solr.createCore(model, 'new-core'); })(); ``` \ No newline at end of file From f59f624328359c0dba73d936bc915184d4c529c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nataniel=20L=C3=B3pez?= Date: Fri, 14 Feb 2020 18:09:09 -0300 Subject: [PATCH 15/15] Applied PR corrections --- CHANGELOG.md | 14 +++++++++++++- lib/helpers/endpoint.js | 7 +++++++ lib/helpers/filters.js | 29 +++++++++++++++++------------ lib/helpers/schema.js | 5 +++++ lib/helpers/utils.js | 16 ++++++++++------ lib/solr.js | 2 +- tests/solr-test.js | 9 ++++----- 7 files changed, 57 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e9a481..6b511f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,4 +7,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added -- Solr DB Driver Package \ No newline at end of file +- Solr DB Driver Package +- `endpoint` helper +- `filters` helper +- `query` helper +- `request` helper +- `response` helper +- `schema` helper +- `utils` helper +- `insert` and `multiInsert` methods +- `remove` and `multiRemove` methods +- `get` and `getTotals` methods +- `distinct` method +- `createSchemas`, `updateSchemas` and `createCore` methods \ No newline at end of file diff --git a/lib/helpers/endpoint.js b/lib/helpers/endpoint.js index 998ec17..2198ff9 100644 --- a/lib/helpers/endpoint.js +++ b/lib/helpers/endpoint.js @@ -16,6 +16,13 @@ class Endpoint { }; } + /** + * Creates a Solr endpoint + * @param {String} endpoint The endpoint to create + * @param {String} url The solr url + * @param {String} core The solr core name + * @param {Object} replacements The replacements to apply to the endpoint + */ static create(endpoint, url, core, replacements) { return this._buildEndpoint(path.join(ENDPOINT_BASE, endpoint), { url, core, ...replacements }); } diff --git a/lib/helpers/filters.js b/lib/helpers/filters.js index 4d18431..877f19b 100644 --- a/lib/helpers/filters.js +++ b/lib/helpers/filters.js @@ -8,6 +8,11 @@ const DEFAULT_FILTER_TYPE = 'equal'; class Filters { + /** + * Builds the filters for Solr legibility + * @param {Object} filters The filters + * @param {Object} modelFields The model fields + */ static build(filters, modelFields) { const filtersGroup = this._parseFilterGroup(filters, modelFields); @@ -58,18 +63,6 @@ class Filters { }); } - static get _formatters() { - - return { - equal: this._formatEq.bind(this), - notEqual: this._formatNe.bind(this), - greater: this._formatGt, - greaterOrEqual: this._formatGte, - lesser: this._formatLt, - lesserOrEqual: this._formatLte - }; - } - static _formatByType(filtersGroup) { return Object.entries(filtersGroup).reduce((filtersByType, [field, filterData]) => { @@ -94,6 +87,18 @@ class Filters { }, {}); } + static get _formatters() { + + return { + equal: this._formatEq.bind(this), + notEqual: this._formatNe.bind(this), + greater: this._formatGt, + greaterOrEqual: this._formatGte, + lesser: this._formatLt, + lesserOrEqual: this._formatLte + }; + } + static _formatEq(field, value) { return `${field}:"${value}"`; } diff --git a/lib/helpers/schema.js b/lib/helpers/schema.js index 7f06665..6c43a69 100644 --- a/lib/helpers/schema.js +++ b/lib/helpers/schema.js @@ -14,6 +14,11 @@ const FIELD_TYPES = { class Schema { + /** + * Builds a schema query for Solr + * @param {String} method The method to use for building the schema api query + * @param {Object} schemas The model schemas + */ static buildQuery(method, schemas) { const builtSchemas = Object.entries(schemas).reduce((fields, [field, schema]) => { diff --git a/lib/helpers/utils.js b/lib/helpers/utils.js index cd6939b..2607538 100644 --- a/lib/helpers/utils.js +++ b/lib/helpers/utils.js @@ -1,12 +1,16 @@ 'use strict'; -function isObject(object) { - return object !== null && typeof object === 'object' && !Array.isArray(object); -} +/** + * Validates if the received object is an JSON object and not an array + * @param {Object} object The object to validate + */ +const isObject = object => (object !== null && typeof object === 'object' && !Array.isArray(object)); -function base64(value) { - return Buffer.from(value).toString('base64'); -} +/** + * Encodes the received value into a base64 string + * @param {String} value value + */ +const base64 = value => (Buffer.from(value).toString('base64')); module.exports = { isObject, diff --git a/lib/solr.js b/lib/solr.js index 7c629fe..970e935 100644 --- a/lib/solr.js +++ b/lib/solr.js @@ -199,7 +199,7 @@ class Solr { * @example * await multiRemove(model, { myField: { type: 'greater', value: 10 } }); */ - async mutliRemove(model, filters) { + async multiRemove(model, filters) { this._validateModel(model); diff --git a/tests/solr-test.js b/tests/solr-test.js index 5f97956..d2fe4f5 100644 --- a/tests/solr-test.js +++ b/tests/solr-test.js @@ -255,7 +255,6 @@ describe('Solr', () => { }); [ - null, undefined, 'string', @@ -677,7 +676,7 @@ describe('Solr', () => { } }); - await assert.doesNotReject(solr.mutliRemove(model, { field: 'value', otherField: { type: 'lesserOrEqual', value: 10 } })); + await assert.doesNotReject(solr.multiRemove(model, { field: 'value', otherField: { type: 'lesserOrEqual', value: 10 } })); request.done(); }); @@ -692,7 +691,7 @@ describe('Solr', () => { } }); - await assert.rejects(solr.mutliRemove(model), { + await assert.rejects(solr.multiRemove(model), { name: 'SolrError', code: SolrError.codes.REQUEST_FAILED }); @@ -712,7 +711,7 @@ describe('Solr', () => { } }); - await assert.rejects(solr.mutliRemove(model, { field: 'value' }), { + await assert.rejects(solr.multiRemove(model, { field: 'value' }), { name: 'SolrError', code: SolrError.codes.INTERNAL_SOLR_ERROR }); @@ -722,7 +721,7 @@ describe('Solr', () => { it('Should throw when the received model is invalid', async () => { - await assert.rejects(solr.mutliRemove(), { + await assert.rejects(solr.multiRemove(), { name: 'SolrError', code: SolrError.codes.INVALID_MODEL });