diff --git a/api/fe.js b/api/fe.js new file mode 100644 index 0000000..374c046 --- /dev/null +++ b/api/fe.js @@ -0,0 +1,5 @@ +const { generateScript } = require('../forward_engineering/api'); + +module.exports = { + generateScript, +}; diff --git a/esbuild.package.js b/esbuild.package.js index 189f557..cc387c6 100644 --- a/esbuild.package.js +++ b/esbuild.package.js @@ -2,6 +2,7 @@ const fs = require('fs'); const path = require('path'); const esbuild = require('esbuild'); const { clean } = require('esbuild-plugin-clean'); +const { copy } = require('esbuild-plugin-copy'); const { copyFolderFiles, addReleaseFlag } = require('@hackolade/hck-esbuild-plugins-pack'); const { EXCLUDED_EXTENSIONS, EXCLUDED_FILES, DEFAULT_RELEASE_FOLDER_PATH } = require('./buildConstants'); @@ -11,6 +12,7 @@ const RELEASE_FOLDER_PATH = path.join(DEFAULT_RELEASE_FOLDER_PATH, `${packageDat esbuild .build({ entryPoints: [ + path.resolve(__dirname, 'api', 'fe.js'), path.resolve(__dirname, 'forward_engineering', 'api.js'), path.resolve(__dirname, 'reverse_engineering', 'api.js'), ], @@ -21,10 +23,17 @@ esbuild outdir: RELEASE_FOLDER_PATH, minify: true, logLevel: 'info', + external: ['lodash'], plugins: [ clean({ patterns: [DEFAULT_RELEASE_FOLDER_PATH], }), + copy({ + assets: { + from: [path.join('node_modules', 'lodash', '**', '*')], + to: [path.join('node_modules', 'lodash')], + }, + }), copyFolderFiles({ fromPath: __dirname, targetFolderPath: RELEASE_FOLDER_PATH, diff --git a/forward_engineering/api.js b/forward_engineering/api.js index a3b0c08..8159099 100644 --- a/forward_engineering/api.js +++ b/forward_engineering/api.js @@ -1,303 +1,43 @@ -const helper = require('../helper/helper.js'); -const schemaHelper = require('../helper/schemaHelper.js'); +const { getFieldsSchema } = require('./helpers/getFieldsSchema'); +const { getMappingScript } = require('./helpers/getMappingScript'); +const { getTypeSchema } = require('./helpers/getTypeSchema'); +const { getCurlScript } = require('./helpers/getCurlScript'); +const { getKibanaScript } = require('./helpers/getKibanaScript'); module.exports = { generateScript(data, logger, cb) { - const { jsonSchema, modelData, entityData, isUpdateScript } = data; - const containerData = data.containerData || {}; + const { + jsonSchema, + modelData, + entityData, + isUpdateScript, + pluginConfiguration, + internalDefinitions, + modelDefinitions, + externalDefinitions, + containerData = {}, + } = data; + let result = ''; - let fieldsSchema = this.getFieldsSchema({ + + const fieldsSchema = getFieldsSchema({ jsonSchema: JSON.parse(jsonSchema), - internalDefinitions: JSON.parse(data.internalDefinitions), - modelDefinitions: JSON.parse(data.modelDefinitions), - externalDefinitions: JSON.parse(data.externalDefinitions), + internalDefinitions: JSON.parse(internalDefinitions), + modelDefinitions: JSON.parse(modelDefinitions), + externalDefinitions: JSON.parse(externalDefinitions), + fieldLevelConfig: pluginConfiguration.fieldLevelConfig, }); - let typeSchema = this.getTypeSchema(entityData, fieldsSchema); - let mappingScript = this.getMappingScript(containerData, typeSchema); + + const typeSchema = getTypeSchema(entityData, fieldsSchema); + + const mappingScript = getMappingScript(containerData, typeSchema, pluginConfiguration.containerLevelConfig); if (isUpdateScript) { - result = this.getCurlScript(mappingScript, modelData, containerData); + result = getCurlScript(mappingScript, modelData, containerData); } else { - result += this.getKibanaScript(mappingScript, containerData); + result += getKibanaScript(mappingScript, containerData); } cb(null, result); }, - - getCurlScript(mapping, modelData, indexData) { - const host = modelData.host || 'localhost'; - const port = modelData.port || 9200; - const indexName = indexData.name || ''; - const majorVersion = +(modelData.dbVersion || '').split('.').shift(); - const includeTypeName = majorVersion >= 7 ? '&include_type_name=true' : ''; - - return `curl -XPUT '${host}:${port}/${indexName.toLowerCase()}?pretty${includeTypeName}' -H 'Content-Type: application/json' -d '\n${JSON.stringify(mapping, null, 4)}\n'`; - }, - - getKibanaScript(mapping, indexData) { - const indexName = indexData.name || ''; - - return `PUT /${indexName.toLowerCase()}\n${JSON.stringify(mapping, null, 4)}`; - }, - - getFieldsSchema(data) { - const { jsonSchema } = data; - let schema = {}; - - if (!(jsonSchema.properties && jsonSchema.properties._source && jsonSchema.properties._source.properties)) { - return schema; - } - - schema = this.getSchemaByItem(jsonSchema.properties._source.properties, data); - - return schema; - }, - - getSchemaByItem(properties, data) { - let schema = {}; - - for (let fieldName in properties) { - let field = properties[fieldName]; - - schema[fieldName] = this.getField(field, data); - } - - return schema; - }, - - getField(field, data) { - let schema = {}; - const fieldProperties = helper.getFieldProperties(field.type, field, {}); - let type = this.getFieldType(field); - - if (type !== 'object' && type !== 'array') { - schema.type = type; - } - - if (type === 'object') { - schema.properties = {}; - } - - this.setProperties(schema, fieldProperties, data); - - if (type === 'alias') { - return { ...schema, ...this.getAliasSchema(field, data) }; - } else if (type === 'join') { - return { ...schema, ...this.getJoinSchema(field) }; - } else if ( - [ - 'completion', - 'sparse_vector', - 'dense_vector', - 'geo_shape', - 'geo_point', - 'rank_feature', - 'rank_features', - ].includes(type) - ) { - return schema; - } else if (field.properties) { - schema.properties = this.getSchemaByItem(field.properties, data); - } else if (field.items) { - let arrData = field.items; - - if (Array.isArray(field.items)) { - arrData = field.items[0]; - } - - schema = { ...schema, ...this.getField(arrData, data) }; - } - - return schema; - }, - - getFieldType(field) { - switch (field.type) { - case 'geo-shape': - return 'geo_shape'; - case 'geo-point': - return 'geo_point'; - case 'number': - return field.mode || 'long'; - case 'string': - return field.mode || 'text'; - case 'range': - return field.mode || 'integer_range'; - case 'null': - return 'long'; - default: - return field.type; - } - }, - - setProperties(schema, properties, data) { - for (let propName in properties) { - if (propName === 'stringfields') { - try { - schema['fields'] = JSON.parse(properties[propName]); - } catch (e) {} - } else if (this.isFieldList(properties[propName])) { - const names = schemaHelper.getNamesByIds( - properties[propName].map(item => item.keyId), - [data.jsonSchema, data.internalDefinitions, data.modelDefinitions, data.externalDefinitions], - ); - if (names.length) { - schema[propName] = names.length === 1 ? names[0] : names; - } - } else { - schema[propName] = properties[propName]; - } - } - - return schema; - }, - - getTypeSchema(typeData, fieldsSchema) { - let script = {}; - - if (typeData.dynamic) { - script.dynamic = typeData.dynamic; - } - - script.properties = fieldsSchema; - - return { - [(typeData.collectionName || '').toLowerCase()]: script, - }; - }, - - getMappingScript(indexData, typeSchema) { - let mappingScript = {}; - let settings = this.getSettings(indexData); - let aliases = this.getAliases(indexData); - - if (settings) { - mappingScript.settings = settings; - } - - if (aliases) { - mappingScript.aliases = aliases; - } - - mappingScript.mappings = typeSchema; - - return mappingScript; - }, - - getSettings(indexData) { - let settings; - let properties = helper.getContainerLevelProperties(); - - properties.forEach(propertyName => { - if (indexData[propertyName]) { - if (!settings) { - settings = {}; - } - - settings[propertyName] = indexData[propertyName]; - } - }); - - return settings; - }, - - getAliases(indexData) { - let aliases; - - if (!indexData.aliases) { - return aliases; - } - - indexData.aliases.forEach(alias => { - if (alias.name) { - if (!aliases) { - aliases = {}; - } - - aliases[alias.name] = {}; - - if (alias.filter) { - let filterData = ''; - try { - filterData = JSON.parse(alias.filter); - } catch (e) {} - - aliases[alias.name].filter = { - term: filterData, - }; - } - - if (alias.routing) { - aliases[alias.name].routing = alias.routing; - } - } - }); - - return aliases; - }, - - isFieldList(property) { - if (!Array.isArray(property)) { - return false; - } - - if (!property[0]) { - return false; - } - - if (property[0].keyId) { - return true; - } - - return false; - }, - - getJoinSchema(field) { - if (!Array.isArray(field.relations)) { - return {}; - } - - const relations = field.relations.reduce((result, item) => { - if (!item.parent) { - return result; - } - - if (!Array.isArray(item.children)) { - return result; - } - - if (item.children.length === 1) { - return { - ...result, - [item.parent]: item.children?.[0]?.name - }; - } - - return { - ...result, - [item.parent]: item.children.map(item => item.name || '') - }; - }, {}); - - return { relations }; - }, - - getAliasSchema(field, data) { - if (!Array.isArray(field.path)) { - return {}; - } - - if (field.path.length === 0) { - return {}; - } - - const pathName = schemaHelper.getPathName(field.path[0].keyId, [ - data.jsonSchema, - data.internalDefinitions, - data.modelDefinitions, - data.externalDefinitions, - ]); - - return { path: pathName }; - }, }; diff --git a/forward_engineering/helpers/getCurlScript.js b/forward_engineering/helpers/getCurlScript.js new file mode 100644 index 0000000..41957c4 --- /dev/null +++ b/forward_engineering/helpers/getCurlScript.js @@ -0,0 +1,13 @@ +const getCurlScript = (mapping, modelData, indexData) => { + const host = modelData.host || 'localhost'; + const port = modelData.port || 9200; + const indexName = indexData.name || ''; + const majorVersion = +(modelData.dbVersion || '').split('.').shift(); + const includeTypeName = majorVersion >= 7 ? '&include_type_name=true' : ''; + + return `curl -XPUT '${host}:${port}/${indexName.toLowerCase()}?pretty${includeTypeName}' -H 'Content-Type: application/json' -d '\n${JSON.stringify(mapping, null, 4)}\n'`; +}; + +module.exports = { + getCurlScript, +}; diff --git a/forward_engineering/helpers/getField.js b/forward_engineering/helpers/getField.js new file mode 100644 index 0000000..c169120 --- /dev/null +++ b/forward_engineering/helpers/getField.js @@ -0,0 +1,167 @@ +const schemaHelper = require('../../shared/schemaHelper'); +const { getFieldProperties } = require('../../shared/getFieldProperties'); + +const getFieldType = field => { + switch (field.type) { + case 'geo-shape': + return 'geo_shape'; + case 'geo-point': + return 'geo_point'; + case 'number': + return field.mode || 'long'; + case 'string': + return field.mode || 'text'; + case 'range': + return field.mode || 'integer_range'; + case 'null': + return 'long'; + default: + return field.type; + } +}; + +const getJoinSchema = field => { + if (!Array.isArray(field.relations)) { + return {}; + } + + const relations = field.relations.reduce((result, item) => { + if (!item.parent) { + return result; + } + + if (!Array.isArray(item.children)) { + return result; + } + + if (item.children.length === 1) { + return { + ...result, + [item.parent]: item.children?.[0]?.name, + }; + } + + return { + ...result, + [item.parent]: item.children.map(item => item.name || ''), + }; + }, {}); + + return { relations }; +}; + +const getAliasSchema = (field, data) => { + if (!Array.isArray(field.path)) { + return {}; + } + + if (field.path.length === 0) { + return {}; + } + + const pathName = schemaHelper.getPathName(field.path[0].keyId, [ + data.jsonSchema, + data.internalDefinitions, + data.modelDefinitions, + data.externalDefinitions, + ]); + + return { path: pathName }; +}; + +const setProperties = (schema, properties, data) => { + for (let propName in properties) { + if (propName === 'stringfields') { + try { + schema['fields'] = JSON.parse(properties[propName]); + } catch (e) {} + } else if (isFieldList(properties[propName])) { + const names = schemaHelper.getNamesByIds( + properties[propName].map(item => item.keyId), + [data.jsonSchema, data.internalDefinitions, data.modelDefinitions, data.externalDefinitions], + ); + if (names.length) { + schema[propName] = names.length === 1 ? names[0] : names; + } + } else { + schema[propName] = properties[propName]; + } + } + + return schema; +}; + +const isFieldList = property => { + if (!Array.isArray(property)) { + return false; + } + + if (!property[0]) { + return false; + } + + return Boolean(property[0].keyId); +}; + +const getSchemaByItem = (properties, data, fieldLevelConfig) => { + let schema = {}; + + for (let fieldName in properties) { + let field = properties[fieldName]; + + schema[fieldName] = getField(field, data, fieldLevelConfig); + } + + return schema; +}; + +const getField = (field, data, fieldLevelConfig) => { + let schema = {}; + const fieldProperties = getFieldProperties(field.type, field, {}, fieldLevelConfig); + let type = getFieldType(field); + + if (type !== 'object' && type !== 'array') { + schema.type = type; + } + + if (type === 'object') { + schema.properties = {}; + } + + setProperties(schema, fieldProperties, data); + + if (type === 'alias') { + return { ...schema, ...getAliasSchema(field, data) }; + } else if (type === 'join') { + return { ...schema, ...getJoinSchema(field) }; + } else if ( + [ + 'completion', + 'sparse_vector', + 'dense_vector', + 'geo_shape', + 'geo_point', + 'rank_feature', + 'rank_features', + ].includes(type) + ) { + return schema; + } else if (field.properties) { + schema.properties = getSchemaByItem(field.properties, data, fieldLevelConfig); + } else if (field.items) { + let arrData = field.items; + + if (Array.isArray(field.items)) { + arrData = field.items[0]; + } + + schema = { ...schema, ...getField(arrData, data, fieldLevelConfig) }; + } + + return schema; +}; + +module.exports = { + getField, + getSchemaByItem, +}; diff --git a/forward_engineering/helpers/getFieldsSchema.js b/forward_engineering/helpers/getFieldsSchema.js new file mode 100644 index 0000000..9eecd21 --- /dev/null +++ b/forward_engineering/helpers/getFieldsSchema.js @@ -0,0 +1,18 @@ +const { getSchemaByItem } = require('./getField'); + +const getFieldsSchema = data => { + const { jsonSchema, fieldLevelConfig } = data; + let schema = {}; + + if (!jsonSchema?.properties?._source?.properties) { + return schema; + } + + schema = getSchemaByItem(jsonSchema.properties._source.properties, data, fieldLevelConfig); + + return schema; +}; + +module.exports = { + getFieldsSchema, +}; diff --git a/forward_engineering/helpers/getKibanaScript.js b/forward_engineering/helpers/getKibanaScript.js new file mode 100644 index 0000000..ec61587 --- /dev/null +++ b/forward_engineering/helpers/getKibanaScript.js @@ -0,0 +1,9 @@ +const getKibanaScript = (mapping, indexData) => { + const indexName = indexData.name || ''; + + return `PUT /${indexName.toLowerCase()}\n${JSON.stringify(mapping, null, 4)}`; +}; + +module.exports = { + getKibanaScript, +}; diff --git a/forward_engineering/helpers/getMappingScript.js b/forward_engineering/helpers/getMappingScript.js new file mode 100644 index 0000000..b50f203 --- /dev/null +++ b/forward_engineering/helpers/getMappingScript.js @@ -0,0 +1,75 @@ +const { getContainerLevelProperties } = require('../../shared/getContainerLevelProperties'); + +const getSettings = (indexData, containerLevelConfig) => { + let settings; + let properties = getContainerLevelProperties(containerLevelConfig); + + properties.forEach(propertyName => { + if (indexData[propertyName]) { + if (!settings) { + settings = {}; + } + + settings[propertyName] = indexData[propertyName]; + } + }); + + return settings; +}; + +const getAliases = indexData => { + let aliases; + + if (!indexData.aliases) { + return aliases; + } + + indexData.aliases.forEach(alias => { + if (alias.name) { + if (!aliases) { + aliases = {}; + } + + aliases[alias.name] = {}; + + if (alias.filter) { + let filterData = ''; + try { + filterData = JSON.parse(alias.filter); + } catch (e) {} + + aliases[alias.name].filter = { + term: filterData, + }; + } + + if (alias.routing) { + aliases[alias.name].routing = alias.routing; + } + } + }); + + return aliases; +}; + +const getMappingScript = (indexData, typeSchema, containerLevelConfig) => { + let mappingScript = {}; + let settings = getSettings(indexData, containerLevelConfig); + let aliases = getAliases(indexData); + + if (settings) { + mappingScript.settings = settings; + } + + if (aliases) { + mappingScript.aliases = aliases; + } + + mappingScript.mappings = typeSchema; + + return mappingScript; +}; + +module.exports = { + getMappingScript, +}; diff --git a/forward_engineering/helpers/getTypeSchema.js b/forward_engineering/helpers/getTypeSchema.js new file mode 100644 index 0000000..f0de70f --- /dev/null +++ b/forward_engineering/helpers/getTypeSchema.js @@ -0,0 +1,17 @@ +const getTypeSchema = (typeData, fieldsSchema) => { + let script = {}; + + if (typeData.dynamic) { + script.dynamic = typeData.dynamic; + } + + script.properties = fieldsSchema; + + return { + [(typeData.collectionName || '').toLowerCase()]: script, + }; +}; + +module.exports = { + getTypeSchema, +}; diff --git a/helper/helper.js b/helper/helper.js deleted file mode 100644 index 5dc8f91..0000000 --- a/helper/helper.js +++ /dev/null @@ -1,69 +0,0 @@ -const fs = require('fs'); -const path = require('path'); -const readConfig = pathToConfig => { - return JSON.parse( - fs - .readFileSync(path.join(__dirname, pathToConfig)) - .toString() - .replace(/\/\*[.\s\S]*?\*\//gi, ''), - ); -}; -const fieldLevelConfig = readConfig('../properties_pane/field_level/fieldLevelConfig.json'); -const containerLevelConfig = readConfig('../properties_pane/container_level/containerLevelConfig.json'); - -module.exports = { - getTargetFieldLevelPropertyNames(type, data) { - if (!fieldLevelConfig.structure[type] || !Array.isArray(fieldLevelConfig.structure[type])) { - return []; - } - - return fieldLevelConfig.structure[type] - .filter(property => { - if (typeof property === 'object' && property.isTargetProperty) { - if (!property.dependency) { - return true; - } else if (data[property.dependency.key] !== property.dependency.value) { - return false; - } else if ( - Array.isArray(property.options) && - !property.options.includes(data[property.propertyName]) - ) { - return false; - } else { - return true; - } - } - - return false; - }) - .map(property => property.propertyKeyword); - }, - - getFieldProperties(type, data, pseudonyms) { - const propertyNames = this.getTargetFieldLevelPropertyNames(type, data); - - return propertyNames.reduce((result, propertyName) => { - if (Object.prototype.hasOwnProperty.call(data, propertyName)) { - result[propertyName] = data[propertyName]; - } else if (Object.prototype.hasOwnProperty.call(data, pseudonyms[propertyName])) { - result[pseudonyms[propertyName]] = data[pseudonyms[propertyName]]; - } - - return result; - }, {}); - }, - - getContainerLevelProperties() { - let properties = []; - - containerLevelConfig.forEach(tab => { - tab.structure.forEach(property => { - if (property.isTargetProperty) { - properties.push(property.propertyKeyword); - } - }); - }); - - return properties; - }, -}; diff --git a/package-lock.json b/package-lock.json index 0ad23f7..7d66aaf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "Elasticsearch", - "version": "0.2.1", + "version": "0.2.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "Elasticsearch", - "version": "0.2.1", + "version": "0.2.2", "dependencies": { "async": "2.6.4", "elasticsearch": "16.1.1", @@ -18,6 +18,7 @@ "@typescript-eslint/parser": "7.11.0", "esbuild": "0.20.2", "esbuild-plugin-clean": "1.0.1", + "esbuild-plugin-copy": "2.1.1", "eslint": "8.57.0", "eslint-config-prettier": "9.1.0", "eslint-formatter-teamcity": "^1.0.0", @@ -865,6 +866,19 @@ "node": ">=0.10.0" } }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -1024,6 +1038,18 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -1089,6 +1115,42 @@ "node": ">=0.10.0" } }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/clean-stack": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", @@ -1589,6 +1651,78 @@ "node": ">=8" } }, + "node_modules/esbuild-plugin-copy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/esbuild-plugin-copy/-/esbuild-plugin-copy-2.1.1.tgz", + "integrity": "sha512-Bk66jpevTcV8KMFzZI1P7MZKZ+uDcrZm2G2egZ2jNIvVnivDpodZI+/KnpL3Jnap0PBdIHU7HwFGB8r+vV5CVw==", + "dev": true, + "dependencies": { + "chalk": "^4.1.2", + "chokidar": "^3.5.3", + "fs-extra": "^10.0.1", + "globby": "^11.0.3" + }, + "peerDependencies": { + "esbuild": ">= 0.14.0" + } + }, + "node_modules/esbuild-plugin-copy/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/esbuild-plugin-copy/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/esbuild-plugin-copy/node_modules/fs-extra": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-plugin-copy/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -2196,6 +2330,20 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -2612,6 +2760,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/is-boolean-object": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", @@ -3175,6 +3335,15 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/npm-run-path": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", @@ -3525,6 +3694,18 @@ } ] }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, "node_modules/regexp.prototype.flags": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz", diff --git a/package.json b/package.json index a4675aa..37313f2 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,8 @@ }, "scripts": { "lint": "eslint . --max-warnings=0", - "package": "node esbuild.package.js" + "package": "node esbuild.package.js", + "postinstall": "npx simple-git-hooks" }, "devDependencies": { "@hackolade/hck-esbuild-plugins-pack": "0.0.1", @@ -46,6 +47,7 @@ "@typescript-eslint/parser": "7.11.0", "esbuild": "0.20.2", "esbuild-plugin-clean": "1.0.1", + "esbuild-plugin-copy": "2.1.1", "eslint": "8.57.0", "eslint-config-prettier": "9.1.0", "eslint-formatter-teamcity": "^1.0.0", @@ -56,4 +58,4 @@ "prettier": "3.2.5", "simple-git-hooks": "2.11.1" } -} \ No newline at end of file +} diff --git a/reverse_engineering/SchemaCreator.js b/reverse_engineering/SchemaCreator.js index dc81340..8b82302 100644 --- a/reverse_engineering/SchemaCreator.js +++ b/reverse_engineering/SchemaCreator.js @@ -1,3 +1,4 @@ +const { getFieldProperties } = require('../shared/getFieldProperties'); const snippetsPath = '../snippets/'; const snippets = { @@ -19,8 +20,6 @@ const snippets = { 'completionObject': require(snippetsPath + 'completionObject.json'), }; -const helper = require('../helper/helper'); - module.exports = { indices: [], types: [], @@ -80,7 +79,7 @@ module.exports = { }; }, - getSchema(elasticMapping, sample) { + getSchema(elasticMapping, sample, fieldLevelConfig) { let schema = this.getSchemaTemplate(); sample = sample || {}; @@ -89,6 +88,7 @@ module.exports = { elasticMapping.properties, sample._source, elasticMapping.properties, + fieldLevelConfig, ); if (elasticMapping.dynamic) { @@ -98,19 +98,19 @@ module.exports = { return schema; }, - getFields(properties, sample, mapping) { + getFields(properties, sample, mapping, fieldLevelConfig) { let schema = {}; for (let fieldName in properties) { const currentSample = sample?.[fieldName]; - schema[fieldName] = this.getField(properties[fieldName], currentSample, mapping); + schema[fieldName] = this.getField(properties[fieldName], currentSample, mapping, fieldLevelConfig); } return schema; }, - getField(fieldData, sample, mapping) { + getField(fieldData, sample, mapping, fieldLevelConfig) { let schema = {}; if (!fieldData) { @@ -126,7 +126,7 @@ module.exports = { let isArrayType = ['nested', 'array', 'geo-point'].indexOf(schema.type) !== -1; if (hasProperties) { - let properties = this.getFields(fieldData.properties, sample, mapping); + let properties = this.getFields(fieldData.properties, sample, mapping, fieldLevelConfig); if (isArrayType) { schema.items = [ @@ -159,7 +159,7 @@ module.exports = { schema = this.handleCompletionSnippet(schema); } - schema = this.setProperties(schema, fieldData); + schema = this.setProperties(schema, fieldData, fieldLevelConfig); return schema; }, @@ -428,13 +428,14 @@ module.exports = { return schema; }, - setProperties(schema, fieldData) { - const properties = helper.getFieldProperties( + setProperties(schema, fieldData, fieldLevelConfig) { + const properties = getFieldProperties( schema.type, { mode: fieldData.type, ...fieldData }, { 'stringfields': 'fields', }, + fieldLevelConfig, ); for (let propName in properties) { diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index b86b61b..81e5775 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -1,9 +1,7 @@ -'use strict'; - -const elasticsearch = require('elasticsearch'); -const fs = require('fs'); const _ = require('lodash'); +const fs = require('fs'); const async = require('async'); +const elasticsearch = require('elasticsearch'); const SchemaCreator = require('./SchemaCreator'); const versions = require('../package.json').contributes.target.versions; @@ -254,6 +252,7 @@ module.exports = { jsonSchemas, fieldInference, client, + fieldLevelConfig: data.pluginConfiguration.fieldLevelConfig, }; if (majorVersion >= 7) { @@ -319,18 +318,11 @@ const shouldPackageBeAdded = (docPackage, includeEmptyCollection) => { return true; } - if ( + return !( docPackage.documents.length === 0 && - docPackage.validation && - docPackage.validation.jsonSchema && - docPackage.validation.jsonSchema.properties && - docPackage.validation.jsonSchema.properties._source && + docPackage.validation?.jsonSchema?.properties?._source && _.isEmpty(docPackage.validation.jsonSchema.properties._source.properties) - ) { - return false; - } - - return true; + ); }; const getSampleDocSize = (count, recordSamplingSettings) => { @@ -345,7 +337,16 @@ const getSampleDocSize = (count, recordSamplingSettings) => { const getIndexTypeData = ( typeName, - { indexName, recordSamplingSettings, containerLevelKeys, bucketInfo, jsonSchemas, fieldInference, client }, + { + indexName, + recordSamplingSettings, + containerLevelKeys, + bucketInfo, + jsonSchemas, + fieldInference, + client, + fieldLevelConfig, + }, ) => new Promise((resolve, reject) => { async.waterfall( @@ -399,16 +400,13 @@ const getIndexTypeData = ( }; const mappingJsonSchema = typeName - ? jsonSchemas && - jsonSchemas[indexName] && - jsonSchemas[indexName].mappings && - jsonSchemas[indexName].mappings[typeName] - : jsonSchemas && jsonSchemas[indexName] && jsonSchemas[indexName].mappings; + ? jsonSchemas?.[indexName]?.mappings?.[typeName] + : jsonSchemas?.[indexName]?.mappings; const hasJsonSchema = Boolean(mappingJsonSchema); if (hasJsonSchema) { documentsPackage.validation = { - jsonSchema: SchemaCreator.getSchema(mappingJsonSchema, documentTemplate), + jsonSchema: SchemaCreator.getSchema(mappingJsonSchema, documentTemplate, fieldLevelConfig), }; } @@ -439,7 +437,7 @@ const getTypesByVersion = (version, types, indexes) => { indexes = Array.isArray(indexes) ? indexes : []; return indexes.reduce((result, indexName) => { - return {...result, [indexName]: [] }; + return { ...result, [indexName]: [] }; }, {}); }; @@ -447,11 +445,7 @@ const getIndexes = (client, includeSystemCollection) => { return client.indices.getMapping().then(data => { return Object.keys(data) .filter(indexName => { - if (!includeSystemCollection && indexName[0] === '.') { - return false; - } else { - return true; - } + return !(!includeSystemCollection && indexName.startsWith('.')); }) .reduce((result, indexName) => { return { diff --git a/shared/getContainerLevelProperties.js b/shared/getContainerLevelProperties.js new file mode 100644 index 0000000..b8b4d1d --- /dev/null +++ b/shared/getContainerLevelProperties.js @@ -0,0 +1,17 @@ +const getContainerLevelProperties = containerLevelConfig => { + let properties = []; + + containerLevelConfig.forEach(tab => { + tab.structure.forEach(property => { + if (property.isTargetProperty) { + properties.push(property.fieldKeyword); + } + }); + }); + + return properties; +}; + +module.exports = { + getContainerLevelProperties, +}; diff --git a/shared/getFieldProperties.js b/shared/getFieldProperties.js new file mode 100644 index 0000000..8c4bf2a --- /dev/null +++ b/shared/getFieldProperties.js @@ -0,0 +1,41 @@ +const getTargetFieldLevelPropertyNames = (type, data, fieldLevelConfig) => { + if (!fieldLevelConfig.structure[type] || !Array.isArray(fieldLevelConfig.structure[type])) { + return []; + } + + return fieldLevelConfig.structure[type] + .filter(property => { + if (typeof property === 'object' && property.isTargetProperty) { + if (!property.dependency) { + return true; + } else if (data[property.dependency.key] !== property.dependency.value) { + return false; + } else if (Array.isArray(property.options) && !property.options.includes(data[property.fieldName])) { + return false; + } else { + return true; + } + } + + return false; + }) + .map(property => property.fieldKeyword); +}; + +const getFieldProperties = (type, data, pseudonyms, fieldLevelConfig) => { + const propertyNames = getTargetFieldLevelPropertyNames(type, data, fieldLevelConfig); + + return propertyNames.reduce((result, propertyName) => { + if (Object.hasOwn(data, propertyName)) { + result[propertyName] = data[propertyName]; + } else if (Object.hasOwn(data, pseudonyms[propertyName])) { + result[pseudonyms[propertyName]] = data[pseudonyms[propertyName]]; + } + + return result; + }, {}); +}; + +module.exports = { + getFieldProperties, +}; diff --git a/helper/schemaHelper.js b/shared/schemaHelper.js similarity index 96% rename from helper/schemaHelper.js rename to shared/schemaHelper.js index 8e28494..e2109ac 100644 --- a/helper/schemaHelper.js +++ b/shared/schemaHelper.js @@ -1,5 +1,3 @@ -'use strict'; - const getPathById = (schema, id, path) => { if (schema.GUID === id) { return path; @@ -83,7 +81,7 @@ const getNameByPath = (schema, path) => { const joinIndex = items => { return items.reduce((result, item) => { - if (/\[\d+\]/.test(item)) { + if (/\[\d+]/.test(item)) { return [...result.slice(0, -1), result[result.length - 1] + item]; } else { return [...result, item]; @@ -112,7 +110,7 @@ const getPathName = (id, sources) => { return name .slice(1) - .filter(item => !/\[\d+\]/.test(item)) + .filter(item => !/\[\d+]/.test(item)) .join('.'); } }