diff --git a/web/.eslintrc.js b/web/.eslintrc.js index fe93fbe50..f499c3610 100644 --- a/web/.eslintrc.js +++ b/web/.eslintrc.js @@ -78,6 +78,7 @@ module.exports = { 'vue/multiline-html-element-content-newline': 0, 'vue/attributes-order': 0, 'vue/html-self-closing': 0, + "vue/valid-v-model": 0, 'no-useless-constructor': 0, 'no-mixed-operators': 0, 'no-new-func': 0, diff --git a/web/README-DEV.md b/web/README-DEV.md index fe46c3a05..c75b99752 100644 --- a/web/README-DEV.md +++ b/web/README-DEV.md @@ -31,7 +31,6 @@ 新增功能模块先确定涉及应用，按照上面目录结构维护代码同时建议遵守以下约束： - 子应用可以配置自己的layout需要在应用router模块导出配置subAppRoutes -- 子应用支持使用自己的header，需要在config.json里配置模块路径 - 各应用需要使用iView作为UI库，并提供路由，国际化等配置写入config.json - 各应用间不要相互直接依赖，确有依赖通过lerna管理 - 可复用组件，资源需要合理放置，packages/shared 共享组件方法，修改需要注意影响 @@ -43,9 +42,12 @@ ### 如何新增一个子应用，如何扩展 -1. config.json 新增应用配置 +1. config.json apps里新增应用配置（apps里配置的字应用模块会一起合并打包） 2. packages 下新建应用目录或者插件目录进行应用开发 +参考packages/demo： +npm run serve --configfile=config.demo.json +http://localhost:8080/#/demoHome ### 前端开发、构建打包 diff --git a/web/config.demo.json b/web/config.demo.json new file mode 100644 index 000000000..1cab33a2d --- /dev/null +++ b/web/config.demo.json @@ -0,0 +1,39 @@ +{ + "apps": { + "demo": { + "routes": "demo/router", + "module": "demo/module", + "i18n": { + "en": "demo/i18n/en.json", + "zh-CN": "demo/i18n/zh.json" + } + }, + "sparketl": { + "routes": "sparketl/router/index", + "i18n": { + "en": "sparketl/utils/i18n/zh.json", + "zh-CN": "sparketl/utils/i18n/zh.json" + } + } + }, + "exts": { + }, + "components": { + }, + "conf": { + "app_name": "DataSphere Studio", + "app_logo": "dss/assets/images/dssLogo.png", + "user_guide": "", + "faq_link": "/_book/", + "copy_project_enable": true, + "error_report": true, + "table_transfer": true, + "watermark": { + "show": true, + "template": "${username} ${time}", + "timeupdate": 60000 + }, + "update_chrome": "/_book/知识库/DSS常见问题/其他/DSS使用推荐的浏览器版本.html" + }, + "version": "1.1.2" +} diff --git a/web/config.json b/web/config.json index ad97708d6..d0d2cc5e3 100644 --- a/web/config.json +++ b/web/config.json @@ -68,7 +68,6 @@ } }, "components": { - "WeEditorLsp": "editorLsp/index.js", "WeEditor": "editor/index.js" }, "conf": { @@ -82,10 +81,7 @@ "template": "${username} ${time}", "timeupdate": 60000 }, - "lsp_service": { - "sql": "${protocol}//${host}/server", - "py": "${protocol}//${host}/python" - } + "env": "opensource" }, "version": "1.1.12" } diff --git a/web/package-lock.json b/web/package-lock.json index 4bd4cd62b..850e83a9e 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -8,6 +8,7 @@ "name": "dataspherestudio", "version": "1.1.6", "dependencies": { + "@codemirror/lang-json": "^6.0.1", "@riophae/vue-treeselect": "0.4.0", "axios": "0.21.1", "babel-plugin-transform-remove-console": "6.9.4", @@ -15,6 +16,7 @@ "bootstrap": "^3.4.1", "butterfly-dag": "4.1.23", "clipboard": "2.0.8", + "codemirror": "^5.65.13", "core-js": "2.6.11", "d3": "3.5.17", "dayjs": "1.10.7", @@ -37,6 +39,7 @@ "svgo": "1.3.0", "vscode-ws-jsonrpc": "0.2.0", "vue": "2.6.12", + "vue-codemirror": "^4.0.6", "vue-i18n": "8.22.1", "vue-router": "3.4.8", "vuedraggable": "2.24.3", @@ -1442,6 +1445,43 @@ "node": ">=6.9.0" } }, + "node_modules/@codemirror/lang-json": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/@codemirror/lang-json/-/lang-json-6.0.1.tgz", + "integrity": "sha512-+T1flHdgpqDDlJZ2Lkil/rLiRy684WMLc74xUnjJH48GQdfJo/pudlTRreZmKwzP8/tGdKf83wlbAdOCzlJOGQ==", + "dependencies": { + "@codemirror/language": "^6.0.0", + "@lezer/json": "^1.0.0" + } + }, + "node_modules/@codemirror/language": { + "version": "6.8.0", + "resolved": "https://registry.npmmirror.com/@codemirror/language/-/language-6.8.0.tgz", + "integrity": "sha512-r1paAyWOZkfY0RaYEZj3Kul+MiQTEbDvYqf8gPGaRvNneHXCmfSaAVFjwRUPlgxS8yflMxw2CTu6uCMp8R8A2g==", + "dependencies": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "@lezer/common": "^1.0.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0", + "style-mod": "^4.0.0" + } + }, + "node_modules/@codemirror/state": { + "version": "6.2.1", + "resolved": "https://registry.npmmirror.com/@codemirror/state/-/state-6.2.1.tgz", + "integrity": "sha512-RupHSZ8+OjNT38zU9fKH2sv+Dnlr8Eb8sl4NOnnqz95mCFTZUaiRP8Xv5MeeaG0px2b8Bnfe7YGwCV3nsBhbuw==" + }, + "node_modules/@codemirror/view": { + "version": "6.14.0", + "resolved": "https://registry.npmmirror.com/@codemirror/view/-/view-6.14.0.tgz", + "integrity": "sha512-I263FPs4In42MNmrdwN2DfmYPFMVMXgT7o/mxdGp4jv5LPs8i0FOxzmxF5yeeQdYSTztb2ZhmPIu0ahveInVTg==", + "dependencies": { + "@codemirror/state": "^6.1.4", + "style-mod": "^4.0.0", + "w3c-keyname": "^2.2.4" + } + }, "node_modules/@gar/promisify": { "version": "1.1.3", "resolved": "https://registry.npmmirror.com/@gar/promisify/-/promisify-1.1.3.tgz", @@ -5430,6 +5470,36 @@ "typedarray-to-buffer": "^3.1.5" } }, + "node_modules/@lezer/common": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/@lezer/common/-/common-1.0.3.tgz", + "integrity": "sha512-JH4wAXCgUOcCGNekQPLhVeUtIqjH0yPBs7vvUdSjyQama9618IOKFJwkv2kcqdhF0my8hQEgCTEJU0GIgnahvA==" + }, + "node_modules/@lezer/highlight": { + "version": "1.1.6", + "resolved": "https://registry.npmmirror.com/@lezer/highlight/-/highlight-1.1.6.tgz", + "integrity": "sha512-cmSJYa2us+r3SePpRCjN5ymCqCPv+zyXmDl0ciWtVaNiORT/MxM7ZgOMQZADD0o51qOaOg24qc/zBViOIwAjJg==", + "dependencies": { + "@lezer/common": "^1.0.0" + } + }, + "node_modules/@lezer/json": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/@lezer/json/-/json-1.0.1.tgz", + "integrity": "sha512-nkVC27qiEZEjySbi6gQRuMwa2sDu2PtfjSgz0A4QF81QyRGm3kb2YRzLcOPcTEtmcwvrX/cej7mlhbwViA4WJw==", + "dependencies": { + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0" + } + }, + "node_modules/@lezer/lr": { + "version": "1.3.9", + "resolved": "https://registry.npmmirror.com/@lezer/lr/-/lr-1.3.9.tgz", + "integrity": "sha512-XPz6dzuTHlnsbA5M2DZgjflNQ+9Hi5Swhic0RULdp3oOs3rh6bqGZolosVqN/fQIT8uNiepzINJDnS39oweTHQ==", + "dependencies": { + "@lezer/common": "^1.0.0" + } + }, "node_modules/@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmmirror.com/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", @@ -9358,6 +9428,11 @@ "node": ">=0.10.0" } }, + "node_modules/codemirror": { + "version": "5.65.13", + "resolved": "https://registry.npmmirror.com/codemirror/-/codemirror-5.65.13.tgz", + "integrity": "sha512-SVWEzKXmbHmTQQWaz03Shrh4nybG0wXx2MEu3FO4ezbPW8IbnZEd5iGHGEffSUaitKYa3i+pHpBsSvw8sPHtzg==" + }, "node_modules/collection-visit": { "version": "1.0.0", "resolved": "https://registry.npmmirror.com/collection-visit/-/collection-visit-1.0.0.tgz", @@ -11825,6 +11900,11 @@ "wrappy": "1" } }, + "node_modules/diff-match-patch": { + "version": "1.0.5", + "resolved": "https://registry.npmmirror.com/diff-match-patch/-/diff-match-patch-1.0.5.tgz", + "integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==" + }, "node_modules/diffie-hellman": { "version": "5.0.3", "resolved": "https://registry.npmmirror.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz", @@ -24795,6 +24875,11 @@ "node": ">=4" } }, + "node_modules/style-mod": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/style-mod/-/style-mod-4.0.3.tgz", + "integrity": "sha512-78Jv8kYJdjbvRwwijtCevYADfsI0lGzYJe4mMFdceO8l75DFFDoqBhR1jVDicDRRaX4//g1u9wKeo+ztc2h1Rw==" + }, "node_modules/stylehacks": { "version": "4.0.3", "resolved": "https://registry.npmmirror.com/stylehacks/-/stylehacks-4.0.3.tgz", @@ -26267,6 +26352,19 @@ "mockjs": "^1.0.1-beta3" } }, + "node_modules/vue-codemirror": { + "version": "4.0.6", + "resolved": "https://registry.npmmirror.com/vue-codemirror/-/vue-codemirror-4.0.6.tgz", + "integrity": "sha512-ilU7Uf0mqBNSSV3KT7FNEeRIxH4s1fmpG4TfHlzvXn0QiQAbkXS9lLfwuZpaBVEnpP5CSE62iGJjoliTuA8poQ==", + "dependencies": { + "codemirror": "^5.41.0", + "diff-match-patch": "^1.0.0" + }, + "engines": { + "node": ">= 4.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/vue-eslint-parser": { "version": "2.0.3", "resolved": "https://registry.npmmirror.com/vue-eslint-parser/-/vue-eslint-parser-2.0.3.tgz", @@ -26402,6 +26500,11 @@ "vue": "^2.0.0" } }, + "node_modules/w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmmirror.com/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==" + }, "node_modules/watch-size": { "version": "2.0.0", "resolved": "https://registry.npmmirror.com/watch-size/-/watch-size-2.0.0.tgz", @@ -28711,6 +28814,43 @@ "to-fast-properties": "^2.0.0" } }, + "@codemirror/lang-json": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/@codemirror/lang-json/-/lang-json-6.0.1.tgz", + "integrity": "sha512-+T1flHdgpqDDlJZ2Lkil/rLiRy684WMLc74xUnjJH48GQdfJo/pudlTRreZmKwzP8/tGdKf83wlbAdOCzlJOGQ==", + "requires": { + "@codemirror/language": "^6.0.0", + "@lezer/json": "^1.0.0" + } + }, + "@codemirror/language": { + "version": "6.8.0", + "resolved": "https://registry.npmmirror.com/@codemirror/language/-/language-6.8.0.tgz", + "integrity": "sha512-r1paAyWOZkfY0RaYEZj3Kul+MiQTEbDvYqf8gPGaRvNneHXCmfSaAVFjwRUPlgxS8yflMxw2CTu6uCMp8R8A2g==", + "requires": { + "@codemirror/state": "^6.0.0", + "@codemirror/view": "^6.0.0", + "@lezer/common": "^1.0.0", + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0", + "style-mod": "^4.0.0" + } + }, + "@codemirror/state": { + "version": "6.2.1", + "resolved": "https://registry.npmmirror.com/@codemirror/state/-/state-6.2.1.tgz", + "integrity": "sha512-RupHSZ8+OjNT38zU9fKH2sv+Dnlr8Eb8sl4NOnnqz95mCFTZUaiRP8Xv5MeeaG0px2b8Bnfe7YGwCV3nsBhbuw==" + }, + "@codemirror/view": { + "version": "6.14.0", + "resolved": "https://registry.npmmirror.com/@codemirror/view/-/view-6.14.0.tgz", + "integrity": "sha512-I263FPs4In42MNmrdwN2DfmYPFMVMXgT7o/mxdGp4jv5LPs8i0FOxzmxF5yeeQdYSTztb2ZhmPIu0ahveInVTg==", + "requires": { + "@codemirror/state": "^6.1.4", + "style-mod": "^4.0.0", + "w3c-keyname": "^2.2.4" + } + }, "@gar/promisify": { "version": "1.1.3", "resolved": "https://registry.npmmirror.com/@gar/promisify/-/promisify-1.1.3.tgz", @@ -31857,6 +31997,36 @@ } } }, + "@lezer/common": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/@lezer/common/-/common-1.0.3.tgz", + "integrity": "sha512-JH4wAXCgUOcCGNekQPLhVeUtIqjH0yPBs7vvUdSjyQama9618IOKFJwkv2kcqdhF0my8hQEgCTEJU0GIgnahvA==" + }, + "@lezer/highlight": { + "version": "1.1.6", + "resolved": "https://registry.npmmirror.com/@lezer/highlight/-/highlight-1.1.6.tgz", + "integrity": "sha512-cmSJYa2us+r3SePpRCjN5ymCqCPv+zyXmDl0ciWtVaNiORT/MxM7ZgOMQZADD0o51qOaOg24qc/zBViOIwAjJg==", + "requires": { + "@lezer/common": "^1.0.0" + } + }, + "@lezer/json": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/@lezer/json/-/json-1.0.1.tgz", + "integrity": "sha512-nkVC27qiEZEjySbi6gQRuMwa2sDu2PtfjSgz0A4QF81QyRGm3kb2YRzLcOPcTEtmcwvrX/cej7mlhbwViA4WJw==", + "requires": { + "@lezer/highlight": "^1.0.0", + "@lezer/lr": "^1.0.0" + } + }, + "@lezer/lr": { + "version": "1.3.9", + "resolved": "https://registry.npmmirror.com/@lezer/lr/-/lr-1.3.9.tgz", + "integrity": "sha512-XPz6dzuTHlnsbA5M2DZgjflNQ+9Hi5Swhic0RULdp3oOs3rh6bqGZolosVqN/fQIT8uNiepzINJDnS39oweTHQ==", + "requires": { + "@lezer/common": "^1.0.0" + } + }, "@mrmlnc/readdir-enhanced": { "version": "2.2.1", "resolved": "https://registry.npmmirror.com/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz", @@ -35150,6 +35320,11 @@ "integrity": "sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==", "dev": true }, + "codemirror": { + "version": "5.65.13", + "resolved": "https://registry.npmmirror.com/codemirror/-/codemirror-5.65.13.tgz", + "integrity": "sha512-SVWEzKXmbHmTQQWaz03Shrh4nybG0wXx2MEu3FO4ezbPW8IbnZEd5iGHGEffSUaitKYa3i+pHpBsSvw8sPHtzg==" + }, "collection-visit": { "version": "1.0.0", "resolved": "https://registry.npmmirror.com/collection-visit/-/collection-visit-1.0.0.tgz", @@ -37199,6 +37374,11 @@ "wrappy": "1" } }, + "diff-match-patch": { + "version": "1.0.5", + "resolved": "https://registry.npmmirror.com/diff-match-patch/-/diff-match-patch-1.0.5.tgz", + "integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==" + }, "diffie-hellman": { "version": "5.0.3", "resolved": "https://registry.npmmirror.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz", @@ -47814,6 +47994,11 @@ "through": "^2.3.4" } }, + "style-mod": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/style-mod/-/style-mod-4.0.3.tgz", + "integrity": "sha512-78Jv8kYJdjbvRwwijtCevYADfsI0lGzYJe4mMFdceO8l75DFFDoqBhR1jVDicDRRaX4//g1u9wKeo+ztc2h1Rw==" + }, "stylehacks": { "version": "4.0.3", "resolved": "https://registry.npmmirror.com/stylehacks/-/stylehacks-4.0.3.tgz", @@ -49047,6 +49232,15 @@ "mockjs": "^1.0.1-beta3" } }, + "vue-codemirror": { + "version": "4.0.6", + "resolved": "https://registry.npmmirror.com/vue-codemirror/-/vue-codemirror-4.0.6.tgz", + "integrity": "sha512-ilU7Uf0mqBNSSV3KT7FNEeRIxH4s1fmpG4TfHlzvXn0QiQAbkXS9lLfwuZpaBVEnpP5CSE62iGJjoliTuA8poQ==", + "requires": { + "codemirror": "^5.41.0", + "diff-match-patch": "^1.0.0" + } + }, "vue-eslint-parser": { "version": "2.0.3", "resolved": "https://registry.npmmirror.com/vue-eslint-parser/-/vue-eslint-parser-2.0.3.tgz", @@ -49161,6 +49355,11 @@ "integrity": "sha512-7fRsG2Yw5Z07LUz/IIu9barpmYiN9q+ZTC+CrVamvCbmsxyhz8mU1OuYFbfORysaUskioNMxTGDo+HOzeDfSyQ==", "requires": {} }, + "w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmmirror.com/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==" + }, "watch-size": { "version": "2.0.0", "resolved": "https://registry.npmmirror.com/watch-size/-/watch-size-2.0.0.tgz", diff --git a/web/package.json b/web/package.json index b89fd4651..30bd9164c 100644 --- a/web/package.json +++ b/web/package.json @@ -3,8 +3,8 @@ "version": "1.1.6", "private": true, "scripts": { - "serve": "cd packages/dss && npm run serve", - "build": "cd packages/dss && npm run build", + "serve": "patch-package && cd packages/dss && npm run serve", + "build": "patch-package && cd packages/dss && npm run build", "serve-sandbox": "patch-package && npm run serve --configfile=config.sandbox.json", "build-sandbox": "patch-package && npm run build --configfile=config.sandbox.json", "serve-scriptis": "patch-package && npm run serve --module=scriptis --micro_module=scriptis", diff --git a/web/packages/demo/i18n/en.json b/web/packages/demo/i18n/en.json new file mode 100644 index 000000000..a6e374d9b --- /dev/null +++ b/web/packages/demo/i18n/en.json @@ -0,0 +1,4 @@ +{ + "message": { + } +} diff --git a/web/packages/demo/i18n/zh.json b/web/packages/demo/i18n/zh.json new file mode 100644 index 000000000..a6e374d9b --- /dev/null +++ b/web/packages/demo/i18n/zh.json @@ -0,0 +1,4 @@ +{ + "message": { + } +} diff --git a/web/packages/demo/module/index.vue b/web/packages/demo/module/index.vue new file mode 100644 index 000000000..adecfb99e --- /dev/null +++ b/web/packages/demo/module/index.vue @@ -0,0 +1,3 @@ + diff --git a/web/packages/demo/router.js b/web/packages/demo/router.js new file mode 100644 index 000000000..5f966b7e2 --- /dev/null +++ b/web/packages/demo/router.js @@ -0,0 +1,26 @@ +// 子应用layout 自定义 header footer 等需要导出subAppRoutes +// export const subAppRoutes = { +// path: '', +// name: 'layout', +// component: () => import('./view/layout.vue'), +// redirect: '/home', +// meta: { +// publicPage: true, // 权限公开 +// }, +// children: [ + +// ] +// } + +export default [ + { + path: 'demoHome', + name: 'DemoHome', + meta: { + title: 'Tests', + publicPage: true, + }, + component: () => + import('./module/index.vue'), + }, +] diff --git a/web/packages/editor/editor.vue b/web/packages/editor/editor.vue index e9c155922..692d5086c 100644 --- a/web/packages/editor/editor.vue +++ b/web/packages/editor/editor.vue @@ -458,19 +458,21 @@ export default { vm.editor.updateOptions({wordWrap: 'on'}); }, }); - - this.editor.addAction({ - id: 'newdbsuggest', - label: this.$t('message.common.monacoMenu.newdbcomplition'), - keybindings: [], - keybindingContext: null, - contextMenuGroupId: 'control', - contextMenuOrder: 2.5, - run() { - localStorage.setItem('scriptis-edditor-type', 'lsp'); - location.reload(); - }, - }); + + if (this.$APP_CONF && this.$APP_CONF.lsp_service) { + this.editor.addAction({ + id: 'newdbsuggest', + label: this.$t('message.common.monacoMenu.newdbcomplition'), + keybindings: [], + keybindingContext: null, + contextMenuGroupId: 'control', + contextMenuOrder: 2.5, + run() { + localStorage.setItem('scriptis-edditor-type', 'lsp'); + location.reload(); + }, + }); + } if (this.language === 'hql') { // 控制语法检查 diff --git a/web/packages/scriptis/module/hiveSidebar/hiveList/index.vue b/web/packages/scriptis/module/hiveSidebar/hiveList/index.vue index 01a37fbe1..96bd7812b 100644 --- a/web/packages/scriptis/module/hiveSidebar/hiveList/index.vue +++ b/web/packages/scriptis/module/hiveSidebar/hiveList/index.vue @@ -264,7 +264,7 @@ export default { toggleShow(e) { const item = e.target.innerText const overDb = e.target.className.indexOf('fi-hivedb') > -1 - if ( overDb && !this.dbnameDesc[item] ) { + if ( overDb && !this.dbnameDesc[item] && this.$APP_CONF && this.$APP_CONF.env === 'webank') { // mouseover 在数据库项上且未获取描述信息 clearTimeout(this.fetchDbInfo) this.fetchDbInfo = setTimeout(() => { diff --git a/web/packages/sparketl/components/Codemirror/index.vue b/web/packages/sparketl/components/Codemirror/index.vue new file mode 100644 index 000000000..9329ca7de --- /dev/null +++ b/web/packages/sparketl/components/Codemirror/index.vue @@ -0,0 +1,82 @@ + + + + + diff --git a/web/packages/sparketl/components/InputStringArray/index.vue b/web/packages/sparketl/components/InputStringArray/index.vue new file mode 100644 index 000000000..85b78d86f --- /dev/null +++ b/web/packages/sparketl/components/InputStringArray/index.vue @@ -0,0 +1,36 @@ + + + + + + + diff --git a/web/packages/sparketl/components/InputStringMap/index.vue b/web/packages/sparketl/components/InputStringMap/index.vue new file mode 100644 index 000000000..1f60bb091 --- /dev/null +++ b/web/packages/sparketl/components/InputStringMap/index.vue @@ -0,0 +1,39 @@ + + + + + diff --git a/web/packages/sparketl/components/Sink/FileSink.vue b/web/packages/sparketl/components/Sink/FileSink.vue new file mode 100644 index 000000000..890843c75 --- /dev/null +++ b/web/packages/sparketl/components/Sink/FileSink.vue @@ -0,0 +1,130 @@ + + + + diff --git a/web/packages/sparketl/components/Sink/HiveSink.vue b/web/packages/sparketl/components/Sink/HiveSink.vue new file mode 100644 index 000000000..b5878d0d2 --- /dev/null +++ b/web/packages/sparketl/components/Sink/HiveSink.vue @@ -0,0 +1,149 @@ + + + + diff --git a/web/packages/sparketl/components/Sink/JdbcSink.vue b/web/packages/sparketl/components/Sink/JdbcSink.vue new file mode 100644 index 000000000..807fa663e --- /dev/null +++ b/web/packages/sparketl/components/Sink/JdbcSink.vue @@ -0,0 +1,187 @@ + + + + diff --git a/web/packages/sparketl/components/Sink/ManagedJdbcSink.vue b/web/packages/sparketl/components/Sink/ManagedJdbcSink.vue new file mode 100644 index 000000000..126e27c20 --- /dev/null +++ b/web/packages/sparketl/components/Sink/ManagedJdbcSink.vue @@ -0,0 +1,164 @@ + + + + diff --git a/web/packages/sparketl/components/Source/FileSource.vue b/web/packages/sparketl/components/Source/FileSource.vue new file mode 100644 index 000000000..f3c9bc158 --- /dev/null +++ b/web/packages/sparketl/components/Source/FileSource.vue @@ -0,0 +1,110 @@ + + + + diff --git a/web/packages/sparketl/components/Source/JdbcSource.vue b/web/packages/sparketl/components/Source/JdbcSource.vue new file mode 100644 index 000000000..3dbfff2e1 --- /dev/null +++ b/web/packages/sparketl/components/Source/JdbcSource.vue @@ -0,0 +1,138 @@ + + + + diff --git a/web/packages/sparketl/components/Source/ManagedJdbcSource.vue b/web/packages/sparketl/components/Source/ManagedJdbcSource.vue new file mode 100644 index 000000000..908937bec --- /dev/null +++ b/web/packages/sparketl/components/Source/ManagedJdbcSource.vue @@ -0,0 +1,104 @@ + + + + diff --git a/web/packages/sparketl/components/Transformation/SqlTransformation.vue b/web/packages/sparketl/components/Transformation/SqlTransformation.vue new file mode 100644 index 000000000..c6ed56a51 --- /dev/null +++ b/web/packages/sparketl/components/Transformation/SqlTransformation.vue @@ -0,0 +1,88 @@ + + + + diff --git a/web/packages/sparketl/components/TypedPlugin.vue b/web/packages/sparketl/components/TypedPlugin.vue new file mode 100644 index 000000000..4e692456e --- /dev/null +++ b/web/packages/sparketl/components/TypedPlugin.vue @@ -0,0 +1,90 @@ + + + + + diff --git a/web/packages/sparketl/package.json b/web/packages/sparketl/package.json new file mode 100644 index 000000000..d8a0b4503 --- /dev/null +++ b/web/packages/sparketl/package.json @@ -0,0 +1,9 @@ +{ + "name": "@dataspherestudio/sparketl", + "version": "1.0.0", + "dependencies": { + "@codemirror/lang-json": "^6.0.1", + "codemirror": "^5.65.13", + "vue-codemirror": "^4.0.6" + } +} diff --git a/web/packages/sparketl/pages/home.vue b/web/packages/sparketl/pages/home.vue new file mode 100644 index 000000000..35d0d71d8 --- /dev/null +++ b/web/packages/sparketl/pages/home.vue @@ -0,0 +1,221 @@ + + + + + + + diff --git a/web/packages/sparketl/router/index.js b/web/packages/sparketl/router/index.js new file mode 100644 index 000000000..00d393cd5 --- /dev/null +++ b/web/packages/sparketl/router/index.js @@ -0,0 +1,10 @@ +const routes = [ + { + path: "sparketl", + name: "Sparketl", + component: () => import("../pages/home.vue"), + }, +]; + + +export default routes; diff --git a/web/packages/sparketl/utils/BaseConstant.js b/web/packages/sparketl/utils/BaseConstant.js new file mode 100644 index 000000000..a68b12e35 --- /dev/null +++ b/web/packages/sparketl/utils/BaseConstant.js @@ -0,0 +1,482 @@ +const getBaseSource = () => ({ + variables: {}, + resultTable: "", + persist: false, + storageLevel: "MEMORY_AND_DISK", + options: {}, +}); + +const getBaseTransformation = () => ({ + variables: {}, + sourceTable: "", + resultTable: "", + persist: false, + storageLevel: "MEMORY_AND_DISK", +}); + +const getBaseSink = () => ({ + variables: {}, + sourceTable: "", + sourceQuery: "", + numPartitions: 0, + options: {}, +}); + +const PluginModels = { + source: { + jdbc: () => ({ + type: "source", + name: "jdbc", + config: { + ...getBaseSource(), + url: "", + driver: "", + user: "", + password: "", + query: "", + }, + }), + managed_jdbc: () => ({ + type: "source", + name: "managed_jdbc", + config: { + ...getBaseSource(), + datasource: "", + query: "", + }, + }), + file: () => ({ + type: "source", + name: "file", + config: { + ...getBaseSource(), + path: "", + serializer: "parquet", + columnNames: [], + }, + }), + }, + transformation: { + sql: () => ({ + name: "sql", + type: "transformation", + config: { + ...getBaseTransformation(), + sql: "", + }, + }), + }, + sink: { + hive: () => ({ + type: "sink", + name: "hive", + config: { + ...getBaseSink(), + targetDatabase: "", + targetTable: "", + saveMode: "overwrite", + strongCheck: false, + writeAsFile: false, + }, + }), + jdbc: () => ({ + type: "sink", + name: "jdbc", + config: { + ...getBaseSink(), + url: "", + driver: "", + user: "", + password: "", + targetDatabase: "", + targetTable: "", + saveMode: "overwrite", + preQueries: [], + }, + }), + managed_jdbc: () => ({ + type: "sink", + name: "managed_jdbc", + config: { + ...getBaseSink(), + targetDatasource: "", + targetDatabase: "", + targetTable: "", + saveMode: "overwrite", + preQueries: [], + }, + }), + file: () => ({ + type: "sink", + name: "file", + config: { + ...getBaseSink(), + path: "", + serializer: "parquet", + saveMode: "overwrite", + partitionBy: [], + }, + }), + }, +}; + +const FileSerializers = ["parquet", "orc", "csv", "text", "json"]; + +const Layout = { + cols: { + small: { xs: 24, sm: 24, md: 12, lg: 12, xl: 8 }, + }, + labelCols: { + small: { xs: 24, sm: 8, md: 10, lg: 10, xl: 9 }, + medium: {}, + large: { xs: 24, sm: 8, md: 5, lg: 5, xl: 3 }, + }, + wrapperColsWithLabel: { + small: { xs: 24, sm: 8, md: 10, lg: 10, xl: 9 }, + medium: {}, + large: { + xs: { span: 24, offset: 0 }, + sm: { span: 16, offset: 8 }, + md: { span: 19, offset: 5 }, + lg: { span: 19, offset: 5 }, + xl: { span: 21, offset: 3 }, + }, + }, +}; + +const StorageLevels = [ + { value: "NONE", label: "NONE" }, + { value: "DISK_ONLY", label: "DISK_ONLY" }, + { value: "DISK_ONLY_2", label: "DISK_ONLY_2" }, + { value: "MEMORY_ONLY", label: "MEMORY_ONLY" }, + { value: "MEMORY_ONLY_2", label: "MEMORY_ONLY_2" }, + { value: "MEMORY_ONLY_SER", label: "MEMORY_ONLY_SER" }, + { value: "MEMORY_ONLY_SER_2", label: "MEMORY_ONLY_SER_2" }, + { value: "MEMORY_AND_DISK", label: "MEMORY_AND_DISK" }, + { value: "MEMORY_AND_DISK_2", label: "MEMORY_AND_DISK_2" }, + { value: "MEMORY_AND_DISK_SER", label: "MEMORY_AND_DISK_SER" }, + { value: "MEMORY_AND_DISK_SER_2", label: "MEMORY_AND_DISK_SER_2" }, + { value: "OFF_HEAP", label: "OFF_HEAP" }, +]; + +const Databases = [ + { + databaseName: "ods", + }, + { + databaseName: "dw", + }, + { + databaseName: "dm", + }, +]; +const Datasource = ["hive:1000", "sr:2000"]; + +const SampleConfig = { + variables: {}, + sources: [ + { + name: "managed_jdbc", + config: { + variables: { + dt: "2022-01-01", + }, + resultTable: "source_table_01", + persist: true, + storageLevel: "MEMORY_AND_DISK", + options: { + connectionCollation: "utf8mb4_unicode_ci", + }, + datasource: "test_mysql", + query: "select * from test_mysql_db.test_tb where id > 0", + }, + }, + { + name: "jdbc", + config: { + variables: { + dt: "2022-01-01", + }, + resultTable: "source_table_02", + persist: false, + storageLevel: "MEMORY_AND_DISK", + options: { + connectionCollation: "utf8mb4_unicode_ci", + }, + url: "jdbc:mysql://localhost:3306/maple_datasource?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&rewriteBatchedStatements=true&useServerPrepStmts=true&allowMultiQueries=true&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=Asia/Shanghai", + driver: "com.mysql.cj.jdbc.Driver", + user: "xi_root", + password: "123456", + query: "select * from test_mysql_db.test_tb where id > 0", + }, + }, + { + name: "file", + config: { + variables: { + dt: "2022-01-01", + }, + resultTable: "source_table_03", + persist: false, + storageLevel: "MEMORY_AND_DISK", + options: {}, + path: "hdfs:///data/tmp_file/${dt}", + serializer: "csv", + columnNames: ["id", "name", "address"], + }, + }, + ], + transformations: [ + { + name: "sql", + config: { + variables: { + dt: "2022-01-01", + }, + sourceTable: "", + resultTable: "transform_table_01", + persist: false, + storageLevel: "MEMORY_AND_DISK", + sql: "select st03.name, st03.address, dtt.test\nfrom source_table_03 st03 join dw.test_tb dtt on st03.id=dtt.tid", + }, + }, + ], + sinks: [ + { + name: "hive", + config: { + variables: { + dt: "2022-01-01", + }, + sourceTable: "transform_table_01", + sourceQuery: "", + numPartitions: 10, + options: {}, + targetDatabase: "dm", + targetTable: "dm_test_table", + saveMode: "overwrite", + strongCheck: false, + writeAsFile: false, + }, + }, + { + name: "jdbc", + config: { + variables: { + dt: "2022-01-01", + }, + sourceTable: "source_table_01", + sourceQuery: "", + numPartitions: 0, + options: { + connectionCollation: "utf8mb4_unicode_ci", + isolationLevel: "NONE", + batchsize: "5000", + }, + url: "jdbc:mysql://localhost:3306/maple_datasource?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&rewriteBatchedStatements=true&useServerPrepStmts=true&allowMultiQueries=true&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=Asia/Shanghai", + driver: "com.mysql.cj.jdbc.Driver", + user: "xi_root", + password: "123456", + targetDatabase: "test_mysql_db", + targetTable: "test_mysql_db_table_0001", + saveMode: "overwrite", + preQueries: ["delete from test_mysql_db.test_mysql_db_table_0001"], + }, + }, + { + name: "managed_jdbc", + config: { + variables: { + dt: "2022-01-01", + }, + sourceTable: "", + sourceQuery: "select * from source_table_01", + numPartitions: 0, + options: {}, + targetDatasource: "test_postgresql", + targetDatabase: "postgresql_test_db", + targetTable: "postgresql_test_tb", + saveMode: "overwrite", + preQueries: ["delete from postgresql_test_tb where id > 0"], + }, + }, + { + name: "file", + config: { + variables: { + dt: "2022-01-01", + }, + sourceTable: "source_table_03", + sourceQuery: "", + numPartitions: 0, + options: {}, + path: "hdfs:///data/tmp_file/${dt}", + serializer: "parquet", + saveMode: "overwrite", + partitionBy: ["type"], + }, + }, + ], +}; + +const SampleArrayConfig = { + variables: {}, + plugins: [ + { + type: "source", + name: "managed_jdbc", + config: { + variables: { + dt: "2022-01-01", + }, + resultTable: "source_table_01", + persist: true, + storageLevel: "MEMORY_AND_DISK", + options: { + connectionCollation: "utf8mb4_unicode_ci", + }, + datasource: "test_mysql", + query: "select * from test_mysql_db.test_tb where id > 0", + }, + }, + { + type: "source", + name: "jdbc", + config: { + variables: { + dt: "2022-01-01", + }, + resultTable: "source_table_02", + persist: false, + storageLevel: "MEMORY_AND_DISK", + options: { + connectionCollation: "utf8mb4_unicode_ci", + }, + url: "jdbc:mysql://localhost:3306/maple_datasource?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&rewriteBatchedStatements=true&useServerPrepStmts=true&allowMultiQueries=true&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=Asia/Shanghai", + driver: "com.mysql.cj.jdbc.Driver", + user: "xi_root", + password: "123456", + query: "select * from test_mysql_db.test_tb where id > 0", + }, + }, + { + type: "source", + name: "file", + config: { + variables: { + dt: "2022-01-01", + }, + resultTable: "source_table_03", + persist: false, + storageLevel: "MEMORY_AND_DISK", + options: {}, + path: "hdfs:///data/tmp_file/${dt}", + serializer: "csv", + columnNames: ["id", "name", "address"], + }, + }, + { + type: "source", + name: "sql", + config: { + variables: { + dt: "2022-01-01", + }, + sourceTable: "", + resultTable: "transform_table_01", + persist: false, + storageLevel: "MEMORY_AND_DISK", + sql: "select st03.name, st03.address, dtt.test\nfrom source_table_03 st03 join dw.test_tb dtt on st03.id=dtt.tid", + }, + }, + { + type: "sink", + name: "hive", + config: { + variables: { + dt: "2022-01-01", + }, + sourceTable: "transform_table_01", + sourceQuery: "", + numPartitions: 10, + options: {}, + targetDatabase: "dm", + targetTable: "dm_test_table", + saveMode: "overwrite", + strongCheck: false, + writeAsFile: false, + }, + }, + { + type: "sink", + name: "jdbc", + config: { + variables: { + dt: "2022-01-01", + }, + sourceTable: "source_table_01", + sourceQuery: "", + numPartitions: 0, + options: { + connectionCollation: "utf8mb4_unicode_ci", + isolationLevel: "NONE", + batchsize: "5000", + }, + url: "jdbc:mysql://localhost:3306/maple_datasource?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&rewriteBatchedStatements=true&useServerPrepStmts=true&allowMultiQueries=true&zeroDateTimeBehavior=convertToNull&useSSL=false&serverTimezone=Asia/Shanghai", + driver: "com.mysql.cj.jdbc.Driver", + user: "xi_root", + password: "123456", + targetDatabase: "test_mysql_db", + targetTable: "test_mysql_db_table_0001", + saveMode: "overwrite", + preQueries: ["delete from test_mysql_db.test_mysql_db_table_0001"], + }, + }, + { + type: "sink", + name: "managed_jdbc", + config: { + variables: { + dt: "2022-01-01", + }, + sourceTable: "", + sourceQuery: "select * from source_table_01", + numPartitions: 0, + options: {}, + targetDatasource: "test_postgresql", + targetDatabase: "postgresql_test_db", + targetTable: "postgresql_test_tb", + saveMode: "overwrite", + preQueries: ["delete from postgresql_test_tb where id > 0"], + }, + }, + { + type: "sink", + name: "file", + config: { + variables: { + dt: "2022-01-01", + }, + sourceTable: "source_table_03", + sourceQuery: "", + numPartitions: 0, + options: {}, + path: "hdfs:///data/tmp_file/${dt}", + serializer: "parquet", + saveMode: "overwrite", + partitionBy: ["type"], + }, + }, + ], +}; + +export { + PluginModels, + FileSerializers, + Layout, + Databases, + StorageLevels, + SampleConfig, + SampleArrayConfig, + Datasource, +}; diff --git a/web/packages/sparketl/utils/i18n/zh.json b/web/packages/sparketl/utils/i18n/zh.json new file mode 100644 index 000000000..c91be6486 --- /dev/null +++ b/web/packages/sparketl/utils/i18n/zh.json @@ -0,0 +1,45 @@ +{ + "plugin" : { + "registryTableName": "注册表名", + "outputTableName": "输出表名", + "parameter": "参数", + "variable": "变量", + "saveMode": "写入模式", + "storageLevel": "缓存级别", + "enableCache": "开启缓存", + "open": "开启", + "close": "关闭", + "driver": "驱动类名", + "jdbcUrl": "jdbc Url", + "querySql": "查询语句", + "username": "用户名", + "password": "密码", + "sql": "sql", + "append": "追加", + "cover": "覆盖", + "sourceSql": "来源语句", + "sourceTable": "来源表", + "partitions": "分区数", + "partitionsColumn": "分区字段", + "filePriority": "文件优先", + "strongCheck": "强校验", + "targetDatabase": "目标库", + "targetTable": "目标表", + "datasource": "数据源", + "preQueries": "预执行SQL", + "addPreQueries": "添加预执行SQL", + "fileFormat": "文件格式", + "path": "文件路径", + "delete": "删除", + "columnNames": "字段名", + "enter": "请输入", + "common": { + "globalVariable": "全局变量", + "addPlugin": "添加插件", + "addPath": "写入路径", + "outTableName": "输出表名", + "registryTableName": "注册表名" + } + } + +} \ No newline at end of file diff --git a/web/packages/sparketl/utils/index.js b/web/packages/sparketl/utils/index.js new file mode 100644 index 000000000..987f2558d --- /dev/null +++ b/web/packages/sparketl/utils/index.js @@ -0,0 +1,8 @@ +export const toJson = (s, separator) => { + let json = {}; + s.split(separator).forEach((line) => { + let option = line.split("=").map((item) => item.trim()); + if (option[0] !== "") json[option[0]] = option[1] || ""; + }); + return json; +}; diff --git a/web/packages/workflows/module/process/module.vue b/web/packages/workflows/module/process/module.vue index 99cc26cd2..9fc213728 100755 --- a/web/packages/workflows/module/process/module.vue +++ b/web/packages/workflows/module/process/module.vue @@ -294,6 +294,7 @@