diff --git a/kystudio/src/components/common/EditExcludeColumnsDialog/EditExcludeColumnsDialog.vue b/kystudio/src/components/common/EditExcludeColumnsDialog/EditExcludeColumnsDialog.vue new file mode 100644 index 00000000000..7f963970264 --- /dev/null +++ b/kystudio/src/components/common/EditExcludeColumnsDialog/EditExcludeColumnsDialog.vue @@ -0,0 +1,332 @@ + + + + + diff --git a/kystudio/src/components/common/EditExcludeColumnsDialog/locales.js b/kystudio/src/components/common/EditExcludeColumnsDialog/locales.js new file mode 100644 index 00000000000..95f17021edd --- /dev/null +++ b/kystudio/src/components/common/EditExcludeColumnsDialog/locales.js @@ -0,0 +1,32 @@ +export default { + en: { + addExcludeColumns: 'Add Exclude Columns', + appendExcludeColumns: 'Append Exclude Columns', + delExcludeColumns: 'Delete Exclude Columns', + limitExcludedTablesTip: 'Only 50 tables would be displayed. Please enter keyword to search.', + selectTable: 'Select Table', + selectColumns: 'Select Columns', + filterByColumns: 'Search by column name', + delAllColumnTitle: 'Are you sure you want to delete all exclude columns?', + addAllColumnTitle: 'Are you sure you want to exclude all columns?', + delAllColumnBtn: 'Delete all columns', + addAllColumnBtn: 'Add all columns', + commentTip: 'Synced from the source table.', + excludeDesc: 'Please pay attention that excluded column can only be set to lookup table,in order to solve SCD problems.
If you want to exclude a whole table,you can exclude all columns of the table.' + }, + 'zh-cn': { + addExcludeColumns: '添加屏蔽列', + appendExcludeColumns: '增加屏蔽列', + delExcludeColumns: '删除屏蔽列', + limitExcludedTablesTip: '默认仅展示 50 张表。请输入关键词进行搜索。', + selectTable: '选择表', + selectColumns: '选择列', + filterByColumns: '搜索列名', + delAllColumnTitle: '确定要删除所有屏蔽列？', + addAllColumnTitle: '确定要屏蔽所有列？', + delAllColumnBtn: '删除所有屏蔽列', + addAllColumnBtn: '屏蔽所有列', + commentTip: '从数据源中同步的注释信息。', + excludeDesc: '请注意屏蔽列仅用于维度表，用于解决缓慢变化维的问题。
如果您想使用旧版本的“屏蔽表”功能，在当前版本等同于屏蔽一张表中所有的列。' + } +} diff --git a/kystudio/src/components/common/EditExcludeColumnsDialog/store.js b/kystudio/src/components/common/EditExcludeColumnsDialog/store.js new file mode 100644 index 00000000000..3b47824edea --- /dev/null +++ b/kystudio/src/components/common/EditExcludeColumnsDialog/store.js @@ -0,0 +1,47 @@ +import * as actionTypes from '../../../store/types' + +export function getInitialState () { + return { + isShow: false, + callback: null, + excludeColumntitle: '', + excludeTable: '' + } +} + +export default { + state: getInitialState(), + mutations: { + [actionTypes.SHOW_MODAL] (state) { + state.isShow = true + }, + [actionTypes.HIDE_MODAL] (state) { + state.isShow = false + }, + [actionTypes.SET_MODAL] (state, payload) { + for (const [key, value] of Object.entries(payload)) { + state[key] = value + } + }, + [actionTypes.INIT_MODAL] (state) { + for (const [key, value] of Object.entries(getInitialState())) { + state[key] = value + } + }, + [actionTypes.SET_MODAL_FORM] (state, payload) { + for (const [key, value] of Object.entries(payload)) { + state.form[key] = value + } + } + }, + actions: { + [actionTypes.CALL_MODAL] ({ commit }, payload) { + return new Promise(resolve => { + commit(actionTypes.INIT_MODAL) + commit(actionTypes.SET_MODAL, { ...payload, callback: resolve }) + commit(actionTypes.SHOW_MODAL) + }) + } + }, + namespaced: true +} diff --git a/kystudio/src/components/setting/SettingBasic/SettingBasic.vue b/kystudio/src/components/setting/SettingBasic/SettingBasic.vue index 0030282513b..7803f822008 100644 --- a/kystudio/src/components/setting/SettingBasic/SettingBasic.vue +++ b/kystudio/src/components/setting/SettingBasic/SettingBasic.vue @@ -137,6 +137,66 @@ + + +
+
+ {{$t('excludeRule')}} + +
+
+
+
+

{{$t('moreDetails')}}

+
+
    +
  1. •{{$t('excludeRuleDetailMsg1')}}
    2. +
  2. •{{$t('excludeRuleDetailMsg2')}}
    3. +
  3. •{{$t('excludeRuleDetailMsg3')}}
    4. +
+
+
+
+ {{$t('addExcludeColumns')}} + + + + + + + + + + +
+
+
+
+ + @@ -146,10 +206,12 @@ import { mapActions, mapGetters, mapMutations, mapState } from 'vuex' import { Component, Watch } from 'vue-property-decorator' import locales from './locales' -import { handleError, handleSuccessAsync, objectClone, ArrayFlat } from '../../../util' -import { projectTypeIcons, lowUsageStorageTypes, autoMergeTypes, volatileTypes, validate, initialFormValue, _getProjectGeneralInfo, _getSegmentSettings, _getPushdownConfig, _getStorageQuota, _getIndexOptimization, _getRetentionRangeScale } from './handler' +import { handleError, handleSuccessAsync, objectClone, ArrayFlat, kylinConfirm } from '../../../util' +import { projectTypeIcons, lowUsageStorageTypes, autoMergeTypes, volatileTypes, validate, initialFormValue, _getProjectGeneralInfo, _getSegmentSettings, _getPushdownConfig, _getExcludeColumnConfig, _getStorageQuota, _getIndexOptimization, _getRetentionRangeScale } from './handler' import { retentionTypes } from '../handler' +import { pageCount, pageRefTags } from '../../../config' import EditableBlock from '../../common/EditableBlock/EditableBlock.vue' +import ExcludeColumnsDialog from '../../common/EditExcludeColumnsDialog/EditExcludeColumnsDialog.vue' import SourceAuthorityForm from '../../common/DataSourceModal/SourceJDBC/SourceAuthorityForm/SourceAuthorityForm.vue' @Component({ @@ -161,7 +223,8 @@ import SourceAuthorityForm from '../../common/DataSourceModal/SourceJDBC/SourceA }, components: { EditableBlock, - SourceAuthorityForm + SourceAuthorityForm, + ExcludeColumnsDialog }, computed: { ...mapGetters([ @@ -184,11 +247,16 @@ import SourceAuthorityForm from '../../common/DataSourceModal/SourceJDBC/SourceA getUserAndGroups: 'GET_USER_AND_GROUPS', updateFavoriteRules: 'UPDATE_FAVORITE_RULES', fetchDBandTables: 'FETCH_DB_AND_TABLES', - checkConnectByGbase: 'CHECK_BASE_CONFIG' + checkConnectByGbase: 'CHECK_BASE_CONFIG', + loadExcludeTables: 'LOAD_EXCLUDE_TABLES', + updateExcludeColumnConfig: 'UPDATE_EXCLUDE_COLUMN_CONFIG' }), ...mapActions('DetailDialogModal', { callGlobalDetailDialog: 'CALL_MODAL' }), + ...mapActions('ExcludeColumnsDialog', { + callExcludeColumnsDialog: 'CALL_MODAL' + }), ...mapMutations({ updateProject: 'UPDATE_PROJECT' }) @@ -213,6 +281,14 @@ export default class SettingBasic extends Vue { JDBCConnectSettingBackup = [] jdbcDatasourceEnabled = false allDatasourceTables = [] + showExcludeRuleDetails = false + pageRefTags = pageRefTags + excludeColumnsTables = [] + filter = { + page_offset: 0, + page_size: +localStorage.getItem(this.pageRefTags.excludeColumnsTablesPager) || pageCount + } + excludeColumnsTablesSize = 1 get projectIcon () { return projectTypeIcons[this.project.maintain_model_type] @@ -313,10 +389,14 @@ export default class SettingBasic extends Vue { this.handleInit('pushdown-settings') this.handleInit('storage-quota') this.handleInit('index-optimization') + this.handleInit('exclude-rule') } async mounted () { this.initForm() this.getAllDatasourceTables() + if (this.$store.state.project.projectExcludeTableConfig) { + this.getExcludeColumns() + } } handleCheckMergeRanges (value) { if (value.length > 0) { @@ -346,6 +426,23 @@ export default class SettingBasic extends Vue { submitData.push_down_enabled = value await this.updatePushdownConfig(submitData); break } + case 'exclude-rule': { + if (!value) { + try { + await kylinConfirm(this.$t('confirmCloseExcludeRule'), {type: 'warning'}, this.$t('closeExcludeRuleTitle')) + } catch (e) { + return + } + } + const submitData = _getExcludeColumnConfig(this.project) + submitData.table_exclusion_enabled = value + await this.updateExcludeColumnConfig(submitData) + this.form.table_exclusion_enabled = value + if (value) { + this.getExcludeColumns() + } + break + } } this.$emit('reload-setting') this.$message({ type: 'success', message: this.$t('kylinLang.common.updateSuccess') }) @@ -474,6 +571,9 @@ export default class SettingBasic extends Vue { case 'index-optimization': { this.form = { ...this.form, ..._getIndexOptimization(this.project) }; break } + case 'exclude-rule': { + this.form = { ...this.form, ..._getExcludeColumnConfig(this.project) }; break + } } } async handleResetForm (type, successCallback, errorCallback) { @@ -554,6 +654,37 @@ export default class SettingBasic extends Vue { } } + async addExcludeColumns (excludeTable) { + const isSubmit = await this.callExcludeColumnsDialog({ excludeColumntitle: !excludeTable ? 'addExcludeColumns' : 'appendExcludeColumns', excludeTable: excludeTable }) + if (isSubmit) { + this.getExcludeColumns() + } + } + + async delExcludeColumns (excludeTable) { + const isSubmit = await this.callExcludeColumnsDialog({ excludeColumntitle: 'delExcludeColumns', excludeTable: excludeTable }) + if (isSubmit) { + this.getExcludeColumns() + } + } + + currentChange (size, count) { + this.filter.page_offset = size + this.filter.page_size = count + this.getExcludeColumns() + } + + async getExcludeColumns () { + try { + const res = await this.loadExcludeTables({ ...this.filter, project: this.currentSelectedProject }) + const { value, total_size } = await handleSuccessAsync(res) + this.excludeColumnsTables = value + this.excludeColumnsTablesSize = total_size + } catch (e) { + handleError(e) + } + } + // 更改数据源设置 modifyDataSourceSetting (index) { const key = Date.now().toString(32) @@ -691,6 +822,26 @@ export default class SettingBasic extends Vue { } } } + .exclude-rule-msg { + font-size: 12px; + .tips { + .review-details { + color: @base-color; + cursor: pointer !important; + position: relative; + } + } + .details { + background: @base-background-color-1; + padding: 10px 10px; + margin-top: 5px; + box-sizing: border-box; + .point { + font-style: inherit; + margin-right: 5px; + } + } + } .ruleSetting { padding: 15px 20px; .conds-title { @@ -716,38 +867,6 @@ export default class SettingBasic extends Vue { .conds { margin-bottom: 16px; } - .exclude-rule-msg { - font-size: 12px; - .tips { - .review-details { - color: @base-color; - cursor: pointer; - position: relative; - .arrow { - transform: rotate(90deg); - font-size: 10px; - margin-left: 5px; - } - } - } - .details { - background: @base-background-color-1; - padding: 10px 10px; - margin-top: 5px; - box-sizing: border-box; - .point { - font-style: inherit; - margin-right: 5px; - } - } - } - .exclude_rule-form { - width: 100%; - .exclude_rule-select { - width: 100%; - margin-top: 10px; - } - } } .rule-setting-input { display: inline-block; @@ -769,12 +888,5 @@ export default class SettingBasic extends Vue { font-size: 12px; } } -.limit-excluded-tables-msg { - height: 32px; - color: @text-normal-color; - line-height: 32px; - text-align: center; - font-size: 12px; -} diff --git a/kystudio/src/components/setting/SettingBasic/handler.js b/kystudio/src/components/setting/SettingBasic/handler.js index 5fbbce315a0..47fe3671824 100644 --- a/kystudio/src/components/setting/SettingBasic/handler.js +++ b/kystudio/src/components/setting/SettingBasic/handler.js @@ -110,6 +110,12 @@ export function _getPushdownConfig (data) { push_down_range_limited: data.push_down_range_limited } } +export function _getExcludeColumnConfig (data) { + return { + project: data.project, + table_exclusion_enabled: data.table_exclusion_enabled + } +} export function _getStorageQuota (data) { return { project: data.project, diff --git a/kystudio/src/components/setting/SettingBasic/locales.js b/kystudio/src/components/setting/SettingBasic/locales.js index 8325ffca1bc..a110ec35468 100644 --- a/kystudio/src/components/setting/SettingBasic/locales.js +++ b/kystudio/src/components/setting/SettingBasic/locales.js @@ -85,12 +85,20 @@ export default { emptySegmentEnableDesc: 'With this switch ON, you may create a segment with no index (reserved segment). Please note that queries would be answered by pushdown engine when they hit reserved segments.', overTimeLimitTip: 'Can\'t exceed 3600 seconds', prevGreaterThanNext: 'The content should be greater than the lower limit', + excludeRuleSettings: 'Exclude Column Rule Setting', excludeRule: 'Exclude Rule', - excludeRuleTip: 'Columns in the following tables will not be included in the recommendations. Suitable for AS-IS analysis. ', + excludeRuleTip: 'Excluded column is used for Slowly Changing Dimension(SCD),especially for the situation that you want to use SCD type 1 for some columns and SCD type 2 for others.', + closeExcludeRuleTitle: 'Close Exclude Column Setting', + confirmCloseExcludeRule: 'If close the exclude rules, please pay attention that the former excluded columns is still in AS-IS (SCD Type1) mode until aggregate index containing these columns is added.', moreDetails: 'More Info', - excludeRuleDetailMsg1: 'If the data of certain columns in a dimension table change frequently, the table can be added into the exclude list, and aggregate indexes containing these columns should be deleted.', - excludeRuleDetailMsg2: 'Then the foreign key in fact table will be included in the recommended index, replacing columns in the following tables. After building the indexes, queries for these tables can be retrieved through the foreign key.', - limitExcludedTablesTip: 'Only 50 tables would be displayed. Please enter keyword to search.', + excludeRuleDetailMsg1: 'If the data of certain columns in a dimension table changes frequently, the columns can be added into the exclude list, and aggregate indexes containing these columns should be deleted. In that way you can apply AS-IS(SCD type1) to these columns', + excludeRuleDetailMsg2: 'Then the foreign key in fact table will be included in the recommended index, replacing excluded columns. After building the indexes, queries for these columns can be retrieved through the foreign key.', + excludeRuleDetailMsg3: 'If you want to use SCD type 2 for some other columns,you can add them to aggregate indexes.', + addExcludeColumns: 'Add Exclude Columns', + delExcludeColumns: 'Delete Exclude Columns', + allColumns: 'All columns', + table: 'Table', + columns: 'Columns', forbidenSwitchJdbcTips: 'Can\'t turn it OFF when there exists JDBC table in the project', hitRules: 'Hit Rules', ruleTips: 'Only the recommendations complying the following rules will be recommended.', diff --git a/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/index.vue b/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/index.vue index 857a7bbcb03..33332393be8 100644 --- a/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/index.vue +++ b/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/index.vue @@ -467,7 +467,10 @@ show-icon>
- {{$t('excludeTableCheckbox')}} +
@@ -736,9 +739,9 @@ export default class AggregateModal extends Vue { } // 是否展示屏蔽表 checkbox - get showExcludedTableCheckBox () { - return this.backUpDimensions.length ? this.backUpDimensions.filter(it => typeof it.depend_lookup_table !== 'undefined' && it.depend_lookup_table).length > 0 : false - } + // get showExcludedTableCheckBox () { + // return this.backUpDimensions.length ? this.backUpDimensions.filter(it => typeof it.excluded !== 'undefined' && it.excluded).length > 0 : false + // } // 是否存在多对多且被屏蔽的表 get hasManyToManyAndAntiTable () { @@ -766,7 +769,7 @@ export default class AggregateModal extends Vue { // 是否为屏蔽表的 column isExistExcludeTable (col) { - return typeof col.depend_lookup_table !== 'undefined' ? col.depend_lookup_table : false + return typeof col.excluded !== 'undefined' ? col.excluded : false } getMultipleCardinality (aggregateIdx, jointRowIdx) { diff --git a/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/locales.js b/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/locales.js index a9c81bec648..c48786c10b2 100644 --- a/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/locales.js +++ b/kystudio/src/components/studio/StudioModel/ModelList/AggregateModal/locales.js @@ -108,7 +108,7 @@ export default { buildIndexTip: 'Successfully saved the aggregate index(es). The building job can\'t be submitted at the moment, as there exists an ongoing building job for this model. Please try submitting the building job until the current one is completed or manually stop it.', disabledConstantMeasureTip: 'Can\'t modify the default measure.', excludeTableCheckbox: 'Display columns excluded from recommendations', - excludeTableCheckboxTip: 'Exclude Rules can be modified in project setting', + excludeTableCheckboxTip: 'If an excluded column is added to indexes, this column will store "historical truth"(SCD Type2, As Was).', excludedTableIconTip: 'Excluded from recommendations', indexTimeRange: 'Index’s Data Range', manyToManyAntiTableTip: 'For the tables excluded from recommendations, if the join relationship of a table is One-to-Many or Many-to-Many, dimensions from this table can\'t be used in indexes. ', diff --git a/kystudio/src/components/studio/StudioModel/TableIndexEdit/locales.js b/kystudio/src/components/studio/StudioModel/TableIndexEdit/locales.js index c7fa4338a43..031267a13ac 100644 --- a/kystudio/src/components/studio/StudioModel/TableIndexEdit/locales.js +++ b/kystudio/src/components/studio/StudioModel/TableIndexEdit/locales.js @@ -23,7 +23,7 @@ export default { tableIndexShardByTips: 'Please select columns for detail query. To enhance the query performance, please move the frequently used dimensions to the top of the list, and set a column with relatively large cardinality as ShardBy.', filterByColumns: 'Search by column name', excludeTableCheckbox: 'Display columns excluded from recommendations', - excludeTableCheckboxTip: 'Exclude Rules can be modified in project setting', + excludeTableCheckboxTip: 'If an excluded column is added to indexes, this column will store "historical truth"(SCD Type2, As Was).
If you want to use "latest status" (SCD Type 1, As Is) ,please don\'t add this column to any index and add foreign key instead.', excludedTableIconTip: 'Excluded from recommendations', manyToManyAntiTableTip: 'For the tables excluded from recommendations, if the join relationship of a table is One-to-Many or Many-to-Many, dimensions from this table can\'t be used in indexes. ', indexTimeRange: 'Index’s Time Range', diff --git a/kystudio/src/components/studio/StudioModel/TableIndexEdit/tableindex_edit.vue b/kystudio/src/components/studio/StudioModel/TableIndexEdit/tableindex_edit.vue index aa0c1c12132..08340e79a7b 100644 --- a/kystudio/src/components/studio/StudioModel/TableIndexEdit/tableindex_edit.vue +++ b/kystudio/src/components/studio/StudioModel/TableIndexEdit/tableindex_edit.vue @@ -36,7 +36,10 @@ @@ -171,9 +174,9 @@ return (this.modelInstance.model_type === 'HYBRID' && this.tableIndexMeta.index_range !== 'STREAMING') || (this.modelInstance.model_type !== 'STREAMING' && this.modelInstance.model_type !== 'HYBRID') } - get showExcludedTableCheckBox () { - return this.allColumns.length ? this.allColumns.filter(it => typeof it.depend_lookup_table !== 'undefined' && it.depend_lookup_table).length > 0 : false - } + // get showExcludedTableCheckBox () { + // return this.allColumns.length ? this.allColumns.filter(it => typeof it.excluded !== 'undefined' && it.excluded).length > 0 : false + // } topRow (col) { let index = this.getRowIndex(col, 'fullName') this.allColumns.splice(0, 0, col) @@ -223,7 +226,7 @@ } // 是否为屏蔽表的 column isExistExcludeTable (col) { - return typeof col.depend_lookup_table !== 'undefined' ? col.depend_lookup_table : false + return typeof col.excluded !== 'undefined' ? col.excluded : false } get filterResult () { if (!this.isShow) { @@ -247,7 +250,7 @@ this.allColumns = [] // let result = [] let result = this.modelInstance.selected_columns.map((c) => { - return { fullName: c.column, cardinality: c.cardinality, depend_lookup_table: typeof c.depend_lookup_table !== 'undefined' ? c.depend_lookup_table : true } + return { fullName: c.column, cardinality: c.cardinality, excluded: typeof c.excluded !== 'undefined' ? c.excluded : true } }) // let modelUsedTables = this.modelInstance && this.modelInstance.getTableColumns() || [] // modelUsedTables.forEach((col) => { @@ -256,7 +259,7 @@ if (this.tableIndexMeta.col_order.length) { const selected = this.tableIndexMeta.col_order.map(item => { const index = result.findIndex(it => it.fullName === item) - return {fullName: item, cardinality: result[index].cardinality, depend_lookup_table: result[index].depend_lookup_table} + return {fullName: item, cardinality: result[index].cardinality, excluded: result[index].excluded} }) const unSort = result.filter(item => !this.tableIndexMeta.col_order.includes(item.fullName)) result = [...selected, ...unSort] @@ -267,7 +270,7 @@ // result.push(col.tableAlias + '.' + col.columnName) // }) result.forEach((ctx, index) => { - let obj = {fullName: ctx.fullName, cardinality: ctx.cardinality, depend_lookup_table: ctx.depend_lookup_table, isUsed: false, isShared: false, colorful: false} + let obj = {fullName: ctx.fullName, cardinality: ctx.cardinality, excluded: ctx.excluded, isUsed: false, isShared: false, colorful: false} if (this.tableIndexMeta.col_order.indexOf(ctx.fullName) >= 0) { obj.isUsed = true } diff --git a/kystudio/src/components/studio/StudioSource/TableColumns/TableColumns.vue b/kystudio/src/components/studio/StudioSource/TableColumns/TableColumns.vue index 63c66238901..1efcefdc9e5 100644 --- a/kystudio/src/components/studio/StudioSource/TableColumns/TableColumns.vue +++ b/kystudio/src/components/studio/StudioSource/TableColumns/TableColumns.vue @@ -35,6 +35,18 @@ show-overflow-tooltip :label="$t('kylinLang.dataSource.dataType')"> + + + {{scope.row.excluded?$t('yes'):$t('no')}} + +