Skip to content

Commit 98ffad3

Browse files
VictorTrapenokVictorTrapenok
authored
feat: sqlAlias attribute for preAggregations and short format for pre-aggregation table names (#1068)
* use seconds instead of ms * add maxLength for identifierRegex * Add preAggregations test * Draft for preAggregation.sqlAlias functionality * added sqlAlias and cat tables last update value * remove test code * tests and fix bug. * Add fail early checks for Postgres and Mysql pre-aggregation table names truncate issue * set oracle limit to 128 characters * add docs * code style fix * code style * use naming_version for preAggregation tables * error text * code style * added tests Fixes #86 Fixes #907
1 parent 26c0420 commit 98ffad3

File tree

14 files changed

+595
-34
lines changed

14 files changed

+595
-34
lines changed

‎docs/Schema/pre-aggregations.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ Pre-aggregation name together with cube name will be used as a prefix for pre-ag
2323
Pre-aggregations names should:
2424
- Be unique within a cube
2525
- Start with a lowercase letter
26+
- For long pre-aggregations names, you can set the `sqlAlias` attribute
2627

2728
You can use `0-9`,`_` and letters when naming pre-aggregations.
2829

‎packages/cubejs-query-orchestrator/orchestrator/PreAggregations.js

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,17 @@ const LocalCacheDriver = require('./LocalCacheDriver');
77
const QueryCache = require('./QueryCache');
88
const ContinueWaitError = require('./ContinueWaitError');
99

10-
function version(cacheKey) {
10+
function encodeTimeStamp(time) {
11+
return Math.floor(time / 1000).toString(32);
12+
}
13+
14+
function version(cacheKey, isDigestKey = false) {
1115
let result = '';
1216
const hashCharset = 'abcdefghijklmnopqrstuvwxyz012345';
13-
const digestBuffer = crypto.createHash('md5').update(JSON.stringify(cacheKey)).digest();
17+
let digestBuffer = Number(cacheKey);
18+
if (!isDigestKey) {
19+
digestBuffer = crypto.createHash('md5').update(JSON.stringify(cacheKey)).digest();
20+
}
1421

1522
let residue = 0;
1623
let shiftCounter = 0;
@@ -35,15 +42,25 @@ const tablesToVersionEntries = (schema, tables) => R.sortBy(
3542
table => -table.last_updated_at,
3643
tables.map(table => {
3744
const match = (table.table_name || table.TABLE_NAME).match(/(.+)_(.+)_(.+)_(.+)/);
38-
if (match) {
39-
return {
40-
table_name: `${schema}.${match[1]}`,
41-
content_version: match[2],
42-
structure_version: match[3],
43-
last_updated_at: parseInt(match[4], 10)
44-
};
45+
46+
if (!match) {
47+
return null;
4548
}
46-
return null;
49+
50+
const entity = {
51+
table_name: `${schema}.${match[1]}`,
52+
content_version: match[2],
53+
structure_version: match[3]
54+
};
55+
56+
if (match[4].length < 13) {
57+
entity.last_updated_at = parseInt(match[4], 32) * 1000;
58+
entity.naming_version = 2;
59+
} else {
60+
entity.last_updated_at = parseInt(match[4], 10);
61+
}
62+
63+
return entity;
4764
}).filter(R.identity)
4865
);
4966

@@ -304,11 +321,13 @@ class PreAggregationLoader {
304321
) || versionEntries.find(
305322
e => e.table_name === this.preAggregation.tableName
306323
);
324+
307325
const newVersionEntry = {
308326
table_name: this.preAggregation.tableName,
309327
structure_version: structureVersion,
310328
content_version: contentVersion,
311-
last_updated_at: new Date().getTime()
329+
last_updated_at: new Date().getTime(),
330+
naming_version: 2,
312331
};
313332

314333
const mostRecentTargetTableName = async () => {
@@ -738,6 +757,10 @@ class PreAggregations {
738757
}
739758

740759
static targetTableName(versionEntry) {
760+
if (versionEntry.naming_version === 2) {
761+
return `${versionEntry.table_name}_${versionEntry.content_version}_${versionEntry.structure_version}_${encodeTimeStamp(versionEntry.last_updated_at)}`;
762+
}
763+
741764
return `${versionEntry.table_name}_${versionEntry.content_version}_${versionEntry.structure_version}_${versionEntry.last_updated_at}`;
742765
}
743766
}

‎packages/cubejs-query-orchestrator/test/PreAggregations.test.js

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,4 +252,86 @@ describe('PreAggregations', () => {
252252
expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il/);
253253
});
254254
});
255+
256+
describe(`naming_version tests`, () => {
257+
let preAggregations = null;
258+
let PreAggregations = null;
259+
260+
beforeEach(async () => {
261+
PreAggregations = require('../orchestrator/PreAggregations');
262+
preAggregations = new PreAggregations(
263+
"TEST",
264+
mockDriverFactory,
265+
(msg, params) => {},
266+
queryCache,
267+
{
268+
queueOptions: {
269+
executionTimeout: 1
270+
},
271+
externalDriverFactory: async () => {
272+
const driver = mockExternalDriver;
273+
driver.createTable("stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1593709044209");
274+
driver.createTable("stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652");
275+
return driver;
276+
},
277+
},
278+
);
279+
});
280+
281+
test('test for function targetTableName', () => {
282+
let result = PreAggregations.targetTableName({
283+
table_name:'orders_number_and_count20191101',
284+
content_version:'kjypcoio',
285+
structure_version:'5yftl5il',
286+
last_updated_at:1600329890789,
287+
});
288+
expect(result).toEqual('orders_number_and_count20191101_kjypcoio_5yftl5il_1600329890789')
289+
290+
result = PreAggregations.targetTableName({
291+
table_name:'orders_number_and_count20191101',
292+
content_version:'kjypcoio',
293+
structure_version:'5yftl5il',
294+
last_updated_at:1600329890789,
295+
naming_version:2
296+
});
297+
expect(result).toEqual('orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652')
298+
});
299+
300+
test('naming_version and sort by last_updated_at', async () => {
301+
const result = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternal);
302+
expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652/);
303+
});
304+
});
305+
306+
describe(`naming_version sort tests`, () => {
307+
let preAggregations = null;
308+
let PreAggregations = null;
309+
310+
beforeEach(async () => {
311+
PreAggregations = require('../orchestrator/PreAggregations');
312+
preAggregations = new PreAggregations(
313+
"TEST",
314+
mockDriverFactory,
315+
(msg, params) => {},
316+
queryCache,
317+
{
318+
queueOptions: {
319+
executionTimeout: 1
320+
},
321+
externalDriverFactory: async () => {
322+
const driver = mockExternalDriver;
323+
driver.createTable("stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1893709044209");
324+
driver.createTable("stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1fm6652");
325+
return driver;
326+
},
327+
},
328+
);
329+
});
330+
331+
test('naming_version and sort by last_updated_at', async () => {
332+
const result = await preAggregations.loadAllPreAggregationsIfNeeded(basicQueryExternal);
333+
expect(result[0][1].targetTableName).toMatch(/stb_pre_aggregations.orders_number_and_count20191101_kjypcoio_5yftl5il_1893709044209/);
334+
});
335+
});
336+
255337
});

‎packages/cubejs-schema-compiler/adapter/BaseQuery.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1738,7 +1738,8 @@ class BaseQuery {
17381738
}
17391739

17401740
preAggregationTableName(cube, preAggregationName, skipSchema) {
1741-
return `${skipSchema ? '' : this.preAggregationSchema() && `${this.preAggregationSchema()}.`}${this.aliasName(`${cube}.${preAggregationName}`, true)}`;
1741+
const tblName = this.aliasName(`${cube}.${preAggregationName}`, true);
1742+
return `${skipSchema ? '' : this.preAggregationSchema() && `${this.preAggregationSchema()}.`}${tblName}`;
17421743
}
17431744

17441745
preAggregationSchema() {

‎packages/cubejs-schema-compiler/adapter/BigqueryQuery.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ class BigqueryQuery extends BaseQuery {
127127
}
128128

129129
concatStringsSql(strings) {
130-
return `CONCAT(${strings.join(", ")})`;
130+
return `CONCAT(${strings.join(', ')})`;
131131
}
132132

133133
defaultRefreshKeyRenewalThreshold() {

‎packages/cubejs-schema-compiler/adapter/MysqlQuery.js

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ const moment = require('moment-timezone');
22

33
const BaseQuery = require('./BaseQuery');
44
const BaseFilter = require('./BaseFilter');
5+
const UserError = require('../compiler/UserError');
56

67
const GRANULARITY_TO_INTERVAL = {
78
day: (date) => `DATE_FORMAT(${date}, '%Y-%m-%dT00:00:00.000')`,
@@ -64,7 +65,7 @@ class MysqlQuery extends BaseQuery {
6465
}
6566

6667
concatStringsSql(strings) {
67-
return `CONCAT(${strings.join(", ")})`;
68+
return `CONCAT(${strings.join(', ')})`;
6869
}
6970

7071
unixTimestampSql() {
@@ -74,6 +75,14 @@ class MysqlQuery extends BaseQuery {
7475
wrapSegmentForDimensionSelect(sql) {
7576
return `IF(${sql}, 1, 0)`;
7677
}
78+
79+
preAggregationTableName(cube, preAggregationName, skipSchema) {
80+
const name = super.preAggregationTableName(cube, preAggregationName, skipSchema);
81+
if (name.length > 64) {
82+
throw new UserError(`MySQL can not work with table names that longer than 64 symbols. Consider using the 'sqlAlias' attribute in your cube and in your pre-aggregation definition for ${name}.`);
83+
}
84+
return name;
85+
}
7786
}
7887

7988
module.exports = MysqlQuery;

‎packages/cubejs-schema-compiler/adapter/OracleQuery.js

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
const BaseQuery = require('./BaseQuery');
22
const BaseFilter = require('./BaseFilter');
3+
const UserError = require('../compiler/UserError');
34

45
const GRANULARITY_VALUE = {
56
day: 'DD',
@@ -54,7 +55,7 @@ class OracleQuery extends BaseQuery {
5455
if (!dimensions.length) {
5556
return '';
5657
}
57-
return ` GROUP BY ${dimensions.map(item => item.dimensionSql()).join(", ")}`;
58+
return ` GROUP BY ${dimensions.map(item => item.dimensionSql()).join(', ')}`;
5859
}
5960

6061
convertTz(field) {
@@ -90,6 +91,14 @@ class OracleQuery extends BaseQuery {
9091
unixTimestampSql() {
9192
return `((cast (systimestamp at time zone 'UTC' as date) - date '1970-01-01') * 86400)`;
9293
}
94+
95+
preAggregationTableName(cube, preAggregationName, skipSchema) {
96+
const name = super.preAggregationTableName(cube, preAggregationName, skipSchema);
97+
if (name.length > 128) {
98+
throw new UserError(`Oracle can not work with table names that longer than 64 symbols. Consider using the 'sqlAlias' attribute in your cube and in your pre-aggregation definition for ${name}.`);
99+
}
100+
return name;
101+
}
93102
}
94103

95104
module.exports = OracleQuery;

‎packages/cubejs-schema-compiler/adapter/PostgresQuery.js

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
const BaseQuery = require('./BaseQuery');
22
const ParamAllocator = require('./ParamAllocator');
3+
const UserError = require('../compiler/UserError');
34

45
const GRANULARITY_TO_INTERVAL = {
56
day: 'day',
@@ -41,6 +42,14 @@ class PostgresQuery extends BaseQuery {
4142
countDistinctApprox(sql) {
4243
return `round(hll_cardinality(hll_add_agg(hll_hash_any(${sql}))))`;
4344
}
45+
46+
preAggregationTableName(cube, preAggregationName, skipSchema) {
47+
const name = super.preAggregationTableName(cube, preAggregationName, skipSchema);
48+
if (name.length > 64) {
49+
throw new UserError(`Postgres can not work with table names that longer than 64 symbols. Consider using the 'sqlAlias' attribute in your cube and in your pre-aggregation definition for ${name}.`);
50+
}
51+
return name;
52+
}
4453
}
4554

4655
module.exports = PostgresQuery;

‎packages/cubejs-schema-compiler/adapter/PreAggregations.js

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,8 @@ class PreAggregations {
9898

9999
preAggregationDescriptionFor(cube, foundPreAggregation) {
100100
const { preAggregationName, preAggregation } = foundPreAggregation;
101-
const tableName = this.preAggregationTableName(cube, preAggregationName, preAggregation);
101+
const name = foundPreAggregation.sqlAlias || preAggregationName;
102+
const tableName = this.preAggregationTableName(cube, name, preAggregation);
102103
const refreshKeyQueries = this.query.preAggregationInvalidateKeyQueries(cube, preAggregation);
103104
return {
104105
preAggregationsSchema: this.query.preAggregationSchema(),
@@ -110,7 +111,7 @@ class PreAggregations {
110111
external: preAggregation.external,
111112
indexesSql: Object.keys(preAggregation.indexes || {}).map(
112113
index => {
113-
const indexName = this.preAggregationTableName(cube, `${preAggregationName}_${index}`, preAggregation, true);
114+
const indexName = this.preAggregationTableName(cube, `${foundPreAggregation.sqlAlias || preAggregationName}_${index}`, preAggregation, true);
114115
return {
115116
indexName,
116117
sql: this.query.indexSql(
@@ -135,9 +136,11 @@ class PreAggregations {
135136
preAggregation.partitionGranularity === 'hour' ? 13 : 10
136137
).replace(/[-T:]/g, '');
137138
}
139+
140+
const name = preAggregation.sqlAlias || preAggregationName;
138141
return this.query.preAggregationTableName(
139142
cube,
140-
preAggregationName + partitionSuffix,
143+
name + partitionSuffix,
141144
skipSchema
142145
);
143146
}
@@ -475,20 +478,27 @@ class PreAggregations {
475478
}
476479

477480
originalSqlPreAggregationTable(preAggregation) {
478-
return this.canPartitionsBeUsed(preAggregation) ?
479-
this.partitionUnion(preAggregation, true) :
480-
this.query.preAggregationTableName(
481-
preAggregation.cube,
482-
preAggregation.preAggregationName
483-
);
481+
if (this.canPartitionsBeUsed(preAggregation)) {
482+
return this.partitionUnion(preAggregation, true);
483+
}
484+
485+
let { preAggregationName } = preAggregation;
486+
if (preAggregation.preAggregation && preAggregation.preAggregation.sqlAlias) {
487+
preAggregationName = preAggregation.preAggregation.sqlAlias;
488+
}
489+
490+
return this.query.preAggregationTableName(
491+
preAggregation.cube,
492+
preAggregationName
493+
);
484494
}
485495

486496
rollupPreAggregation(preAggregationForQuery) {
487497
const table = this.canPartitionsBeUsed(preAggregationForQuery) ?
488498
this.partitionUnion(preAggregationForQuery) :
489499
this.query.preAggregationTableName(
490500
preAggregationForQuery.cube,
491-
preAggregationForQuery.preAggregationName
501+
preAggregationForQuery.sqlAlias || preAggregationForQuery.preAggregationName
492502
);
493503
const segmentFilters = this.query.segments.map(
494504
s => this.query.newFilter({ dimension: s.segment, operator: 'equals', values: [true] })
@@ -542,7 +552,7 @@ class PreAggregations {
542552
const preAggregation = this.addPartitionRangeTo(preAggregationForQuery, dimension, range);
543553
return this.preAggregationTableName(
544554
preAggregationForQuery.cube,
545-
preAggregationForQuery.preAggregationName,
555+
preAggregationForQuery.sqlAlias || preAggregationForQuery.preAggregationName,
546556
preAggregation.preAggregation
547557
);
548558
});

‎packages/cubejs-schema-compiler/compiler/CubeValidator.js

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
const Joi = require('@hapi/joi');
22

33
const identifierRegex = /^[_a-zA-Z][_a-zA-Z0-9]*$/;
4-
const identifier = Joi.string().regex(/^[_a-zA-Z][_a-zA-Z0-9]*$/, 'identifier');
4+
5+
const identifier = Joi.string().regex(identifierRegex, 'identifier');
56
const timeInterval =
67
Joi.alternatives([
78
Joi.string().regex(/^(-?\d+) (minute|hour|day|week|month|year)$/, 'time interval'),
@@ -77,6 +78,7 @@ const BasePreAggregationWithoutPartitionGranularity = {
7778
updateWindow: timeInterval
7879
})
7980
),
81+
sqlAlias: Joi.string().optional(),
8082
useOriginalSqlPreAggregations: Joi.boolean(),
8183
external: Joi.boolean(),
8284
scheduledRefresh: Joi.boolean(),
@@ -194,7 +196,7 @@ const cubeSchema = Joi.object().keys({
194196
preAggregations: Joi.object().pattern(identifierRegex, Joi.alternatives().try(
195197
Joi.object().keys(Object.assign({}, BasePreAggregation, {
196198
type: Joi.any().valid('autoRollup').required(),
197-
maxPreAggregations: Joi.number()
199+
maxPreAggregations: Joi.number(),
198200
})),
199201
Joi.object().keys(Object.assign({}, BasePreAggregation, {
200202
type: Joi.any().valid('originalSql').required(),

0 commit comments

Comments
 (0)