From 73d128dbf4e68021ec1ea237d0ea6f3117349d68 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 5 May 2025 12:42:55 +0300 Subject: [PATCH 1/8] code polish --- .../src/adapter/CubeStoreQuery.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts index afb51ee45fbc8..e4c677c3d1074 100644 --- a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts @@ -134,13 +134,12 @@ export class CubeStoreQuery extends BaseQuery { } public seriesSql(timeDimension) { - const values = timeDimension.timeSeries().map( + return timeDimension.timeSeries().map( ([from, to]) => `select to_timestamp('${from}') date_from, to_timestamp('${to}') date_to` ).join(' UNION ALL '); - return values; } - public concatStringsSql(strings) { + public concatStringsSql(strings: string[]) { return `CONCAT(${strings.join(', ')})`; } @@ -148,7 +147,7 @@ export class CubeStoreQuery extends BaseQuery { return 'UNIX_TIMESTAMP()'; } - public wrapSegmentForDimensionSelect(sql) { + public wrapSegmentForDimensionSelect(sql: string) { return `IF(${sql}, 1, 0)`; } @@ -156,19 +155,19 @@ export class CubeStoreQuery extends BaseQuery { return `merge(${sql})`; } - public hllCardinalityMerge(sql) { + public hllCardinalityMerge(sql: string) { return `cardinality(merge(${sql}))`; } - public hllCardinality(sql) { + public hllCardinality(sql: string) { return `cardinality(${sql})`; } - public castToString(sql) { + public castToString(sql: string) { return `CAST(${sql} as VARCHAR)`; } - public countDistinctApprox(sql) { + public countDistinctApprox(sql: string) { // TODO: We should throw an error, but this gets called even when only `hllMerge` result is used. return `approx_distinct_is_unsupported_in_cubestore(${sql}))`; } From 465d6fd15eeb9b1846f69d462f4946020fd2fe98 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 5 May 2025 13:46:20 +0300 Subject: [PATCH 2/8] Refactor a bit --- .../src/orchestrator/PreAggregations.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts index 453e9546ff017..c45d9156c7d34 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts @@ -718,7 +718,7 @@ export class PreAggregations { public async getVersionEntries(preAggregations: PreAggregationDescription[], requestId): Promise { const loadCacheByDataSource = {}; - const getLoadCacheByDataSource = (dataSource = 'default', preAggregationSchema) => { + const getLoadCacheByDataSource = (preAggregationSchema, dataSource = 'default') => { if (!loadCacheByDataSource[`${dataSource}_${preAggregationSchema}`]) { loadCacheByDataSource[`${dataSource}_${preAggregationSchema}`] = new PreAggregationLoadCache( @@ -740,9 +740,9 @@ export class PreAggregations { preAggregations.map( async preAggregation => { const { dataSource, preAggregationsSchema } = preAggregation; - const cacheKey = getLoadCacheByDataSource(dataSource, preAggregationsSchema).tablesCachePrefixKey(preAggregation); + const cacheKey = getLoadCacheByDataSource(preAggregationsSchema, dataSource).tablesCachePrefixKey(preAggregation); if (!firstByCacheKey[cacheKey]) { - firstByCacheKey[cacheKey] = getLoadCacheByDataSource(dataSource, preAggregationsSchema).getVersionEntries(preAggregation); + firstByCacheKey[cacheKey] = getLoadCacheByDataSource(preAggregationsSchema, dataSource).getVersionEntries(preAggregation); const res = await firstByCacheKey[cacheKey]; return res.versionEntries; } From a3a2999ff346b41059e273ab1dd04a37ff855cea Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 5 May 2025 17:02:53 +0300 Subject: [PATCH 3/8] Add tests for daily partitions --- .../test/unit/PreAggregations.test.js | 138 +++++++++++++++++- 1 file changed, 135 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.js b/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.js index b5ae8ff871023..aa0bd850baa61 100644 --- a/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.js +++ b/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.js @@ -534,7 +534,7 @@ describe('PreAggregations', () => { }); describe('partitionPreAggregations', () => { - test('should construct correct partitionPreAggregations for dateRange in UTC', async () => { + test('should construct correct partitionPreAggregations for dateRange in UTC (Day partitions)', async () => { const loader = createLoader({ timezone: 'UTC', }); @@ -576,7 +576,7 @@ describe('PreAggregations', () => { expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); }); - test('should construct correct partitionPreAggregations for dateRange in America/New_York', async () => { + test('should construct correct partitionPreAggregations for dateRange in America/New_York (Day partitions)', async () => { const loader = createLoader({ timezone: 'America/New_York', // UTC-5 }); @@ -618,7 +618,7 @@ describe('PreAggregations', () => { expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-04T04:59:59.999'); }); - test('should construct correct partitionPreAggregations for dateRange in Asia/Tokyo', async () => { + test('should construct correct partitionPreAggregations for dateRange in Asia/Tokyo (Day partitions)', async () => { const loader = createLoader({ timezone: 'Asia/Tokyo', // UTC+9 }); @@ -659,6 +659,138 @@ describe('PreAggregations', () => { expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-02T15:00:00.000'); expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T14:59:59.999'); }); + + test('should construct correct partitionPreAggregations for dateRange in UTC (Hour partitions)', async () => { + const loader = createLoader({ + partitionGranularity: 'hour', + timezone: 'UTC', + }); + + const results = await loader.partitionPreAggregations(); + expect(results.length).toEqual(72); + + let [preAggDesc] = results; + expect(preAggDesc.tableName).toEqual('test_table2024010100'); // Partition tables are the same for all time zones + expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); // buildRange is the same for all time zones + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T00:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010100')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T00:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010100')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T00:59:59.999'); + + [, preAggDesc] = results; + expect(preAggDesc.tableName).toEqual('test_table2024010101'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T01:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T01:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010101')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T01:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T01:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010101')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T01:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T01:59:59.999'); + + // eslint-disable-next-line prefer-destructuring + preAggDesc = results[71]; + expect(preAggDesc.tableName).toEqual('test_table2024010323'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T23:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010323')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T23:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010323')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T23:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); + }); + + test('should construct correct partitionPreAggregations for dateRange in America/New_York (Hour partitions)', async () => { + const loader = createLoader({ + partitionGranularity: 'hour', + timezone: 'America/New_York', // UTC-5 + }); + + const results = await loader.partitionPreAggregations(); + expect(results.length).toEqual(72); + + let [preAggDesc] = results; + expect(preAggDesc.tableName).toEqual('test_table2024010100'); // Partition tables are the same for all time zones + expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); // buildRange is the same for all time zones + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T00:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010100')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T05:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T05:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010100')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T05:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T05:59:59.999'); + + [, preAggDesc] = results; + expect(preAggDesc.tableName).toEqual('test_table2024010101'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T01:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T01:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010101')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T06:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T06:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010101')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T06:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T06:59:59.999'); + + // eslint-disable-next-line prefer-destructuring + preAggDesc = results[71]; + expect(preAggDesc.tableName).toEqual('test_table2024010323'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T23:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010323')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-04T04:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-04T04:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010323')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-04T04:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-04T04:59:59.999'); + }); + + test('should construct correct partitionPreAggregations for dateRange in Asia/Tokyo (Hour partitions)', async () => { + const loader = createLoader({ + partitionGranularity: 'hour', + timezone: 'Asia/Tokyo', // UTC+9 + }); + + const results = await loader.partitionPreAggregations(); + expect(results.length).toEqual(72); + + let [preAggDesc] = results; + expect(preAggDesc.tableName).toEqual('test_table2024010100'); // Partition tables are the same for all time zones + expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); // buildRange is the same for all time zones + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T00:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010100')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2023-12-31T15:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2023-12-31T15:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010100')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2023-12-31T15:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2023-12-31T15:59:59.999'); + + [, preAggDesc] = results; + expect(preAggDesc.tableName).toEqual('test_table2024010101'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T01:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T01:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010101')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2023-12-31T16:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2023-12-31T16:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010101')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2023-12-31T16:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2023-12-31T16:59:59.999'); + + // eslint-disable-next-line prefer-destructuring + preAggDesc = results[71]; + expect(preAggDesc.tableName).toEqual('test_table2024010323'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T23:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010323')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T14:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T14:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010323')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T14:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T14:59:59.999'); + }); }); describe('partitionPreAggregations', () => { From b101851ee2ee485748d3ecb33c0d7dc7dfebd84a Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 5 May 2025 22:10:51 +0300 Subject: [PATCH 4/8] refactor parseLocalDate to use UTC during parsing and then input timezone for result shifting --- packages/cubejs-api-gateway/src/gateway.ts | 6 +++--- packages/cubejs-backend-shared/src/time.ts | 9 +++++---- .../orchestrator/PreAggregationPartitionRangeLoader.ts | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/cubejs-api-gateway/src/gateway.ts b/packages/cubejs-api-gateway/src/gateway.ts index 0f89cba369c53..d5d24f19387a7 100644 --- a/packages/cubejs-api-gateway/src/gateway.ts +++ b/packages/cubejs-api-gateway/src/gateway.ts @@ -9,7 +9,7 @@ import structuredClone from '@ungap/structured-clone'; import { getEnv, getRealType, - parseLocalDate, + parseUtcIntoLocalDate, QueryAlias, } from '@cubejs-backend/shared'; import { @@ -921,8 +921,8 @@ class ApiGateway { // It's expected that selector.dateRange is provided in local time (without timezone) // At the same time it is ok to get timestamps with `Z` (in UTC). if (selector.dateRange) { - const start = parseLocalDate([{ val: selector.dateRange[0] }], 'UTC'); - const end = parseLocalDate([{ val: selector.dateRange[1] }], 'UTC'); + const start = parseUtcIntoLocalDate([{ val: selector.dateRange[0] }], 'UTC'); + const end = parseUtcIntoLocalDate([{ val: selector.dateRange[1] }], 'UTC'); if (!start || !end) { throw new UserError(`Cannot parse selector date range ${selector.dateRange}`); } diff --git a/packages/cubejs-backend-shared/src/time.ts b/packages/cubejs-backend-shared/src/time.ts index 62ab33b93c3c7..84dd47a119766 100644 --- a/packages/cubejs-backend-shared/src/time.ts +++ b/packages/cubejs-backend-shared/src/time.ts @@ -251,7 +251,7 @@ export const utcToLocalTimeZone = (timezone: string, timestampFormat: string, ti return moment.tz(timestamp, 'UTC').tz(timezone).format(timestampFormat); }; -export const parseLocalDate = (data: { [key: string]: string }[] | null | undefined, timezone: string, timestampFormat: string = 'YYYY-MM-DDTHH:mm:ss.SSS'): string | null => { +export const parseUtcIntoLocalDate = (data: { [key: string]: string }[] | null | undefined, timezone: string, timestampFormat: string = 'YYYY-MM-DDTHH:mm:ss.SSS'): string | null => { if (!data) { return null; } @@ -278,11 +278,12 @@ export const parseLocalDate = (data: { [key: string]: string }[] | null | undefi let parsedMoment; if (value.includes('Z') || /([+-]\d{2}:?\d{2})$/.test(value.trim())) { - // We have timezone info + // We have timezone info encoded in the value string parsedMoment = moment(value, formats, true); } else { - // If no tz info - use provided timezone - parsedMoment = moment.tz(value, formats, true, timezone); + // If no tz info - use UTC as cube expects data source connection to be in UTC timezone + // and so date functions (e.g. `now()`) would return timestamps in UTC. + parsedMoment = moment.tz(value, formats, true, 'UTC'); } if (!parsedMoment.isValid()) { diff --git a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts index f76fd9cc61a99..c81625a1e102f 100644 --- a/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts +++ b/packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts @@ -9,7 +9,7 @@ import { utcToLocalTimeZone, timeSeries, localTimestampToUtc, - parseLocalDate, + parseUtcIntoLocalDate, } from '@cubejs-backend/shared'; import { InlineTable, TableStructure } from '@cubejs-backend/base-driver'; import { DriverFactory } from './DriverFactory'; @@ -526,7 +526,7 @@ export class PreAggregationPartitionRangeLoader { } public static extractDate(data: any, timezone: string, timestampFormat: string = DEFAULT_TS_FORMAT): string { - return parseLocalDate(data, timezone, timestampFormat); + return parseUtcIntoLocalDate(data, timezone, timestampFormat); } public static readonly FROM_PARTITION_RANGE = FROM_PARTITION_RANGE; From 4511ccef14cbc4f9ec39c1b87fb806917fd4bd89 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 5 May 2025 22:20:48 +0300 Subject: [PATCH 5/8] tests --- .../cubejs-backend-shared/test/time.test.ts | 28 +-- .../test/unit/PreAggregations.test.js | 198 ++++++++++-------- 2 files changed, 125 insertions(+), 101 deletions(-) diff --git a/packages/cubejs-backend-shared/test/time.test.ts b/packages/cubejs-backend-shared/test/time.test.ts index 65174395044a9..1599ac2a5a1db 100644 --- a/packages/cubejs-backend-shared/test/time.test.ts +++ b/packages/cubejs-backend-shared/test/time.test.ts @@ -4,7 +4,7 @@ import { timeSeries, isPredefinedGranularity, timeSeriesFromCustomInterval, - parseLocalDate, + parseUtcIntoLocalDate, utcToLocalTimeZone, addSecondsToLocalTimestamp, reformatInIsoLocal, @@ -264,37 +264,37 @@ describe('extractDate', () => { const timezone = 'Europe/Kiev'; it('should return null if data is empty', () => { - expect(parseLocalDate(null, timezone)).toBeNull(); - expect(parseLocalDate(undefined, timezone)).toBeNull(); - expect(parseLocalDate([], timezone)).toBeNull(); + expect(parseUtcIntoLocalDate(null, timezone)).toBeNull(); + expect(parseUtcIntoLocalDate(undefined, timezone)).toBeNull(); + expect(parseUtcIntoLocalDate([], timezone)).toBeNull(); }); it('should return null if no valid date is found in data', () => { - expect(parseLocalDate([{}], timezone)).toBeNull(); - expect(parseLocalDate([{ someKey: 'invalid date' }], timezone)).toBeNull(); + expect(parseUtcIntoLocalDate([{}], timezone)).toBeNull(); + expect(parseUtcIntoLocalDate([{ someKey: 'invalid date' }], timezone)).toBeNull(); }); it('should throw an error for unknown timezone', () => { const input = [{ date: '2025-02-28T12:00:00+03:00' }]; - expect(() => parseLocalDate(input, 'Invalid/Timezone')) + expect(() => parseUtcIntoLocalDate(input, 'Invalid/Timezone')) .toThrowError('Unknown timezone: Invalid/Timezone'); }); it('should parse a date with UTC timezone', () => { const input = [{ date: '2025-02-28T12:00:00Z' }]; - const result = parseLocalDate(input, timezone); + const result = parseUtcIntoLocalDate(input, timezone); expect(result).toBe('2025-02-28T14:00:00.000'); }); it('should parse a date with an offset timezone', () => { const input = [{ date: '2025-02-28T12:00:00+03:00' }]; - const result = parseLocalDate(input, timezone); + const result = parseUtcIntoLocalDate(input, timezone); expect(result).toBe('2025-02-28T11:00:00.000'); }); it('should parse a date without timezone as UTC', () => { const input = [{ date: '2025-02-28 12:00:00' }]; - const result = parseLocalDate(input, timezone); + const result = parseUtcIntoLocalDate(input, timezone); expect(result).toBe('2025-02-28T12:00:00.000'); }); @@ -304,10 +304,10 @@ describe('extractDate', () => { const input3 = [{ date: '2025-02-28T12:00:00Z' }]; const input4 = [{ date: '2025-02-28T12:00:00+03:00' }]; - expect(parseLocalDate(input1, timezone)).toBe('2025-02-28T12:00:00.000'); - expect(parseLocalDate(input2, timezone)).toBe('2025-02-28T12:00:00.000'); - expect(parseLocalDate(input3, timezone)).toBe('2025-02-28T14:00:00.000'); - expect(parseLocalDate(input4, timezone)).toBe('2025-02-28T11:00:00.000'); + expect(parseUtcIntoLocalDate(input1, timezone)).toBe('2025-02-28T12:00:00.000'); + expect(parseUtcIntoLocalDate(input2, timezone)).toBe('2025-02-28T12:00:00.000'); + expect(parseUtcIntoLocalDate(input3, timezone)).toBe('2025-02-28T14:00:00.000'); + expect(parseUtcIntoLocalDate(input4, timezone)).toBe('2025-02-28T11:00:00.000'); }); }); diff --git a/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.js b/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.js index aa0bd850baa61..9f7c5ff8cc9f0 100644 --- a/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.js +++ b/packages/cubejs-query-orchestrator/test/unit/PreAggregations.test.js @@ -80,11 +80,14 @@ const mockPreAggregation = (overrides = {}) => ({ timestampFormat: 'YYYY-MM-DDTHH:mm:ss.SSS', timestampPrecision: 3, dataSource: 'default', + partitionInvalidateKeyQueries: [], preAggregationStartEndQueries: [ ['SELECT MIN(ts)', [], {}], ['SELECT MAX(ts)', [], {}] ], loadSql: ['CREATE TABLE test_table AS SELECT * FROM source_table WHERE ts >= $1 and ts <= $2', [FROM_PARTITION_RANGE, TO_PARTITION_RANGE]], + sql: ['SELECT * FROM source_table WHERE ts >= $1 and ts <= $2', [FROM_PARTITION_RANGE, TO_PARTITION_RANGE]], + previewSql: ['SELECT * FROM SELECT * FROM dev_pre_aggregations.test_table__daily LIMIT 1000', []], ...overrides, }); @@ -533,6 +536,27 @@ describe('PreAggregations', () => { }); }); + describe('PreAggregations without partitions', () => { + test('should construct correct preAggregation without partitions', async () => { + const loader = createLoader({ + timezone: 'UTC', + partitionGranularity: undefined, + }); + + const results = await loader.partitionPreAggregations(); + expect(results.length).toEqual(1); + + const [preAggDesc] = results; + expect(preAggDesc.tableName).toEqual('test_table'); + expect(preAggDesc.buildRangeStart).toBeUndefined(); + expect(preAggDesc.buildRangeEnd).toBeUndefined(); + expect(preAggDesc.loadSql[0].includes('test_table')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual(FROM_PARTITION_RANGE); + expect(preAggDesc.loadSql[1][1]).toEqual(TO_PARTITION_RANGE); + expect(preAggDesc.structureVersionLoadSql).toBeUndefined(); + }); + }); + describe('partitionPreAggregations', () => { test('should construct correct partitionPreAggregations for dateRange in UTC (Day partitions)', async () => { const loader = createLoader({ @@ -543,8 +567,8 @@ describe('PreAggregations', () => { expect(results.length).toEqual(3); let [preAggDesc] = results; - expect(preAggDesc.tableName).toEqual('test_table20240101'); // Partition tables are the same for all time zones - expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); // buildRange is the same for all time zones + expect(preAggDesc.tableName).toEqual('test_table20240101'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T23:59:59.999'); expect(preAggDesc.loadSql[0].includes('test_table20240101')).toBeTruthy(); expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); @@ -582,11 +606,22 @@ describe('PreAggregations', () => { }); const results = await loader.partitionPreAggregations(); - expect(results.length).toEqual(3); + expect(results.length).toEqual(4); let [preAggDesc] = results; - expect(preAggDesc.tableName).toEqual('test_table20240101'); // Partition tables are the same for all time zones - expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); // buildRange is the same for all time zones + expect(preAggDesc.tableName).toEqual('test_table20231231'); + expect(preAggDesc.buildRangeStart).toEqual('2023-12-31T00:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2023-12-31T23:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table20231231')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2023-12-31T05:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T04:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20231231')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2023-12-31T05:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T04:59:59.999'); + + [, preAggDesc] = results; + expect(preAggDesc.tableName).toEqual('test_table20240101'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T23:59:59.999'); expect(preAggDesc.loadSql[0].includes('test_table20240101')).toBeTruthy(); expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T05:00:00.000'); @@ -595,24 +630,13 @@ describe('PreAggregations', () => { expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T05:00:00.000'); expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-02T04:59:59.999'); - [, preAggDesc] = results; - expect(preAggDesc.tableName).toEqual('test_table20240102'); - expect(preAggDesc.buildRangeStart).toEqual('2024-01-02T00:00:00.000'); - expect(preAggDesc.buildRangeEnd).toEqual('2024-01-02T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table20240102')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-02T05:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T04:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240102')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-02T05:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T04:59:59.999'); - - [,, preAggDesc] = results; + [,,, preAggDesc] = results; expect(preAggDesc.tableName).toEqual('test_table20240103'); expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999'); expect(preAggDesc.loadSql[0].includes('test_table20240103')).toBeTruthy(); expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T05:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-04T04:59:59.999'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-04T04:59:59.999'); // Because DateRangeEnd Mock Query returns it expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240103')).toBeTruthy(); expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T05:00:00.000'); expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-04T04:59:59.999'); @@ -624,11 +648,11 @@ describe('PreAggregations', () => { }); const results = await loader.partitionPreAggregations(); - expect(results.length).toEqual(3); + expect(results.length).toEqual(4); let [preAggDesc] = results; - expect(preAggDesc.tableName).toEqual('test_table20240101'); // Partition tables are the same for all time zones - expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); // buildRange is the same for all time zones + expect(preAggDesc.tableName).toEqual('test_table20240101'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T23:59:59.999'); expect(preAggDesc.loadSql[0].includes('test_table20240101')).toBeTruthy(); expect(preAggDesc.loadSql[1][0]).toEqual('2023-12-31T15:00:00.000'); @@ -648,16 +672,16 @@ describe('PreAggregations', () => { expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T15:00:00.000'); expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-02T14:59:59.999'); - [,, preAggDesc] = results; - expect(preAggDesc.tableName).toEqual('test_table20240103'); - expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T00:00:00.000'); - expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table20240103')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-02T15:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T14:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240103')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-02T15:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T14:59:59.999'); + [,,, preAggDesc] = results; + expect(preAggDesc.tableName).toEqual('test_table20240104'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-04T00:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-04T23:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table20240104')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T15:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-04T14:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table20240104')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T15:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-04T14:59:59.999'); }); test('should construct correct partitionPreAggregations for dateRange in UTC (Hour partitions)', async () => { @@ -670,8 +694,8 @@ describe('PreAggregations', () => { expect(results.length).toEqual(72); let [preAggDesc] = results; - expect(preAggDesc.tableName).toEqual('test_table2024010100'); // Partition tables are the same for all time zones - expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); // buildRange is the same for all time zones + expect(preAggDesc.tableName).toEqual('test_table2024010100'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T00:59:59.999'); expect(preAggDesc.loadSql[0].includes('test_table2024010100')).toBeTruthy(); expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); @@ -714,38 +738,38 @@ describe('PreAggregations', () => { expect(results.length).toEqual(72); let [preAggDesc] = results; - expect(preAggDesc.tableName).toEqual('test_table2024010100'); // Partition tables are the same for all time zones - expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); // buildRange is the same for all time zones - expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T00:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010100')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T05:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T05:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010100')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T05:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T05:59:59.999'); + expect(preAggDesc.tableName).toEqual('test_table2023123119'); + expect(preAggDesc.buildRangeStart).toEqual('2023-12-31T19:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2023-12-31T19:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2023123119')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T00:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2023123119')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T00:59:59.999'); [, preAggDesc] = results; - expect(preAggDesc.tableName).toEqual('test_table2024010101'); - expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T01:00:00.000'); - expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T01:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010101')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T06:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T06:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010101')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T06:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T06:59:59.999'); + expect(preAggDesc.tableName).toEqual('test_table2023123120'); + expect(preAggDesc.buildRangeStart).toEqual('2023-12-31T20:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2023-12-31T20:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2023123120')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T01:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T01:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2023123120')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T01:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T01:59:59.999'); // eslint-disable-next-line prefer-destructuring preAggDesc = results[71]; - expect(preAggDesc.tableName).toEqual('test_table2024010323'); - expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T23:00:00.000'); - expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010323')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-04T04:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-04T04:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010323')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-04T04:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-04T04:59:59.999'); + expect(preAggDesc.tableName).toEqual('test_table2024010318'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T18:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T18:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010318')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T23:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010318')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T23:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); }); test('should construct correct partitionPreAggregations for dateRange in Asia/Tokyo (Hour partitions)', async () => { @@ -758,38 +782,38 @@ describe('PreAggregations', () => { expect(results.length).toEqual(72); let [preAggDesc] = results; - expect(preAggDesc.tableName).toEqual('test_table2024010100'); // Partition tables are the same for all time zones - expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T00:00:00.000'); // buildRange is the same for all time zones - expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T00:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010100')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2023-12-31T15:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2023-12-31T15:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010100')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2023-12-31T15:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2023-12-31T15:59:59.999'); + expect(preAggDesc.tableName).toEqual('test_table2024010109'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T09:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T09:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010109')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T00:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010109')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T00:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T00:59:59.999'); [, preAggDesc] = results; - expect(preAggDesc.tableName).toEqual('test_table2024010101'); - expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T01:00:00.000'); - expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T01:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010101')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2023-12-31T16:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2023-12-31T16:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010101')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2023-12-31T16:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2023-12-31T16:59:59.999'); + expect(preAggDesc.tableName).toEqual('test_table2024010110'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-01T10:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-01T10:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010110')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-01T01:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-01T01:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010110')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-01T01:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-01T01:59:59.999'); // eslint-disable-next-line prefer-destructuring preAggDesc = results[71]; - expect(preAggDesc.tableName).toEqual('test_table2024010323'); - expect(preAggDesc.buildRangeStart).toEqual('2024-01-03T23:00:00.000'); - expect(preAggDesc.buildRangeEnd).toEqual('2024-01-03T23:59:59.999'); - expect(preAggDesc.loadSql[0].includes('test_table2024010323')).toBeTruthy(); - expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T14:00:00.000'); - expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T14:59:59.999'); - expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010323')).toBeTruthy(); - expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T14:00:00.000'); - expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T14:59:59.999'); + expect(preAggDesc.tableName).toEqual('test_table2024010408'); + expect(preAggDesc.buildRangeStart).toEqual('2024-01-04T08:00:00.000'); + expect(preAggDesc.buildRangeEnd).toEqual('2024-01-04T08:59:59.999'); + expect(preAggDesc.loadSql[0].includes('test_table2024010408')).toBeTruthy(); + expect(preAggDesc.loadSql[1][0]).toEqual('2024-01-03T23:00:00.000'); + expect(preAggDesc.loadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); + expect(preAggDesc.structureVersionLoadSql[0].includes('test_table2024010408')).toBeTruthy(); + expect(preAggDesc.structureVersionLoadSql[1][0]).toEqual('2024-01-03T23:00:00.000'); + expect(preAggDesc.structureVersionLoadSql[1][1]).toEqual('2024-01-03T23:59:59.999'); }); }); From 4f2d86e04bd2ea1f5bfcf8946f85434363d6b7c8 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 5 May 2025 22:39:37 +0300 Subject: [PATCH 6/8] fix tests --- packages/cubejs-backend-shared/test/time.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-backend-shared/test/time.test.ts b/packages/cubejs-backend-shared/test/time.test.ts index 1599ac2a5a1db..02ada380a88bb 100644 --- a/packages/cubejs-backend-shared/test/time.test.ts +++ b/packages/cubejs-backend-shared/test/time.test.ts @@ -295,7 +295,7 @@ describe('extractDate', () => { it('should parse a date without timezone as UTC', () => { const input = [{ date: '2025-02-28 12:00:00' }]; const result = parseUtcIntoLocalDate(input, timezone); - expect(result).toBe('2025-02-28T12:00:00.000'); + expect(result).toBe('2025-02-28T14:00:00.000'); }); it('should handle multiple formats', () => { @@ -304,8 +304,8 @@ describe('extractDate', () => { const input3 = [{ date: '2025-02-28T12:00:00Z' }]; const input4 = [{ date: '2025-02-28T12:00:00+03:00' }]; - expect(parseUtcIntoLocalDate(input1, timezone)).toBe('2025-02-28T12:00:00.000'); - expect(parseUtcIntoLocalDate(input2, timezone)).toBe('2025-02-28T12:00:00.000'); + expect(parseUtcIntoLocalDate(input1, timezone)).toBe('2025-02-28T14:00:00.000'); + expect(parseUtcIntoLocalDate(input2, timezone)).toBe('2025-02-28T14:00:00.000'); expect(parseUtcIntoLocalDate(input3, timezone)).toBe('2025-02-28T14:00:00.000'); expect(parseUtcIntoLocalDate(input4, timezone)).toBe('2025-02-28T11:00:00.000'); }); From ee0057552670fdc7b2414b078ab15a5a8d15e37f Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 5 May 2025 23:17:03 +0300 Subject: [PATCH 7/8] fix test --- .../test/unit/RefreshScheduler.test.ts | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts index fa0bd8cc0e8ac..d4225b75ae2f1 100644 --- a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts +++ b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts @@ -697,63 +697,63 @@ describe('Refresh Scheduler', () => { { tableName: 'stb_pre_aggregations.foo_second20201231', timezone: 'UTC', fromTable: 'foo_tenant1' }, { tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'UTC', fromTable: 'bar' }, { - tableName: 'stb_pre_aggregations.foo_first20201231', + tableName: 'stb_pre_aggregations.foo_first20201230', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1', }, - { tableName: 'stb_pre_aggregations.foo_orphaned20201231', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' }, + { tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' }, { - tableName: 'stb_pre_aggregations.foo_second20201231', + tableName: 'stb_pre_aggregations.foo_second20201230', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1', }, - { tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'America/Los_Angeles', fromTable: 'bar' }, + { tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'America/Los_Angeles', fromTable: 'bar' }, { tableName: 'stb_pre_aggregations.foo_first20201230', timezone: 'UTC', fromTable: 'foo_tenant1' }, { tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'UTC', fromTable: 'foo_tenant1' }, { tableName: 'stb_pre_aggregations.foo_second20201230', timezone: 'UTC', fromTable: 'foo_tenant1' }, { tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'UTC', fromTable: 'bar' }, { - tableName: 'stb_pre_aggregations.foo_first20201230', + tableName: 'stb_pre_aggregations.foo_first20201229', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1', }, - { tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' }, + { tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' }, { - tableName: 'stb_pre_aggregations.foo_second20201230', + tableName: 'stb_pre_aggregations.foo_second20201229', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1', }, - { tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'America/Los_Angeles', fromTable: 'bar' }, + { tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'America/Los_Angeles', fromTable: 'bar' }, { tableName: 'stb_pre_aggregations.foo_first20201229', timezone: 'UTC', fromTable: 'foo_tenant1' }, { tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'UTC', fromTable: 'foo_tenant1' }, { tableName: 'stb_pre_aggregations.foo_second20201229', timezone: 'UTC', fromTable: 'foo_tenant1' }, { tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'UTC', fromTable: 'bar' }, { - tableName: 'stb_pre_aggregations.foo_first20201229', + tableName: 'stb_pre_aggregations.foo_first20201228', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1', }, - { tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' }, + { tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' }, { - tableName: 'stb_pre_aggregations.foo_second20201229', + tableName: 'stb_pre_aggregations.foo_second20201228', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1', }, - { tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'America/Los_Angeles', fromTable: 'bar' }, + { tableName: 'stb_pre_aggregations.bar_first20201228', timezone: 'America/Los_Angeles', fromTable: 'bar' }, { tableName: 'stb_pre_aggregations.foo_first20201228', timezone: 'UTC', fromTable: 'foo_tenant1' }, { tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'UTC', fromTable: 'foo_tenant1' }, { tableName: 'stb_pre_aggregations.foo_second20201228', timezone: 'UTC', fromTable: 'foo_tenant1' }, { - tableName: 'stb_pre_aggregations.foo_first20201228', + tableName: 'stb_pre_aggregations.foo_first20201227', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1', }, - { tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' }, + { tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' }, { - tableName: 'stb_pre_aggregations.foo_second20201228', + tableName: 'stb_pre_aggregations.foo_second20201227', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1', }, @@ -762,13 +762,13 @@ describe('Refresh Scheduler', () => { { tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'UTC', fromTable: 'foo_tenant1' }, { tableName: 'stb_pre_aggregations.foo_second20201227', timezone: 'UTC', fromTable: 'foo_tenant1' }, { - tableName: 'stb_pre_aggregations.foo_first20201227', + tableName: 'stb_pre_aggregations.foo_first20201226', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1', }, - { tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' }, + { tableName: 'stb_pre_aggregations.foo_orphaned20201226', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' }, { - tableName: 'stb_pre_aggregations.foo_second20201227', + tableName: 'stb_pre_aggregations.foo_second20201226', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1', }, From ee087002e4f7a04b9d4af9beaf2fa605169b0c4c Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 5 May 2025 23:36:02 +0300 Subject: [PATCH 8/8] another test fix --- packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts index d4225b75ae2f1..3a4366b37672a 100644 --- a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts +++ b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts @@ -980,7 +980,7 @@ describe('Refresh Scheduler', () => { clearInterval(interval); expect(mockDriver.createdTables.filter(o => o.tableName.includes('foo_first') && o.timezone === 'UTC').length).toEqual(3); - expect(mockDriver.createdTables.filter(o => o.tableName.includes('foo_first') && o.timezone === 'America/Los_Angeles').length).toEqual(3); + expect(mockDriver.createdTables.filter(o => o.tableName.includes('foo_first') && o.timezone === 'America/Los_Angeles').length).toEqual(2); expect(mockDriver.createdTables.filter(o => o.tableName.includes('foo_orphaned')).length).toEqual(0); expect(mockDriver.createdTables.filter(o => o.tableName.includes('foo_second')).length).toEqual(0); });