fix: Sanity check for silent truncate name problem during pre-aggregation creation

Author: paveltiunov

packages/cubejs-query-orchestrator/orchestrator/PreAggregations.js

@@ -260,11 +260,13 @@ class PreAggregationLoader {
 
     const mostRecentTargetTableName = async () => {
       await this.loadCache.reset(this.preAggregation);
-      return this.targetTableName(
-        getVersionEntryByContentVersion(
-          await this.loadCache.getVersionEntries(this.preAggregation)
-        )
+      const lastVersion = getVersionEntryByContentVersion(
+        await this.loadCache.getVersionEntries(this.preAggregation)
       );
+      if (!lastVersion) {
+        throw new Error(`Pre-aggregation table is not found for ${this.preAggregation.tableName} after it was successfully created. It usually means database silently truncates table names due to max name length.`);
+      }
+      return this.targetTableName(lastVersion);
     };
 
     if (versionEntry) {

packages/cubejs-query-orchestrator/test/QueryOrchestratorTest.js

@@ -22,7 +22,11 @@ class MockDriver {
   }
 
   async loadPreAggregationIntoTable(preAggregationTableName) {
-    this.tables.push(preAggregationTableName);
+    this.tables.push(preAggregationTableName.substring(0, 100));
+  }
+
+  async dropTable(tableName) {
+    this.tables = this.tables.filter(t => t !== tableName.split('.')[1]);
   }
 }
 
@@ -83,4 +87,30 @@ describe('QueryOrchestrator', () => {
     should(result.data[0]).match(/orders_number_and_count20191101_l3kvjcmu_khbemovd/);
     should(mockDriver.executedQueries).matchAny(/CREATE INDEX orders_number_and_count_week20191101_l3kvjcmu_khbemovd/);
   });
+
+  it('silent truncate', async () => {
+    const query = {
+      query: "SELECT \"orders__created_at_week\" \"orders__created_at_week\", sum(\"orders__count\") \"orders__count\" FROM (SELECT * FROM stb_pre_aggregations.orders_number_and_count_and_very_very_very_very_very_very_long20191101) as partition_union  WHERE (\"orders__created_at_week\" >= ($1::timestamptz::timestamptz AT TIME ZONE 'UTC') AND \"orders__created_at_week\" <= ($2::timestamptz::timestamptz AT TIME ZONE 'UTC')) GROUP BY 1 ORDER BY 1 ASC LIMIT 10000",
+      values: ["2019-11-01T00:00:00Z", "2019-11-30T23:59:59Z"],
+      cacheKeyQueries: {
+        renewalThreshold: 21600,
+        queries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]]
+      },
+      preAggregations: [{
+        preAggregationsSchema: "stb_pre_aggregations",
+        tableName: "stb_pre_aggregations.orders_number_and_count_and_very_very_very_very_very_very_long20191101",
+        loadSql: ["CREATE TABLE stb_pre_aggregations.orders_number_and_count_and_very_very_very_very_very_very_long20191101 AS SELECT\n      date_trunc('week', (\"orders\".created_at::timestamptz AT TIME ZONE 'UTC')) \"orders__created_at_week\", count(\"orders\".id) \"orders__count\", sum(\"orders\".number) \"orders__number\"\n    FROM\n      public.orders AS \"orders\"\n  WHERE (\"orders\".created_at >= $1::timestamptz AND \"orders\".created_at <= $2::timestamptz) GROUP BY 1", ["2019-11-01T00:00:00Z", "2019-11-30T23:59:59Z"]],
+        invalidateKeyQueries: [["SELECT date_trunc('hour', (NOW()::timestamptz AT TIME ZONE 'UTC')) as current_hour", []]],
+      }],
+      renewQuery: true
+    };
+    let thrown = true;
+    try {
+      await queryOrchestrator.fetchQuery(query);
+      thrown = false;
+    } catch (e) {
+      should(e.message).match(/Pre-aggregation table is not found/);
+    }
+    should(thrown).equal(true);
+  });
 });