feat(mssql-driver): add readonly aggregation for mssql sources (#920) Thanks to @JoshMentzer!

JoshMentzer · mentzerj · web-flow · commit dfeccca18c78 · 2020-08-10T18:17:10.000-07:00
Co-authored-by: Joshua D. Mentzer &lt;mentzerj@trinity-health.org&gt;
diff --git a/packages/cubejs-mssql-driver/driver/MSSqlDriver.js b/packages/cubejs-mssql-driver/driver/MSSqlDriver.js
@@ -14,7 +14,8 @@ class MSSqlDriver extends BaseDriver {
         process.env.CUBEJS_DB_DOMAIN : undefined,
       requestTimeout: 10 * 60 * 1000, // 10 minutes
       options: {
-        encrypt: !!process.env.CUBEJS_DB_SSL || false
+        encrypt: !!process.env.CUBEJS_DB_SSL || false,
+        useUTC: false
       },
       pool: {
         max: 8,
@@ -72,6 +73,23 @@ class MSSqlDriver extends BaseDriver {
       return null;
     });
   }
+
+  async downloadQueryResults(query, values) {
+    const result = await this.query(query, values);
+    const types = Object.keys(result.columns).map((key) => ({
+      name: result.columns[key].name,
+      type: this.toGenericType(result.columns[key].type.declaration),
+    }));
+
+    return {
+      rows: result,
+      types,
+    };
+  }
+
+  readOnly() {
+    return !!this.config.readOnly;
+  }
 }
 
 module.exports = MSSqlDriver;
diff --git a/packages/cubejs-schema-compiler/adapter/MssqlQuery.js b/packages/cubejs-schema-compiler/adapter/MssqlQuery.js
@@ -29,13 +29,13 @@ class MssqlParamAllocator extends ParamAllocator {
 }
 
 const GRANULARITY_TO_INTERVAL = {
-  day: (date) => `FORMAT(${date}, 'yyyy-MM-ddT00:00:00.000')`,
-  week: (date) => `FORMAT(dateadd(week, DATEDIFF(week, '1900-01-01', ${date}), '1900-01-01'), 'yyyy-MM-ddT00:00:00.000')`,
-  hour: (date) => `FORMAT(${date}, 'yyyy-MM-ddTHH:00:00.000')`,
-  minute: (date) => `FORMAT(${date}, 'yyyy-MM-ddTHH:mm:00.000')`,
-  second: (date) => `FORMAT(${date}, 'yyyy-MM-ddTHH:mm:ss.000')`,
-  month: (date) => `FORMAT(${date}, 'yyyy-MM-01T00:00:00.000')`,
-  year: (date) => `FORMAT(${date}, 'yyyy-01-01T00:00:00.000')`
+  day: (date) => `dateadd(day, DATEDIFF(day, 0, ${date}), 0)`,
+  week: (date) => `dateadd(week, DATEDIFF(week, 0, ${date}), 0)`,
+  hour: (date) => `dateadd(hour, DATEDIFF(hour, 0, ${date}), 0)`,
+  minute: (date) => `dateadd(minute, DATEDIFF(minute, 0, ${date}), 0)`,
+  second: (date) => `CAST(FORMAT(${date}, 'yyyy-MM-ddTHH:mm:ss.000') AS datetime)`, // until SQL 2016, this causes an int overflow; in SQL 2016 these calls can be changed to DATEDIFF_BIG
+  month: (date) => `dateadd(month, DATEDIFF(month, 0, ${date}), 0)`,
+  year: (date) => `dateadd(year, DATEDIFF(year, 0, ${date}), 0)`,
 };
 
 class MssqlFilter extends BaseFilter {
diff --git a/packages/cubejs-schema-compiler/test/MSSqlPreAggregationsTest.js b/packages/cubejs-schema-compiler/test/MSSqlPreAggregationsTest.js
@@ -178,155 +178,177 @@ describe('MSSqlPreAggregations', function test() {
     preAggregation = Array.isArray(preAggregation) ? preAggregation : [preAggregation];
     return [
       preAggregation.reduce(
-        (replacedQuery, desc) => replacedQuery.replace(new RegExp(desc.tableName, 'g'), `##${desc.tableName}_${suffix}`), toReplace
+        (replacedQuery, desc) =>
+          replacedQuery.replace(new RegExp(desc.tableName, 'g'), `##${desc.tableName}_${suffix}`),
+        toReplace
       ),
-      params
+      params,
     ];
   }
 
   function tempTablePreAggregations(preAggregationsDescriptions) {
-    return R.unnest(preAggregationsDescriptions.map(
-      desc => desc.invalidateKeyQueries.concat([[desc.loadSql[0], desc.loadSql[1]]])
-    ));
+    return R.unnest(
+      preAggregationsDescriptions.map((desc) => desc.invalidateKeyQueries.concat([[desc.loadSql[0], desc.loadSql[1]]]))
+    );
   }
 
-  it('simple pre-aggregation', () => compiler.compile().then(() => {
-    const query = new MSSqlQuery({ joinGraph, cubeEvaluator, compiler }, {
-      measures: [
-        'visitors.count'
-      ],
-      timeDimensions: [{
-        dimension: 'visitors.createdAt',
-        granularity: 'day',
-        dateRange: ['2017-01-01', '2017-01-30']
-      }],
-      timezone: 'UTC',
-      order: [{
-        id: 'visitors.createdAt'
-      }],
-      preAggregationsSchema: ''
-    });
+  it('simple pre-aggregation', () =>
+    compiler.compile().then(() => {
+      const query = new MSSqlQuery(
+        { joinGraph, cubeEvaluator, compiler },
+        {
+          measures: ['visitors.count'],
+          timeDimensions: [
+            {
+              dimension: 'visitors.createdAt',
+              granularity: 'day',
+              dateRange: ['2017-01-01', '2017-01-30'],
+            },
+          ],
+          timezone: 'UTC',
+          order: [
+            {
+              id: 'visitors.createdAt',
+            },
+          ],
+          preAggregationsSchema: '',
+        }
+      );
 
-    const queryAndParams = query.buildSqlAndParams();
-    console.log(queryAndParams);
-    const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
-    console.log(preAggregationsDescription);
+      const queryAndParams = query.buildSqlAndParams();
+      console.log(queryAndParams);
+      const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
+      console.log(preAggregationsDescription);
 
-    return dbRunner.testQueries(tempTablePreAggregations(preAggregationsDescription).concat([
-      query.buildSqlAndParams()
-    ]).map(q => replaceTableName(q, preAggregationsDescription, 1))).then(res => {
-      res.should.be.deepEqual(
-        [
-          {
-            "visitors__created_at_day": "2017-01-03T00:00:00.000",
-            "visitors__count": 1
-          },
-          {
-            "visitors__created_at_day": "2017-01-05T00:00:00.000",
-            "visitors__count": 1
-          },
-          {
-            "visitors__created_at_day": "2017-01-06T00:00:00.000",
-            "visitors__count": 1
-          },
-          {
-            "visitors__created_at_day": "2017-01-07T00:00:00.000",
-            "visitors__count": 2
-          }
-        ]
+      return dbRunner
+        .testQueries(
+          tempTablePreAggregations(preAggregationsDescription)
+            .concat([query.buildSqlAndParams()])
+            .map((q) => replaceTableName(q, preAggregationsDescription, 1))
+        )
+        .then((res) => {
+          res.should.be.deepEqual([
+            {
+              visitors__created_at_day: new Date('2017-01-03T00:00:00.000Z'),
+              visitors__count: 1,
+            },
+            {
+              visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z'),
+              visitors__count: 1,
+            },
+            {
+              visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z'),
+              visitors__count: 1,
+            },
+            {
+              visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z'),
+              visitors__count: 2,
+            },
+          ]);
+        });
+    }));
+
+  it('leaf measure pre-aggregation', () =>
+    compiler.compile().then(() => {
+      const query = new MSSqlQuery(
+        { joinGraph, cubeEvaluator, compiler },
+        {
+          measures: ['visitors.ratio'],
+          timeDimensions: [
+            {
+              dimension: 'visitors.createdAt',
+              granularity: 'day',
+              dateRange: ['2017-01-01', '2017-01-30'],
+            },
+          ],
+          timezone: 'UTC',
+          order: [
+            {
+              id: 'visitors.createdAt',
+            },
+          ],
+          preAggregationsSchema: '',
+        }
       );
-    });
-  }));
 
-  it('leaf measure pre-aggregation', () => compiler.compile().then(() => {
-    const query = new MSSqlQuery({ joinGraph, cubeEvaluator, compiler }, {
-      measures: [
-        'visitors.ratio'
-      ],
-      timeDimensions: [{
-        dimension: 'visitors.createdAt',
-        granularity: 'day',
-        dateRange: ['2017-01-01', '2017-01-30']
-      }],
-      timezone: 'UTC',
-      order: [{
-        id: 'visitors.createdAt'
-      }],
-      preAggregationsSchema: ''
-    });
+      const queryAndParams = query.buildSqlAndParams();
+      console.log(queryAndParams);
+      const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
+      console.log(preAggregationsDescription);
+      preAggregationsDescription[0].loadSql[0].should.match(/visitors_ratio/);
 
-    const queryAndParams = query.buildSqlAndParams();
-    console.log(queryAndParams);
-    const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
-    console.log(preAggregationsDescription);
-    preAggregationsDescription[0].loadSql[0].should.match(/visitors_ratio/);
+      return dbRunner
+        .testQueries(
+          tempTablePreAggregations(preAggregationsDescription)
+            .concat([query.buildSqlAndParams()])
+            .map((q) => replaceTableName(q, preAggregationsDescription, 10))
+        )
+        .then((res) => {
+          res.should.be.deepEqual([
+            {
+              visitors__created_at_day: new Date('2017-01-03T00:00:00.000Z'),
+              visitors__ratio: 0.333333333333,
+            },
+            {
+              visitors__created_at_day: new Date('2017-01-05T00:00:00.000Z'),
+              visitors__ratio: 0.5,
+            },
+            {
+              visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z'),
+              visitors__ratio: 1,
+            },
+            {
+              visitors__created_at_day: new Date('2017-01-07T00:00:00.000Z'),
+              visitors__ratio: null,
+            },
+          ]);
+        });
+    }));
 
-    return dbRunner.testQueries(tempTablePreAggregations(preAggregationsDescription).concat([
-      query.buildSqlAndParams()
-    ]).map(q => replaceTableName(q, preAggregationsDescription, 10))).then(res => {
-      res.should.be.deepEqual(
-        [
-          {
-            "visitors__created_at_day": "2017-01-03T00:00:00.000",
-            "visitors__ratio": 0.333333333333
-          },
-          {
-            "visitors__created_at_day": "2017-01-05T00:00:00.000",
-            "visitors__ratio": 0.5
-          },
-          {
-            "visitors__created_at_day": "2017-01-06T00:00:00.000",
-            "visitors__ratio": 1
-          },
-          {
-            "visitors__created_at_day": "2017-01-07T00:00:00.000",
-            "visitors__ratio": null
-          }
-        ]
+  it('segment', () =>
+    compiler.compile().then(() => {
+      const query = new MSSqlQuery(
+        { joinGraph, cubeEvaluator, compiler },
+        {
+          measures: ['visitors.checkinsTotal'],
+          dimensions: [],
+          segments: ['visitors.google'],
+          timezone: 'UTC',
+          preAggregationsSchema: '',
+          timeDimensions: [
+            {
+              dimension: 'visitors.createdAt',
+              granularity: 'day',
+              dateRange: ['2016-12-30', '2017-01-06'],
+            },
+          ],
+          order: [
+            {
+              id: 'visitors.createdAt',
+            },
+          ],
+        }
       );
-    });
-  }));
-
-  it('segment', () => compiler.compile().then(() => {
-    const query = new MSSqlQuery({ joinGraph, cubeEvaluator, compiler }, {
-      measures: [
-        'visitors.checkinsTotal'
-      ],
-      dimensions: [],
-      segments: ['visitors.google'],
-      timezone: 'UTC',
-      preAggregationsSchema: '',
-      timeDimensions: [{
-        dimension: 'visitors.createdAt',
-        granularity: 'day',
-        dateRange: ['2016-12-30', '2017-01-06']
-      }],
-      order: [{
-        id: 'visitors.createdAt'
-      }],
-    });
 
-    const queryAndParams = query.buildSqlAndParams();
-    console.log(queryAndParams);
-    const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
-    console.log(preAggregationsDescription);
+      const queryAndParams = query.buildSqlAndParams();
+      console.log(queryAndParams);
+      const preAggregationsDescription = query.preAggregations.preAggregationsDescription();
+      console.log(preAggregationsDescription);
 
-    const queries = tempTablePreAggregations(preAggregationsDescription);
+      const queries = tempTablePreAggregations(preAggregationsDescription);
 
-    console.log(JSON.stringify(queries.concat(queryAndParams)));
+      console.log(JSON.stringify(queries.concat(queryAndParams)));
 
-    return dbRunner.testQueries(
-      queries.concat([queryAndParams]).map(q => replaceTableName(q, preAggregationsDescription, 142))
-    ).then(res => {
-      console.log(JSON.stringify(res));
-      res.should.be.deepEqual(
-        [
-          {
-            "visitors__created_at_day": "2017-01-06T00:00:00.000",
-            "visitors__checkins_total": 1
-          }
-        ]
-      );
-    });
-  }));
+      return dbRunner
+        .testQueries(queries.concat([queryAndParams]).map((q) => replaceTableName(q, preAggregationsDescription, 142)))
+        .then((res) => {
+          console.log(JSON.stringify(res));
+          res.should.be.deepEqual([
+            {
+              visitors__created_at_day: new Date('2017-01-06T00:00:00.000Z'),
+              visitors__checkins_total: 1,
+            },
+          ]);
+        });
+    }));
 });
