feat: Allow null filter values

paveltiunov · paveltiunov · commit 9e339f79a803 · 2020-02-03T15:59:12.000-08:00
Fixes #362
diff --git a/packages/cubejs-api-gateway/index.js b/packages/cubejs-api-gateway/index.js
@@ -127,7 +127,7 @@ const querySchema = Joi.object().keys({
     dimension: id,
     member: id,
     operator: Joi.valid(operators).required(),
-    values: Joi.array().items(Joi.string().allow(''))
+    values: Joi.array().items(Joi.string().allow('', null))
   }).xor('dimension', 'member')),
   timeDimensions: Joi.array().items(Joi.object().keys({
     dimension: id.required(),
diff --git a/packages/cubejs-api-gateway/index.test.js b/packages/cubejs-api-gateway/index.test.js
@@ -5,16 +5,37 @@ const express = require('express');
 const ApiGateway = require('./index');
 
 const compilerApi = jest.fn().mockImplementation(() => ({
-  getSql() {
-    return 'SELECT * FROM test';
+  async getSql() {
+    return {
+      sql: ['SELECT * FROM test', []],
+      aliasNameToMember: {
+        foo__bar: 'Foo.bar'
+      }
+    };
   },
 
-  metaConfig() {
-    return [];
+  async metaConfig() {
+    return [{
+      config: {
+        name: 'Foo',
+        measures: [{
+          name: 'Foo.bar'
+        }],
+        dimensions: [{
+          name: 'Foo.id'
+        }]
+      }
+    }];
   }
 }));
-const adapterApi = jest.fn();
-const logger = jest.fn();
+const adapterApi = jest.fn().mockImplementation(() => ({
+  async executeQuery() {
+    return {
+      data: [{ foo__bar: 42 }]
+    };
+  }
+}));
+const logger = (type, message) => console.log({ type, ...message });
 
 describe(`API Gateway`, () => {
   process.env.NODE_ENV = 'production';
@@ -45,4 +66,13 @@ describe(`API Gateway`, () => {
       .expect(400);
     expect(res.body && res.body.error).toStrictEqual("Query should contain either measures, dimensions or timeDimensions with granularities in order to be valid");
   });
+
+  test(`null filter values`, async () => {
+    const res = await request(app)
+      .get('/cubejs-api/v1/load?query={"measures":["Foo.bar"],"filters":[{"dimension":"Foo.id","operator":"equals","values":[null]}]}')
+      .set('Authorization', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.e30.t-IDcSemACt8x4iTMCda8Yhe3iZaWbvV5XKSTbuAn0M')
+      .expect(200);
+    console.log(res.body);
+    expect(res.body && res.body.data).toStrictEqual([{ 'Foo.bar': 42 }]);
+  });
 });
diff --git a/packages/cubejs-schema-compiler/adapter/BaseFilter.js b/packages/cubejs-schema-compiler/adapter/BaseFilter.js
@@ -1,7 +1,9 @@
 const inlection = require('inflection');
 const momentRange = require('moment-range');
 const moment = momentRange.extendMoment(require('moment-timezone'));
-const { repeat, join, map, contains } = require('ramda');
+const {
+  repeat, join, map, contains
+} = require('ramda');
 
 const BaseDimension = require('./BaseDimension');
 
@@ -40,8 +42,9 @@ class BaseFilter extends BaseDimension {
   }
 
   conditionSql(columnSql) {
-    const operatorMethod = `${inlection.camelize(this.operator).replace(/[A-Z]/, (c) =>
-      (c != null ? c : '').toLowerCase()
+    const operatorMethod = `${inlection.camelize(this.operator).replace(
+      /[A-Z]/,
+      (c) => (c != null ? c : '').toLowerCase()
     )}Where`;
     const sql = this[operatorMethod](columnSql);
     return this.query.paramAllocator.allocateParamsForQuestionString(sql, this.filterParams());
@@ -67,9 +70,7 @@ class BaseFilter extends BaseDimension {
 
   // noinspection JSMethodCanBeStatic
   escapeWildcardChars(param) {
-    return typeof param === 'string'
-      ? param.replace(/([_%])/gi, '\\$1')
-      : param;
+    return typeof param === 'string' ? param.replace(/([_%])/gi, '\\$1') : param;
   }
 
   isWildcardOperator() {
@@ -83,7 +84,7 @@ class BaseFilter extends BaseDimension {
     if (this.operator === 'set' || this.operator === 'not_set' || this.operator === 'expressionEquals') {
       return [];
     }
-    const params = Array.isArray(this.values) ? this.values : [this.values];
+    const params = this.valuesArray().filter(v => v != null);
 
     if (this.isWildcardOperator()) {
       return map(this.escapeWildcardChars, params);
@@ -92,6 +93,14 @@ class BaseFilter extends BaseDimension {
     return params;
   }
 
+  valuesArray() {
+    return Array.isArray(this.values) ? this.values : [this.values];
+  }
+
+  valuesContainNull() {
+    return this.valuesArray().indexOf(null) !== -1;
+  }
+
   castParameter() {
     return '?';
   }
@@ -110,8 +119,15 @@ class BaseFilter extends BaseDimension {
 
   likeOr(column, not) {
     const basePart = this.likeIgnoreCase(column, not);
-    const nullCheck = `${not ? ` OR ${column} IS NULL` : ''}`;
-    return `${join(not ? ' AND ' : ' OR ', repeat(basePart, this.values.length))}${nullCheck}`;
+    return `${join(not ? ' AND ' : ' OR ', repeat(basePart, this.filterParams().length))}${this.orIsNullCheck(column, not)}`;
+  }
+
+  orIsNullCheck(column, not) {
+    return `${this.shouldAddOrIsNull(not) ? ` OR ${column} IS NULL` : ''}`;
+  }
+
+  shouldAddOrIsNull(not) {
+    return not ? !this.valuesContainNull() : this.valuesContainNull();
   }
 
   likeIgnoreCase(column, not) {
@@ -123,27 +139,35 @@ class BaseFilter extends BaseDimension {
       return this.inWhere(column);
     }
 
-    return `${column} = ${this.castParameter()}`;
+    if (this.valuesContainNull()) {
+      return this.notSetWhere(column);
+    }
+
+    return `${column} = ${this.castParameter()}${this.orIsNullCheck(column, false)}`;
   }
 
   inPlaceholders() {
-    return `(${join(', ', repeat(this.castParameter(), this.values.length || 1))})`;
+    return `(${join(', ', repeat(this.castParameter(), this.filterParams().length || 1))})`;
   }
 
   inWhere(column) {
-    return `${column} IN ${this.inPlaceholders()}`;
+    return `${column} IN ${this.inPlaceholders()}${this.orIsNullCheck(column, false)}`;
   }
 
   notEqualsWhere(column) {
     if (this.isArrayValues()) {
       return this.notInWhere(column);
     }
 
-    return `${column} <> ${this.castParameter()}`;
+    if (this.valuesContainNull()) {
+      return this.setWhere(column);
+    }
+
+    return `${column} <> ${this.castParameter()}${this.orIsNullCheck(column, true)}`;
   }
 
   notInWhere(column) {
-    return `${column} NOT IN ${this.inPlaceholders()}`;
+    return `${column} NOT IN ${this.inPlaceholders()}${this.orIsNullCheck(column, true)}`;
   }
 
   setWhere(column) {
diff --git a/packages/cubejs-schema-compiler/test/SQLGenerationTest.js b/packages/cubejs-schema-compiler/test/SQLGenerationTest.js
@@ -1305,6 +1305,75 @@ describe('SQL Generation', function test() {
     ])
   );
 
+  it(
+    'contains null filter',
+    () => runQueryTest({
+      measures: [],
+      dimensions: [
+        'visitors.source'
+      ],
+      timeDimensions: [],
+      timezone: 'America/Los_Angeles',
+      filters: [{
+        dimension: 'visitors.source',
+        operator: 'contains',
+        values: ['goo', null]
+      }],
+      order: [{
+        id: 'visitors.source'
+      }]
+    }, [
+      { "visitors__source": 'google' },
+      { "visitors__source": null }
+    ])
+  );
+
+  it(
+    'null filter',
+    () => runQueryTest({
+      measures: [],
+      dimensions: [
+        'visitors.source'
+      ],
+      timeDimensions: [],
+      timezone: 'America/Los_Angeles',
+      filters: [{
+        dimension: 'visitors.source',
+        operator: 'equals',
+        values: ['google', null]
+      }],
+      order: [{
+        id: 'visitors.source'
+      }]
+    }, [
+      { "visitors__source": 'google' },
+      { "visitors__source": null },
+    ])
+  );
+
+  it(
+    'not equals filter',
+    () => runQueryTest({
+      measures: [],
+      dimensions: [
+        'visitors.source'
+      ],
+      timeDimensions: [],
+      timezone: 'America/Los_Angeles',
+      filters: [{
+        dimension: 'visitors.source',
+        operator: 'notEquals',
+        values: ['google']
+      }],
+      order: [{
+        id: 'visitors.source'
+      }]
+    }, [
+      { "visitors__source": 'some' },
+      { "visitors__source": null },
+    ])
+  );
+
   it('year granularity', () =>
     runQueryTest({
       measures: [
