diff --git a/docs/pages/product/data-modeling/syntax.mdx b/docs/pages/product/data-modeling/syntax.mdx index 2939745e1b67e..46865cd47b7cc 100644 --- a/docs/pages/product/data-modeling/syntax.mdx +++ b/docs/pages/product/data-modeling/syntax.mdx @@ -602,6 +602,73 @@ the [template context](/reference/python/cube#templatecontext-class). +### Curly braces and escaping + +As you can see in the examples above, within [SQL expressions][self-sql-expressions], +curly braces are used to reference cubes and members. + +In YAML data models, use `{reference}`: + +```yaml +cubes: + - name: orders + sql: > + SELECT id, created_at + FROM {other_cube.sql()} + + dimensions: + - name: status + sql: status + type: string + + - name: status_x2 + sql: "{status} || ' ' || {status}" + type: string +``` + +In JavaScript data models, use `${reference}` in [JavaScript template +literals][link-js-template-literals] (mind the dollar sign): + +```javascript +cube(`orders`, { + sql: ` + SELECT id, created_at + FROM ${other_cube.sql()} + `, + + dimensions: { + status: { + sql: `status`, + type: `string` + }, + + status_x2: { + sql: `${status} || ' ' || ${status}`, + type: `string` + } + } +}) +``` + +If you need to use literal, non-referential curly braces in YAML, e.g., +to define a JSON object, you can escape them with a backslash: + +```yaml +cubes: + - name: json_object_in_postgres + sql: SELECT CAST('\{"key":"value"\}'::JSON AS TEXT) AS json_column + + - name: csv_from_s3_in_duckdb + sql: > + SELECT * + FROM read_csv( + 's3://bbb/aaa.csv', + delim = ',', + header = true, + columns=\{'time':'DATE','count':'NUMERIC'\} + ) +``` + ### Non-SQL references Outside [SQL expressions][self-sql-expressions], `column` is not recognized @@ -693,4 +760,5 @@ defining dynamic data models. [ref-custom-granularities]: /reference/data-model/dimensions#granularities [ref-style-guide]: /guides/style-guide [ref-polymorphism]: /product/data-modeling/concepts/polymorphic-cubes -[ref-data-blending]: /product/data-modeling/concepts/data-blending \ No newline at end of file +[ref-data-blending]: /product/data-modeling/concepts/data-blending +[link-js-template-literals]: https://developer.mozilla.org/en-US/docs/Learn_web_development/Core/Scripting/Strings#embedding_javascript \ No newline at end of file diff --git a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts index 30933af64df3d..bfcaab5d0cd28 100644 --- a/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts +++ b/packages/cubejs-schema-compiler/src/compiler/YamlCompiler.ts @@ -230,11 +230,11 @@ export class YamlCompiler { } else if (str[i] === '`' && peek().inStr) { result.push(str[i]); stateStack.pop(); - } else if (str[i] === '{' && str[i + 1] === '{' && peek()?.inFormattedStr) { - result.push('{{'); + } else if (str[i] === '\\' && str[i + 1] === '{' && stateStack.length === 0) { + result.push('\\{'); i += 1; - } else if (str[i] === '}' && str[i + 1] === '}' && peek()?.inFormattedStr) { - result.push('}}'); + } else if (str[i] === '\\' && str[i + 1] === '}' && stateStack.length === 0) { + result.push('\\}'); i += 1; } else if (str[i] === '{' && peek()?.inFormattedStr) { result.push(str[i]); diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts index a6abb2eae3a56..9695454ca3077 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/yaml-compiler.test.ts @@ -57,6 +57,40 @@ cubes: ); }); + it('simple with json/curly in sql', async () => { + const { compiler, joinGraph, cubeEvaluator } = prepareYamlCompiler(` +cubes: + - name: ActiveUsers + sql: SELECT 1 as user_id, '2022-01-01'::TIMESTAMP as timestamp, CAST('\\{"key":"value"\\}'::JSON AS TEXT) AS json_col + + dimensions: + - name: time + sql: "{CUBE}.timestamp" + type: time + - name: json_col + sql: json_col + type: string + `); + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['ActiveUsers.time', 'ActiveUsers.json_col'], + timezone: 'UTC' + }); + + console.log(query.buildSqlAndParams()); + + const res = await dbRunner.testQuery(query.buildSqlAndParams()); + console.log(JSON.stringify(res)); + + expect(res).toEqual( + [{ + active_users__time: '2022-01-01T00:00:00.000Z', + active_users__json_col: '{"key":"value"}', + }] + ); + }); + it('missed sql', async () => { const { compiler } = prepareYamlCompiler(` cubes: diff --git a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts index 5fbe4f18f1100..cf04ff44961a4 100644 --- a/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/yaml-schema.test.ts @@ -77,30 +77,33 @@ describe('Yaml Schema Testing', () => { await compiler.compile(); }); - it('escapes backticks', async () => { + it('empty file', async () => { const { compiler } = prepareYamlCompiler( - `cubes: - - name: Users - sql: SELECT * FROM e2e.users - dimensions: - - name: id - sql: id - type: number - primaryKey: true - - name: c2 - sql: "{CUBE}.\`C2\`" - type: string - ` + ' ' ); await compiler.compile(); }); - it('empty string - issue#7126', async () => { + it('empty cubes in file', async () => { const { compiler } = prepareYamlCompiler( - `cubes: - - name: Users - title: ''` + 'cubes: ' + ); + + await compiler.compile(); + }); + + it('empty views in file', async () => { + const { compiler } = prepareYamlCompiler( + 'views: ' + ); + + await compiler.compile(); + }); + + it('Unexpected keys', async () => { + const { compiler } = prepareYamlCompiler( + 'circles: ' ); try { @@ -108,15 +111,17 @@ describe('Yaml Schema Testing', () => { throw new Error('compile must return an error'); } catch (e: any) { - expect(e.message).toContain('Users cube: "title" must be a string'); + expect(e.message).toContain('Unexpected YAML key'); } }); - it('null for string field', async () => { + it('can\'t parse error', async () => { const { compiler } = prepareYamlCompiler( `cubes: - - name: Users - title: null` + - name: Products + sql: select { "string"+123 } from tbl + dimensions: + ` ); try { @@ -124,106 +129,196 @@ describe('Yaml Schema Testing', () => { throw new Error('compile must return an error'); } catch (e: any) { - expect(e.message).toContain('Unexpected input during yaml transpiling: null'); + expect(e.message).toContain('Can\'t parse python expression'); } }); - it('empty (null) dimensions', async () => { + it('unnamed measure', async () => { const { compiler } = prepareYamlCompiler( `cubes: - name: Users sql: SELECT * FROM e2e.users dimensions: - ` + - sql: id + type: number + primaryKey: true + ` ); - await compiler.compile(); + try { + await compiler.compile(); + + throw new Error('compile must return an error'); + } catch (e: any) { + expect(e.message).toContain('name isn\'t defined for dimension: '); + } }); - it('empty (null) measures', async () => { - const { compiler } = prepareYamlCompiler( - `cubes: + describe('Escaping and quoting', () => { + it('escapes backticks', async () => { + const { compiler } = prepareYamlCompiler( + `cubes: - name: Users sql: SELECT * FROM e2e.users - measures: - ` - ); + dimensions: + - name: id + sql: id + type: number + primaryKey: true + - name: c2 + sql: "{CUBE}.\`C2\`" + type: string + ` + ); - await compiler.compile(); - }); + await compiler.compile(); + }); - it('empty (null) segments', async () => { - const { compiler } = prepareYamlCompiler( - `cubes: + it('escape double quotes', async () => { + const { compiler } = prepareYamlCompiler( + `cubes: - name: Users sql: SELECT * FROM e2e.users - segments: - ` - ); + dimensions: + - name: id + sql: id + type: number + primaryKey: true + - name: id_str + sql: "ID" + type: string + ` + ); - await compiler.compile(); - }); + await compiler.compile(); + }); - it('empty (null) preAggregations', async () => { - const { compiler } = prepareYamlCompiler( - `cubes: + it('escape curly braces', async () => { + const { compiler } = prepareYamlCompiler( + `cubes: - name: Users - sql: SELECT * FROM e2e.users - dimensions: [] - measures: [] - segments: [] - preAggregations: - joins: [] - hierarchies: [] - ` - ); + sql: SELECT 1 AS id, CAST('\\{"key":"value"\\}'::JSON AS TEXT) AS json_col + dimensions: + - name: id + sql: id + type: number + primaryKey: true + ` + ); - await compiler.compile(); + await compiler.compile(); + }); }); - it('empty (null) joins', async () => { - const { compiler } = prepareYamlCompiler( - `cubes: - - name: Users - sql: SELECT * FROM e2e.users - joins: - ` - ); + describe('Parsing edge cases: ', () => { + it('empty string - issue#7126', async () => { + const { compiler } = prepareYamlCompiler( + `cubes: + - name: Users + title: ''` + ); - await compiler.compile(); - }); + try { + await compiler.compile(); - it('empty (null) hierarchies', async () => { - const { compiler } = prepareYamlCompiler( - `cubes: - - name: Users - sql: SELECT * FROM e2e.users - hierarchies: - ` - ); + throw new Error('compile must return an error'); + } catch (e: any) { + expect(e.message).toContain('Users cube: "title" must be a string'); + } + }); - await compiler.compile(); - }); + it('null for string field', async () => { + const { compiler } = prepareYamlCompiler( + `cubes: + - name: Users + title: null` + ); - it('unnamed measure', async () => { - const { compiler } = prepareYamlCompiler( - `cubes: - - name: Users - sql: SELECT * FROM e2e.users - dimensions: - - sql: id - type: number - primaryKey: true + try { + await compiler.compile(); + + throw new Error('compile must return an error'); + } catch (e: any) { + expect(e.message).toContain('Unexpected input during yaml transpiling: null'); + } + }); + + it('empty (null) dimensions', async () => { + const { compiler } = prepareYamlCompiler( + `cubes: + - name: Users + sql: SELECT * FROM e2e.users + dimensions: ` - ); + ); - try { await compiler.compile(); + }); - throw new Error('compile must return an error'); - } catch (e: any) { - expect(e.message).toContain('name isn\'t defined for dimension: '); - } + it('empty (null) measures', async () => { + const { compiler } = prepareYamlCompiler( + `cubes: + - name: Users + sql: SELECT * FROM e2e.users + measures: + ` + ); + + await compiler.compile(); + }); + + it('empty (null) segments', async () => { + const { compiler } = prepareYamlCompiler( + `cubes: + - name: Users + sql: SELECT * FROM e2e.users + segments: + ` + ); + + await compiler.compile(); + }); + + it('empty (null) preAggregations', async () => { + const { compiler } = prepareYamlCompiler( + `cubes: + - name: Users + sql: SELECT * FROM e2e.users + dimensions: [] + measures: [] + segments: [] + preAggregations: + joins: [] + hierarchies: [] + ` + ); + + await compiler.compile(); + }); + + it('empty (null) joins', async () => { + const { compiler } = prepareYamlCompiler( + `cubes: + - name: Users + sql: SELECT * FROM e2e.users + joins: + ` + ); + + await compiler.compile(); + }); + + it('empty (null) hierarchies', async () => { + const { compiler } = prepareYamlCompiler( + `cubes: + - name: Users + sql: SELECT * FROM e2e.users + hierarchies: + ` + ); + + await compiler.compile(); + }); }); it('accepts cube meta', async () => { @@ -425,6 +520,7 @@ describe('Yaml Schema Testing', () => { await compiler.compile(); }); }); + describe('Access policy: ', () => { it('defines a correct accessPolicy', async () => { const { compiler } = prepareYamlCompiler( @@ -439,6 +535,9 @@ describe('Yaml Schema Testing', () => { - name: status sql: status type: string + - name: is_true + sql: is_true + type: boolean measures: - name: count type: count @@ -461,6 +560,14 @@ describe('Yaml Schema Testing', () => { operator: equals values: - "{ securityContext.currentDate }" + - member: "count" + operator: equals + values: + - 123 + - member: "is_true" + operator: equals + values: + - true memberLevel: includes: - status