diff --git a/docs/pages/product/configuration/data-sources/pinot.mdx b/docs/pages/product/configuration/data-sources/pinot.mdx index 85a4e0b6a4801..70075af23afa4 100644 --- a/docs/pages/product/configuration/data-sources/pinot.mdx +++ b/docs/pages/product/configuration/data-sources/pinot.mdx @@ -5,7 +5,8 @@ redirect_from: # Pinot -Apache Pinot is a real-time distributed OLAP datastore purpose-built for low-latency, high-throughput analytics, and perfect for user-facing analytical workloads +Apache Pinot is a real-time distributed OLAP datastore purpose-built for low-latency, high-throughput analytics, +and perfect for user-facing analytical workloads. ## Prerequisites @@ -28,12 +29,12 @@ CUBEJS_DB_PASS=********** ## Environment Variables -| Environment Variable | Description | Possible Values | Required | -| -------------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------- | :------: | -| `CUBEJS_DB_HOST` | The host URL for your Pinot broker | A valid host URL | ✅ | -| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ✅ | -| `CUBEJS_DB_USER` | The username used to connect to the broker | A valid username | ❌ | -| `CUBEJS_DB_PASS` | The password used to connect to the broker | A valid password | ❌ | +| Environment Variable | Description | Possible Values | Required | +|----------------------|--------------------------------------------|---------------------|:--------:| +| `CUBEJS_DB_HOST` | The host URL for your Pinot broker | A valid host URL | ✅ | +| `CUBEJS_DB_PORT` | The port for the database connection | A valid port number | ✅ | +| `CUBEJS_DB_USER` | The username used to connect to the broker | A valid username | ❌ | +| `CUBEJS_DB_PASS` | The password used to connect to the broker | A valid password | ❌ | ## Pre-Aggregation Feature Support @@ -55,8 +56,8 @@ here][ref-caching-using-preaggs-build-strats]. | Feature | Works with read-only mode? | Is default? | -| ------------- | :------------------------: | :---------: | -| Simple | ✅ | ✅ | +|---------------|:--------------------------:|:-----------:| +| Simple | ✅ | ✅ | | Batching | - | - | | Export bucket | - | - | @@ -77,7 +78,7 @@ Pinot does not support export buckets. ## SSL -Cube does not require any additional configuration to enable SSL as Elasticsearch connections are made over HTTPS. +Cube does not require any additional configuration to enable SSL as Pinot connections are made over HTTPS. [pinot]: https://docs.pinot.apache.org/ [pinot-docs-approx-agg-fns]: diff --git a/packages/cubejs-pinot-driver/CHANGELOG.md b/packages/cubejs-pinot-driver/CHANGELOG.md index e69de29bb2d1d..293c5681ad929 100644 --- a/packages/cubejs-pinot-driver/CHANGELOG.md +++ b/packages/cubejs-pinot-driver/CHANGELOG.md @@ -0,0 +1,2 @@ +# Change Log + diff --git a/packages/cubejs-pinot-driver/README.md b/packages/cubejs-pinot-driver/README.md index a9b7a5d4e76ca..1bded317e4699 100644 --- a/packages/cubejs-pinot-driver/README.md +++ b/packages/cubejs-pinot-driver/README.md @@ -11,36 +11,6 @@ Cube.js Pinot driver. [Learn more](https://github.com/cube-js/cube.js#getting-started) -### Project Status +# License -In Review. [#8689](https://github.com/cube-js/cube/pull/8689) - -### Installation - -`npm i @inthememory/pinot-driver` - -### Usage -#### For Docker - -Build development [docker image](https://github.com/cube-js/cube/blob/master/packages/cubejs-docker/DEVELOPMENT.md). - -Assuming the built image is tagged `cubejs/cube:dev` - -``` -FROM cubejs/cube:dev -RUN npm i @inthememory/pinot-driver -``` - -``` - environment: - - CUBEJS_DB_TYPE=pinot - - CUBEJS_DB_HOST= #broker_host - - CUBEJS_DB_PORT= #broker_port - - CUBEJS_DB_USER= #database user - - CUBEJS_DB_PASS= #database password - - CUBEJS_DEV_MODE=true #if running locally -``` - -### License - -Cube.js Pinot Database Driver is [Apache 2.0 licensed](./LICENSE). \ No newline at end of file +Cube.js Pinot Database Driver is [Apache 2.0 licensed](./LICENSE). diff --git a/packages/cubejs-pinot-driver/package.json b/packages/cubejs-pinot-driver/package.json index d6d866d9f2b65..1d7662c044dfc 100644 --- a/packages/cubejs-pinot-driver/package.json +++ b/packages/cubejs-pinot-driver/package.json @@ -1,12 +1,12 @@ { - "name": "@inthememory/pinot-driver", + "name": "@cubejs-backend/pinot-driver", "description": "Cube.js Pinot database driver", - "author": "Julian Ronsse, InTheMemory", - "version": "0.35.80", + "author": "Julian Ronsse, InTheMemory, Cube Dev, Inc.", + "version": "1.1.0", "repository": { "type": "git", "url": "https://github.com/cube-js/cube.git", - "directory": "packages/cubejs-pinotdb-driver" + "directory": "packages/cubejs-pinot-driver" }, "engines": { "node": "^14.0.0 || ^16.0.0 || >=17.0.0" @@ -21,17 +21,17 @@ "build": "rm -rf dist && npm run tsc", "tsc": "tsc", "watch": "tsc -w", - "integration": "jest dist/test", - "integration:pinot": "jest dist/test", + "integration": "npm run integration:pinot", + "integration:pinot": "jest --verbose dist/test", "lint": "eslint src/* --ext .ts", "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { - "@cubejs-backend/base-driver": "^0.36.0", - "@cubejs-backend/schema-compiler": "^0.36.0", - "@cubejs-backend/shared": "^0.36.0", + "@cubejs-backend/base-driver": "1.0.3", + "@cubejs-backend/schema-compiler": "1.1.0", + "@cubejs-backend/shared": "1.0.3", "node-fetch": "^2.6.1", - "ramda": "^0.27.0", + "ramda": "^0.27.2", "sqlstring": "^2.3.3" }, "license": "Apache-2.0", @@ -39,11 +39,11 @@ "access": "public" }, "devDependencies": { - "@cubejs-backend/linter": "^0.36.0", + "@cubejs-backend/linter": "^1.0.0", "@types/jest": "^27", "jest": "^27", "should": "^13.2.3", - "testcontainers": "^10.13.0", + "testcontainers": "^10.10.4", "typescript": "~5.2.2" }, "jest": { diff --git a/packages/cubejs-pinot-driver/pinot-resources/controller.conf b/packages/cubejs-pinot-driver/pinot-resources/controller.conf index 2651bb565a182..5270dfb623ed5 100644 --- a/packages/cubejs-pinot-driver/pinot-resources/controller.conf +++ b/packages/cubejs-pinot-driver/pinot-resources/controller.conf @@ -6,4 +6,4 @@ controller.data.dir=/var/pinot/controller/data controller.zk.str=zookeeper:2181 pinot.set.instance.id.to.hostname=true controller.admin.access.control.principals=admin -controller.admin.access.control.principals.admin.password=mysecret \ No newline at end of file +controller.admin.access.control.principals.admin.password=mysecret diff --git a/packages/cubejs-pinot-driver/src/PinotDriver.ts b/packages/cubejs-pinot-driver/src/PinotDriver.ts index e997650dcc0fa..60fa891c62607 100644 --- a/packages/cubejs-pinot-driver/src/PinotDriver.ts +++ b/packages/cubejs-pinot-driver/src/PinotDriver.ts @@ -1,7 +1,7 @@ /** * @copyright Cube Dev, Inc. * @license Apache-2.0 - * @fileoverview The `PrestoDriver` and related types declaration. + * @fileoverview The `PinotDriver` and related types declaration. */ import { @@ -146,7 +146,10 @@ export class PinotDriver extends BaseDriver implements DriverInterface { 'Content-Type': 'application/json', ...this.authorizationHeaders() }), - body: JSON.stringify({ sql: query, queryOptions: `useMultistageEngine=true;timeoutMs=${this.config.queryTimeout}` }) + body: JSON.stringify({ + sql: query, + queryOptions: `useMultistageEngine=true;timeoutMs=${this.config.queryTimeout}` + }) }); return new Promise((resolve, reject) => { diff --git a/packages/cubejs-pinot-driver/src/PinotQuery.ts b/packages/cubejs-pinot-driver/src/PinotQuery.ts index fcf82809a348f..e5a07c99f8a5c 100644 --- a/packages/cubejs-pinot-driver/src/PinotQuery.ts +++ b/packages/cubejs-pinot-driver/src/PinotQuery.ts @@ -89,7 +89,8 @@ export class PinotQuery extends BaseQuery { const values = timeDimension.timeSeries().map( ([from, to]) => `select '${from}' f, '${to}' t` ).join(' UNION ALL '); - return `SELECT ${this.timeStampCast('dates.f')} date_from, ${this.timeStampCast('dates.t')} date_to FROM (${values}) AS dates`; + return `SELECT ${this.timeStampCast('dates.f')} date_from, ${this.timeStampCast('dates.t')} date_to + FROM (${values}) AS dates`; } public applyMeasureFilters(evaluateSql: '*' | string, symbol: any, cubeName: string) { @@ -123,14 +124,6 @@ export class PinotQuery extends BaseQuery { }; } - public hllInit(sql: string) { - return this.countDistinctApprox(sql); // todo: ensure the correct way to do so in pinot - } - - public hllMerge(sql: string) { - return this.countDistinctApprox(sql); // todo: ensure the correct way to do so in pinot - } - public countDistinctApprox(sql: string) { return `DistinctCountHLLPlus(${sql})`; } diff --git a/packages/cubejs-server-core/src/core/DriverDependencies.js b/packages/cubejs-server-core/src/core/DriverDependencies.js index 47317889a285a..ba89f6c81a3a9 100644 --- a/packages/cubejs-server-core/src/core/DriverDependencies.js +++ b/packages/cubejs-server-core/src/core/DriverDependencies.js @@ -26,7 +26,7 @@ module.exports = { ksql: '@cubejs-backend/ksql-driver', questdb: '@cubejs-backend/questdb-driver', materialize: '@cubejs-backend/materialize-driver', - pinot: '@inthememory/pinot-driver', + pinot: '@cubejs-backend/pinot-driver', // List for JDBC drivers 'databricks-jdbc': '@cubejs-backend/databricks-jdbc-driver', };