feat(cubesql): Group By Rollup support #16869

Workflow file for this run

	name: Build

	on:
	push:
	paths:
	- '.github/actions/smoke.sh'
	- '.github/workflows/push.yml'
	- '.github/workflows/master.yml'
	- 'packages/**'
	- 'rust/cubestore/**'
	- 'rust/cubesql/**'
	- '.eslintrc.js'
	- '.prettierrc'
	- 'package.json'
	- 'lerna.json'
	- 'rollup.config.js'
	- 'yarn.lock'
	branches:
	- 'master'
	pull_request:
	paths:
	- '.github/workflows/push.yml'
	- '.github/workflows/master.yml'
	- 'packages/**'
	- 'rust/cubestore/**'
	- 'rust/cubesql/**'
	- '.eslintrc.js'
	- '.prettierrc'
	- 'package.json'
	- 'lerna.json'
	- 'rollup.config.js'
	- 'yarn.lock'

	jobs:
	unit:
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	needs: latest-tag-sha
	if: (needs['latest-tag-sha'].outputs.sha != github.sha)

	strategy:
	matrix:
	node-version: [18.x, 20.x]
	fail-fast: false

	steps:
	- id: get-tag-out
	run: echo "$OUT"
	env:
	OUT: ${{ needs['latest-tag-sha'].outputs.sha }}
	- name: Checkout
	uses: actions/checkout@v4
	with:
	# pulls all commits (needed for codecov)
	fetch-depth: 2
	- name: Install Rust
	uses: actions-rs/toolchain@v1
	with:
	toolchain: nightly-2023-12-13
	override: true
	components: rustfmt
	- name: Install Node.js ${{ matrix.node-version }}
	uses: actions/setup-node@v3
	with:
	node-version: ${{ matrix.node-version }}
	- name: Get yarn cache directory path
	id: yarn-cache-dir-path
	run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
	shell: bash
	- name: Restore yarn cache
	uses: actions/cache@v3
	with:
	path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
	key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
	restore-keys: \|
	${{ runner.os }}-yarn-
	- name: Set Yarn version
	run: yarn policies set-version v1.22.19
	- name: Yarn install
	uses: nick-invision/retry@v2
	env:
	CUBESTORE_SKIP_POST_INSTALL: true
	with:
	max_attempts: 3
	retry_on: error
	retry_wait_seconds: 15
	timeout_minutes: 20
	command: yarn install --frozen-lockfile
	- name: Lerna tsc
	run: yarn tsc
	- name: Build client
	run: yarn build
	- name: Lerna test
	run: yarn lerna run --concurrency 1 --stream --no-prefix unit
	# - uses: codecov/codecov-action@v1
	# if: (matrix.node-version == '18.x')
	# with:
	# files: ./packages/*/coverage/clover.xml
	# flags: cube-backend
	# verbose: true # optional (default = false)

	lint:
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	needs: latest-tag-sha
	if: (needs['latest-tag-sha'].outputs.sha != github.sha)

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Install Rust
	uses: actions-rs/toolchain@v1
	with:
	toolchain: nightly-2023-12-13
	override: true
	components: rustfmt
	- name: Install Node.js 18.x
	uses: actions/setup-node@v3
	with:
	node-version: 18.x
	- name: Get yarn cache directory path
	id: yarn-cache-dir-path
	run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
	shell: bash
	- name: Restore yarn cache
	uses: actions/cache@v3
	with:
	path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
	key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
	restore-keys: \|
	${{ runner.os }}-yarn-
	- name: Set Yarn version
	run: yarn policies set-version v1.22.19
	- name: Yarn install
	uses: nick-invision/retry@v2
	env:
	CUBESTORE_SKIP_POST_INSTALL: true
	with:
	max_attempts: 3
	retry_on: error
	retry_wait_seconds: 15
	timeout_minutes: 20
	command: yarn install --frozen-lockfile
	- name: NPM lint
	run: yarn lint:npm
	- name: Lerna lint
	run: yarn lerna run --concurrency 1 lint

	build:
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	needs: latest-tag-sha
	if: (needs['latest-tag-sha'].outputs.sha != github.sha)

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Install Rust
	uses: actions-rs/toolchain@v1
	with:
	toolchain: nightly-2023-12-13
	override: true
	components: rustfmt
	- name: Install Node.js 18.x
	uses: actions/setup-node@v3
	with:
	node-version: 18.x
	- name: Get yarn cache directory path
	id: yarn-cache-dir-path
	run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
	shell: bash
	- name: Restore yarn cache
	uses: actions/cache@v3
	with:
	path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
	key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
	restore-keys: \|
	${{ runner.os }}-yarn-
	- name: Set Yarn version
	run: yarn policies set-version v1.22.19
	- name: Yarn install
	uses: nick-invision/retry@v2
	env:
	CUBESTORE_SKIP_POST_INSTALL: true
	with:
	max_attempts: 3
	retry_on: error
	retry_wait_seconds: 15
	timeout_minutes: 20
	command: yarn install --frozen-lockfile
	- name: Check Yarn lock wasn't modified
	run: if [ "$(git status \| grep nothing)x" = "x" ]; then echo "Non empty changeset after lerna bootstrap"; git status; exit 1; else echo "Nothing to commit. Proceeding"; fi;
	- name: Build Core Client libraries
	run: yarn build
	- name: Build other packages
	run: yarn lerna run --concurrency 1 build
	env:
	NODE_OPTIONS: --max_old_space_size=4096

	build-cubestore:
	needs: [latest-tag-sha]
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	if: (needs['latest-tag-sha'].outputs.sha != github.sha)
	steps:
	- name: Maximize build space (disk space limitations)
	run: \|
	echo "Before"
	df -h
	sudo apt-get remove -y 'php.*'
	sudo apt-get remove -y '^mongodb-.*'
	sudo apt-get remove -y '^mysql-.*'
	sudo apt-get autoremove -y
	sudo apt-get clean

	sudo rm -rf /usr/share/dotnet
	sudo rm -rf /usr/local/lib/android
	sudo rm -rf /opt/ghc
	sudo rm -rf /opt/hostedtoolcache/CodeQL
	echo "After"
	df -h
	- name: Checkout
	uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	with:
	workspaces: ./rust/cubestore -> target
	# Separate path for release key to protect cache bloating
	shared-key: cubestore-release
	key: ubuntu-20.04
	- name: Build Cube Store
	run: \|
	cd rust/cubestore
	cargo build --release -j 4
	- name: 'Upload cubestored-x86_64-unknown-linux-gnu-release artifact'
	uses: actions/upload-artifact@v3
	with:
	name: cubestored-x86_64-unknown-linux-gnu-release
	path: ./rust/cubestore/target/release/cubestored
	retention-days: 5

	integration-cubestore:
	needs: [latest-tag-sha, build-cubestore]
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	if: (needs['latest-tag-sha'].outputs.sha != github.sha)

	strategy:
	matrix:
	node-version: [18.x]
	fail-fast: false

	steps:
	- name: Maximize build space (disk space limitations)
	run: \|
	echo "Before"
	df -h
	sudo apt-get remove -y 'php.*'
	sudo apt-get remove -y '^mongodb-.*'
	sudo apt-get remove -y '^mysql-.*'
	sudo apt-get autoremove -y
	sudo apt-get clean

	sudo rm -rf /usr/share/dotnet
	sudo rm -rf /usr/local/lib/android
	sudo rm -rf /opt/ghc
	sudo rm -rf /opt/hostedtoolcache/CodeQL
	echo "After"
	df -h
	- name: Checkout
	uses: actions/checkout@v4
	- name: Install Node.js ${{ matrix.node-version }}
	uses: actions/setup-node@v3
	with:
	node-version: ${{ matrix.node-version }}
	- name: Get yarn cache directory path
	id: yarn-cache-dir-path
	run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
	shell: bash
	- name: Restore yarn cache
	uses: actions/cache@v3
	with:
	path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
	key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
	restore-keys: \|
	${{ runner.os }}-yarn-
	- name: Set Yarn version
	run: yarn policies set-version v1.22.19
	- name: Yarn install
	uses: nick-invision/retry@v2
	env:
	CUBESTORE_SKIP_POST_INSTALL: true
	with:
	max_attempts: 3
	retry_on: error
	retry_wait_seconds: 15
	timeout_minutes: 20
	command: yarn install --frozen-lockfile
	- name: Lerna tsc
	run: yarn tsc
	- name: Download cubestored-x86_64-unknown-linux-gnu-release artifact
	uses: actions/download-artifact@v3
	with:
	path: ./rust/cubestore/target/release/
	name: cubestored-x86_64-unknown-linux-gnu-release
	- name: Run Cube Store in background
	run: \|
	chmod +x ./rust/cubestore/target/release/cubestored
	RUNNER_TRACKING_ID="" && ./rust/cubestore/target/release/cubestored &
	- name: Run Cubestore Integration
	timeout-minutes: 10
	run: \|
	yarn lerna run --concurrency 1 --stream --no-prefix integration:cubestore

	integration:
	needs: [unit, lint, latest-tag-sha]
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	if: (needs['latest-tag-sha'].outputs.sha != github.sha)

	strategy:
	matrix:
	node-version: [18.x]
	db: [
	'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb',
	'mysql-aurora-serverless', 'crate', 'mongobi'
	]
	fail-fast: false

	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Install Rust
	uses: actions-rs/toolchain@v1
	with:
	toolchain: nightly-2023-12-13
	override: true
	components: rustfmt
	- name: Install Node.js ${{ matrix.node-version }}
	uses: actions/setup-node@v3
	with:
	node-version: ${{ matrix.node-version }}
	- name: Get yarn cache directory path
	id: yarn-cache-dir-path
	run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
	shell: bash
	- name: Restore yarn cache
	uses: actions/cache@v3
	with:
	path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
	key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
	restore-keys: \|
	${{ runner.os }}-yarn-
	- name: Set Yarn version
	run: yarn policies set-version v1.22.19
	- name: Yarn install
	uses: nick-invision/retry@v2
	env:
	CUBESTORE_SKIP_POST_INSTALL: true
	with:
	max_attempts: 3
	retry_on: error
	retry_wait_seconds: 15
	timeout_minutes: 20
	command: yarn install --frozen-lockfile
	- name: Lerna tsc
	run: yarn tsc
	- name: Run Integration tests for ${{ matrix.db }} matrix
	timeout-minutes: 30
	run: ./.github/actions/integration/${{ matrix.db }}.sh

	integration-smoke:
	needs: [ latest-tag-sha, build-cubestore ]
	runs-on: ubuntu-20.04
	timeout-minutes: 90
	if: (needs['latest-tag-sha'].outputs.sha != github.sha)

	strategy:
	matrix:
	node-version: [ 18.x ]
	fail-fast: false

	steps:
	- name: Maximize build space (disk space limitations)
	run: \|
	echo "Before"
	df -h
	sudo apt-get remove -y 'php.*'
	sudo apt-get remove -y '^mongodb-.*'
	sudo apt-get remove -y '^mysql-.*'
	sudo apt-get autoremove -y
	sudo apt-get clean

	sudo rm -rf /usr/share/dotnet
	sudo rm -rf /usr/local/lib/android
	sudo rm -rf /opt/ghc
	sudo rm -rf /opt/hostedtoolcache/CodeQL
	echo "After"
	df -h
	- name: Checkout
	uses: actions/checkout@v4
	- name: Install Node.js ${{ matrix.node-version }}
	uses: actions/setup-node@v3
	with:
	node-version: ${{ matrix.node-version }}
	- name: Get yarn cache directory path
	id: yarn-cache-dir-path
	run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
	shell: bash
	- name: Restore yarn cache
	uses: actions/cache@v3
	with:
	path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
	key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
	restore-keys: \|
	${{ runner.os }}-yarn-
	- name: Set Yarn version
	run: yarn policies set-version v1.22.19
	- name: Yarn install
	uses: nick-invision/retry@v2
	env:
	CUBESTORE_SKIP_POST_INSTALL: true
	with:
	max_attempts: 3
	retry_on: error
	retry_wait_seconds: 15
	timeout_minutes: 20
	command: yarn install --frozen-lockfile
	- name: Install instant client for Oracle
	uses: GoodManWEN/oracle-client-action@main
	- name: Build client
	run: yarn build
	- name: Lerna tsc
	run: yarn tsc
	- name: Download cubestored-x86_64-unknown-linux-gnu-release artifact
	uses: actions/download-artifact@v3
	with:
	path: rust/cubestore/downloaded/latest/bin/
	name: cubestored-x86_64-unknown-linux-gnu-release
	- name: Chmod +x for cubestored
	run: \|
	chmod +x ./rust/cubestore/downloaded/latest/bin/cubestored
	- name: Run Integration smoke tests
	timeout-minutes: 30
	run: ./.github/actions/smoke.sh

	docker-image-latest-set-tag:
	# At least git should be completed pushed up until this moment
	needs: [lint, latest-tag-sha]
	if: (needs['latest-tag-sha'].outputs.sha != github.sha)
	runs-on: ubuntu-20.04
	outputs:
	tag: ${{ steps.get-tag.outputs.tag }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- id: get-tag
	run: echo "::set-output name=tag::$(git tag --contains $GITHUB_SHA)"
	env:
	GITHUB_SHA: ${{ github.sha }}

	latest-tag-sha:
	runs-on: ubuntu-20.04
	outputs:
	sha: ${{ steps.get-tag.outputs.sha }}
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	- id: git-log
	run: git log HEAD~30..HEAD
	- id: get-tag-test
	run: echo "$SHA $(git rev-list -n 1 $(git tag --contains $SHA))"
	env:
	SHA: ${{ github.sha }}
	- id: get-tag
	run: echo "::set-output name=sha::$(git rev-list -n 1 $(git tag --contains $SHA))"
	env:
	SHA: ${{ github.sha }}
	- id: get-tag-out
	run: echo "$OUT"
	env:
	OUT: ${{ steps.get-tag.outputs.sha }}

	docker-dev:
	needs: [latest-tag-sha]
	if: (needs['latest-tag-sha'].outputs.sha != github.sha)
	name: Build & Test :dev for ${{ matrix.name }} without pushing
	runs-on: ubuntu-22.04
	timeout-minutes: 60
	services:
	registry:
	image: registry:2
	ports:
	- 5000:5000
	strategy:
	matrix:
	dockerfile:
	- dev.Dockerfile
	include:
	- dockerfile: dev.Dockerfile
	name: Debian
	tag: tmp-dev
	fail-fast: false
	steps:
	- name: Maximize build space (disk space limitations)
	run: \|
	echo "Before"
	df -h
	sudo apt-get remove -y 'php.*'
	sudo apt-get remove -y '^mongodb-.*'
	sudo apt-get remove -y '^mysql-.*'
	sudo apt-get autoremove -y
	sudo apt-get clean

	sudo rm -rf /usr/share/dotnet
	sudo rm -rf /usr/local/lib/android
	sudo rm -rf /opt/ghc
	sudo rm -rf /opt/hostedtoolcache/CodeQL
	echo "After"
	df -h
	- name: Checkout
	uses: actions/checkout@v4
	- name: Set up QEMU
	uses: docker/setup-qemu-action@v2
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v2
	with:
	version: v0.9.1
	driver-opts: network=host
	- name: Build image
	uses: docker/build-push-action@v3
	timeout-minutes: 30
	with:
	context: .
	file: ./packages/cubejs-docker/${{ matrix.dockerfile }}
	platforms: linux/amd64
	push: true
	tags: localhost:5000/cubejs/cube:${{ matrix.tag }}
	- name: Use Node.js 18.x
	uses: actions/setup-node@v3
	with:
	node-version: 18.x
	- name: Get yarn cache directory path
	id: yarn-cache-dir-path
	run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
	shell: bash
	- name: Restore yarn cache
	uses: actions/cache@v3
	with:
	path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
	key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
	restore-keys: \|
	${{ runner.os }}-yarn-
	- name: Set Yarn version
	run: yarn policies set-version v1.22.19
	- name: Yarn install
	uses: nick-invision/retry@v2
	env:
	CUBESTORE_SKIP_POST_INSTALL: true
	with:
	max_attempts: 3
	retry_on: error
	retry_wait_seconds: 15
	timeout_minutes: 20
	command: yarn install --frozen-lockfile
	- name: Build client
	run: yarn build
	- name: Lerna tsc
	run: yarn tsc
	- name: Testing CubeJS (container mode) via BirdBox
	run: \|
	cd packages/cubejs-testing/
	export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
	export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
	export DEBUG=testcontainers
	yarn run dataset:minimal
	yarn run birdbox:postgresql
	yarn run birdbox:postgresql-pre-aggregations
	# - name: Testing Athena driver (container mode) via BirdBox
	# env:
	# CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }}
	# CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }}
	# CUBEJS_AWS_REGION: us-east-1
	# CUBEJS_AWS_S3_OUTPUT_LOCATION: s3://cubejs-opensource/testing/output
	# CUBEJS_DB_EXPORT_BUCKET: s3://cubejs-opensource/testing/export
	# run: \|
	# cd packages/cubejs-testing/
	# export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
	# export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
	# export DEBUG=testcontainers
	# yarn run driver:athena --log=ignore --mode=docker
	# - name: Testing BigQuery driver (container mode) via BirdBox
	# env:
	# CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.CUBEJS_DB_BQ_CREDENTIALS }}
	# CUBEJS_DB_BQ_PROJECT_ID: cube-open-source
	# CUBEJS_DB_EXPORT_BUCKET: cube-open-source-export-bucket
	# run: \|
	# cd packages/cubejs-testing/
	# export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
	# export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
	# export DEBUG=testcontainers
	# yarn run driver:bigquery --log=ignore --mode=docker
	- name: Testing PostgreSQL driver (container mode) via BirdBox
	env:
	CUBEJS_DB_TYPE: postgres
	CUBEJS_DB_USER: postgres
	CUBEJS_DB_PASS: postgres
	run: \|
	cd packages/cubejs-testing/
	export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
	export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
	export DEBUG=testcontainers
	yarn run driver:postgres --log=ignore --mode=docker
	- name: Testing Docker image via Cypress (Chrome)
	env:
	CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
	BIRDBOX_CYPRESS_UPDATE_SCREENSHOTS: ${{ contains(github.event.head_commit.message, '[update screenshots]') }}
	run: \|
	cd packages/cubejs-testing/
	export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }}
	export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/
	export BIRDBOX_CYPRESS_BROWSER=chrome
	export BIRDBOX_CYPRESS_TARGET=postgresql
	export DEBUG=testcontainers
	yarn run cypress:install
	yarn run cypress:birdbox
	- name: Upload screenshots on failure
	uses: actions/upload-artifact@v2
	if: failure()
	with:
	name: cypress-screenshots-docker-dev-${{ matrix.name }}
	path: packages/cubejs-testing/cypress/screenshots

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat(cubesql): Group By Rollup support #16869

Workflow file

feat(cubesql): Group By Rollup support #16869

Jobs

Run details

Workflow file for this run