From 969061588c91f4aac61ccef9228d5363048e4530 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Tue, 31 Oct 2023 10:12:29 +0100 Subject: [PATCH 01/20] add v8 to ci --- .github/workflows/ci-build.yml | 9 ++++++-- .github/workflows/dependencies_check.yml | 23 +++++++++++++++---- dependencies.md | 10 -------- doc/changes/changelog.md | 1 + doc/changes/changes_2.2.3.md | 20 ++++++++++++++++ doc/user_guide/postgresql_user_guide.md | 2 +- pk_generated_parent.pom | 14 +++++------ pom.xml | 6 ++--- ...gresVirtualSchemaIntegrationTestSetup.java | 2 +- 9 files changed, 59 insertions(+), 28 deletions(-) create mode 100644 doc/changes/changes_2.2.3.md diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 47d57c5..4088dc1 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -10,8 +10,12 @@ jobs: build: runs-on: ubuntu-20.04 # UDFs fail with "VM error: Internal error: VM crashed" on ubuntu-latest concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.docker_db_version }} cancel-in-progress: true + strategy: + fail-fast: false + matrix: + docker_db_version: ["7.1.24", "8.23.0"] steps: - name: Checkout the repository uses: actions/checkout@v4 @@ -37,7 +41,8 @@ jobs: run: | JAVA_HOME=$JAVA_HOME_11_X64 mvn --batch-mode clean verify \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ - -DtrimStackTrace=false + -DtrimStackTrace=false \ + -Dcom.exasol.dockerdb.image=${{ matrix.docker_db_version }} - name: Publish Test Report uses: scacap/action-surefire-report@v1 if: ${{ always() && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} diff --git a/.github/workflows/dependencies_check.yml b/.github/workflows/dependencies_check.yml index 3059964..4b6eadf 100644 --- a/.github/workflows/dependencies_check.yml +++ b/.github/workflows/dependencies_check.yml @@ -1,12 +1,15 @@ -name: Dependencies Check +name: Report Security Issues for Repository on: + workflow_dispatch: schedule: - cron: "0 2 * * *" jobs: - build: + report_security_issues: runs-on: ubuntu-latest + permissions: + issues: write steps: - uses: actions/checkout@v4 @@ -16,5 +19,17 @@ jobs: distribution: "temurin" java-version: 11 cache: "maven" - - name: Checking dependencies for vulnerabilities - run: mvn --batch-mode org.sonatype.ossindex.maven:ossindex-maven-plugin:audit -f pom.xml + + - name: Generate ossindex report + run: | + mvn org.sonatype.ossindex.maven:ossindex-maven-plugin:audit \ + org.sonatype.ossindex.maven:ossindex-maven-plugin:audit-aggregate \ + -Dossindex.reportFile=$(pwd)/ossindex-report.json \ + -Dossindex.fail=false + + - name: Report Security Issues + uses: exasol/python-toolbox/.github/actions/security-issues@main + with: + format: "maven" + command: "cat ossindex-report.json" + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/dependencies.md b/dependencies.md index 93bfb92..f6dd43b 100644 --- a/dependencies.md +++ b/dependencies.md @@ -49,11 +49,6 @@ | [JaCoCo :: Maven Plugin][48] | [Eclipse Public License 2.0][49] | | [error-code-crawler-maven-plugin][50] | [MIT License][51] | | [Reproducible Build Maven Plugin][52] | [Apache 2.0][23] | -| [Maven Clean Plugin][53] | [The Apache Software License, Version 2.0][23] | -| [Maven Resources Plugin][54] | [The Apache Software License, Version 2.0][23] | -| [Maven Install Plugin][55] | [The Apache Software License, Version 2.0][23] | -| [Maven Deploy Plugin][56] | [The Apache Software License, Version 2.0][23] | -| [Maven Site Plugin 3][57] | [The Apache Software License, Version 2.0][23] | [0]: https://github.com/exasol/virtual-schema-common-jdbc/ [1]: https://github.com/exasol/virtual-schema-common-jdbc/blob/main/LICENSE @@ -108,8 +103,3 @@ [50]: https://github.com/exasol/error-code-crawler-maven-plugin/ [51]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE [52]: http://zlika.github.io/reproducible-build-maven-plugin -[53]: http://maven.apache.org/plugins/maven-clean-plugin/ -[54]: http://maven.apache.org/plugins/maven-resources-plugin/ -[55]: http://maven.apache.org/plugins/maven-install-plugin/ -[56]: http://maven.apache.org/plugins/maven-deploy-plugin/ -[57]: http://maven.apache.org/plugins/maven-site-plugin/ diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index abe8faa..f9f049f 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [2.2.3](changes_2.2.3.md) * [2.2.2](changes_2.2.2.md) * [2.2.1](changes_2.2.1.md) * [2.2.0](changes_2.2.0.md) diff --git a/doc/changes/changes_2.2.3.md b/doc/changes/changes_2.2.3.md new file mode 100644 index 0000000..e28c2fe --- /dev/null +++ b/doc/changes/changes_2.2.3.md @@ -0,0 +1,20 @@ +# Virtual Schema for PostgreSQL 2.2.3, released 2023-??-?? + +Code name: + +## Summary + +## Features + +* ISSUE_NUMBER: description + +## Dependency Updates + +### Plugin Dependency Updates + +* Updated `com.exasol:error-code-crawler-maven-plugin:1.3.0` to `1.3.1` +* Updated `com.exasol:project-keeper-maven-plugin:2.9.12` to `2.9.15` +* Updated `org.apache.maven.plugins:maven-enforcer-plugin:3.4.0` to `3.4.1` +* Updated `org.codehaus.mojo:versions-maven-plugin:2.16.0` to `2.16.1` +* Updated `org.jacoco:jacoco-maven-plugin:0.8.10` to `0.8.11` +* Updated `org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184` to `3.10.0.2594` diff --git a/doc/user_guide/postgresql_user_guide.md b/doc/user_guide/postgresql_user_guide.md index 3b0ae79..0c40375 100644 --- a/doc/user_guide/postgresql_user_guide.md +++ b/doc/user_guide/postgresql_user_guide.md @@ -47,7 +47,7 @@ The SQL statement below creates the adapter script, defines the Java class that --/ CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-11.0.2-postgresql-2.2.2.jar; + %jar /buckets///virtual-schema-dist-11.0.2-postgresql-2.2.3.jar; %jar /buckets///postgresql-.jar; / ``` diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index 71b66ab..25448a5 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -3,7 +3,7 @@ 4.0.0 com.exasol postgresql-virtual-schema-generated-parent - 2.2.2 + 2.2.3 pom UTF-8 @@ -37,7 +37,7 @@ org.sonarsource.scanner.maven sonar-maven-plugin - 3.9.1.2184 + 3.10.0.2594 org.apache.maven.plugins @@ -51,7 +51,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.4.0 + 3.4.1 enforce-maven @@ -61,7 +61,7 @@ - [3.8.7,3.9.0) + 3.6.3 @@ -121,7 +121,7 @@ org.codehaus.mojo versions-maven-plugin - 2.16.0 + 2.16.1 display-updates @@ -235,7 +235,7 @@ org.jacoco jacoco-maven-plugin - 0.8.10 + 0.8.11 prepare-agent @@ -276,7 +276,7 @@ com.exasol error-code-crawler-maven-plugin - 1.3.0 + 1.3.1 verify diff --git a/pom.xml b/pom.xml index d6faa36..68a35f2 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 postgresql-virtual-schema - 2.2.2 + 2.2.3 Virtual Schema for PostgreSQL Virtual Schema for connecting PostgreSQL as data source to Exasol https://github.com/exasol/postgresql-virtual-schema/ @@ -120,7 +120,7 @@ com.exasol project-keeper-maven-plugin - 2.9.12 + 2.9.15 @@ -184,7 +184,7 @@ postgresql-virtual-schema-generated-parent com.exasol - 2.2.2 + 2.2.3 pk_generated_parent.pom diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java index 588dac2..e03d6a0 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java @@ -26,7 +26,7 @@ * This class contains the common integration test setup for all PostgreSQL virtual schemas. */ public class PostgresVirtualSchemaIntegrationTestSetup implements Closeable { - private static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-11.0.2-postgresql-2.2.2.jar"; + private static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-11.0.2-postgresql-2.2.3.jar"; private static final Path PATH_TO_VIRTUAL_SCHEMAS_JAR = Path.of("target", VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION); private static final String SCHEMA_EXASOL = "SCHEMA_EXASOL"; private static final String ADAPTER_SCRIPT_EXASOL = "ADAPTER_SCRIPT_EXASOL"; From b1b39f5faae3e20235ee20c8b930136d0fcea63f Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Wed, 31 Jan 2024 15:28:17 +0100 Subject: [PATCH 02/20] update vsjdbc & pk + run pk fix --- .gitattributes | 3 +- .github/workflows/broken_links_checker.yml | 2 + .github/workflows/ci-build-next-java.yml | 8 +- .github/workflows/ci-build.yml | 46 ++++++--- .github/workflows/dependencies_check.yml | 11 ++- ...elease_droid_prepare_original_checksum.yml | 18 +++- .../release_droid_print_quick_checksum.yml | 11 ++- ...ase_droid_upload_github_release_assets.yml | 11 ++- .project-keeper.yml | 15 +-- dependencies.md | 94 ++++++++++--------- doc/changes/changelog.md | 1 + doc/changes/changes_3.0.0.md | 33 +++++++ doc/user_guide/postgresql_user_guide.md | 2 +- pk_generated_parent.pom | 81 ++++++++++++++-- pom.xml | 8 +- ...gresVirtualSchemaIntegrationTestSetup.java | 2 +- 16 files changed, 245 insertions(+), 101 deletions(-) create mode 100644 doc/changes/changes_3.0.0.md diff --git a/.gitattributes b/.gitattributes index f1e6c1c..c67f857 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,10 +2,11 @@ pk_generated_parent.pom linguist-genera dependencies.md linguist-generated=true doc/changes/changelog.md linguist-generated=true .github/workflows/broken_links_checker.yml linguist-generated=true +.github/workflows/ci-build.yml linguist-generated=true .github/workflows/ci-build-next-java.yml linguist-generated=true .github/workflows/dependencies_check.yml linguist-generated=true .github/workflows/release_droid_print_quick_checksum.yml linguist-generated=true .github/workflows/release_droid_upload_github_release_assets.yml linguist-generated=true - +.github/workflows/release_droid_prepare_original_checksum.yml linguist-generated=true .settings/org.eclipse.jdt.core.prefs linguist-generated=true .settings/org.eclipse.jdt.ui.prefs linguist-generated=true diff --git a/.github/workflows/broken_links_checker.yml b/.github/workflows/broken_links_checker.yml index 82ec1cd..0fbcad5 100644 --- a/.github/workflows/broken_links_checker.yml +++ b/.github/workflows/broken_links_checker.yml @@ -1,3 +1,5 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/broken_links_checker.yml name: Broken Links Checker on: diff --git a/.github/workflows/ci-build-next-java.yml b/.github/workflows/ci-build-next-java.yml index 7cbab08..e3acdb7 100644 --- a/.github/workflows/ci-build-next-java.yml +++ b/.github/workflows/ci-build-next-java.yml @@ -1,5 +1,6 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/ci-build-next-java.yml name: CI Build next Java - on: push: branches: @@ -18,7 +19,7 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: "temurin" java-version: 17 @@ -26,8 +27,9 @@ jobs: - name: Run tests and build with Maven run: | mvn --batch-mode --update-snapshots clean package -DtrimStackTrace=false \ + -Djava.version=17 \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - - name: Publish Test Report + - name: Publish Test Report for Java 17 uses: scacap/action-surefire-report@v1 if: ${{ always() && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} with: diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 4088dc1..76adff4 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -1,5 +1,6 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml name: CI Build - on: push: branches: @@ -7,27 +8,34 @@ on: pull_request: jobs: - build: - runs-on: ubuntu-20.04 # UDFs fail with "VM error: Internal error: VM crashed" on ubuntu-latest + matrix-build: + runs-on: ubuntu-20.04 concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.docker_db_version }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.exasol_db_version }} cancel-in-progress: true strategy: fail-fast: false matrix: - docker_db_version: ["7.1.24", "8.23.0"] + exasol_db_version: ["7.1.25", "8.24.0"] + env: + DEFAULT_EXASOL_DB_VERSION: "7.1.25" steps: + - name: Free Disk Space + if: ${{ false }} + run: | + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet - name: Checkout the repository uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up JDK 11 & 17 - uses: actions/setup-java@v3 + - name: Set up JDKs + uses: actions/setup-java@v4 with: distribution: "temurin" java-version: | - 17 11 + 17 cache: "maven" - name: Cache SonarCloud packages uses: actions/cache@v3 @@ -39,24 +47,32 @@ jobs: run: echo 'testcontainers.reuse.enable=true' > "$HOME/.testcontainers.properties" - name: Run tests and build with Maven run: | - JAVA_HOME=$JAVA_HOME_11_X64 mvn --batch-mode clean verify \ + mvn --batch-mode clean verify \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ -DtrimStackTrace=false \ - -Dcom.exasol.dockerdb.image=${{ matrix.docker_db_version }} - - name: Publish Test Report + -Dcom.exasol.dockerdb.image=${{ matrix.exasol_db_version }} + env: + # Set additional environment variable as in scala projects the scalatest plugin does not forward + # the system property -Dcom.exasol.dockerdb.image to the test's implementation. + EXASOL_DB_VERSION: ${{ matrix.exasol_db_version }} + - name: Publish Test Report for Exasol ${{ matrix.exasol_db_version }} uses: scacap/action-surefire-report@v1 if: ${{ always() && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} with: github_token: ${{ secrets.GITHUB_TOKEN }} - name: Sonar analysis - if: ${{ env.SONAR_TOKEN != null }} + if: ${{ env.SONAR_TOKEN != null && matrix.exasol_db_version == env.DEFAULT_EXASOL_DB_VERSION }} run: | - JAVA_HOME=$JAVA_HOME_17_X64 mvn --batch-mode org.sonarsource.scanner.maven:sonar-maven-plugin:sonar \ + mvn --batch-mode org.sonarsource.scanner.maven:sonar-maven-plugin:sonar \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ -DtrimStackTrace=false \ - -Dsonar.organization=exasol \ - -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.token=$SONAR_TOKEN env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + build: + needs: matrix-build + runs-on: ubuntu-latest + steps: + - run: echo "Build successful" diff --git a/.github/workflows/dependencies_check.yml b/.github/workflows/dependencies_check.yml index 4b6eadf..87b64ba 100644 --- a/.github/workflows/dependencies_check.yml +++ b/.github/workflows/dependencies_check.yml @@ -1,5 +1,6 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/dependencies_check.yml name: Report Security Issues for Repository - on: workflow_dispatch: schedule: @@ -13,11 +14,13 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up JDK 11 - uses: actions/setup-java@v3 + - name: Set up JDKs + uses: actions/setup-java@v4 with: distribution: "temurin" - java-version: 11 + java-version: | + 11 + 17 cache: "maven" - name: Generate ossindex report diff --git a/.github/workflows/release_droid_prepare_original_checksum.yml b/.github/workflows/release_droid_prepare_original_checksum.yml index e7381af..413274b 100644 --- a/.github/workflows/release_droid_prepare_original_checksum.yml +++ b/.github/workflows/release_droid_prepare_original_checksum.yml @@ -1,21 +1,29 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/release_droid_prepare_original_checksum.yml name: Release Droid - Prepare Original Checksum - on: workflow_dispatch: jobs: build: - runs-on: ubuntu-20.04 # UDFs fail with "VM error: Internal error: VM crashed" on ubuntu-latest + runs-on: ubuntu-20.04 steps: + - name: Free Disk Space + if: ${{ false }} + run: | + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet - name: Checkout the repository uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up JDK 11 - uses: actions/setup-java@v3 + - name: Set up JDKs + uses: actions/setup-java@v4 with: distribution: "temurin" - java-version: 11 + java-version: | + 11 + 17 cache: "maven" - name: Enable testcontainer reuse run: echo 'testcontainers.reuse.enable=true' > "$HOME/.testcontainers.properties" diff --git a/.github/workflows/release_droid_print_quick_checksum.yml b/.github/workflows/release_droid_print_quick_checksum.yml index aed4444..86979cd 100644 --- a/.github/workflows/release_droid_print_quick_checksum.yml +++ b/.github/workflows/release_droid_print_quick_checksum.yml @@ -1,5 +1,6 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/release_droid_print_quick_checksum.yml name: Release Droid - Print Quick Checksum - on: workflow_dispatch: @@ -11,11 +12,13 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up JDK 11 - uses: actions/setup-java@v3 + - name: Set up JDKs + uses: actions/setup-java@v4 with: distribution: "temurin" - java-version: 11 + java-version: | + 11 + 17 cache: "maven" - name: Build with Maven skipping tests run: mvn --batch-mode clean verify -DskipTests diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml index 7ae8bbb..b19f7cf 100644 --- a/.github/workflows/release_droid_upload_github_release_assets.yml +++ b/.github/workflows/release_droid_upload_github_release_assets.yml @@ -1,5 +1,6 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/release_droid_upload_github_release_assets.yml name: Release Droid - Upload GitHub Release Assets - on: workflow_dispatch: inputs: @@ -15,11 +16,13 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up JDK 11 - uses: actions/setup-java@v3 + - name: Set up JDKs + uses: actions/setup-java@v4 with: distribution: "temurin" - java-version: 11 + java-version: | + 11 + 17 cache: "maven" - name: Build with Maven skipping tests run: mvn --batch-mode clean verify -DskipTests diff --git a/.project-keeper.yml b/.project-keeper.yml index 77d59f0..7ae66b2 100644 --- a/.project-keeper.yml +++ b/.project-keeper.yml @@ -2,13 +2,16 @@ sources: - type: maven path: pom.xml modules: - - jar_artifact - integration_tests - advertise: true + - udf_coverage + - jar_artifact version: fromSource: pom.xml +build: + runnerOs: ubuntu-20.04 + freeDiskSpace: false + exasolDbVersions: + - "7.1.25" + - "8.24.0" linkReplacements: - - "https://jdbc.postgresql.org/about/license.html|https://jdbc.postgresql.org/license/" -excludes: - - "E-PK-CORE-18: Outdated content: '.github/workflows/ci-build.yml'" - - "E-PK-CORE-18: Outdated content: '.github/workflows/release_droid_prepare_original_checksum.yml'" + - "https://jdbc.postgresql.org/about/license.html|https://jdbc.postgresql.org/license/" \ No newline at end of file diff --git a/dependencies.md b/dependencies.md index f6dd43b..81e478b 100644 --- a/dependencies.md +++ b/dependencies.md @@ -26,29 +26,31 @@ | [Markdown Generator][22] | [The Apache Software License, Version 2.0][23] | | [Autogenerated resource verifier][24] | [MIT License][25] | | [virtual-schema-shared-integration-tests][26] | [MIT License][27] | +| [JaCoCo :: Agent][28] | [Eclipse Public License 2.0][29] | ## Plugin Dependencies -| Dependency | License | -| ------------------------------------------------------- | ---------------------------------------------- | -| [SonarQube Scanner for Maven][28] | [GNU LGPL 3][29] | -| [Apache Maven Compiler Plugin][30] | [Apache-2.0][31] | -| [Apache Maven Enforcer Plugin][32] | [Apache-2.0][31] | -| [Maven Flatten Plugin][33] | [Apache Software Licenese][31] | -| [org.sonatype.ossindex.maven:ossindex-maven-plugin][34] | [ASL2][23] | -| [Maven Surefire Plugin][35] | [Apache-2.0][31] | -| [Versions Maven Plugin][36] | [Apache License, Version 2.0][31] | -| [duplicate-finder-maven-plugin Maven Mojo][37] | [Apache License 2.0][38] | -| [Apache Maven Assembly Plugin][39] | [Apache-2.0][31] | -| [Apache Maven JAR Plugin][40] | [Apache License, Version 2.0][31] | -| [Artifact reference checker and unifier][41] | [MIT License][42] | -| [Project keeper maven plugin][43] | [The MIT License][44] | -| [Maven Dependency Plugin][45] | [The Apache Software License, Version 2.0][23] | -| [Exec Maven Plugin][46] | [Apache License 2][31] | -| [Maven Failsafe Plugin][47] | [Apache-2.0][31] | -| [JaCoCo :: Maven Plugin][48] | [Eclipse Public License 2.0][49] | -| [error-code-crawler-maven-plugin][50] | [MIT License][51] | -| [Reproducible Build Maven Plugin][52] | [Apache 2.0][23] | +| Dependency | License | +| ------------------------------------------------------- | --------------------------------- | +| [SonarQube Scanner for Maven][30] | [GNU LGPL 3][31] | +| [Apache Maven Toolchains Plugin][32] | [Apache License, Version 2.0][33] | +| [Apache Maven Compiler Plugin][34] | [Apache-2.0][33] | +| [Apache Maven Enforcer Plugin][35] | [Apache-2.0][33] | +| [Maven Flatten Plugin][36] | [Apache Software Licenese][33] | +| [org.sonatype.ossindex.maven:ossindex-maven-plugin][37] | [ASL2][23] | +| [Maven Surefire Plugin][38] | [Apache-2.0][33] | +| [Versions Maven Plugin][39] | [Apache License, Version 2.0][33] | +| [duplicate-finder-maven-plugin Maven Mojo][40] | [Apache License 2.0][41] | +| [Apache Maven Assembly Plugin][42] | [Apache-2.0][33] | +| [Apache Maven JAR Plugin][43] | [Apache License, Version 2.0][33] | +| [Artifact reference checker and unifier][44] | [MIT License][45] | +| [Project Keeper Maven plugin][46] | [The MIT License][47] | +| [Apache Maven Dependency Plugin][48] | [Apache-2.0][33] | +| [Exec Maven Plugin][49] | [Apache License 2][33] | +| [Maven Failsafe Plugin][50] | [Apache-2.0][33] | +| [JaCoCo :: Maven Plugin][51] | [Eclipse Public License 2.0][29] | +| [error-code-crawler-maven-plugin][52] | [MIT License][53] | +| [Reproducible Build Maven Plugin][54] | [Apache 2.0][23] | [0]: https://github.com/exasol/virtual-schema-common-jdbc/ [1]: https://github.com/exasol/virtual-schema-common-jdbc/blob/main/LICENSE @@ -78,28 +80,30 @@ [25]: https://github.com/exasol/autogenerated-resource-verifier-java/blob/main/LICENSE [26]: https://github.com/exasol/virtual-schema-shared-integration-tests/ [27]: https://github.com/exasol/virtual-schema-shared-integration-tests/blob/main/LICENSE -[28]: http://sonarsource.github.io/sonar-scanner-maven/ -[29]: http://www.gnu.org/licenses/lgpl.txt -[30]: https://maven.apache.org/plugins/maven-compiler-plugin/ -[31]: https://www.apache.org/licenses/LICENSE-2.0.txt -[32]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ -[33]: https://www.mojohaus.org/flatten-maven-plugin/ -[34]: https://sonatype.github.io/ossindex-maven/maven-plugin/ -[35]: https://maven.apache.org/surefire/maven-surefire-plugin/ -[36]: https://www.mojohaus.org/versions/versions-maven-plugin/ -[37]: https://basepom.github.io/duplicate-finder-maven-plugin -[38]: http://www.apache.org/licenses/LICENSE-2.0.html -[39]: https://maven.apache.org/plugins/maven-assembly-plugin/ -[40]: https://maven.apache.org/plugins/maven-jar-plugin/ -[41]: https://github.com/exasol/artifact-reference-checker-maven-plugin/ -[42]: https://github.com/exasol/artifact-reference-checker-maven-plugin/blob/main/LICENSE -[43]: https://github.com/exasol/project-keeper/ -[44]: https://github.com/exasol/project-keeper/blob/main/LICENSE -[45]: http://maven.apache.org/plugins/maven-dependency-plugin/ -[46]: https://www.mojohaus.org/exec-maven-plugin -[47]: https://maven.apache.org/surefire/maven-failsafe-plugin/ -[48]: https://www.jacoco.org/jacoco/trunk/doc/maven.html -[49]: https://www.eclipse.org/legal/epl-2.0/ -[50]: https://github.com/exasol/error-code-crawler-maven-plugin/ -[51]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE -[52]: http://zlika.github.io/reproducible-build-maven-plugin +[28]: https://www.eclemma.org/jacoco/index.html +[29]: https://www.eclipse.org/legal/epl-2.0/ +[30]: http://sonarsource.github.io/sonar-scanner-maven/ +[31]: http://www.gnu.org/licenses/lgpl.txt +[32]: https://maven.apache.org/plugins/maven-toolchains-plugin/ +[33]: https://www.apache.org/licenses/LICENSE-2.0.txt +[34]: https://maven.apache.org/plugins/maven-compiler-plugin/ +[35]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ +[36]: https://www.mojohaus.org/flatten-maven-plugin/ +[37]: https://sonatype.github.io/ossindex-maven/maven-plugin/ +[38]: https://maven.apache.org/surefire/maven-surefire-plugin/ +[39]: https://www.mojohaus.org/versions/versions-maven-plugin/ +[40]: https://basepom.github.io/duplicate-finder-maven-plugin +[41]: http://www.apache.org/licenses/LICENSE-2.0.html +[42]: https://maven.apache.org/plugins/maven-assembly-plugin/ +[43]: https://maven.apache.org/plugins/maven-jar-plugin/ +[44]: https://github.com/exasol/artifact-reference-checker-maven-plugin/ +[45]: https://github.com/exasol/artifact-reference-checker-maven-plugin/blob/main/LICENSE +[46]: https://github.com/exasol/project-keeper/ +[47]: https://github.com/exasol/project-keeper/blob/main/LICENSE +[48]: https://maven.apache.org/plugins/maven-dependency-plugin/ +[49]: https://www.mojohaus.org/exec-maven-plugin +[50]: https://maven.apache.org/surefire/maven-failsafe-plugin/ +[51]: https://www.jacoco.org/jacoco/trunk/doc/maven.html +[52]: https://github.com/exasol/error-code-crawler-maven-plugin/ +[53]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE +[54]: http://zlika.github.io/reproducible-build-maven-plugin diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index f9f049f..8939758 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [3.0.0](changes_3.0.0.md) * [2.2.3](changes_2.2.3.md) * [2.2.2](changes_2.2.2.md) * [2.2.1](changes_2.2.1.md) diff --git a/doc/changes/changes_3.0.0.md b/doc/changes/changes_3.0.0.md new file mode 100644 index 0000000..0f14195 --- /dev/null +++ b/doc/changes/changes_3.0.0.md @@ -0,0 +1,33 @@ +# Virtual Schema for PostgreSQL 3.0.0, released 2024-??-?? + +Code name: + +## Summary + +## Features + +* ISSUE_NUMBER: description + +## Dependency Updates + +### Compile Dependency Updates + +* Updated `com.exasol:virtual-schema-common-jdbc:11.0.2` to `12.0.0` + +### Test Dependency Updates + +* Updated `com.exasol:virtual-schema-common-jdbc:11.0.2` to `12.0.0` +* Added `org.jacoco:org.jacoco.agent:0.8.11` + +### Plugin Dependency Updates + +* Updated `com.exasol:error-code-crawler-maven-plugin:1.3.0` to `1.3.1` +* Updated `com.exasol:project-keeper-maven-plugin:2.9.12` to `3.0.1` +* Updated `org.apache.maven.plugins:maven-dependency-plugin:2.8` to `3.6.1` +* Updated `org.apache.maven.plugins:maven-enforcer-plugin:3.4.0` to `3.4.1` +* Updated `org.apache.maven.plugins:maven-failsafe-plugin:3.1.2` to `3.2.3` +* Updated `org.apache.maven.plugins:maven-surefire-plugin:3.1.2` to `3.2.3` +* Added `org.apache.maven.plugins:maven-toolchains-plugin:3.1.0` +* Updated `org.codehaus.mojo:versions-maven-plugin:2.16.0` to `2.16.2` +* Updated `org.jacoco:jacoco-maven-plugin:0.8.10` to `0.8.11` +* Updated `org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184` to `3.10.0.2594` diff --git a/doc/user_guide/postgresql_user_guide.md b/doc/user_guide/postgresql_user_guide.md index 0c40375..0bf401f 100644 --- a/doc/user_guide/postgresql_user_guide.md +++ b/doc/user_guide/postgresql_user_guide.md @@ -47,7 +47,7 @@ The SQL statement below creates the adapter script, defines the Java class that --/ CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets///virtual-schema-dist-11.0.2-postgresql-2.2.3.jar; + %jar /buckets///virtual-schema-dist-12.0.0-postgresql-3.0.0.jar; %jar /buckets///postgresql-.jar; / ``` diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index 25448a5..1e0eb55 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -3,12 +3,14 @@ 4.0.0 com.exasol postgresql-virtual-schema-generated-parent - 2.2.3 + 3.0.0 pom UTF-8 UTF-8 11 + exasol + https://sonarcloud.io @@ -31,7 +33,15 @@ scm:git:https://github.com/exasol/postgresql-virtual-schema.git https://github.com/exasol/postgresql-virtual-schema/ - + + + org.jacoco + org.jacoco.agent + 0.8.11 + test + runtime + + @@ -39,6 +49,25 @@ sonar-maven-plugin 3.10.0.2594 + + org.apache.maven.plugins + maven-toolchains-plugin + 3.1.0 + + + + toolchain + + + + + + + ${java.version} + + + + org.apache.maven.plugins maven-compiler-plugin @@ -46,6 +75,12 @@ ${java.version} ${java.version} + true + + + -Xlint:all,-processing + + @@ -63,6 +98,9 @@ 3.6.3 + + 17 + @@ -110,7 +148,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.1.2 + 3.2.3 @@ -121,7 +159,7 @@ org.codehaus.mojo versions-maven-plugin - 2.16.1 + 2.16.2 display-updates @@ -211,15 +249,36 @@ + + org.apache.maven.plugins + maven-dependency-plugin + 3.6.1 + + + copy-jacoco + + copy-dependencies + + compile + + org.jacoco.agent + runtime + ${project.build.directory}/jacoco-agent + true + + + + org.apache.maven.plugins maven-failsafe-plugin - 3.1.2 + 3.2.3 - + -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} - + + true + ${test.excludeTags} @@ -243,6 +302,12 @@ prepare-agent + + prepare-agent-integration + + prepare-agent-integration + + merge-results verify diff --git a/pom.xml b/pom.xml index 68a35f2..bf53f9c 100644 --- a/pom.xml +++ b/pom.xml @@ -2,12 +2,12 @@ 4.0.0 postgresql-virtual-schema - 2.2.3 + 3.0.0 Virtual Schema for PostgreSQL Virtual Schema for connecting PostgreSQL as data source to Exasol https://github.com/exasol/postgresql-virtual-schema/ - 11.0.2 + 12.0.0 1.19.0 @@ -120,7 +120,7 @@ com.exasol project-keeper-maven-plugin - 2.9.15 + 3.0.1 @@ -184,7 +184,7 @@ postgresql-virtual-schema-generated-parent com.exasol - 2.2.3 + 3.0.0 pk_generated_parent.pom diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java index e03d6a0..92fcb6d 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java @@ -26,7 +26,7 @@ * This class contains the common integration test setup for all PostgreSQL virtual schemas. */ public class PostgresVirtualSchemaIntegrationTestSetup implements Closeable { - private static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-11.0.2-postgresql-2.2.3.jar"; + private static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-12.0.0-postgresql-3.0.0.jar"; private static final Path PATH_TO_VIRTUAL_SCHEMAS_JAR = Path.of("target", VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION); private static final String SCHEMA_EXASOL = "SCHEMA_EXASOL"; private static final String ADAPTER_SCRIPT_EXASOL = "ADAPTER_SCRIPT_EXASOL"; From 535921895776fa11a48e75a7d019db93ea66f4c2 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Wed, 31 Jan 2024 16:56:43 +0100 Subject: [PATCH 03/20] updated remaining dependencies --- dependencies.md | 4 ++-- doc/changes/changelog.md | 1 - doc/changes/changes_2.2.3.md | 20 ------------------- doc/changes/changes_3.0.0.md | 9 +++++++++ pom.xml | 16 +++++++-------- ...gresVirtualSchemaIntegrationTestSetup.java | 2 +- 6 files changed, 20 insertions(+), 32 deletions(-) delete mode 100644 doc/changes/changes_2.2.3.md diff --git a/dependencies.md b/dependencies.md index 81e478b..48210b0 100644 --- a/dependencies.md +++ b/dependencies.md @@ -16,7 +16,7 @@ | [Virtual Schema Common JDBC][0] | [MIT License][1] | | [Hamcrest][6] | [BSD License 3][7] | | [JUnit Jupiter (Aggregator)][8] | [Eclipse Public License v2.0][9] | -| [mockito-junit-jupiter][10] | [The MIT License][11] | +| [mockito-junit-jupiter][10] | [MIT][11] | | [Test containers for Exasol on Docker][12] | [MIT License][13] | | [Testcontainers :: JUnit Jupiter Extension][14] | [MIT][15] | | [Testcontainers :: JDBC :: PostgreSQL][14] | [MIT][15] | @@ -63,7 +63,7 @@ [8]: https://junit.org/junit5/ [9]: https://www.eclipse.org/legal/epl-v20.html [10]: https://github.com/mockito/mockito -[11]: https://github.com/mockito/mockito/blob/main/LICENSE +[11]: https://opensource.org/licenses/MIT [12]: https://github.com/exasol/exasol-testcontainers/ [13]: https://github.com/exasol/exasol-testcontainers/blob/main/LICENSE [14]: https://java.testcontainers.org diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 8939758..2eabc8e 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,7 +1,6 @@ # Changes * [3.0.0](changes_3.0.0.md) -* [2.2.3](changes_2.2.3.md) * [2.2.2](changes_2.2.2.md) * [2.2.1](changes_2.2.1.md) * [2.2.0](changes_2.2.0.md) diff --git a/doc/changes/changes_2.2.3.md b/doc/changes/changes_2.2.3.md deleted file mode 100644 index e28c2fe..0000000 --- a/doc/changes/changes_2.2.3.md +++ /dev/null @@ -1,20 +0,0 @@ -# Virtual Schema for PostgreSQL 2.2.3, released 2023-??-?? - -Code name: - -## Summary - -## Features - -* ISSUE_NUMBER: description - -## Dependency Updates - -### Plugin Dependency Updates - -* Updated `com.exasol:error-code-crawler-maven-plugin:1.3.0` to `1.3.1` -* Updated `com.exasol:project-keeper-maven-plugin:2.9.12` to `2.9.15` -* Updated `org.apache.maven.plugins:maven-enforcer-plugin:3.4.0` to `3.4.1` -* Updated `org.codehaus.mojo:versions-maven-plugin:2.16.0` to `2.16.1` -* Updated `org.jacoco:jacoco-maven-plugin:0.8.10` to `0.8.11` -* Updated `org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184` to `3.10.0.2594` diff --git a/doc/changes/changes_3.0.0.md b/doc/changes/changes_3.0.0.md index 0f14195..7227a36 100644 --- a/doc/changes/changes_3.0.0.md +++ b/doc/changes/changes_3.0.0.md @@ -13,11 +13,20 @@ Code name: ### Compile Dependency Updates * Updated `com.exasol:virtual-schema-common-jdbc:11.0.2` to `12.0.0` +* Updated `org.postgresql:postgresql:42.6.0` to `42.7.1` ### Test Dependency Updates +* Updated `com.exasol:exasol-testcontainers:6.6.2` to `7.0.0` +* Updated `com.exasol:hamcrest-resultset-matcher:1.6.1` to `1.6.4` +* Updated `com.exasol:test-db-builder-java:3.5.1` to `3.5.3` * Updated `com.exasol:virtual-schema-common-jdbc:11.0.2` to `12.0.0` +* Updated `com.exasol:virtual-schema-shared-integration-tests:2.2.5` to `2.2.6` * Added `org.jacoco:org.jacoco.agent:0.8.11` +* Updated `org.junit.jupiter:junit-jupiter:5.10.0` to `5.10.1` +* Updated `org.mockito:mockito-junit-jupiter:5.5.0` to `5.10.0` +* Updated `org.testcontainers:junit-jupiter:1.19.0` to `1.19.4` +* Updated `org.testcontainers:postgresql:1.19.0` to `1.19.4` ### Plugin Dependency Updates diff --git a/pom.xml b/pom.xml index bf53f9c..36d923c 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ https://github.com/exasol/postgresql-virtual-schema/ 12.0.0 - 1.19.0 + 1.19.4 @@ -24,7 +24,7 @@ org.postgresql postgresql - 42.6.0 + 42.7.1 @@ -43,20 +43,20 @@ org.junit.jupiter junit-jupiter - 5.10.0 + 5.10.1 test org.mockito mockito-junit-jupiter - 5.5.0 + 5.10.0 test com.exasol exasol-testcontainers - 6.6.2 + 7.0.0 test @@ -74,13 +74,13 @@ com.exasol test-db-builder-java - 3.5.1 + 3.5.3 test com.exasol hamcrest-resultset-matcher - 1.6.1 + 1.6.4 test @@ -104,7 +104,7 @@ com.exasol virtual-schema-shared-integration-tests - 2.2.5 + 2.2.6 test diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java index 92fcb6d..c1f3ea3 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java @@ -30,7 +30,7 @@ public class PostgresVirtualSchemaIntegrationTestSetup implements Closeable { private static final Path PATH_TO_VIRTUAL_SCHEMAS_JAR = Path.of("target", VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION); private static final String SCHEMA_EXASOL = "SCHEMA_EXASOL"; private static final String ADAPTER_SCRIPT_EXASOL = "ADAPTER_SCRIPT_EXASOL"; - private static final String EXASOL_DOCKER_IMAGE_REFERENCE = "7.1.6"; + private static final String EXASOL_DOCKER_IMAGE_REFERENCE = "8.24.0"; private static final String POSTGRES_CONTAINER_NAME = "postgres:14.2"; private static final String JDBC_DRIVER_NAME = "postgresql.jar"; static final Path JDBC_DRIVER_PATH = Path.of("target/postgresql-driver/" + JDBC_DRIVER_NAME); From d255dcd651aebd87b2238da01c377c42c6ca05f8 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Fri, 2 Feb 2024 10:32:25 +0100 Subject: [PATCH 04/20] fix integration tests for later Exasol DB versions : Register the JDBC driver for ExaLoader --- ...stgresVirtualSchemaIntegrationTestSetup.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java index c1f3ea3..e495338 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java @@ -32,8 +32,17 @@ public class PostgresVirtualSchemaIntegrationTestSetup implements Closeable { private static final String ADAPTER_SCRIPT_EXASOL = "ADAPTER_SCRIPT_EXASOL"; private static final String EXASOL_DOCKER_IMAGE_REFERENCE = "8.24.0"; private static final String POSTGRES_CONTAINER_NAME = "postgres:14.2"; + private static final String JDBC_DRIVER_NAME = "postgresql.jar"; - static final Path JDBC_DRIVER_PATH = Path.of("target/postgresql-driver/" + JDBC_DRIVER_NAME); + private static final String JDBC_DRIVER_CONFIGURATION_FILE_NAME = "settings.cfg"; + private static final Path JDBC_DRIVER_PATH = Path.of("target/postgresql-driver/" + JDBC_DRIVER_NAME); + private static final String JDBC_DRIVER_CONFIGURATION_FILE_CONTENT = "DRIVERNAME=POSTGRES_JDBC_DRIVER\n" // + + "JAR=" + JDBC_DRIVER_NAME + "\n" // + + "DRIVERMAIN=org.postgresql.Driver\n" // + + "PREFIX=jdbc:postgresql:\n" // + + "FETCHSIZE=100000\n" // + + "INSERTSIZE=-1\n"; + private static final int POSTGRES_PORT = 5432; private final Statement postgresStatement; private final PostgreSQLContainer> postgresqlContainer = new PostgreSQLContainer<>( @@ -86,10 +95,12 @@ public class PostgresVirtualSchemaIntegrationTestSetup implements Closeable { private static void uploadDriverToBucket(final Bucket bucket) throws InterruptedException, TimeoutException, BucketAccessException { try { - bucket.uploadFile(JDBC_DRIVER_PATH, JDBC_DRIVER_NAME); + bucket.uploadStringContent(JDBC_DRIVER_CONFIGURATION_FILE_CONTENT, + "drivers/jdbc/" + JDBC_DRIVER_CONFIGURATION_FILE_NAME); + bucket.uploadFile(JDBC_DRIVER_PATH, "drivers/jdbc/" + JDBC_DRIVER_NAME); } catch (final BucketAccessException | FileNotFoundException exception) { throw new IllegalStateException( - ExaError.messageBuilder("F-VSPG-8") + ExaError.messageBuilder("F-VSMYSQL-2") .message("An error occurred while uploading the jdbc driver to the bucket.") .mitigation("Make sure the {{JDBC_DRIVER_PATH}} file exists.") .parameter("JDBC_DRIVER_PATH", JDBC_DRIVER_PATH) From a56b1b165f7404a751ad190712bd4a196bb10826 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Thu, 8 Feb 2024 14:59:34 +0100 Subject: [PATCH 05/20] fix testcountall for V8 --- .../postgresql/PostgreSQLSqlDialectIT.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java index 16f4445..3fc6de7 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java @@ -257,9 +257,8 @@ void testFullOuterJoinWithComplexCondition() throws SQLException { void testCountAll() throws SQLException { final String qualifiedExpectedTableName = virtualSchemaPostgres.getName() + "." + TABLE_POSTGRES_SIMPLE; final String query = "SELECT COUNT(*) FROM " + qualifiedExpectedTableName; - final ResultSet expected = getExpectedResultSet(List.of("x DECIMAL(19,0)"), // - List.of("1.00000")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + final ResultSet actualResultSet = getActualResultSet(query); + assertThat(actualResultSet, table("BIGINT").row(1L).matches()); } @Test @@ -383,12 +382,12 @@ void testDatatypeBytea() throws SQLException { void testDatatypeCharacter() throws SQLException { final String empty = " "; final String expected = "hajksdf" + String.join("", Collections.nCopies(993, empty)); - assertSingleValue("myCharacter", "CHAR(1000) ASCII", expected); + assertSingleValue("myCharacter", "CHAR(1000) UTF8", expected); } @Test void testDatatypeCharacterVar() throws SQLException { - assertSingleValue("myCharactervar", "VARCHAR(1000) ASCII", "hjkdhjgfh"); + assertSingleValue("myCharactervar", "VARCHAR(1000) UTF8", "hjkdhjgfh"); } @Test @@ -491,7 +490,7 @@ void testDatatypeSmallInt() throws SQLException { @Test void testDatatypeText() throws SQLException { - assertSingleValue("myText", "VARCHAR(2000000) ASCII", "This cat is super cute"); + assertSingleValue("myText", "VARCHAR(2000000) UTF8", "This cat is super cute"); } @Test @@ -552,10 +551,14 @@ private ResultSet getExpectedResultSet(final List expectedColumns, final final String expectedValues = expectedRows.stream().map(row -> "(" + row + ")") .collect(Collectors.joining(",")); final String qualifiedExpectedTableName = exasolSchema.getName() + ".EXPECTED"; - statementExasol.execute("CREATE OR REPLACE TABLE " + qualifiedExpectedTableName + "(" - + String.join(", ", expectedColumns) + ")"); - statementExasol.execute("INSERT INTO " + qualifiedExpectedTableName + " VALUES" + expectedValues); - return statementExasol.executeQuery("SELECT * FROM " + qualifiedExpectedTableName); + final String createTableStatement = "CREATE OR REPLACE TABLE " + qualifiedExpectedTableName + "(" + + String.join(", ", expectedColumns) + ");"; + statementExasol.execute(createTableStatement); + final String insertIntoTableStatement = "INSERT INTO " + qualifiedExpectedTableName + " VALUES " + + expectedValues + ";"; + statementExasol.execute(insertIntoTableStatement); + final String selectStatement = "SELECT * FROM " + qualifiedExpectedTableName + ";"; + return statementExasol.executeQuery(selectStatement); } private ResultSet getActualResultSet(final String query) throws SQLException { From a80d7a1726e97bc064cf77853af0658e6a6f8423 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Thu, 8 Feb 2024 15:13:31 +0100 Subject: [PATCH 06/20] fix testQueryUpperCaseTableQuotedThrowsException test for v8 --- .../dialects/postgresql/PostgreSQLSqlDialectIT.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java index 3fc6de7..9e4f8d1 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java @@ -270,10 +270,11 @@ void testCreateSchemaWithUpperCaseTablesThrowsException() { @Test void testQueryUpperCaseTableQuotedThrowsException() { - final Exception exception = assertThrows(SQLException.class, () -> statementExasol.execute("SELECT x FROM " - + virtualSchemaPostgresUppercaseTable.getName() + ".\"" + TABLE_POSTGRES_MIXED_CASE + "\"")); - assertThat(exception.getMessage(), containsString("object \"" + virtualSchemaPostgresUppercaseTable.getName() - + "\".\"" + TABLE_POSTGRES_MIXED_CASE + "\" not found")); + final String selectStatement = "SELECT x FROM " + virtualSchemaPostgresUppercaseTable.getName() + ".\"" + + TABLE_POSTGRES_MIXED_CASE + "\""; + final Exception exception = assertThrows(SQLException.class, () -> statementExasol.execute(selectStatement)); + assertThat(exception.getMessage(), containsString("object " + virtualSchemaPostgresUppercaseTable.getName() + + ".\"" + TABLE_POSTGRES_MIXED_CASE + "\" not found")); } @Test From 00d8cd7183ae7fbea39aaca59b7dcac769c369bb Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Wed, 21 Feb 2024 15:14:02 +0100 Subject: [PATCH 07/20] add specific tests for all new failing 'unsolved' v8 shared integration tests --- .../postgresql/PostgreSQLSqlDialectIT.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java index 9e4f8d1..198ce80 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java @@ -253,6 +253,38 @@ void testFullOuterJoinWithComplexCondition() throws SQLException { assertThat(getActualResultSet(query), matchesResultSet(expected)); } + @Test + void testYearScalarFunctionFromTimeStamp() throws SQLException { + final String query = "SELECT year(\"MYTIMESTAMP\") FROM " + virtualSchemaPostgres.getName() + "." + + TABLE_POSTGRES_ALL_DATA_TYPES; + final ResultSet actualResultSet = getActualResultSet(query); + assertThat(actualResultSet, table("BIGINT").row(1L).matches()); + } + + @Test + void testYearScalarFunctionFromDate() throws SQLException { + final String query = "SELECT year(\"MYDATE\") FROM " + virtualSchemaPostgres.getName() + "." + + TABLE_POSTGRES_ALL_DATA_TYPES; + final ResultSet actualResultSet = getActualResultSet(query); + assertThat(actualResultSet, table("BIGINT").row(1L).matches()); + } + + @Test + void testCurrentSchemaScalarFunction() throws SQLException { + final String query = " SELECT current_schema FROM " + virtualSchemaPostgres.getName() + "." + + TABLE_POSTGRES_ALL_DATA_TYPES; + final ResultSet actualResultSet = getActualResultSet(query); + assertThat(actualResultSet, table("BIGINT").row(1L).matches()); + } + + @Test + void testFloatDivFunction() throws SQLException { + final String query = " SELECT MYINTEGER / MYINTEGER FROM " + virtualSchemaPostgres.getName() + "." + + TABLE_POSTGRES_ALL_DATA_TYPES; + final ResultSet actualResultSet = getActualResultSet(query); + assertThat(actualResultSet, table("BIGINT").row(1L).matches()); + } + @Test void testCountAll() throws SQLException { final String qualifiedExpectedTableName = virtualSchemaPostgres.getName() + "." + TABLE_POSTGRES_SIMPLE; From 9d8bd226aa9bc0258d6b42d914c2018136e224a6 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Tue, 27 Feb 2024 12:43:05 +0100 Subject: [PATCH 08/20] updated dependencies and ran last version of project-keeper --- .github/workflows/ci-build.yml | 2 +- .github/workflows/dependencies_check.yml | 28 ++++- .github/workflows/dependencies_update.yml | 103 ++++++++++++++++++ ...elease_droid_prepare_original_checksum.yml | 2 +- .gitignore | 1 + doc/changes/changes_3.0.0.md | 18 +-- pk_generated_parent.pom | 21 +++- pom.xml | 8 +- 8 files changed, 162 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/dependencies_update.yml diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 76adff4..73fcdcb 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -38,7 +38,7 @@ jobs: 17 cache: "maven" - name: Cache SonarCloud packages - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar diff --git a/.github/workflows/dependencies_check.yml b/.github/workflows/dependencies_check.yml index 87b64ba..61b3baf 100644 --- a/.github/workflows/dependencies_check.yml +++ b/.github/workflows/dependencies_check.yml @@ -10,10 +10,13 @@ jobs: report_security_issues: runs-on: ubuntu-latest permissions: + contents: read issues: write - + outputs: + created-issues: ${{ steps.security-issues.outputs.created-issues }} steps: - uses: actions/checkout@v4 + - name: Set up JDKs uses: actions/setup-java@v4 with: @@ -25,14 +28,35 @@ jobs: - name: Generate ossindex report run: | - mvn org.sonatype.ossindex.maven:ossindex-maven-plugin:audit \ + mvn --batch-mode org.sonatype.ossindex.maven:ossindex-maven-plugin:audit \ org.sonatype.ossindex.maven:ossindex-maven-plugin:audit-aggregate \ -Dossindex.reportFile=$(pwd)/ossindex-report.json \ -Dossindex.fail=false - name: Report Security Issues + id: security-issues uses: exasol/python-toolbox/.github/actions/security-issues@main with: format: "maven" command: "cat ossindex-report.json" github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Output security issues (Debugging) + run: | + echo "$CREATED_ISSUES" > test.jsonl + cat test.jsonl + env: + CREATED_ISSUES: ${{ steps.security-issues.outputs.created-issues }} + + start_dependency_udpate: + needs: report_security_issues + if: ${{ needs.report_security_issues.outputs.created-issues }} + concurrency: + cancel-in-progress: true + group: "dependency_update" + permissions: + contents: write + pull-requests: write + uses: ./.github/workflows/dependencies_update.yml + with: + vulnerability_issues: ${{ needs.report_security_issues.outputs.created-issues }} diff --git a/.github/workflows/dependencies_update.yml b/.github/workflows/dependencies_update.yml new file mode 100644 index 0000000..d19b11c --- /dev/null +++ b/.github/workflows/dependencies_update.yml @@ -0,0 +1,103 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/dependencies_update.yml +name: Update dependencies +on: + workflow_call: + inputs: + vulnerability_issues: + description: "GitHub issues for vulnerable dependencies as JSONL" + required: false + type: string + workflow_dispatch: + +jobs: + update_dependencies: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - uses: actions/checkout@v4 + + - name: Set up JDKs + uses: actions/setup-java@v4 + with: + distribution: "temurin" + java-version: | + 11 + 17 + cache: "maven" + + - name: Print issues + run: | + echo "Issues from Action input: $ISSUES" + env: + ISSUES: ${{ inputs.vulnerability_issues }} + + - name: Fail if not running on a branch + if: ${{ !startsWith(github.ref, 'refs/heads/') }} + uses: actions/github-script@v7 + with: + script: | + core.setFailed('Not running on a branch, github.ref is ${{ github.ref }}. Please start this workflow only on main or a branch') + + - name: Update dependencies + run: | + mvn --batch-mode com.exasol:project-keeper-maven-plugin:update-dependencies --projects . \ + -Dproject-keeper:vulnerabilities="$CREATED_ISSUES" + env: + CREATED_ISSUES: ${{ inputs.vulnerability_issues }} + + - name: Project Keeper Fix + run: | + mvn --batch-mode com.exasol:project-keeper-maven-plugin:fix --projects . + + - name: Project Keeper Fix for updated Project Keeper version + # Calling PK fix a second time is necessary because the first invocation potentially updated PK itself. + # So we need to run PK fix again with the latest PK version. + run: | + mvn --batch-mode com.exasol:project-keeper-maven-plugin:fix --projects . + + - name: Generate PR comment + id: pr-comment + run: | + echo 'comment<> "$GITHUB_OUTPUT" + echo 'This Pull Request was created by `dependencies_update.yml` workflow' >> "$GITHUB_OUTPUT" + echo $CREATED_ISSUES | jq --raw-output '. | "Closes " + .issue_url + " (" + .cve + ")"' >> "$GITHUB_OUTPUT" + echo 'EOF' >> "$GITHUB_OUTPUT" + env: + CREATED_ISSUES: ${{ inputs.vulnerability_issues }} + + - name: Configure git + run: | + git config --global user.email "opensource@exasol.com" + git config --global user.name "Automatic Dependency Updater" + + - name: Create branch + if: ${{ github.ref == 'refs/heads/main' }} + run: | + branch_name="dependency-update/$(date "+%Y%m%d%H%M%S")" + echo "Creating branch $branch_name" + git checkout -b "$branch_name" + + - name: Commit changes & push + if: ${{ startsWith(github.ref, 'refs/heads/' ) }} + run: | + branch_name=$(git rev-parse --abbrev-ref HEAD) + echo "Current branch: $branch_name, local changes:" + git diff --stat + git diff --numstat + echo "Committing changes..." + git commit --all --message "Update dependencies" + echo "Pushing branch $branch_name..." + git push --set-upstream origin $branch_name + echo "Done." + + - name: Create pull request + if: ${{ github.ref == 'refs/heads/main' }} + run: | + gh pr create --base main --title "Update dependencies" --body "$COMMENT" + env: + COMMENT: ${{ steps.pr-comment.outputs.comment }} + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/release_droid_prepare_original_checksum.yml b/.github/workflows/release_droid_prepare_original_checksum.yml index 413274b..2ff28b3 100644 --- a/.github/workflows/release_droid_prepare_original_checksum.yml +++ b/.github/workflows/release_droid_prepare_original_checksum.yml @@ -32,7 +32,7 @@ jobs: - name: Prepare checksum run: find target -maxdepth 1 -name *.jar -exec sha256sum "{}" + > original_checksum - name: Upload checksum to the artifactory - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: original_checksum retention-days: 5 diff --git a/.gitignore b/.gitignore index 1b788aa..b1e8103 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ venv/ /.apt_generated/ /.apt_generated_tests/ *.flattened-pom.xml +/bin/ diff --git a/doc/changes/changes_3.0.0.md b/doc/changes/changes_3.0.0.md index 7227a36..dde0d37 100644 --- a/doc/changes/changes_3.0.0.md +++ b/doc/changes/changes_3.0.0.md @@ -6,22 +6,22 @@ Code name: ## Features -* ISSUE_NUMBER: description +- ISSUE_NUMBER: description ## Dependency Updates ### Compile Dependency Updates * Updated `com.exasol:virtual-schema-common-jdbc:11.0.2` to `12.0.0` -* Updated `org.postgresql:postgresql:42.6.0` to `42.7.1` +* Updated `org.postgresql:postgresql:42.6.0` to `42.7.2` ### Test Dependency Updates -* Updated `com.exasol:exasol-testcontainers:6.6.2` to `7.0.0` +* Updated `com.exasol:exasol-testcontainers:6.6.2` to `7.0.1` * Updated `com.exasol:hamcrest-resultset-matcher:1.6.1` to `1.6.4` * Updated `com.exasol:test-db-builder-java:3.5.1` to `3.5.3` * Updated `com.exasol:virtual-schema-common-jdbc:11.0.2` to `12.0.0` -* Updated `com.exasol:virtual-schema-shared-integration-tests:2.2.5` to `2.2.6` +* Updated `com.exasol:virtual-schema-shared-integration-tests:2.2.5` to `3.0.0` * Added `org.jacoco:org.jacoco.agent:0.8.11` * Updated `org.junit.jupiter:junit-jupiter:5.10.0` to `5.10.1` * Updated `org.mockito:mockito-junit-jupiter:5.5.0` to `5.10.0` @@ -30,13 +30,15 @@ Code name: ### Plugin Dependency Updates -* Updated `com.exasol:error-code-crawler-maven-plugin:1.3.0` to `1.3.1` -* Updated `com.exasol:project-keeper-maven-plugin:2.9.12` to `3.0.1` +* Updated `com.exasol:error-code-crawler-maven-plugin:1.3.0` to `2.0.0` +* Updated `com.exasol:project-keeper-maven-plugin:2.9.12` to `4.0.0` +* Updated `org.apache.maven.plugins:maven-compiler-plugin:3.11.0` to `3.12.1` * Updated `org.apache.maven.plugins:maven-dependency-plugin:2.8` to `3.6.1` * Updated `org.apache.maven.plugins:maven-enforcer-plugin:3.4.0` to `3.4.1` -* Updated `org.apache.maven.plugins:maven-failsafe-plugin:3.1.2` to `3.2.3` -* Updated `org.apache.maven.plugins:maven-surefire-plugin:3.1.2` to `3.2.3` +* Updated `org.apache.maven.plugins:maven-failsafe-plugin:3.1.2` to `3.2.5` +* Updated `org.apache.maven.plugins:maven-surefire-plugin:3.1.2` to `3.2.5` * Added `org.apache.maven.plugins:maven-toolchains-plugin:3.1.0` +* Updated `org.codehaus.mojo:flatten-maven-plugin:1.5.0` to `1.6.0` * Updated `org.codehaus.mojo:versions-maven-plugin:2.16.0` to `2.16.2` * Updated `org.jacoco:jacoco-maven-plugin:0.8.10` to `0.8.11` * Updated `org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184` to `3.10.0.2594` diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index 1e0eb55..d53d4fc 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -71,7 +71,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.11.0 + 3.12.1 ${java.version} ${java.version} @@ -109,7 +109,7 @@ org.codehaus.mojo flatten-maven-plugin - 1.5.0 + 1.6.0 true oss @@ -148,7 +148,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.2.3 + 3.2.5 @@ -172,6 +172,17 @@ file:///${project.basedir}/versionsMavenPluginRules.xml + false + true + true + true + false + true + true + true + false + true + true @@ -272,7 +283,7 @@ org.apache.maven.plugins maven-failsafe-plugin - 3.2.3 + 3.2.5 -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} @@ -341,7 +352,7 @@ com.exasol error-code-crawler-maven-plugin - 1.3.1 + 2.0.0 verify diff --git a/pom.xml b/pom.xml index 36d923c..7000a2e 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ org.postgresql postgresql - 42.7.1 + 42.7.2 @@ -56,7 +56,7 @@ com.exasol exasol-testcontainers - 7.0.0 + 7.0.1 test @@ -104,7 +104,7 @@ com.exasol virtual-schema-shared-integration-tests - 2.2.6 + 3.0.0 test @@ -120,7 +120,7 @@ com.exasol project-keeper-maven-plugin - 3.0.1 + 4.0.0 From 026d7af2266d695028d7dc64a48ca77411c7a4e7 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Tue, 27 Feb 2024 12:57:12 +0100 Subject: [PATCH 09/20] Fixes for - time scalar functions: add cast to decimal - division: add cast to double precision before divide --- .../PostgresSQLSqlGenerationVisitor.java | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitor.java b/src/main/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitor.java index 7c4e014..a22999f 100644 --- a/src/main/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitor.java +++ b/src/main/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitor.java @@ -85,11 +85,18 @@ public String visit(final SqlFunctionScalar function) throws AdapterException { return getDateTime(argumentsSql, scalarFunction); case POSIX_TIME: return getPosixTime(argumentsSql); + case FLOAT_DIV: + return getCastToDoublePrecisionAndDivide(argumentsSql); default: return super.visit(function); } } + private String getCastToDoublePrecisionAndDivide(final List sqlArguments) { + return "( CAST (" + sqlArguments.get(0) + " AS DOUBLE PRECISION) / CAST (" + sqlArguments.get(1) + + " AS DOUBLE PRECISION))"; + } + private String getAddDateTime(final List argumentsSql, final String unit) { final StringBuilder builder = new StringBuilder(); builder.append(argumentsSql.get(0)); @@ -104,7 +111,7 @@ private String buildInterval(final List argumentsSql, final String unit) private String getDateTime(final List argumentsSql, final ScalarFunction scalarFunction) { final StringBuilder builder = new StringBuilder(); - builder.append("DATE_PART("); + builder.append("CAST(DATE_PART("); switch (scalarFunction) { case SECOND: builder.append("'SECOND'"); @@ -129,7 +136,30 @@ private String getDateTime(final List argumentsSql, final ScalarFunction } builder.append(","); builder.append(argumentsSql.get(0)); - builder.append(")"); + builder.append(") AS DECIMAL("); + switch (scalarFunction) { + case SECOND: + builder.append("2"); + break; + case MINUTE: + builder.append("2"); + break; + case DAY: + builder.append("2"); + break; + case WEEK: + builder.append("2"); + break; + case MONTH: + builder.append("2"); + break; + case YEAR: + builder.append("4"); + break; + default: + break; + } + builder.append(",0))"); return builder.toString(); } From d20ac872f9a2b045a052bf5aba5c3d9fe8b2cb5c Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Tue, 27 Feb 2024 13:08:44 +0100 Subject: [PATCH 10/20] adapted, added tests --- .../PostgreSQLScalarFunctionsIT.java | 21 ++- .../postgresql/PostgreSQLSqlDialectIT.java | 14 +- .../PostgresSQLSqlGenerationVisitorTest.java | 158 +++++++++--------- .../scalarFunctionsParameterCache.yml | 16 +- 4 files changed, 114 insertions(+), 95 deletions(-) diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLScalarFunctionsIT.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLScalarFunctionsIT.java index ff5b17c..355c7fb 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLScalarFunctionsIT.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLScalarFunctionsIT.java @@ -4,7 +4,6 @@ import java.sql.SQLException; import java.util.*; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.extension.ExtendWith; import com.exasol.adapter.commontests.scalarfunction.ScalarFunctionsTestBase; @@ -46,7 +45,8 @@ public VirtualSchemaTestSetupProvider getVirtualSchemaTestSetupProvider() { } } - final VirtualSchema virtualSchema = SETUP.createVirtualSchema(postgresSchema.getName(), Collections.emptyMap()); + final VirtualSchema virtualSchema = SETUP.createVirtualSchema(postgresSchema.getName(), + Collections.emptyMap()); return new PostgreSQLSingleTableVirtualSchemaTestSetup(virtualSchema, postgresSchema); }; @@ -87,7 +87,12 @@ public Set getDialectSpecificExcludes() { "add_days", // expected was a value close to <1972-01-01> (tolerance: +/- <0.00010>) but was // "1972-01-01T00:00:00Z" - "add_years"); + "add_years", + // Check 'current_schema' functionality, re-enable tests after resolution #79 + // currently a bug in the compiler, compiler always expects 'VARCHAR(1) ASCII' see + // https://github.com/exasol/postgresql-virtual-schema/issues/79 + // https://exasol.atlassian.net/browse/SPOT-19716 + "current_schema"); } @Override @@ -119,8 +124,14 @@ public void close() { } } - @BeforeAll - static void beforeAll() { + // protected virtual method, must be overridden + @Override + protected void beforeAllSetup() throws SQLException { TimeZone.setDefault(TimeZone.getTimeZone("UTC")); } + + // protected virtual method, must be overridden + @Override + protected void afterAllTeardown() throws SQLException { + } } diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java index 198ce80..601beed 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java @@ -258,7 +258,8 @@ void testYearScalarFunctionFromTimeStamp() throws SQLException { final String query = "SELECT year(\"MYTIMESTAMP\") FROM " + virtualSchemaPostgres.getName() + "." + TABLE_POSTGRES_ALL_DATA_TYPES; final ResultSet actualResultSet = getActualResultSet(query); - assertThat(actualResultSet, table("BIGINT").row(1L).matches()); + final Short yearShort = 2010; + assertThat(actualResultSet, table().row(yearShort).matches()); } @Test @@ -266,15 +267,20 @@ void testYearScalarFunctionFromDate() throws SQLException { final String query = "SELECT year(\"MYDATE\") FROM " + virtualSchemaPostgres.getName() + "." + TABLE_POSTGRES_ALL_DATA_TYPES; final ResultSet actualResultSet = getActualResultSet(query); - assertThat(actualResultSet, table("BIGINT").row(1L).matches()); + final Short yearShort = 2010; + assertThat(actualResultSet, table().row(yearShort).matches()); } + // Check 'current_schema' functionality, re-enable tests after resolution + // currently a bug in the compiler, compiler always expects 'VARCHAR(1) ASCII' see + // https://github.com/exasol/postgresql-virtual-schema/issues/79 + // https://exasol.atlassian.net/browse/SPOT-19716 + @Disabled @Test void testCurrentSchemaScalarFunction() throws SQLException { final String query = " SELECT current_schema FROM " + virtualSchemaPostgres.getName() + "." + TABLE_POSTGRES_ALL_DATA_TYPES; final ResultSet actualResultSet = getActualResultSet(query); - assertThat(actualResultSet, table("BIGINT").row(1L).matches()); } @Test @@ -282,7 +288,7 @@ void testFloatDivFunction() throws SQLException { final String query = " SELECT MYINTEGER / MYINTEGER FROM " + virtualSchemaPostgres.getName() + "." + TABLE_POSTGRES_ALL_DATA_TYPES; final ResultSet actualResultSet = getActualResultSet(query); - assertThat(actualResultSet, table("BIGINT").row(1L).matches()); + assertThat(actualResultSet, table("DOUBLE PRECISION").row(1.0).matches()); } @Test diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitorTest.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitorTest.java index da9402c..8d8b8c0 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitorTest.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitorTest.java @@ -29,90 +29,92 @@ @ExtendWith(MockitoExtension.class) class PostgresSQLSqlGenerationVisitorTest { - private PostgresSQLSqlGenerationVisitor visitor; + private PostgresSQLSqlGenerationVisitor visitor; - @BeforeEach - void beforeEach(@Mock final ConnectionFactory connectionFactoryMock) { - final SqlDialect dialect = new PostgreSQLSqlDialect(connectionFactoryMock, AdapterProperties.emptyProperties()); - final SqlGenerationContext context = new SqlGenerationContext("test_catalog", "test_schema", false); - this.visitor = new PostgresSQLSqlGenerationVisitor(dialect, context); - } + @BeforeEach + void beforeEach(@Mock final ConnectionFactory connectionFactoryMock) { + final SqlDialect dialect = new PostgreSQLSqlDialect(connectionFactoryMock, + AdapterProperties.emptyProperties()); + final SqlGenerationContext context = new SqlGenerationContext("test_catalog", "test_schema", false); + this.visitor = new PostgresSQLSqlGenerationVisitor(dialect, context); + } - @CsvSource({ "ADD_DAYS, days", // - "ADD_HOURS, hours", // - "ADD_MINUTES, mins", // - "ADD_SECONDS, secs", // - "ADD_YEARS, years", // - "ADD_WEEKS, weeks", // - "ADD_MONTHS, months" }) - @ParameterizedTest - void testVisitSqlFunctionScalarAddDate(final ScalarFunction scalarFunction, final String expected) - throws AdapterException { - final SqlFunctionScalar sqlFunctionScalar = createSqlFunctionScalarForDateTest(scalarFunction, 10); - assertThat(this.visitor.visit(sqlFunctionScalar), - equalTo("\"test_column\" + make_interval(" + expected + " => 10)")); - } + @CsvSource({ "ADD_DAYS, days", // + "ADD_HOURS, hours", // + "ADD_MINUTES, mins", // + "ADD_SECONDS, secs", // + "ADD_YEARS, years", // + "ADD_WEEKS, weeks", // + "ADD_MONTHS, months" }) + @ParameterizedTest + void testVisitSqlFunctionScalarAddDate(final ScalarFunction scalarFunction, final String expected) + throws AdapterException { + final SqlFunctionScalar sqlFunctionScalar = createSqlFunctionScalarForDateTest(scalarFunction, 10); + assertThat(this.visitor.visit(sqlFunctionScalar), + equalTo("\"test_column\" + make_interval(" + expected + " => 10)")); + } - private SqlFunctionScalar createSqlFunctionScalarForDateTest(final ScalarFunction scalarFunction, - final int numericValue) { - final List arguments = new ArrayList<>(); - arguments.add(new SqlColumn(1, - ColumnMetadata.builder().name("test_column") - .adapterNotes("{\"jdbcDataType\":93, " + "\"typeName\":\"TIMESTAMP\"}") - .type(DataType.createChar(20, DataType.ExaCharset.UTF8)).build())); - arguments.add(new SqlLiteralExactnumeric(new BigDecimal(numericValue))); - return new SqlFunctionScalar(scalarFunction, arguments); - } + private SqlFunctionScalar createSqlFunctionScalarForDateTest(final ScalarFunction scalarFunction, + final int numericValue) { + final List arguments = new ArrayList<>(); + arguments.add(new SqlColumn(1, + ColumnMetadata.builder().name("test_column") + .adapterNotes("{\"jdbcDataType\":93, " + "\"typeName\":\"TIMESTAMP\"}") + .type(DataType.createChar(20, DataType.ExaCharset.UTF8)).build())); + arguments.add(new SqlLiteralExactnumeric(new BigDecimal(numericValue))); + return new SqlFunctionScalar(scalarFunction, arguments); + } - @CsvSource({ "SECOND, SECOND", // - "MINUTE, MINUTE", // - "DAY, DAY", // - "WEEK, WEEK", // - "MONTH, MONTH", // - "YEAR, YEAR" }) - @ParameterizedTest - void testVisitSqlFunctionScalarDatetime(final ScalarFunction scalarFunction, final String expected) - throws AdapterException { - final SqlFunctionScalar sqlFunctionScalar = createSqlFunctionScalarForDateTest(scalarFunction, 0); - assertThat(this.visitor.visit(sqlFunctionScalar), equalTo("DATE_PART('" + expected + "',\"test_column\")")); - } + @CsvSource({ "SECOND, SECOND, 2", // + "MINUTE, MINUTE, 2", // + "DAY, DAY, 2", // + "WEEK, WEEK, 2", // + "MONTH, MONTH, 2", // + "YEAR, YEAR, 4" }) + @ParameterizedTest + void testVisitSqlFunctionScalarDatetime(final ScalarFunction scalarFunction, final String expected, + final String decimalSize) throws AdapterException { + final SqlFunctionScalar sqlFunctionScalar = createSqlFunctionScalarForDateTest(scalarFunction, 0); + assertThat(this.visitor.visit(sqlFunctionScalar), equalTo("CAST(DATE_PART('" + expected + + "',\"test_column\") AS DECIMAL(" + decimalSize + ",0))")); + } - @Test - void testVisitSqlFunctionScalarPosixTime() throws AdapterException { - final SqlFunctionScalar sqlFunctionScalar = createSqlFunctionScalarForDateTest(POSIX_TIME, 0); - assertThat(this.visitor.visit(sqlFunctionScalar), equalTo("EXTRACT(EPOCH FROM \"test_column\")")); - } + @Test + void testVisitSqlFunctionScalarPosixTime() throws AdapterException { + final SqlFunctionScalar sqlFunctionScalar = createSqlFunctionScalarForDateTest(POSIX_TIME, 0); + assertThat(this.visitor.visit(sqlFunctionScalar), equalTo("EXTRACT(EPOCH FROM \"test_column\")")); + } - @Test - void testVisitSqlSelectListAnyValue() throws AdapterException { - final SqlSelectList sqlSelectList = SqlSelectList.createAnyValueSelectList(); - assertSqlNodeConvertedToOne(sqlSelectList, this.visitor); - } + @Test + void testVisitSqlSelectListAnyValue() throws AdapterException { + final SqlSelectList sqlSelectList = SqlSelectList.createAnyValueSelectList(); + assertSqlNodeConvertedToOne(sqlSelectList, this.visitor); + } - @Test - void testVisitSqlStatementSelect() throws AdapterException { - final SqlStatementSelect select = (SqlStatementSelect) DialectTestData.getTestSqlNode(); - assertThat(this.visitor.visit(select), // - equalTo("SELECT \"user_id\", " // - + "COUNT(\"url\") FROM \"test_schema\".\"clicks\" " // - + "WHERE 1 < \"user_id\" " // - + "GROUP BY \"user_id\" " // - + "HAVING 1 < COUNT(\"url\") " // - + "ORDER BY \"user_id\" LIMIT 10")); - } + @Test + void testVisitSqlStatementSelect() throws AdapterException { + final SqlStatementSelect select = (SqlStatementSelect) DialectTestData.getTestSqlNode(); + assertThat(this.visitor.visit(select), // + equalTo("SELECT \"user_id\", " // + + "COUNT(\"url\") FROM \"test_schema\".\"clicks\" " // + + "WHERE 1 < \"user_id\" " // + + "GROUP BY \"user_id\" " // + + "HAVING 1 < COUNT(\"url\") " // + + "ORDER BY \"user_id\" LIMIT 10")); + } - @Test - void testVisitSqlFunctionAggregateGroupConcat() throws AdapterException { - final SqlLiteralString argument = new SqlLiteralString("test"); - final ColumnMetadata columnMetadata = ColumnMetadata.builder().name("test_column").type(DataType.createBool()) - .build(); - final ColumnMetadata columnMetadata2 = ColumnMetadata.builder().name("test_column2") - .type(DataType.createDouble()).build(); - final List orderByArguments = List.of(new SqlColumn(1, columnMetadata), - new SqlColumn(2, columnMetadata2)); - final SqlOrderBy orderBy = new SqlOrderBy(orderByArguments, List.of(false, true), List.of(false, true)); - final SqlFunctionAggregateGroupConcat sqlFunctionAggregateGroupConcat = SqlFunctionAggregateGroupConcat - .builder(argument).separator(new SqlLiteralString("'")).orderBy(orderBy).build(); - assertThat(this.visitor.visit(sqlFunctionAggregateGroupConcat), equalTo("STRING_AGG(E'test', E'''') ")); - } + @Test + void testVisitSqlFunctionAggregateGroupConcat() throws AdapterException { + final SqlLiteralString argument = new SqlLiteralString("test"); + final ColumnMetadata columnMetadata = ColumnMetadata.builder().name("test_column") + .type(DataType.createBool()).build(); + final ColumnMetadata columnMetadata2 = ColumnMetadata.builder().name("test_column2") + .type(DataType.createDouble()).build(); + final List orderByArguments = List.of(new SqlColumn(1, columnMetadata), + new SqlColumn(2, columnMetadata2)); + final SqlOrderBy orderBy = new SqlOrderBy(orderByArguments, List.of(false, true), List.of(false, true)); + final SqlFunctionAggregateGroupConcat sqlFunctionAggregateGroupConcat = SqlFunctionAggregateGroupConcat + .builder(argument).separator(new SqlLiteralString("'")).orderBy(orderBy).build(); + assertThat(this.visitor.visit(sqlFunctionAggregateGroupConcat), equalTo("STRING_AGG(E'test', E'''') ")); + } } \ No newline at end of file diff --git a/src/test/resources/integration/scalarFunctionsParameterCache.yml b/src/test/resources/integration/scalarFunctionsParameterCache.yml index c828970..cfeb94b 100644 --- a/src/test/resources/integration/scalarFunctionsParameterCache.yml +++ b/src/test/resources/integration/scalarFunctionsParameterCache.yml @@ -15,8 +15,8 @@ atan2: ['"DOUBLE_PRECISION_C0", "DOUBLE_PRECISION_C0"', '"DOUBLE_PRECISION_C0", "DECIMAL18__3_C2"'] bit_Length: ['"VARCHAR2_C4"'] bit_length: ['"VARCHAR2_C4"'] -ceil: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"'] -ceiling: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"'] +ceil: ['"DOUBLE_PRECISION_C0"', '"DECIMAL18__3_C2"'] +ceiling: ['"DOUBLE_PRECISION_C0"', '"DECIMAL18__3_C2"'] char: ['"INTEGER_C1"'] char_Length: ['"VARCHAR2_C4"'] char_length: ['"VARCHAR2_C4"'] @@ -264,7 +264,7 @@ dump: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"', '"BOOLEAN_C "DECIMAL18__3_C2", "INTEGER_C1"', '"VARCHAR2_C4", "DECIMAL18__3_C2", "DECIMAL18__3_C2"', '"VARCHAR2_C4", "BOOLEAN_C3", "INTEGER_C1"', '"VARCHAR2_C4", "BOOLEAN_C3", "DECIMAL18__3_C2"'] exp: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"'] -floor: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"'] +floor: ['"DOUBLE_PRECISION_C0"', '"DECIMAL18__3_C2"'] from_posix_time: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"', '"BOOLEAN_C3"'] hour: ['"DATE_C5"', '"TIMESTAMP_C6"'] hours_between: ['"DATE_C5", "DATE_C5"', '"DATE_C5", "TIMESTAMP_C6"', '"TIMESTAMP_C6", @@ -1480,7 +1480,7 @@ second: ['"DATE_C5"', '"TIMESTAMP_C6"', '"DATE_C5", "DOUBLE_PRECISION_C0"', '"DA '"TIMESTAMP_C6", "BOOLEAN_C3"'] seconds_between: ['"DATE_C5", "DATE_C5"', '"DATE_C5", "TIMESTAMP_C6"', '"TIMESTAMP_C6", "DATE_C5"', '"TIMESTAMP_C6", "TIMESTAMP_C6"'] -sign: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"'] +sign: ['"DECIMAL18__3_C2"'] sin: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"'] sinh: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"'] soundex: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"', '"BOOLEAN_C3"', @@ -1502,10 +1502,10 @@ to_timestamp: ['"DATE_C5"', '"TIMESTAMP_C6"', '"TIMESTAMP_C6", "DECIMAL18__3_C2" '"TIMESTAMP_C6", "VARCHAR2_C4"'] translate: ['"VARCHAR2_C4", "VARCHAR2_C4", "VARCHAR2_C4"'] trim: ['"VARCHAR2_C4"', '"VARCHAR2_C4", "VARCHAR2_C4"'] -trunc: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"', '"INTEGER_C1", - "INTEGER_C1"', '"DECIMAL18__3_C2", "INTEGER_C1"'] -truncate: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"', '"INTEGER_C1", - "INTEGER_C1"', '"DECIMAL18__3_C2", "INTEGER_C1"'] +trunc: ['"DOUBLE_PRECISION_C0"', '"DECIMAL18__3_C2"', '"INTEGER_C1", "INTEGER_C1"', + '"DECIMAL18__3_C2", "INTEGER_C1"'] +truncate: ['"DOUBLE_PRECISION_C0"', '"DECIMAL18__3_C2"', '"INTEGER_C1", "INTEGER_C1"', + '"DECIMAL18__3_C2", "INTEGER_C1"'] ucase: ['"VARCHAR2_C4"'] unicode: ['"VARCHAR2_C4"'] unicodechr: ['"DOUBLE_PRECISION_C0"', '"INTEGER_C1"', '"DECIMAL18__3_C2"', '"BOOLEAN_C3"'] From 68879e29f344bd95fb8e40965909c7ac36c6ee40 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Tue, 27 Feb 2024 15:26:14 +0100 Subject: [PATCH 11/20] fixed test for both exasol versions --- .../adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java index 601beed..92adf7d 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java @@ -311,8 +311,7 @@ void testQueryUpperCaseTableQuotedThrowsException() { final String selectStatement = "SELECT x FROM " + virtualSchemaPostgresUppercaseTable.getName() + ".\"" + TABLE_POSTGRES_MIXED_CASE + "\""; final Exception exception = assertThrows(SQLException.class, () -> statementExasol.execute(selectStatement)); - assertThat(exception.getMessage(), containsString("object " + virtualSchemaPostgresUppercaseTable.getName() - + ".\"" + TABLE_POSTGRES_MIXED_CASE + "\" not found")); + assertThat(exception.getMessage(), containsString(".\"" + TABLE_POSTGRES_MIXED_CASE + "\" not found")); } @Test From a8dd916bbe0315f8303cfcb656fcd2b3db85a700 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Tue, 27 Feb 2024 15:32:35 +0100 Subject: [PATCH 12/20] use driver manager --- ...gresVirtualSchemaIntegrationTestSetup.java | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java index e495338..d75850a 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java @@ -18,6 +18,7 @@ import com.exasol.containers.ExasolService; import com.exasol.dbbuilder.dialects.exasol.*; import com.exasol.dbbuilder.dialects.postgres.PostgreSqlObjectFactory; +import com.exasol.drivers.JdbcDriver; import com.exasol.errorreporting.ExaError; import com.exasol.udfdebugging.UdfTestSetup; import com.github.dockerjava.api.model.ContainerNetwork; @@ -34,14 +35,7 @@ public class PostgresVirtualSchemaIntegrationTestSetup implements Closeable { private static final String POSTGRES_CONTAINER_NAME = "postgres:14.2"; private static final String JDBC_DRIVER_NAME = "postgresql.jar"; - private static final String JDBC_DRIVER_CONFIGURATION_FILE_NAME = "settings.cfg"; private static final Path JDBC_DRIVER_PATH = Path.of("target/postgresql-driver/" + JDBC_DRIVER_NAME); - private static final String JDBC_DRIVER_CONFIGURATION_FILE_CONTENT = "DRIVERNAME=POSTGRES_JDBC_DRIVER\n" // - + "JAR=" + JDBC_DRIVER_NAME + "\n" // - + "DRIVERMAIN=org.postgresql.Driver\n" // - + "PREFIX=jdbc:postgresql:\n" // - + "FETCHSIZE=100000\n" // - + "INSERTSIZE=-1\n"; private static final int POSTGRES_PORT = 5432; private final Statement postgresStatement; @@ -64,7 +58,7 @@ public class PostgresVirtualSchemaIntegrationTestSetup implements Closeable { this.exasolContainer.start(); this.postgresqlContainer.start(); final Bucket bucket = this.exasolContainer.getDefaultBucket(); - uploadDriverToBucket(bucket); + uploadDriverToBucket(this.exasolContainer); uploadVsJarToBucket(bucket); this.exasolConnection = this.exasolContainer.createConnection(""); this.exasolStatement = this.exasolConnection.createStatement(); @@ -92,20 +86,25 @@ public class PostgresVirtualSchemaIntegrationTestSetup implements Closeable { } } - private static void uploadDriverToBucket(final Bucket bucket) + private static void uploadDriverToBucket(final ExasolContainer> container) throws InterruptedException, TimeoutException, BucketAccessException { try { - bucket.uploadStringContent(JDBC_DRIVER_CONFIGURATION_FILE_CONTENT, - "drivers/jdbc/" + JDBC_DRIVER_CONFIGURATION_FILE_NAME); - bucket.uploadFile(JDBC_DRIVER_PATH, "drivers/jdbc/" + JDBC_DRIVER_NAME); - } catch (final BucketAccessException | FileNotFoundException exception) { + container.getDriverManager().install( // + JdbcDriver.builder("POSTGRES_JDBC_DRIVER") // + .enableSecurityManager(false) // + .mainClass("org.postgresql.Driver") // + .prefix("jdbc:postgresql:") // + .sourceFile(JDBC_DRIVER_PATH) // + .build()); + + } catch (final Exception ex) { throw new IllegalStateException( ExaError.messageBuilder("F-VSMYSQL-2") .message("An error occurred while uploading the jdbc driver to the bucket.") .mitigation("Make sure the {{JDBC_DRIVER_PATH}} file exists.") .parameter("JDBC_DRIVER_PATH", JDBC_DRIVER_PATH) .mitigation("You can generate it by executing the integration test with maven.").toString(), - exception); + ex); } } From bce010fe4c3d170b0d7595d89b4e04d6b8e5aaf6 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Tue, 27 Feb 2024 15:34:57 +0100 Subject: [PATCH 13/20] fix exasol error code to correct VS --- .../postgresql/PostgresVirtualSchemaIntegrationTestSetup.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java index d75850a..e0ff510 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java @@ -99,7 +99,7 @@ private static void uploadDriverToBucket(final ExasolContainer Date: Tue, 27 Feb 2024 16:16:52 +0100 Subject: [PATCH 14/20] shorten switch --- .../postgresql/PostgresSQLSqlGenerationVisitor.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitor.java b/src/main/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitor.java index a22999f..fcc5e0e 100644 --- a/src/main/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitor.java +++ b/src/main/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitor.java @@ -139,17 +139,9 @@ private String getDateTime(final List argumentsSql, final ScalarFunction builder.append(") AS DECIMAL("); switch (scalarFunction) { case SECOND: - builder.append("2"); - break; case MINUTE: - builder.append("2"); - break; case DAY: - builder.append("2"); - break; case WEEK: - builder.append("2"); - break; case MONTH: builder.append("2"); break; From dbd0299c30b81d3ea0537f6b82496a4a5bb7822d Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Tue, 27 Feb 2024 16:21:01 +0100 Subject: [PATCH 15/20] fix code smells (1) --- .../adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java index 92adf7d..5ceb7d1 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgreSQLSqlDialectIT.java @@ -275,12 +275,13 @@ void testYearScalarFunctionFromDate() throws SQLException { // currently a bug in the compiler, compiler always expects 'VARCHAR(1) ASCII' see // https://github.com/exasol/postgresql-virtual-schema/issues/79 // https://exasol.atlassian.net/browse/SPOT-19716 - @Disabled + @Disabled("Currently a bug in the compiler, compiler always expects 'VARCHAR(1) ASCII'") @Test void testCurrentSchemaScalarFunction() throws SQLException { final String query = " SELECT current_schema FROM " + virtualSchemaPostgres.getName() + "." + TABLE_POSTGRES_ALL_DATA_TYPES; final ResultSet actualResultSet = getActualResultSet(query); + assertThat(actualResultSet, table().row(TABLE_POSTGRES_ALL_DATA_TYPES).matches()); } @Test From 27a5b2e6051e6c2e06208474791d6d4d94c0702e Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Wed, 28 Feb 2024 12:58:42 +0100 Subject: [PATCH 16/20] refactoring for readability --- .../PostgresSQLSqlGenerationVisitor.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitor.java b/src/main/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitor.java index fcc5e0e..f58a138 100644 --- a/src/main/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitor.java +++ b/src/main/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitor.java @@ -112,6 +112,16 @@ private String buildInterval(final List argumentsSql, final String unit) private String getDateTime(final List argumentsSql, final ScalarFunction scalarFunction) { final StringBuilder builder = new StringBuilder(); builder.append("CAST(DATE_PART("); + appendDatePart(scalarFunction, builder); + builder.append(","); + builder.append(argumentsSql.get(0)); + builder.append(") AS DECIMAL("); + appendDecimalSize(scalarFunction, builder); + builder.append(",0))"); + return builder.toString(); + } + + private static void appendDatePart(ScalarFunction scalarFunction, StringBuilder builder) { switch (scalarFunction) { case SECOND: builder.append("'SECOND'"); @@ -134,9 +144,9 @@ private String getDateTime(final List argumentsSql, final ScalarFunction default: break; } - builder.append(","); - builder.append(argumentsSql.get(0)); - builder.append(") AS DECIMAL("); + } + + private static void appendDecimalSize(ScalarFunction scalarFunction, StringBuilder builder) { switch (scalarFunction) { case SECOND: case MINUTE: @@ -151,8 +161,6 @@ private String getDateTime(final List argumentsSql, final ScalarFunction default: break; } - builder.append(",0))"); - return builder.toString(); } private String getPosixTime(final List argumentsSql) { From 1e0c8fa1d9fbcbe27da5cc9d4f670703f028b662 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Wed, 28 Feb 2024 12:58:59 +0100 Subject: [PATCH 17/20] changed exception variable name (PR feedback) --- .../postgresql/PostgresVirtualSchemaIntegrationTestSetup.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java index e0ff510..6ae21a5 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresVirtualSchemaIntegrationTestSetup.java @@ -97,14 +97,14 @@ private static void uploadDriverToBucket(final ExasolContainer Date: Wed, 28 Feb 2024 12:59:50 +0100 Subject: [PATCH 18/20] Add 'dependencies_update' workflow to generated files list (PR feedback) --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index c67f857..94e0086 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,6 +5,7 @@ doc/changes/changelog.md linguist-genera .github/workflows/ci-build.yml linguist-generated=true .github/workflows/ci-build-next-java.yml linguist-generated=true .github/workflows/dependencies_check.yml linguist-generated=true +.github/workflows/dependencies_update.yml linguist-generated=true .github/workflows/release_droid_print_quick_checksum.yml linguist-generated=true .github/workflows/release_droid_upload_github_release_assets.yml linguist-generated=true .github/workflows/release_droid_prepare_original_checksum.yml linguist-generated=true From 3b31a899df0b1219caf9ba0bd3afab83a8a8b857 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Wed, 28 Feb 2024 13:22:50 +0100 Subject: [PATCH 19/20] Wrote version doc notes --- doc/changes/changes_3.0.0.md | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/doc/changes/changes_3.0.0.md b/doc/changes/changes_3.0.0.md index dde0d37..600c31b 100644 --- a/doc/changes/changes_3.0.0.md +++ b/doc/changes/changes_3.0.0.md @@ -1,12 +1,35 @@ -# Virtual Schema for PostgreSQL 3.0.0, released 2024-??-?? +# Virtual Schema for PostgreSQL 3.0.0, released 2024-02-28 + +Code name: Char set is always `utf-8`, deprecated IMPORT_DATA_TYPES `FROM_RESULT_SET` value -Code name: ## Summary +The behaviour when it comes to character sets is now simplified, +The target char set is now always UTF-8. +The `IMPORT_DATA_TYPES` property (and value `FROM_RESULT_SET`) are now deprecated (change in vs-common-jdbc): +An exception will be thrown when users use `FROM_RESULT_SET`. The exception message warns the user that the value is no longer supported and the property itself is also deprecated. + +Various broken scalar time-related extraction functions for dates and timestamps in the virtual schema are now fixed: `year`,`month`,`day`,`hour`,`minute`,`second`. + +Scalar `division` (`/`) which was broken in some cases now also works correctly. + +Tests for `current_schema` are currently disabled, this is because of a discovered compiler bug: https://github.com/exasol/postgresql-virtual-schema/issues/79 . +These tests will be re-evaluated later when there is more clarity about this issue. + +We also updated dependencies and resolved the following 2 CVEs in test dependency `org.apache.commons:commons-compress`: +- CVE-2024-26308 +- CVE-2024-25710 +We also updated dependencies and resolved the following CVE in test dependency `org.postgresql:postgresql:`: +- CVE-2024-1597 + ## Features -- ISSUE_NUMBER: description +- #68 : Update tests to V8 VSPG refactoring + +## Security + +- #78 : Fix vulnerabilities in org.postgresql:postgresql:jar:42.6.0:compile & org.apache.commons:commons-compress:jar:1.24.0:test ## Dependency Updates From 6f7dc48b0586a6ec960ed4315e3d7d7de4374356 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Wed, 28 Feb 2024 13:30:12 +0100 Subject: [PATCH 20/20] fix formatting --- .../PostgresSQLSqlGenerationVisitorTest.java | 159 +++++++++--------- 1 file changed, 79 insertions(+), 80 deletions(-) diff --git a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitorTest.java b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitorTest.java index 8d8b8c0..9728974 100644 --- a/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitorTest.java +++ b/src/test/java/com/exasol/adapter/dialects/postgresql/PostgresSQLSqlGenerationVisitorTest.java @@ -29,92 +29,91 @@ @ExtendWith(MockitoExtension.class) class PostgresSQLSqlGenerationVisitorTest { - private PostgresSQLSqlGenerationVisitor visitor; + private PostgresSQLSqlGenerationVisitor visitor; - @BeforeEach - void beforeEach(@Mock final ConnectionFactory connectionFactoryMock) { - final SqlDialect dialect = new PostgreSQLSqlDialect(connectionFactoryMock, - AdapterProperties.emptyProperties()); - final SqlGenerationContext context = new SqlGenerationContext("test_catalog", "test_schema", false); - this.visitor = new PostgresSQLSqlGenerationVisitor(dialect, context); - } + @BeforeEach + void beforeEach(@Mock final ConnectionFactory connectionFactoryMock) { + final SqlDialect dialect = new PostgreSQLSqlDialect(connectionFactoryMock, AdapterProperties.emptyProperties()); + final SqlGenerationContext context = new SqlGenerationContext("test_catalog", "test_schema", false); + this.visitor = new PostgresSQLSqlGenerationVisitor(dialect, context); + } - @CsvSource({ "ADD_DAYS, days", // - "ADD_HOURS, hours", // - "ADD_MINUTES, mins", // - "ADD_SECONDS, secs", // - "ADD_YEARS, years", // - "ADD_WEEKS, weeks", // - "ADD_MONTHS, months" }) - @ParameterizedTest - void testVisitSqlFunctionScalarAddDate(final ScalarFunction scalarFunction, final String expected) - throws AdapterException { - final SqlFunctionScalar sqlFunctionScalar = createSqlFunctionScalarForDateTest(scalarFunction, 10); - assertThat(this.visitor.visit(sqlFunctionScalar), - equalTo("\"test_column\" + make_interval(" + expected + " => 10)")); - } + @CsvSource({ "ADD_DAYS, days", // + "ADD_HOURS, hours", // + "ADD_MINUTES, mins", // + "ADD_SECONDS, secs", // + "ADD_YEARS, years", // + "ADD_WEEKS, weeks", // + "ADD_MONTHS, months" }) + @ParameterizedTest + void testVisitSqlFunctionScalarAddDate(final ScalarFunction scalarFunction, final String expected) + throws AdapterException { + final SqlFunctionScalar sqlFunctionScalar = createSqlFunctionScalarForDateTest(scalarFunction, 10); + assertThat(this.visitor.visit(sqlFunctionScalar), + equalTo("\"test_column\" + make_interval(" + expected + " => 10)")); + } - private SqlFunctionScalar createSqlFunctionScalarForDateTest(final ScalarFunction scalarFunction, - final int numericValue) { - final List arguments = new ArrayList<>(); - arguments.add(new SqlColumn(1, - ColumnMetadata.builder().name("test_column") - .adapterNotes("{\"jdbcDataType\":93, " + "\"typeName\":\"TIMESTAMP\"}") - .type(DataType.createChar(20, DataType.ExaCharset.UTF8)).build())); - arguments.add(new SqlLiteralExactnumeric(new BigDecimal(numericValue))); - return new SqlFunctionScalar(scalarFunction, arguments); - } + private SqlFunctionScalar createSqlFunctionScalarForDateTest(final ScalarFunction scalarFunction, + final int numericValue) { + final List arguments = new ArrayList<>(); + arguments.add(new SqlColumn(1, + ColumnMetadata.builder().name("test_column") + .adapterNotes("{\"jdbcDataType\":93, " + "\"typeName\":\"TIMESTAMP\"}") + .type(DataType.createChar(20, DataType.ExaCharset.UTF8)).build())); + arguments.add(new SqlLiteralExactnumeric(new BigDecimal(numericValue))); + return new SqlFunctionScalar(scalarFunction, arguments); + } - @CsvSource({ "SECOND, SECOND, 2", // - "MINUTE, MINUTE, 2", // - "DAY, DAY, 2", // - "WEEK, WEEK, 2", // - "MONTH, MONTH, 2", // - "YEAR, YEAR, 4" }) - @ParameterizedTest - void testVisitSqlFunctionScalarDatetime(final ScalarFunction scalarFunction, final String expected, - final String decimalSize) throws AdapterException { - final SqlFunctionScalar sqlFunctionScalar = createSqlFunctionScalarForDateTest(scalarFunction, 0); - assertThat(this.visitor.visit(sqlFunctionScalar), equalTo("CAST(DATE_PART('" + expected - + "',\"test_column\") AS DECIMAL(" + decimalSize + ",0))")); - } + @CsvSource({ "SECOND, SECOND, 2", // + "MINUTE, MINUTE, 2", // + "DAY, DAY, 2", // + "WEEK, WEEK, 2", // + "MONTH, MONTH, 2", // + "YEAR, YEAR, 4" }) + @ParameterizedTest + void testVisitSqlFunctionScalarDatetime(final ScalarFunction scalarFunction, final String expected, + final String decimalSize) throws AdapterException { + final SqlFunctionScalar sqlFunctionScalar = createSqlFunctionScalarForDateTest(scalarFunction, 0); + assertThat(this.visitor.visit(sqlFunctionScalar), + equalTo("CAST(DATE_PART('" + expected + "',\"test_column\") AS DECIMAL(" + decimalSize + ",0))")); + } - @Test - void testVisitSqlFunctionScalarPosixTime() throws AdapterException { - final SqlFunctionScalar sqlFunctionScalar = createSqlFunctionScalarForDateTest(POSIX_TIME, 0); - assertThat(this.visitor.visit(sqlFunctionScalar), equalTo("EXTRACT(EPOCH FROM \"test_column\")")); - } + @Test + void testVisitSqlFunctionScalarPosixTime() throws AdapterException { + final SqlFunctionScalar sqlFunctionScalar = createSqlFunctionScalarForDateTest(POSIX_TIME, 0); + assertThat(this.visitor.visit(sqlFunctionScalar), equalTo("EXTRACT(EPOCH FROM \"test_column\")")); + } - @Test - void testVisitSqlSelectListAnyValue() throws AdapterException { - final SqlSelectList sqlSelectList = SqlSelectList.createAnyValueSelectList(); - assertSqlNodeConvertedToOne(sqlSelectList, this.visitor); - } + @Test + void testVisitSqlSelectListAnyValue() throws AdapterException { + final SqlSelectList sqlSelectList = SqlSelectList.createAnyValueSelectList(); + assertSqlNodeConvertedToOne(sqlSelectList, this.visitor); + } - @Test - void testVisitSqlStatementSelect() throws AdapterException { - final SqlStatementSelect select = (SqlStatementSelect) DialectTestData.getTestSqlNode(); - assertThat(this.visitor.visit(select), // - equalTo("SELECT \"user_id\", " // - + "COUNT(\"url\") FROM \"test_schema\".\"clicks\" " // - + "WHERE 1 < \"user_id\" " // - + "GROUP BY \"user_id\" " // - + "HAVING 1 < COUNT(\"url\") " // - + "ORDER BY \"user_id\" LIMIT 10")); - } + @Test + void testVisitSqlStatementSelect() throws AdapterException { + final SqlStatementSelect select = (SqlStatementSelect) DialectTestData.getTestSqlNode(); + assertThat(this.visitor.visit(select), // + equalTo("SELECT \"user_id\", " // + + "COUNT(\"url\") FROM \"test_schema\".\"clicks\" " // + + "WHERE 1 < \"user_id\" " // + + "GROUP BY \"user_id\" " // + + "HAVING 1 < COUNT(\"url\") " // + + "ORDER BY \"user_id\" LIMIT 10")); + } - @Test - void testVisitSqlFunctionAggregateGroupConcat() throws AdapterException { - final SqlLiteralString argument = new SqlLiteralString("test"); - final ColumnMetadata columnMetadata = ColumnMetadata.builder().name("test_column") - .type(DataType.createBool()).build(); - final ColumnMetadata columnMetadata2 = ColumnMetadata.builder().name("test_column2") - .type(DataType.createDouble()).build(); - final List orderByArguments = List.of(new SqlColumn(1, columnMetadata), - new SqlColumn(2, columnMetadata2)); - final SqlOrderBy orderBy = new SqlOrderBy(orderByArguments, List.of(false, true), List.of(false, true)); - final SqlFunctionAggregateGroupConcat sqlFunctionAggregateGroupConcat = SqlFunctionAggregateGroupConcat - .builder(argument).separator(new SqlLiteralString("'")).orderBy(orderBy).build(); - assertThat(this.visitor.visit(sqlFunctionAggregateGroupConcat), equalTo("STRING_AGG(E'test', E'''') ")); - } + @Test + void testVisitSqlFunctionAggregateGroupConcat() throws AdapterException { + final SqlLiteralString argument = new SqlLiteralString("test"); + final ColumnMetadata columnMetadata = ColumnMetadata.builder().name("test_column").type(DataType.createBool()) + .build(); + final ColumnMetadata columnMetadata2 = ColumnMetadata.builder().name("test_column2") + .type(DataType.createDouble()).build(); + final List orderByArguments = List.of(new SqlColumn(1, columnMetadata), + new SqlColumn(2, columnMetadata2)); + final SqlOrderBy orderBy = new SqlOrderBy(orderByArguments, List.of(false, true), List.of(false, true)); + final SqlFunctionAggregateGroupConcat sqlFunctionAggregateGroupConcat = SqlFunctionAggregateGroupConcat + .builder(argument).separator(new SqlLiteralString("'")).orderBy(orderBy).build(); + assertThat(this.visitor.visit(sqlFunctionAggregateGroupConcat), equalTo("STRING_AGG(E'test', E'''') ")); + } } \ No newline at end of file