From ace810f2b66445d5fe8761b84293df70a195b8f6 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 28 Nov 2022 23:30:45 -0800 Subject: [PATCH 1/2] Test Apache ORC 1.8.1-SNAPSHOT --- .github/workflows/flink-ci.yml | 6 ++++++ .github/workflows/hive-ci.yml | 6 ++++++ .github/workflows/java-ci.yml | 9 +++++++++ .github/workflows/spark-ci.yml | 9 +++++++++ versions.props | 2 +- 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml index 24a11fb10a62..33aa077339f6 100644 --- a/.github/workflows/flink-ci.yml +++ b/.github/workflows/flink-ci.yml @@ -80,6 +80,9 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: ${{ runner.os }}-gradle- - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + - run: | + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive - run: ./gradlew -DsparkVersions= -DhiveVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:iceberg-flink-${{ matrix.flink }}:check :iceberg-flink:iceberg-flink-runtime-${{ matrix.flink }}:check -DscalaVersion=2.11 -DknownScalaVersions=2.11 -Pquick=true -x javadoc - uses: actions/upload-artifact@v3 if: failure() @@ -111,6 +114,9 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: ${{ runner.os }}-gradle- - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + - run: | + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive - run: ./gradlew -DsparkVersions= -DhiveVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:iceberg-flink-${{ matrix.flink }}:check :iceberg-flink:iceberg-flink-runtime-${{ matrix.flink }}:check -Pquick=true -x javadoc - uses: actions/upload-artifact@v3 if: failure() diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml index ba6cc1535ed7..c5fd7e597920 100644 --- a/.github/workflows/hive-ci.yml +++ b/.github/workflows/hive-ci.yml @@ -75,6 +75,9 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: ${{ runner.os }}-gradle- - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + - run: | + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive - run: ./gradlew -DsparkVersions= -DhiveVersions=2 -DflinkVersions= -Pquick=true :iceberg-mr:check :iceberg-hive-runtime:check -x javadoc - uses: actions/upload-artifact@v3 if: failure() @@ -101,6 +104,9 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: ${{ runner.os }}-gradle- - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + - run: | + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive - run: ./gradlew -DsparkVersions= -DhiveVersions=3 -DflinkVersions= -Pquick=true :iceberg-hive3-orc-bundle:check :iceberg-hive3:check :iceberg-hive-runtime:check -x javadoc - uses: actions/upload-artifact@v3 if: failure() diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index 5b8e0356f4e6..f6c51272aab6 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -72,6 +72,9 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: ${{ runner.os }}-gradle- - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + - run: | + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive - run: ./gradlew check -DsparkVersions= -DhiveVersions= -DflinkVersions= -Pquick=true -x javadoc - uses: actions/upload-artifact@v3 if: failure() @@ -88,6 +91,9 @@ jobs: with: distribution: zulu java-version: 8 + - run: | + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive - run: ./gradlew -DallVersions build -x test -x javadoc -x integrationTest build-javadoc: @@ -98,4 +104,7 @@ jobs: with: distribution: zulu java-version: 8 + - run: | + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive - run: ./gradlew -Pquick=true javadoc diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index eb8abb61039d..ad1601a5b20c 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -74,6 +74,9 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: ${{ runner.os }}-gradle- - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + - run: | + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive - run: ./gradlew -DsparkVersions=2.4 -DhiveVersions= -DflinkVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark-2.4:check :iceberg-spark:iceberg-spark-runtime-2.4:check -Pquick=true -x javadoc - uses: actions/upload-artifact@v3 if: failure() @@ -104,6 +107,9 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: ${{ runner.os }}-gradle- - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + - run: | + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.12 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.12:check -Pquick=true -x javadoc - uses: actions/upload-artifact@v3 if: failure() @@ -134,6 +140,9 @@ jobs: key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} restore-keys: ${{ runner.os }}-gradle- - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts + - run: | + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.13 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.13:check -Pquick=true -x javadoc - uses: actions/upload-artifact@v3 if: failure() diff --git a/versions.props b/versions.props index 5ff68e57214b..967fe8829e1b 100644 --- a/versions.props +++ b/versions.props @@ -4,7 +4,7 @@ org.apache.calcite:* = 1.10.0 org.apache.hadoop:* = 2.7.3 org.apache.hive:* = 2.3.8 org.apache.httpcomponents.client5:* = 5.1 -org.apache.orc:* = 1.8.0 +org.apache.orc:* = 1.8.1-SNAPSHOT org.apache.parquet:* = 1.12.3 org.apache.pig:pig = 0.14.0 com.fasterxml.jackson.*:* = 2.11.4 From 49799f6a8423068218f42e1674a810dc56ec61d1 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 29 Nov 2022 03:05:40 -0800 Subject: [PATCH 2/2] Add orc-tools --- .github/workflows/hive-ci.yml | 2 ++ .github/workflows/java-ci.yml | 2 ++ .github/workflows/spark-ci.yml | 3 +++ 3 files changed, 7 insertions(+) diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml index c5fd7e597920..5c1884ef98ac 100644 --- a/.github/workflows/hive-ci.yml +++ b/.github/workflows/hive-ci.yml @@ -78,6 +78,7 @@ jobs: - run: | mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-tools -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots - run: ./gradlew -DsparkVersions= -DhiveVersions=2 -DflinkVersions= -Pquick=true :iceberg-mr:check :iceberg-hive-runtime:check -x javadoc - uses: actions/upload-artifact@v3 if: failure() @@ -107,6 +108,7 @@ jobs: - run: | mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-tools -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots - run: ./gradlew -DsparkVersions= -DhiveVersions=3 -DflinkVersions= -Pquick=true :iceberg-hive3-orc-bundle:check :iceberg-hive3:check :iceberg-hive-runtime:check -x javadoc - uses: actions/upload-artifact@v3 if: failure() diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index f6c51272aab6..58a87ba2405e 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -75,6 +75,7 @@ jobs: - run: | mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-tools -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots - run: ./gradlew check -DsparkVersions= -DhiveVersions= -DflinkVersions= -Pquick=true -x javadoc - uses: actions/upload-artifact@v3 if: failure() @@ -94,6 +95,7 @@ jobs: - run: | mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-tools -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots - run: ./gradlew -DallVersions build -x test -x javadoc -x integrationTest build-javadoc: diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index ad1601a5b20c..71cf32a1c151 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -77,6 +77,7 @@ jobs: - run: | mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-tools -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots - run: ./gradlew -DsparkVersions=2.4 -DhiveVersions= -DflinkVersions= :iceberg-spark:check :iceberg-spark:iceberg-spark-2.4:check :iceberg-spark:iceberg-spark-runtime-2.4:check -Pquick=true -x javadoc - uses: actions/upload-artifact@v3 if: failure() @@ -110,6 +111,7 @@ jobs: - run: | mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-tools -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.12 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.12:check -Pquick=true -x javadoc - uses: actions/upload-artifact@v3 if: failure() @@ -143,6 +145,7 @@ jobs: - run: | mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-core -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots -Dclassifier=nohive + mvn dependency:get -DgroupId=org.apache.orc -DartifactId=orc-tools -Dversion=1.8.1-SNAPSHOT -Dpackaging=jar -D=remoteRepositories=https://repository.apache.org/snapshots - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.13 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.13:check -Pquick=true -x javadoc - uses: actions/upload-artifact@v3 if: failure()