Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Test #5793

Closed
wants to merge 21 commits into from
Closed

Test #5793

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
443cd06
Test
liujinhui1994 Jun 8, 2022
de6cd3d
fix dep
xushiyan Jun 8, 2022
ad2c096
test tool versions
xushiyan Jun 8, 2022
c018120
fix jupiter dep in hudi-spark2
xushiyan Jun 8, 2022
7f11611
temp disable hudi-spark2
xushiyan Jun 8, 2022
e42346d
fix junit deps
xushiyan Jun 8, 2022
0af417a
Fix hudi-spark2 module tests
codope Jun 8, 2022
0db278b
UT in integ test modul
xushiyan Jun 8, 2022
6e35b7d
fix dep for cli and integ module
xushiyan Jun 9, 2022
3603086
skip tests for bundles
xushiyan Jun 9, 2022
80f6fbc
fix dep for hudi flink modules
xushiyan Jun 9, 2022
f88cb2f
fix cli test name to avoid test detection
xushiyan Jun 9, 2022
66d5a28
fix azure pipeline yml setup
xushiyan Jun 9, 2022
21c71f6
fix utilities test init problems
xushiyan Jun 9, 2022
b7c61bf
fix integ test usage
xushiyan Jun 9, 2022
f04ca32
fix cli dep and IT SPARK_HOME
xushiyan Jun 9, 2022
767ba37
fix cli log4j dep
xushiyan Jun 9, 2022
65ef79f
fix azp commands
xushiyan Jun 9, 2022
e6c0b34
fix azp env vars
xushiyan Jun 9, 2022
2078b70
fix azp mvn opts
xushiyan Jun 9, 2022
4d22016
fix integ module dep
xushiyan Jun 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 84 additions & 32 deletions azure-pipelines.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,68 @@ trigger:
pool:
vmImage: 'ubuntu-18.04'

parameters:
- name: job1Modules
type: object
default:
- 'hudi-common'
- 'hudi-flink-datasource/hudi-flink'
- 'hudi-flink-datasource/hudi-flink1.13.x'
- 'hudi-flink-datasource/hudi-flink1.14.x'
- name: job2Modules
type: object
default:
- 'hudi-client/hudi-spark-client'
- name: job3Modules
type: object
default:
- 'hudi-cli'
- 'hudi-client/hudi-client-common'
- 'hudi-client/hudi-flink-client'
- 'hudi-client/hudi-java-client'
- 'hudi-sync/hudi-adb-sync'
- 'hudi-sync/hudi-datahub-sync'
- 'hudi-sync/hudi-hive-sync'
- 'hudi-sync/hudi-sync-common'
- 'hudi-utilities'
- name: job4Modules
type: object
default:
- '!hudi-cli'
- '!hudi-client'
- '!hudi-client/hudi-client-common'
- '!hudi-client/hudi-flink-client'
- '!hudi-client/hudi-java-client'
- '!hudi-client/hudi-spark-client'
- '!hudi-common'
- '!hudi-examples'
- '!hudi-examples/hudi-examples-common'
- '!hudi-examples/hudi-examples-flink'
- '!hudi-examples/hudi-examples-java'
- '!hudi-examples/hudi-examples-spark'
- '!hudi-flink-datasource'
- '!hudi-flink-datasource/hudi-flink'
- '!hudi-flink-datasource/hudi-flink1.13.x'
- '!hudi-flink-datasource/hudi-flink1.14.x'
- '!hudi-sync'
- '!hudi-sync/hudi-adb-sync'
- '!hudi-sync/hudi-datahub-sync'
- '!hudi-sync/hudi-hive-sync'
- '!hudi-sync/hudi-sync-common'
- '!hudi-utilities'

variables:
MAVEN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true'
BUILD_PROFILES: '-Dscala-2.11 -Dspark2.4 -Dflink1.14'
PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true'
MVN_OPTS_INSTALL: '-T 2.5C -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS)'
MVN_OPTS_TEST: '-fae $(BUILD_PROFILES) $(PLUGIN_OPTS)'
SPARK_VERSION: '2.4.4'
HADOOP_VERSION: '2.7'
SPARK_ARCHIVE: spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)
EXCLUDE_TESTED_MODULES: '!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-spark,!hudi-common,!hudi-flink-datasource/hudi-flink,!hudi-client/hudi-spark-client,!hudi-client/hudi-client-common,!hudi-client/hudi-flink-client,!hudi-client/hudi-java-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync'
JOB1_MODULES: ${{ join(',',parameters.job1Modules) }}
JOB2_MODULES: ${{ join(',',parameters.job2Modules) }}
JOB3_MODULES: ${{ join(',',parameters.job3Modules) }}
JOB4_MODULES: ${{ join(',',parameters.job4Modules) }}

stages:
- stage: test
Expand All @@ -40,28 +96,27 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'clean install'
options: -T 2.5C -DskipTests
options: $(MVN_OPTS_INSTALL)
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
- task: Maven@3
displayName: UT common flink client/spark-client
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Punit-tests -pl hudi-common,hudi-flink-datasource/hudi-flink,hudi-client/hudi-spark-client
options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_MODULES),hudi-client/hudi-spark-client
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
mavenOptions: '-Xmx4g'
- task: Maven@3
displayName: FT common flink
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Pfunctional-tests -pl hudi-common,hudi-flink-datasource/hudi-flink
options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB1_MODULES)
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
mavenOptions: '-Xmx4g'
- job: UT_FT_2
displayName: FT client/spark-client
timeoutInMinutes: '120'
Expand All @@ -71,50 +126,48 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'clean install'
options: -T 2.5C -DskipTests
options: $(MVN_OPTS_INSTALL)
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
- task: Maven@3
displayName: FT client/spark-client
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Pfunctional-tests -pl hudi-client/hudi-spark-client
options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB2_MODULES)
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
mavenOptions: '-Xmx4g'
- job: UT_FT_3
displayName: UT FT clients & cli & utilities & sync/hive-sync
displayName: UT FT clients & cli & utilities & sync
timeoutInMinutes: '120'
steps:
- task: Maven@3
displayName: maven install
inputs:
mavenPomFile: 'pom.xml'
goals: 'clean install'
options: -T 2.5C -DskipTests
options: $(MVN_OPTS_INSTALL)
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
- task: Maven@3
displayName: UT clients & cli & utilities & sync/hive-sync
displayName: UT clients & cli & utilities & sync
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Punit-tests -pl hudi-client/hudi-client-common,hudi-client/hudi-flink-client,hudi-client/hudi-java-client,hudi-cli,hudi-utilities,hudi-sync/hudi-hive-sync
options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES)
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
mavenOptions: '-Xmx4g'
- task: Maven@3
displayName: FT clients & cli & utilities & sync/hive-sync
displayName: FT clients & cli & utilities & sync
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Pfunctional-tests -pl hudi-client/hudi-client-common,hudi-client/hudi-flink-client,hudi-client/hudi-java-client,hudi-cli,hudi-utilities,hudi-sync/hudi-hive-sync
options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB3_MODULES)
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
mavenOptions: '-Xmx4g'
- job: UT_FT_4
displayName: UT FT other modules
timeoutInMinutes: '120'
Expand All @@ -124,28 +177,27 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'clean install'
options: -T 2.5C -DskipTests
options: $(MVN_OPTS_INSTALL)
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
- task: Maven@3
displayName: UT other modules
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Punit-tests -pl $(EXCLUDE_TESTED_MODULES)
options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_MODULES)
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
mavenOptions: '-Xmx4g'
- task: Maven@3
displayName: FT other modules
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Pfunctional-tests -pl $(EXCLUDE_TESTED_MODULES)
options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_MODULES)
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
mavenOptions: '-Xmx4g'
- job: IT
displayName: IT modules
timeoutInMinutes: '120'
Expand All @@ -155,19 +207,18 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'clean install'
options: -T 2.5C -Pintegration-tests -DskipTests
options: $(MVN_OPTS_INSTALL) -Pintegration-tests
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
- task: Maven@3
displayName: UT integ-test
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
options: -Pintegration-tests -DskipUTs=false -DskipITs=true -pl hudi-integ-test test
options: $(MVN_OPTS_TEST) -Pintegration-tests -DskipUTs=false -DskipITs=true -pl hudi-integ-test
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
mavenOptions: '-Xmx4g'
- task: AzureCLI@2
displayName: Prepare for IT
inputs:
Expand All @@ -180,5 +231,6 @@ stages:
tar -xvf $(Pipeline.Workspace)/$(SPARK_ARCHIVE).tgz -C $(Pipeline.Workspace)/
mkdir /tmp/spark-events/
- script: |
mvn $(MAVEN_OPTS) -Pintegration-tests verify
export SPARK_HOME=$(Pipeline.Workspace)/$(SPARK_ARCHIVE)
mvn $(MVN_OPTS_TEST) -Pintegration-tests verify
displayName: IT
30 changes: 30 additions & 0 deletions hudi-aws/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,11 +144,41 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-runner</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-suite-api</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
71 changes: 67 additions & 4 deletions hudi-cli/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,24 @@
<artifactId>log4j</artifactId>
</dependency>

<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
Expand Down Expand Up @@ -259,36 +277,81 @@
<artifactId>hadoop-hdfs</artifactId>
</dependency>

<!-- HDFS test dependencies -->

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<classifier>tests</classifier>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<classifier>tests</classifier>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<artifactId>netty</artifactId>
<groupId>io.netty</groupId>
</exclusion>
<exclusion>
<artifactId>netty-all</artifactId>
<groupId>io.netty</groupId>
</exclusion>
</exclusions>
</dependency>

<!-- Test -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-runner</artifactId>
Expand Down
Loading