Migrate github actions to use container images (#7603)

* Add server bind address to proxy webserver * Force executor shutdown in mocks * Add config for surefire reuseForks and forkCount * Fix ProxyTest#testRegexSubscription * Migrate github actions to use container images * Unit test and go function tests are not consistent in runtime behavior Co-authored-by: Ali Ahmed <alia@splunk.com>
apache · Jul 23, 2020 · 67bd345 · 67bd345
1 parent 9366a32
commit 67bd345
Show file tree

Hide file tree

Showing 28 changed files with 308 additions and 1,065 deletions.
diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml
@@ -31,56 +31,25 @@ jobs:
   cpp-tests:
     name:
     runs-on: ubuntu-latest
+    container:
+      image: apachepulsar/pulsar-build:latest
     timeout-minutes: 120
 
     steps:
-
       - name: checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Cache local Maven repository
-        uses: actions/cache@v2
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
-
-      - name: Check if this pull request only changes documentation
-        id:   docs
-        uses: apache/pulsar-test-infra/diff-only@master
-        with:
-          args: site2 .github deployment .asf.yaml .ci ct.yaml
-
-      - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
-        if: steps.docs.outputs.changed_only == 'no'
-        with:
-          java-version: 1.8
-
-      - name: clean disk
-        if: steps.docs.outputs.changed_only == 'no'
-        run: |
-          sudo swapoff -a
-          sudo rm -f /swapfile
-          sudo apt clean
-          docker rmi $(docker images -q) -f
-          df -h
-
       - name: build package
-        if: steps.docs.outputs.changed_only == 'no'
-        run: mvn -B -ntp -Dorg.slf4j.simpleLogger.defaultLogLevel=ERROR -pl distribution/server package -am package -DskipTests
+        run: mvn -B -ntp -q package -pl distribution/server -am -DskipTests
 
       - name: build cpp artifacts
-        if: steps.docs.outputs.changed_only == 'no'
         run: |
           echo "Build C++ client library"
           export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug -DBUILD_DYNAMIC_LIB=OFF"
           pulsar-client-cpp/docker-build.sh
 
       - name: run c++ tests
-        if: steps.docs.outputs.changed_only == 'no'
         run: pulsar-client-cpp/docker-tests.sh
diff --git a/.github/workflows/ci-go-functions-test.yaml b/.github/workflows/ci-go-functions-test.yaml
@@ -35,11 +35,12 @@ on:
 jobs:
 
   go-functions-tests:
-    name:
-    runs-on: ubuntu-latest
+    name: Run Go Tests
     strategy:
       matrix:
-        go-version: [1.11, 1.12, 1.13, 1.14]
+        go-version: [1.11.x, 1.12.x, 1.13.x, 1.14.x]
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
     timeout-minutes: 120
 
     steps:
@@ -66,4 +67,4 @@ jobs:
         if: steps.docs.outputs.changed_only == 'no'
         run: |
           cd pulsar-function-go
-          go test -v $(go list ./... | grep -v examples) ${{ matrix.go-version }}
+          go test -v $(go list ./... | grep -v examples)
diff --git a/.github/workflows/ci-license.yaml b/.github/workflows/ci-license.yaml
@@ -61,6 +61,6 @@ jobs:
 
       - name: build and check license
         run: mvn -B -ntp -DskipTests license:check install
-          
+
       - name: license check
         run: src/check-binary-license ./distribution/server/target/apache-pulsar-*-bin.tar.gz
diff --git a/.github/workflows/ci-pulsarbot.yaml b/.github/workflows/ci-pulsarbot.yaml
@@ -35,6 +35,7 @@ jobs:
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}
+
       - name: Execute pulsarbot command
         id:   pulsarbot
         env:

diff --git a/.github/workflows/ci-unit-broker-sasl.yml → .../workflows/ci-unit-broker-broker-gp1.yaml b/.github/workflows/ci-unit-broker-sasl.yml → .../workflows/ci-unit-broker-broker-gp1.yaml
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: CI - Unit - Broker Auth SASL
+name: CI - Unit - Brokers - Broker Group 1
 on:
   pull_request:
     branches:
@@ -31,6 +31,8 @@ jobs:
   unit-tests:
     name:
     runs-on: ubuntu-latest
+    container:
+      image: apachepulsar/pulsar-build:latest
     timeout-minutes: 120
 
     steps:
@@ -40,39 +42,14 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Cache local Maven repository
-        uses: actions/cache@v2
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
-
-      - name: Check if this pull request only changes documentation
-        id:   docs
-        uses: apache/pulsar-test-infra/diff-only@master
-        with:
-          args: site2 .github deployment .asf.yaml .ci ct.yaml
-
-      - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
-        if: steps.docs.outputs.changed_only == 'no'
-        with:
-          java-version: 1.8
-
-      - name: Set up Maven
-        uses: apache/pulsar-test-infra/setup-maven@master
-        if: steps.docs.outputs.changed_only == 'no'
-        with:
-          maven-version: 3.6.1
+      - name: build modules
+        run: mvn -B -q -ntp install -pl pulsar-broker -am -DskipTests
 
-      - name: run unit tests install by skip tests
-        if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests -pl pulsar-broker-auth-sasl -am
+      - name: run unit tests for pulsar-broker
+        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/broker/**/*.java' -Dexclude='org/apache/pulsar/broker/zookeeper/**/*.java,org/apache/pulsar/broker/loadbalance/**/*.java,org/apache/pulsar/broker/service/**/*.java,**/AdminApiOffloadTest.java'
 
-      - name: run unit tests pulsar auth sasl
-        if: steps.docs.outputs.changed_only == 'no'
-        run: mvn test -DfailIfNoTests=false -pl pulsar-broker-auth-sasl
+      - name: run flaky test
+        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/AdminApiOffloadTest.java' -DtestForkCount=1 -DtestReuseFork=true
 
       - name: package surefire artifacts
         if: failure()

diff --git a/.github/workflows/ci-unit-adaptors.yml → .../workflows/ci-unit-broker-broker-gp2.yaml b/.github/workflows/ci-unit-adaptors.yml → .../workflows/ci-unit-broker-broker-gp2.yaml
@@ -17,7 +17,7 @@
 # under the License.
 #
 
-name: CI - Unit - Adaptors
+name: CI - Unit - Brokers - Broker Group 2
 on:
   pull_request:
     branches:
@@ -31,6 +31,8 @@ jobs:
   unit-tests:
     name:
     runs-on: ubuntu-latest
+    container:
+      image: apachepulsar/pulsar-build:latest
     timeout-minutes: 120
 
     steps:
@@ -40,43 +42,24 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Cache local Maven repository
-        uses: actions/cache@v2
-        with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            ${{ runner.os }}-maven-
+      - name: build modules
+        run: mvn -B -q -ntp install -pl pulsar-broker -am -DskipTests
 
-      - name: Check if this pull request only changes documentation
-        id:   docs
-        uses: apache/pulsar-test-infra/diff-only@master
-        with:
-          args: site2 .github deployment .asf.yaml .ci ct.yaml
+      - name: run flaky test
+        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/MessagePublishBufferThrottleTest.java' -DtestForkCount=1 -DtestReuseFork=true
 
-      - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
-        if: steps.docs.outputs.changed_only == 'no'
-        with:
-          java-version: 1.8
+      - name: run flaky test
+        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/ReplicatorTest.java' -DtestForkCount=1 -DtestReuseFork=true
 
-      - name: Set up Maven
-        uses: apache/pulsar-test-infra/setup-maven@master
-        if: steps.docs.outputs.changed_only == 'no'
-        with:
-          maven-version: 3.6.1
+      - name: run flaky test
+        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/TopicOwnerTest.java' -DtestForkCount=1 -DtestReuseFork=true
 
-      - name: run unit tests install by skip tests
-        if: steps.docs.outputs.changed_only == 'no'
-        run: mvn clean install -DskipTests
+      - name: run flaky test
+        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='**/AntiAffinityNamespaceGroupTest.java' -DtestForkCount=1 -DtestReuseFork=true
 
-      - name: run unit tests pulsar io kafka connect adaptor
-        if: steps.docs.outputs.changed_only == 'no'
-        run: mvn test -pl pulsar-io/kafka-connect-adaptor
+      - name: run unit tests for pulsar-broker "org/apache/pulsar/broker/"
+        run: mvn -B -ntp test -pl pulsar-broker -Dinclude='org/apache/pulsar/broker/zookeeper/**/*.java,org/apache/pulsar/broker/loadbalance/**/*.java,org/apache/pulsar/broker/service/**/*.java' -Dexclude='**/ReplicatorTest.java,**/MessagePublishBufferThrottleTest.java,**/TopicOwnerTest.java,**/AntiAffinityNamespaceGroupTest.java'
 
-      - name: run unit tests pulsar storm tests
-        if: steps.docs.outputs.changed_only == 'no'
-        run: mvn test -pl tests/pulsar-storm-test
 
       - name: package surefire artifacts
         if: failure()

diff --git a/.github/workflows/ci-unit-broker-broker.yml b/.github/workflows/ci-unit-broker-broker.yml