From 5da47eba79a55ce2b486cda0c0c62ae7b380d9a4 Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Fri, 18 Feb 2022 02:50:53 -0800 Subject: [PATCH 01/14] Matrix build for Spark and Scala versions --- .github/workflows/java-continuous-integration.yml | 5 +++++ .github/workflows/python-continuous-integration.yml | 3 +-- java/spark-bundle/build.gradle.kts | 4 ++-- java/spark/build.gradle.kts | 4 ++-- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/java-continuous-integration.yml b/.github/workflows/java-continuous-integration.yml index f99ed017fb..fba5f52d0f 100644 --- a/.github/workflows/java-continuous-integration.yml +++ b/.github/workflows/java-continuous-integration.yml @@ -10,6 +10,8 @@ jobs: strategy: matrix: os: [ubuntu-latest] + scala-version: [2.11, 2.12] + spark-version: [2.4.7] runs-on: ${{ matrix.os }} defaults: run: @@ -37,6 +39,9 @@ jobs: with: build-root-directory: java arguments: build + env: + SCALA_VERSION : ${{ matrix.scala-version }} + SPARK_VERSION : ${{ matrix.spark-version }} - name: Smoke test jar in client environment uses: gradle/gradle-build-action@v2 with: diff --git a/.github/workflows/python-continuous-integration.yml b/.github/workflows/python-continuous-integration.yml index ac038fde2c..589fa4b108 100644 --- a/.github/workflows/python-continuous-integration.yml +++ b/.github/workflows/python-continuous-integration.yml @@ -15,8 +15,7 @@ jobs: max-parallel: 6 matrix: os: [ubuntu-latest, macOS-latest] - python-version: [ 3.6, 3.7, 3.8, 3.9] - + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/java/spark-bundle/build.gradle.kts b/java/spark-bundle/build.gradle.kts index ec07a32e10..07cde8da23 100644 --- a/java/spark-bundle/build.gradle.kts +++ b/java/spark-bundle/build.gradle.kts @@ -14,8 +14,8 @@ plugins { id("com.github.johnrengelman.shadow") version ("5.2.0") } -val scalaVersion = project.properties.getOrDefault("scalaVersion", "2.12") -val sparkVersion = project.properties.getOrDefault("sparkVersion", "3.1.1") as String +val scalaVersion = System.getenv("SCALA_VERSION") +val sparkVersion = System.getenv("SPARK_VERSION") val artifactBaseName = "whylogs-spark-bundle_${sparkVersion}-scala_$scalaVersion" val versionString = rootProject.version diff --git a/java/spark/build.gradle.kts b/java/spark/build.gradle.kts index 84e1eeedf7..627ab64016 100644 --- a/java/spark/build.gradle.kts +++ b/java/spark/build.gradle.kts @@ -19,8 +19,8 @@ spotless { } } -val scalaVersion = project.properties.getOrDefault("scalaVersion", "2.12") -val sparkVersion = project.properties.getOrDefault("sparkVersion", "3.1.1") as String +val scalaVersion = System.getenv("SCALA_VERSION") +val sparkVersion = System.getenv("SPARK_VERSION") val artifactBaseName = "${rootProject.name}-spark_$sparkVersion-scala_$scalaVersion" tasks.jar { From 1dd24fb58509458478c7ee453f9a23203f96dbd9 Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Mon, 21 Feb 2022 04:07:19 -0800 Subject: [PATCH 02/14] add spark 3.1.1 --- .github/workflows/java-continuous-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/java-continuous-integration.yml b/.github/workflows/java-continuous-integration.yml index fba5f52d0f..19b8e214ee 100644 --- a/.github/workflows/java-continuous-integration.yml +++ b/.github/workflows/java-continuous-integration.yml @@ -11,7 +11,7 @@ jobs: matrix: os: [ubuntu-latest] scala-version: [2.11, 2.12] - spark-version: [2.4.7] + spark-version: [2.4.7, 3.1.1] runs-on: ${{ matrix.os }} defaults: run: From 6f8669664d7a8cc3152bfa02fb7fbd878b9a0125 Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Mon, 21 Feb 2022 04:53:12 -0800 Subject: [PATCH 03/14] exclude scala 2.11 & spark 3.1.1 --- .github/workflows/java-continuous-integration.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/java-continuous-integration.yml b/.github/workflows/java-continuous-integration.yml index 19b8e214ee..bff5709989 100644 --- a/.github/workflows/java-continuous-integration.yml +++ b/.github/workflows/java-continuous-integration.yml @@ -12,6 +12,9 @@ jobs: os: [ubuntu-latest] scala-version: [2.11, 2.12] spark-version: [2.4.7, 3.1.1] + exclude: + - scala-version: 2.11 + spark-version: 3.1.1 runs-on: ${{ matrix.os }} defaults: run: From 442facfd2234b4207960ae7d5dc1d34415d3238e Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Mon, 21 Feb 2022 05:23:43 -0800 Subject: [PATCH 04/14] publish matrix builds --- .github/workflows/push_release.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push_release.yml b/.github/workflows/push_release.yml index ba85dc1bf5..a1f7e0731b 100644 --- a/.github/workflows/push_release.yml +++ b/.github/workflows/push_release.yml @@ -3,13 +3,18 @@ name: Upload Whylogs Packages on: release: - types: [ released, prereleased] + types: [released, prereleased] jobs: deploy: - + strategy: + matrix: + scala-version: [2.11, 2.12] + spark-version: [2.4.7, 3.1.1] + exclude: + - scala-version: 2.11 + spark-version: 3.1.1 runs-on: ubuntu-latest - steps: - uses: actions/checkout@v2 - name: Set up Python @@ -69,6 +74,8 @@ jobs: env: ORG_GRADLE_PROJECT_signingKey: ${{ secrets.SIGNING_KEY_ASCII }} ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.SIGNING_KEY_PASSWORD }} + SCALA_VERSION : ${{ matrix.scala-version }} + SPARK_VERSION : ${{ matrix.spark-version }} run: ./gradlew publish -PossrhUsername=${{ secrets.NEXUS_TOKEN_USERNAME }} -PossrhPassword=${{ secrets.NEXUS_TOKEN_PASSWORD }} working-directory: java From 95d228bc74777fbf1e59d24725da83ed64e0a41d Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Mon, 21 Feb 2022 10:04:36 -0800 Subject: [PATCH 05/14] include Spark 3.2 & Scala 2.13 --- .github/workflows/java-continuous-integration.yml | 3 +++ .github/workflows/push_release.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/java-continuous-integration.yml b/.github/workflows/java-continuous-integration.yml index bff5709989..f1f3a98073 100644 --- a/.github/workflows/java-continuous-integration.yml +++ b/.github/workflows/java-continuous-integration.yml @@ -15,6 +15,9 @@ jobs: exclude: - scala-version: 2.11 spark-version: 3.1.1 + include: + - scala-verison: 2.13 + spark-version: 3.2.0 runs-on: ${{ matrix.os }} defaults: run: diff --git a/.github/workflows/push_release.yml b/.github/workflows/push_release.yml index a1f7e0731b..5f5aa9743e 100644 --- a/.github/workflows/push_release.yml +++ b/.github/workflows/push_release.yml @@ -14,6 +14,9 @@ jobs: exclude: - scala-version: 2.11 spark-version: 3.1.1 + include: + - scala-verison: 2.13 + spark-version: 3.2.0 runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From 412eb263aaa05a63c8fa2d5b97ad6cb158b8caba Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Mon, 21 Feb 2022 11:44:06 -0800 Subject: [PATCH 06/14] need os in included matrix entry --- .github/workflows/java-continuous-integration.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/java-continuous-integration.yml b/.github/workflows/java-continuous-integration.yml index f1f3a98073..8803a4b3e6 100644 --- a/.github/workflows/java-continuous-integration.yml +++ b/.github/workflows/java-continuous-integration.yml @@ -16,7 +16,8 @@ jobs: - scala-version: 2.11 spark-version: 3.1.1 include: - - scala-verison: 2.13 + - os: ubuntu-latest + scala-verison: 2.13 spark-version: 3.2.0 runs-on: ${{ matrix.os }} defaults: From 6c5ee0ad1d6f887685a8c3605cd1a5414d6645f6 Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Mon, 21 Feb 2022 18:45:50 -0800 Subject: [PATCH 07/14] experiment --- .github/workflows/java-continuous-integration.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/java-continuous-integration.yml b/.github/workflows/java-continuous-integration.yml index 8803a4b3e6..b06621f5ae 100644 --- a/.github/workflows/java-continuous-integration.yml +++ b/.github/workflows/java-continuous-integration.yml @@ -11,14 +11,15 @@ jobs: matrix: os: [ubuntu-latest] scala-version: [2.11, 2.12] - spark-version: [2.4.7, 3.1.1] - exclude: - - scala-version: 2.11 - spark-version: 3.1.1 + spark-version: [2.4.7, 2.4.8] include: - os: ubuntu-latest - scala-verison: 2.13 - spark-version: 3.2.0 + scala-verison: 2.12 + spark-version: [3.0.3, 3.1.1, 3.1.3] + include: + - os: ubuntu-latest + scala-verison: [2.12, 2.13] + spark-version: [3.2.0, 3.2.1] runs-on: ${{ matrix.os }} defaults: run: From d8d08d922da22ce5b83e1008388feeeebfc2fe9a Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Mon, 21 Feb 2022 18:47:16 -0800 Subject: [PATCH 08/14] experiment --- .github/workflows/java-continuous-integration.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/java-continuous-integration.yml b/.github/workflows/java-continuous-integration.yml index b06621f5ae..ba1bcb8578 100644 --- a/.github/workflows/java-continuous-integration.yml +++ b/.github/workflows/java-continuous-integration.yml @@ -16,7 +16,6 @@ jobs: - os: ubuntu-latest scala-verison: 2.12 spark-version: [3.0.3, 3.1.1, 3.1.3] - include: - os: ubuntu-latest scala-verison: [2.12, 2.13] spark-version: [3.2.0, 3.2.1] From 9a7d28477b2f38975e1d5420b5ca3d1533d448df Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Mon, 21 Feb 2022 18:53:27 -0800 Subject: [PATCH 09/14] experiment failed :( --- .../workflows/java-continuous-integration.yml | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/java-continuous-integration.yml b/.github/workflows/java-continuous-integration.yml index ba1bcb8578..ca9f78592e 100644 --- a/.github/workflows/java-continuous-integration.yml +++ b/.github/workflows/java-continuous-integration.yml @@ -15,10 +15,25 @@ jobs: include: - os: ubuntu-latest scala-verison: 2.12 - spark-version: [3.0.3, 3.1.1, 3.1.3] + spark-version: 3.0.3 - os: ubuntu-latest - scala-verison: [2.12, 2.13] - spark-version: [3.2.0, 3.2.1] + scala-verison: 2.12 + spark-version: 3.1.1 + - os: ubuntu-latest + scala-verison: 2.12 + spark-version: 3.1.3 + - os: ubuntu-latest + scala-verison: 2.12 + spark-version: 3.2.0 + - os: ubuntu-latest + scala-verison: 2.12 + spark-version: 3.2.1 + - os: ubuntu-latest + scala-verison: 2.13 + spark-version: 3.2.0 + - os: ubuntu-latest + scala-verison: 2.13 + spark-version: 3.2.1 runs-on: ${{ matrix.os }} defaults: run: From 72f3d3b59ff795ff113be34f3e61203cbdaab756 Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Mon, 21 Feb 2022 20:24:16 -0800 Subject: [PATCH 10/14] syntax fix --- java/spark/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/spark/build.gradle.kts b/java/spark/build.gradle.kts index 627ab64016..573dd85508 100644 --- a/java/spark/build.gradle.kts +++ b/java/spark/build.gradle.kts @@ -32,7 +32,7 @@ tasks.withType { } fun scalaPackage(groupId: String, name: String, version: String) = - "$groupId:${name}_$scalaVersion:$version" + "$groupId:${name}_${scalaVersion}:${version}" sourceSets { main { From f70954f2f17d4582b3103d2c1d7f7111f073d9f7 Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Mon, 21 Feb 2022 20:36:43 -0800 Subject: [PATCH 11/14] typo --- .github/workflows/java-continuous-integration.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/java-continuous-integration.yml b/.github/workflows/java-continuous-integration.yml index ca9f78592e..4a44f37489 100644 --- a/.github/workflows/java-continuous-integration.yml +++ b/.github/workflows/java-continuous-integration.yml @@ -14,25 +14,25 @@ jobs: spark-version: [2.4.7, 2.4.8] include: - os: ubuntu-latest - scala-verison: 2.12 + scala-version: 2.12 spark-version: 3.0.3 - os: ubuntu-latest - scala-verison: 2.12 + scala-version: 2.12 spark-version: 3.1.1 - os: ubuntu-latest - scala-verison: 2.12 + scala-version: 2.12 spark-version: 3.1.3 - os: ubuntu-latest - scala-verison: 2.12 + scala-version: 2.12 spark-version: 3.2.0 - os: ubuntu-latest - scala-verison: 2.12 + scala-version: 2.12 spark-version: 3.2.1 - os: ubuntu-latest - scala-verison: 2.13 + scala-version: 2.13 spark-version: 3.2.0 - os: ubuntu-latest - scala-verison: 2.13 + scala-version: 2.13 spark-version: 3.2.1 runs-on: ${{ matrix.os }} defaults: From d070065ea9c9fda131d2780924963a25e1450341 Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Mon, 21 Feb 2022 22:03:56 -0800 Subject: [PATCH 12/14] Scala 2.13 will need code changes --- .github/workflows/java-continuous-integration.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/java-continuous-integration.yml b/.github/workflows/java-continuous-integration.yml index 4a44f37489..73d20bb96e 100644 --- a/.github/workflows/java-continuous-integration.yml +++ b/.github/workflows/java-continuous-integration.yml @@ -28,12 +28,13 @@ jobs: - os: ubuntu-latest scala-version: 2.12 spark-version: 3.2.1 - - os: ubuntu-latest - scala-version: 2.13 - spark-version: 3.2.0 - - os: ubuntu-latest - scala-version: 2.13 - spark-version: 3.2.1 + // s/JavaConversions/CollectionConverts/ deprecation in Scala 2.13 + //- os: ubuntu-latest + // scala-version: 2.13 + // spark-version: 3.2.0 + //- os: ubuntu-latest + // scala-version: 2.13 + // spark-version: 3.2.1 runs-on: ${{ matrix.os }} defaults: run: From dfe341bc2b8a403144833a3f38897d9848ea961a Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Mon, 21 Feb 2022 22:13:18 -0800 Subject: [PATCH 13/14] skip problematic Spark/Scala combinations --- .../workflows/java-continuous-integration.yml | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/java-continuous-integration.yml b/.github/workflows/java-continuous-integration.yml index 73d20bb96e..7a16886ccb 100644 --- a/.github/workflows/java-continuous-integration.yml +++ b/.github/workflows/java-continuous-integration.yml @@ -22,19 +22,21 @@ jobs: - os: ubuntu-latest scala-version: 2.12 spark-version: 3.1.3 - - os: ubuntu-latest - scala-version: 2.12 - spark-version: 3.2.0 - - os: ubuntu-latest - scala-version: 2.12 - spark-version: 3.2.1 - // s/JavaConversions/CollectionConverts/ deprecation in Scala 2.13 - //- os: ubuntu-latest - // scala-version: 2.13 - // spark-version: 3.2.0 - //- os: ubuntu-latest - // scala-version: 2.13 - // spark-version: 3.2.1 + # build failures + #- os: ubuntu-latest + # scala-version: 2.12 + # spark-version: 3.2.0 + #- os: ubuntu-latest + # scala-version: 2.12 + # spark-version: 3.2.1 + + # s/JavaConversions/CollectionConverts/ deprecation in Scala 2.13 + #- os: ubuntu-latest + # scala-version: 2.13 + # spark-version: 3.2.0 + #- os: ubuntu-latest + # scala-version: 2.13 + # spark-version: 3.2.1 runs-on: ${{ matrix.os }} defaults: run: From 166f9eb7022cb4eee6e39c68eea5166a4a706e0c Mon Sep 17 00:00:00 2001 From: Richard Rogers Date: Tue, 22 Feb 2022 02:49:13 -0800 Subject: [PATCH 14/14] match publish matrix to build matrix --- .github/workflows/push_release.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push_release.yml b/.github/workflows/push_release.yml index 5f5aa9743e..176bd6a763 100644 --- a/.github/workflows/push_release.yml +++ b/.github/workflows/push_release.yml @@ -10,13 +10,14 @@ jobs: strategy: matrix: scala-version: [2.11, 2.12] - spark-version: [2.4.7, 3.1.1] - exclude: - - scala-version: 2.11 - spark-version: 3.1.1 + spark-version: [2.4.7, 2.4.8] include: - - scala-verison: 2.13 - spark-version: 3.2.0 + - scala-version: 2.12 + spark-version: 3.0.3 + - scala-version: 2.12 + spark-version: 3.1.1 + - scala-version: 2.12 + spark-version: 3.1.3 runs-on: ubuntu-latest steps: - uses: actions/checkout@v2