Release 2.1.0

.github/workflows/main.yml

@@ -1,12 +1,12 @@
 # Continuous integration, including test and integration test
 name: CI
 
-# Run in master and dev branches and in all pull requests to those branches
+# Run in main and dev branches and in all pull requests to those branches
 on:
   push:
-    branches: [ master, dev ]
+    branches: [ main, dev ]
   pull_request:
-    branches: [ master, dev ]
+    branches: [ main, dev ]
 
 env:
   DOCKER_IMAGE: radarbase/radar-output-restructure
@@ -22,21 +22,13 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
 
-      - uses: actions/setup-java@v1
+      - uses: actions/setup-java@v2
         with:
+          distribution: zulu
           java-version: 17
 
-      - name: Gradle cache
-        uses: actions/cache@v2
-        with:
-          # Cache gradle directories
-          path: |
-            ~/.gradle/caches
-            ~/.gradle/wrapper
-          # An explicit key for restoring and saving the cache
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle.kts', 'gradle.properties') }}
-          restore-keys: |
-            ${{ runner.os }}-gradle-
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
 
       # Compile the code
       - name: Compile code

.github/workflows/publish_snapshots.yml

@@ -21,21 +21,13 @@ jobs:
         id: is-snapshot
         run: grep 'version = ".*-SNAPSHOT"' build.gradle.kts
 
-      - uses: actions/setup-java@v1
+      - uses: actions/setup-java@v2
         with:
+          distribution: zulu
           java-version: 17
 
-      - name: Cache
-        uses: actions/cache@v2
-        with:
-          # Cache gradle directories
-          path: |
-            ~/.gradle/caches
-            ~/.gradle/wrapper
-          # Key for restoring and saving the cache
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle.kts', 'gradle.properties') }}
-          restore-keys: |
-            ${{ runner.os }}-gradle
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
 
       - name: Install gpg secret key
         run: |

.github/workflows/release.yml

@@ -17,21 +17,14 @@ jobs:
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
-      - uses: actions/setup-java@v1
+
+      - uses: actions/setup-java@v2
         with:
+          distribution: zulu
           java-version: 17
 
-      - name: Gradle cache
-        uses: actions/cache@v2
-        with:
-          # Cache gradle directories
-          path: |
-            ~/.gradle/caches
-            ~/.gradle/wrapper
-          # An explicit key for restoring and saving the cache
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle.kts', 'gradle.properties') }}
-          restore-keys: |
-            ${{ runner.os }}-gradle-
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
 
       # Compile code
       - name: Compile code

.github/workflows/scheduled_snyk.yaml

@@ -9,19 +9,21 @@ jobs:
       REPORT_FILE: test.json
     steps:
       - uses: actions/checkout@master
+      - uses: snyk/actions/setup@master
 
-      - name: Run Snyk to check for vulnerabilities
-        uses: snyk/actions/gradle-jdk11@master
-        env:
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+      - uses: actions/setup-java@v2
         with:
-          command: test
-          args: --json-file-output=${{ env.REPORT_FILE }}
+          distribution: zulu
+          java-version: 17
 
-      - name: Report new vulnerabilities
-        uses: thehyve/report-vulnerability@master
-        with:
-          report-file: ${{ env.REPORT_FILE }}
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+
+      - name: Run Snyk
         env:
-          TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        if: ${{ failure() }}
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+        run: >
+          snyk test
+          --configuration-matching='^runtimeClasspath$'
+          --json-file-output=${{ env.REPORT_FILE }}
+          --org=radar-base

.github/workflows/snyk.yaml

@@ -8,9 +8,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master
+      - uses: snyk/actions/setup@master
+
+      - uses: actions/setup-java@v2
+        with:
+          distribution: zulu
+          java-version: 17
+
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+
       - name: Run Snyk to check for vulnerabilities
-        uses: snyk/actions/gradle-jdk11@master
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          args: --severity-threshold=high
+        run: >
+          snyk test
+          --severity-threshold=high
+          --configuration-matching='^runtimeClasspath$'
+          --fail-on=upgradable
+          --org=radar-base

Dockerfile

@@ -10,7 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM gradle:7.3-jdk17 AS builder
+FROM --platform=$BUILDPLATFORM gradle:7.4-jdk17 AS builder
 
 RUN mkdir /code
 WORKDIR /code

README.md

@@ -70,7 +70,7 @@ When upgrading to version 0.6.0 from version 0.5.x or earlier, please follow the
 This package is available as docker image [`radarbase/radar-output-restructure`](https://hub.docker.com/r/radarbase/radar-output-restructure). The entrypoint of the image is the current application. So in all the commands listed in usage, replace `radar-output-restructure` with for example:
 
 ```shell
-docker run --rm -t --network s3 -v "$PWD/output:/output" radarbase/radar-output-restructure:2.0.2 -o /output /myTopic
+docker run --rm -t --network s3 -v "$PWD/output:/output" radarbase/radar-output-restructure:2.1.0 -o /output /myTopic
 ```
 
 ## Command line usage
@@ -182,7 +182,7 @@ This package requires at least Java JDK 8. Build the distribution with
 and install the package into `/usr/local` with for example
 ```shell
 sudo mkdir -p /usr/local
-sudo tar -xzf build/distributions/radar-output-restructure-2.0.2.tar.gz -C /usr/local --strip-components=1
+sudo tar -xzf build/distributions/radar-output-restructure-2.1.0.tar.gz -C /usr/local --strip-components=1
 ```
 
 Now the `radar-output-restructure` command should be available.

build.gradle.kts

@@ -15,7 +15,7 @@ plugins {
 }
 
 group = "org.radarbase"
-version = "2.0.2"
+version = "2.1.0"
 
 repositories {
     mavenCentral()
@@ -299,5 +299,5 @@ tasks.named<DependencyUpdatesTask>("dependencyUpdates").configure {
 }
 
 tasks.wrapper {
-    gradleVersion = "7.3.3"
+    gradleVersion = "7.4.1"
 }

gradle.properties

@@ -2,26 +2,26 @@ kotlin.code.style=official
 
 kotlinVersion=1.6.10
 dokkaVersion=1.6.10
-dockerComposeVersion=0.14.11
-dependencyUpdateVersion=0.40.0
+dockerComposeVersion=0.15.2
+dependencyUpdateVersion=0.42.0
 nexusPublishVersion=1.1.0
 jsoupVersion=1.14.3
 
 avroVersion=1.11.0
 snappyVersion=1.1.8.4
-jacksonVersion=2.13.1
-jCommanderVersion=1.81
+jacksonVersion=2.13.2
+jCommanderVersion=1.82
 almworksVersion=1.1.2
-minioVersion=8.3.4
-guavaVersion=31.0.1-jre
-opencsvVersion=5.5.2
+minioVersion=8.3.7
+guavaVersion=31.1-jre
+opencsvVersion=5.6
 okhttpVersion=4.9.3
-jedisVersion=4.0.1
-slf4jVersion=1.7.32
-log4jVersion=2.17.1
-azureStorageVersion=12.14.2
-nettyVersion=4.1.72.Final
+jedisVersion=4.1.1
+slf4jVersion=1.7.36
+log4jVersion=2.17.2
+azureStorageVersion=12.15.0
+nettyVersion=4.1.75.Final
 
 junitVersion=5.8.2
 mockitoKotlinVersion=4.0.0
-radarSchemasVersion=0.7.5
+radarSchemasVersion=0.7.7

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

src/main/java/org/radarbase/output/path/RecordPathFactory.kt

@@ -16,7 +16,9 @@
 
 package org.radarbase.output.path
 
+import org.apache.avro.Schema
 import org.apache.avro.generic.GenericRecord
+import org.apache.avro.generic.GenericRecordBuilder
 import org.radarbase.output.Plugin
 import org.radarbase.output.util.TimeUtil
 import org.slf4j.LoggerFactory
@@ -58,19 +60,29 @@ abstract class RecordPathFactory : Plugin {
         record: GenericRecord,
         attempt: Int,
     ): RecordOrganization {
-        val keyField = record.get("key") as? GenericRecord
+        val keyField = record.get("key")
         val valueField = record.get("value") as? GenericRecord
 
         if (keyField == null || valueField == null) {
             logger.error("Failed to process {}", record)
             throw IllegalArgumentException("Failed to process $record; no key or value")
         }
 
-        val time = TimeUtil.getDate(keyField, valueField)
+        val keyRecord: GenericRecord = if (keyField is GenericRecord) {
+            keyField
+        } else {
+            GenericRecordBuilder(observationKeySchema)
+                .set("projectId", valueField.getOrNull("projectId"))
+                .set("userId", keyField.toString())
+                .set("sourceId", valueField.getOrNull("sourceId") ?: "unknown")
+                .build()
+        }
+
+        val time = TimeUtil.getDate(keyRecord, valueField)
 
-        val relativePath = getRelativePath(topic, keyField, valueField, time, attempt)
+        val relativePath = getRelativePath(topic, keyRecord, valueField, time, attempt)
         val outputPath = root.resolve(relativePath)
-        val category = getCategory(keyField, valueField)
+        val category = getCategory(keyRecord, valueField)
         return RecordOrganization(outputPath, category, time)
     }
 
@@ -127,5 +139,27 @@ abstract class RecordPathFactory : Plugin {
                 ?.let { ILLEGAL_CHARACTER_PATTERN.matcher(it.toString()).replaceAll("") }
                 ?.takeIf { it.isNotEmpty() }
                 ?: defaultValue
+
+        private val observationKeySchema = Schema.Parser().parse("""
+            {
+              "namespace": "org.radarcns.kafka",
+              "type": "record",
+              "name": "ObservationKey",
+              "doc": "Key of an observation.",
+              "fields": [
+                {"name": "projectId", "type": ["null", "string"], "doc": "Project identifier. Null if unknown or the user is not enrolled in a project.", "default": null},
+                {"name": "userId", "type": "string", "doc": "User Identifier created during the enrolment."},
+                {"name": "sourceId", "type": "string", "doc": "Unique identifier associated with the source."}
+              ]
+            }
+        """.trimIndent())
+
+        fun GenericRecord.getFieldOrNull(fieldName: String): Schema.Field? {
+            return schema.fields
+                .find { it.name().equals(fieldName, ignoreCase = true) }
+        }
+
+        fun GenericRecord.getOrNull(fieldName: String): Any? = getFieldOrNull(fieldName)
+                ?.let { get(it.pos()) }
     }
 }