Feature/create maven artifacts (#108)

* added publications to many modules

* added workflow to release packages to maven repo

* improved module artifact generation

* minor fixes

* do not publish an "all" lilb

* trasnfertype has default contenttype

* use snapshot version to enable replacing

* added better logging, and a CollectionUtils.isAnyOf() method

* extracted blobstore api into a separate common lib

* fixed custom processor dockerfile

* bumped snapshot version

Co-authored-by: beardyinc <paul.latzelsperger@beardyinc.com>

.github/workflows/publish.yaml

@@ -0,0 +1,27 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Publish package to GitHub Packages
+on:
+  release:
+    types: [created]
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v2
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+      - name: Validate Gradle wrapper
+        uses: gradle/wrapper-validation-action@e6e38bacfdf1a337459f332974bb2327a31aaf4b
+      - name: Publish package
+        run: gradle publishAllPublicationToGitHubPackagesRepository
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

build.gradle.kts

@@ -5,6 +5,7 @@
 
 plugins {
     `java-library`
+    `maven-publish`
 }
 
 repositories {
@@ -30,7 +31,10 @@ subprojects {
 }
 
 allprojects {
+    apply(plugin = "maven-publish")
     pluginManager.withPlugin("java-library") {
+        group = "com.microsoft"
+        version = "0.0.1-SNAPSHOT.1"
         dependencies {
             api("org.jetbrains:annotations:${jetBrainsAnnotationsVersion}")
             api("com.fasterxml.jackson.core:jackson-core:${jacksonVersion}")
@@ -45,6 +49,19 @@ allprojects {
 
         }
 
+        publishing {
+            repositories {
+                maven {
+                    name = "GitHubPackages"
+                    url = uri("https://maven.pkg.github.com/microsoft/data-appliance-gx")
+                    credentials {
+                        username = System.getenv("GITHUB_ACTOR")
+                        password = System.getenv("GITHUB_TOKEN")
+                    }
+                }
+            }
+        }
+
     }
 
     tasks.withType<Test> {
@@ -61,4 +78,4 @@ allprojects {
 
 val test by tasks.getting(Test::class) {
     useJUnitPlatform()
-}
+}

common/azure/build.gradle.kts

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) Microsoft Corporation.
+ * All rights reserved.
+ */
+
+plugins {
+    `java-library`
+    `java-test-fixtures`
+    `maven-publish`
+}
+
+val storageBlobVersion: String by project;
+
+dependencies {
+    api(project(":spi"))
+    api(project(":common:util"))
+    api("com.azure:azure-storage-blob:${storageBlobVersion}")
+
+
+    testFixturesImplementation("org.junit.jupiter:junit-jupiter-api:5.5.2")
+    testFixturesRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.5.2")
+}
+
+publishing {
+    publications {
+        create<MavenPublication>("common.azure") {
+            artifactId = "edc.common.azure"
+            from(components["java"])
+        }
+    }
+}

extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/provider/BlobStoreApi.java → common/azure/src/main/java/com/microsoft/dagx/common/azure/BlobStoreApi.java

@@ -4,7 +4,7 @@
  *
  */
 
-package com.microsoft.dagx.transfer.provision.azure.provider;
+package com.microsoft.dagx.common.azure;
 
 import com.azure.storage.blob.models.BlobItem;
 

extensions/transfer/transfer-provision-azure/src/main/java/com/microsoft/dagx/transfer/provision/azure/provider/BlobStoreApiImpl.java → common/azure/src/main/java/com/microsoft/dagx/common/azure/BlobStoreApiImpl.java

@@ -4,7 +4,7 @@
  *
  */
 
-package com.microsoft.dagx.transfer.provision.azure.provider;
+package com.microsoft.dagx.common.azure;
 
 import com.azure.core.util.BinaryData;
 import com.azure.storage.blob.BlobServiceClient;

common/build.gradle.kts → common/util/build.gradle.kts

@@ -6,6 +6,7 @@
 plugins {
     `java-library`
     `java-test-fixtures`
+    `maven-publish`
 }
 
 val storageBlobVersion: String by project;
@@ -21,3 +22,11 @@ dependencies {
     testFixturesRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.5.2")
 }
 
+publishing {
+    publications {
+        create<MavenPublication>("common.util") {
+            artifactId = "edc.common.util"
+            from(components["java"])
+        }
+    }
+}

common/src/main/java/com/microsoft/dagx/common/collection/CollectionUtil.java → common/util/src/main/java/com/microsoft/dagx/common/collection/CollectionUtil.java

@@ -6,6 +6,7 @@
 
 package com.microsoft.dagx.common.collection;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Map;
 
@@ -27,4 +28,11 @@ private static <V, K> boolean isEmpty(Map<K, V> map) {
         return map == null || map.isEmpty();
     }
 
+    public static <T> boolean isAnyOf(T t, T... collection) {
+        if (collection == null) {
+            return false;
+        }
+
+        return Arrays.asList(collection).contains(t);
+    }
 }

common/src/main/java/com/microsoft/dagx/common/settings/SettingsHelper.java → common/util/src/main/java/com/microsoft/dagx/common/settings/SettingsHelper.java

@@ -7,7 +7,9 @@
 package com.microsoft.dagx.common.settings;
 
 import com.microsoft.dagx.common.string.StringUtils;
+import com.microsoft.dagx.spi.DagxException;
 import com.microsoft.dagx.spi.system.ServiceExtensionContext;
+import org.jetbrains.annotations.NotNull;
 
 import java.util.UUID;
 
@@ -25,4 +27,16 @@ public static synchronized String getConnectorId(ServiceExtensionContext context
         }
         return connectorId;
     }
+
+    /**
+     * Convenience method to either get a particular setting or throw a {@link DagxException}
+     */
+    @NotNull
+    public static String getSettingOrThrow(ServiceExtensionContext serviceContext, String setting) {
+        var value = serviceContext.getSetting(setting, null);
+        if (value == null) {
+            throw new DagxException("could not find setting " + setting + " or it was null");
+        }
+        return value;
+    }
 }

common/src/test/java/com/microsoft/dagx/common/collection/CollectionUtilTest.java → common/util/src/test/java/com/microsoft/dagx/common/collection/CollectionUtilTest.java

@@ -46,4 +46,13 @@ void isNotEmpty_map() {
         assertThat(CollectionUtil.isNotEmpty(Collections.singletonMap("testkey", "testValue"))).isTrue();
         assertThat(CollectionUtil.isNotEmpty(Map.of("foo", "bar", "foo2", "baz"))).isTrue();
     }
+
+    @Test
+    void isAnyOf() {
+        int[] nullArray = null;
+        assertThat(CollectionUtil.isAnyOf(15, nullArray)).isFalse();
+        assertThat(CollectionUtil.isAnyOf(15, 15, 16, 17)).isTrue();
+        assertThat(CollectionUtil.isAnyOf("foobar", "foo", "bar", "baz", "foobar")).isTrue();
+        assertThat(CollectionUtil.isAnyOf("foobar", "bar", "baz")).isFalse();
+    }
 }

core/build.gradle.kts

@@ -7,10 +7,19 @@ val slf4jVersion: String by project
 
 plugins {
     `java-library`
+    `maven-publish`
 }
 
 dependencies {
     api(project(":spi"))
     api("org.slf4j:slf4j-api:${slf4jVersion}")
 }
 
+publishing {
+    publications {
+        create<MavenPublication>("core") {
+            artifactId = "edc.core"
+            from(components["java"])
+        }
+    }
+}

extensions/catalog/catalog-atlas/build.gradle.kts

@@ -10,7 +10,7 @@ plugins {
 
 dependencies {
     api(project(":spi"))
-    api(project(":common"))
+    api(project(":common:util"))
     api(project(":extensions:schema"))
 
     testImplementation(project(":distributions:junit"))

extensions/configuration/configuration-fs/build.gradle.kts

@@ -5,10 +5,18 @@
 
 plugins {
     `java-library`
+    `maven-publish`
 }
 
 dependencies {
     api(project(":spi"))
 }
 
-
+publishing {
+    publications {
+        create<MavenPublication>("configuration-fs") {
+            artifactId = "edc.configuration-fs"
+            from(components["java"])
+        }
+    }
+}

extensions/control-http/build.gradle.kts

@@ -7,6 +7,7 @@ val rsApi: String by project
 
 plugins {
     `java-library`
+    `maven-publish`
 }
 
 dependencies {
@@ -17,3 +18,11 @@ dependencies {
 }
 
 
+publishing {
+    publications {
+        create<MavenPublication>("control-http") {
+            artifactId = "edc.control-http"
+            from(components["java"])
+        }
+    }
+}

extensions/dataseed/dataseed-aws/build.gradle.kts

@@ -11,7 +11,7 @@ val awsVersion: String by project
 
 dependencies {
     api(project(":spi"))
-    implementation(project(":common"))
+    implementation(project(":common:util"))
 
     implementation(project(":extensions:transfer:transfer-provision-aws"))
 

extensions/dataseed/dataseed-azure/build.gradle.kts

@@ -11,7 +11,7 @@ val storageBlobVersion: String by project
 
 dependencies {
     api(project(":spi"))
-    implementation(project(":common"))
+    implementation(project(":common:util"))
 
     implementation(project(":extensions:transfer:transfer-provision-azure"))
 }

extensions/dataseed/dataseed-azure/src/main/java/com/microsoft/dagx/dataseed/azure/AzureBlobDataseedExtension.java

@@ -6,13 +6,13 @@
 
 package com.microsoft.dagx.dataseed.azure;
 
+import com.microsoft.dagx.common.azure.BlobStoreApi;
 import com.microsoft.dagx.spi.monitor.Monitor;
 import com.microsoft.dagx.spi.security.Vault;
 import com.microsoft.dagx.spi.security.VaultResponse;
 import com.microsoft.dagx.spi.system.ServiceExtension;
 import com.microsoft.dagx.spi.system.ServiceExtensionContext;
 import com.microsoft.dagx.transfer.provision.azure.AzureSasToken;
-import com.microsoft.dagx.transfer.provision.azure.provider.BlobStoreApi;
 
 import java.io.IOException;
 import java.io.InputStream;

extensions/dataseed/dataseed-nifi/build.gradle.kts

@@ -10,5 +10,5 @@ plugins {
 
 dependencies {
     api(project(":spi"))
-    implementation(project(":common"))
+    implementation(project(":common:util"))
 }

extensions/dataseed/dataseed-policy/build.gradle.kts

@@ -10,5 +10,5 @@ plugins {
 
 dependencies {
     api(project(":spi"))
-    api(project(":common"))
+    api(project(":common:util"))
 }

extensions/events/events-azure/build.gradle.kts

@@ -12,8 +12,16 @@ val eventGridSdkVersion: String by project
 dependencies {
     api(project(":spi"))
     implementation(project(":extensions:schema"))
-    implementation(project(":common"))
+    implementation(project(":common:util"))
     implementation("com.azure:azure-messaging-eventgrid:${eventGridSdkVersion}")
 }
 
 
+publishing {
+    publications {
+        create<MavenPublication>("events-azure") {
+            artifactId = "edc.events-azure"
+            from(components["java"])
+        }
+    }
+}

extensions/iam/iam-mock/build.gradle.kts

@@ -10,3 +10,11 @@ dependencies {
 }
 
 
+publishing {
+    publications {
+        create<MavenPublication>("iam-mock") {
+            artifactId = "edc.iam-mock"
+            from(components["java"])
+        }
+    }
+}

extensions/iam/oauth2/build.gradle.kts

@@ -14,4 +14,11 @@ dependencies {
     implementation("com.auth0:java-jwt:${jwtVersion}")
 }
 
-
+publishing {
+    publications {
+        create<MavenPublication>("oauth2") {
+            artifactId = "edc.iam.oauth2"
+            from(components["java"])
+        }
+    }
+}

extensions/ids/build.gradle.kts

@@ -14,6 +14,14 @@ dependencies {
     api(project(":extensions:ids:ids-core"))
     api(project(":extensions:ids:ids-api-catalog"))
     api(project(":extensions:ids:ids-api-transfer"))
+    api(project(":extensions:ids:ids-policy-mock"))
 }
 
-
+publishing {
+    publications {
+        create<MavenPublication>("ids") {
+            artifactId = "edc.ids"
+            from(components["java"])
+        }
+    }
+}