Merge 43e11b5 into 7670d32

.github/start_environment.sh

@@ -7,8 +7,7 @@ if [ -z ${GITHUB_WORKSPACE+x} ]; then
     exit 1
 fi
 
-IGLUCTL_ZIP="igluctl_0.7.2_rc1.zip"
-IGLUCTL_URI="http://dl.bintray.com/snowplow/snowplow-generic/$IGLUCTL_ZIP"
+IGLUCTL_URI="https://github.com/snowplow-incubator/igluctl/releases/download/0.7.2/igluctl"
 IGLUCENTRAL_PATH="$GITHUB_WORKSPACE/iglu-central"
 SCHEMAS_PATH="$IGLUCENTRAL_PATH/schemas/" 
 TEST_SCHEMAS="$GITHUB_WORKSPACE/.github/schemas/"
@@ -20,14 +19,14 @@ docker run \
     -p 8080:8080 \
     -v $GITHUB_WORKSPACE/.github:/iglu \
     --rm -d \
-    snowplow-docker-registry.bintray.io/snowplow/iglu-server:0.6.1 \
+    snowplow/iglu-server:0.6.1 \
     --config /iglu/server.conf
 
 echo "Waiting for Iglu Server..."
 sleep 5
 
 wget $IGLUCTL_URI
-unzip -j $IGLUCTL_ZIP
+chmod 755 igluctl
 
 ./igluctl static push \
     $SCHEMAS_PATH \

.github/workflows/test.yml

@@ -1,6 +1,13 @@
-name: Test
+name: CI
 
-on: [push, pull_request]
+on:
+  push:
+    tags:
+      - '*'
+    branches:
+      - master
+      - develop
+  pull_request:
 
 jobs:
   test:
@@ -21,10 +28,12 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
-      uses: actions/setup-java@v1
+    - uses: coursier/cache-action@v6
+    - name: Set up JDK 11
+      uses: actions/setup-java@v2
       with:
-        java-version: 1.8
+        java-version: 11
+        distribution: adopt
     - name: Prepare test environment
       run: $GITHUB_WORKSPACE/.github/start_environment.sh
     - name: Run tests
@@ -44,36 +53,79 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
-      uses: actions/setup-java@v1
+    - uses: coursier/cache-action@v6
+    - name: Set up JDK 11
+      uses: actions/setup-java@v2
       with:
-        java-version: 1.8
-    - name: Compare SBT version with git tag
-      run: .github/check_tag.sh ${GITHUB_REF##*/}
+        java-version: 11
+        distribution: adopt
+    - name: Get current version
+      id: ver
+      run: echo "::set-output name=tag::${GITHUB_REF#refs/tags/}"
     - name: Docker login
       run: docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
       env:
         DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
         DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-    - name: Build and publish Docker image
-      run: sbt "project loader" docker:publish
+    - name: Stage the Docker build
+      run: sbt "project loader" docker:stage
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+    - name: Build image
+      uses: docker/build-push-action@v2
+      with:
+        context: modules/loader/target/docker/stage
+        file: modules/loader/target/docker/stage/Dockerfile
+        platforms: linux/amd64,linux/arm64/v8
+        tags: snowplow/snowplow-postgres-loader:${{ steps.ver.outputs.tag }}
+        push: true
 
   deploy_common:
     needs: test
     if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
-      uses: actions/setup-java@v1
+    - name: Set up JDK 11
+      uses: actions/setup-java@v2
+      with:
+        java-version: 11
+        distribution: adopt
+    - name: Deploy common on Bintray Maven and Maven Central
+      run: sbt "project common" ci-release
+      env:
+        PGP_PASSPHRASE: ${{ secrets.SONA_PGP_PASSPHRASE }}
+        PGP_SECRET: ${{ secrets.SONA_PGP_SECRET }}
+        SONATYPE_USERNAME: ${{ secrets.SONA_USER }}
+        SONATYPE_PASSWORD: ${{ secrets.SONA_PASS }}
+
+  github_release:
+    needs: test
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: coursier/cache-action@v6
+    - name: Set up JDK 11
+      uses: actions/setup-java@v2
+      with:
+        java-version: 11
+        distribution: adopt
+    - name: Get current version
+      id: ver
+      run: echo "::set-output name=tag::${GITHUB_REF#refs/tags/}"
+    - name: Assemble fat jars
+      run: sbt "project loader; set assembly / test := {}; assembly"
+    - name: Create GitHub release and attach artifacts
+      uses: softprops/action-gh-release@v1
       with:
-        java-version: 1.8
-    - name: Compare SBT version with git tag
-      run: .github/check_tag.sh ${GITHUB_REF##*/}
-    - name: Deploy SCE on Bintray Maven and Maven Central
-      run: .github/deploy_common.sh
+        draft: true
+        prerelease: ${{ contains(steps.ver.outputs.tag, 'rc') }}
+        name: ${{ steps.ver.outputs.tag }}
+        tag_name: ${{ steps.ver.outputs.tag }}
+        files: |
+          modules/loader/target/scala-2.13/snowplow-postgres-loader-${{ steps.ver.outputs.tag }}.jar
       env:
-        SONA_USER: snowplow
-        SONA_PASS: ${{ secrets.SONA_PASS }}
-        BINTRAY_SNOWPLOW_MAVEN_USER: ${{ secrets.BINTRAY_SNOWPLOW_MAVEN_USER }}
-        BINTRAY_SNOWPLOW_MAVEN_API_KEY: ${{ secrets.BINTRAY_SNOWPLOW_MAVEN_API_KEY }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

build.sbt

@@ -22,16 +22,15 @@ lazy val common = project
   .settings(BuildSettings.projectSettings)
   .settings(BuildSettings.scoverageSettings)
   .settings(BuildSettings.mavenSettings)
+  .settings(BuildSettings.dynVerSettings)
   .settings(
-    resolvers += Dependencies.SnowplowBintray,
     libraryDependencies ++= Seq(
       Dependencies.postgres,
       Dependencies.catsEffect,
       Dependencies.circe,
       Dependencies.circeGeneric,
       Dependencies.circeExtras,
       Dependencies.circeParser,
-      Dependencies.circeLiteral,
       Dependencies.doobie,
       Dependencies.doobiePg,
       Dependencies.doobiePgCirce,
@@ -41,6 +40,7 @@ lazy val common = project
       Dependencies.analyticsSdk,
       Dependencies.badRows,
       Dependencies.schemaDdl,
+      Dependencies.circeLiteral % Test,
       Dependencies.specs2,
       Dependencies.specs2Check,
       Dependencies.scalaCheck
@@ -54,12 +54,16 @@ lazy val loader = project
   .settings(BuildSettings.dockerSettings)
   .settings(BuildSettings.buildInfoSettings)
   .settings(BuildSettings.addExampleConfToTestCp)
+  .settings(BuildSettings.assemblySettings)
+  .settings(BuildSettings.dynVerSettings)
   .settings(
     libraryDependencies ++= Seq(
+      Dependencies.circeConfig,
       Dependencies.commons,
       Dependencies.fs2Aws,
       Dependencies.fs2PubSub,
       Dependencies.decline,
+      Dependencies.config,
       Dependencies.specs2
     )
   )

config/config.hocon.sample

@@ -0,0 +1,62 @@
+{
+  # Human-readable storage target name, used only for logging
+  "name": "Acme Ltd. Snowplow Postgres"
+  # Machine-readable unique identifier
+  "id": "5c5e4353-4eeb-43da-98f8-2de6dc7fa947"
+
+  "input": {
+    # Enable the Kinesis source
+    "type": "Kinesis"
+    # Name of the application which the KCL daemon should assume
+    "appName": "acme-postgres-loader"
+    # Name of the Kinesis stream to read from
+    "streamName": "enriched-events"
+    # AWS region in which the Kinesis stream resides.
+    "region": "eu-central-1"
+    # Either TRIM_HORIZON or LATEST
+    "initialPosition": "TRIM_HORIZON"
+
+    # Optional, set the  polling mode for retrieving records. Default is FanOut
+    # "retrievalMode": "FanOut"
+    # "retrievalMode": {
+    #   "type": "Polling"
+    #   "maxRecords": 1000
+    # }
+  }
+
+  # Alternative input block for PubSub source
+  # "input": {
+  #   "type": "PubSub"
+  #   "projectId": "my-project"
+  #   "subscriptionId": "my-subscription"
+  # }
+
+  "output" : {
+    "type": "Postgres"
+    # PostgreSQL host ('localhost' for enabled SSH Tunnel)
+    "host": "localhost"
+    # PostgreSQL database port
+    "port": 5432
+    # PostgreSQL database name
+    "database": "snowplow"
+    # PostgreSQL user to load data
+    "username": "postgres"
+    # PostgreSQL password, either plain text or from an environment variable
+    "password": "mysecretpassword"
+    "password": ${?POSTGRES_PASSWORD}
+    # PostgreSQL database schema
+    "schema": "atomic"
+    # JDBC ssl mode
+    "sslMode": "REQUIRE"
+  }
+
+  # Kind of data stored in this instance. Either ENRICHED_EVENTS or JSON
+  "purpose": "ENRICHED_EVENTS"
+
+  "monitoring": {
+    "metrics": {
+      # Optional, cloudwatch metrics are enabled by default.
+      # "cloudWatch": false
+    }
+  }
+}