Merge ef075a8 into 5d98780

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,23 @@
+<!--
+
+** Please read the guidelines below. **
+
+GitHub issues are reserved for bug reports.
+The best place to ask general questions or ask for community support is
+[on our Discourse](https://discourse.snowplowanalytics.com/).
+
+-->
+
+**Version**:
+
+**Expected behavior**:
+
+**Actual behavior**:
+
+**Steps to reproduce**:
+
+<!-- Please try to be as detailed as possible so that we can reproduce and fix the issue
+as quickly as possible. -->
+ 1.
+ 2.
+

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,12 @@
+<!--
+Thank you for contributing to Snowplow Postgres Loader!
+
+You'll find a small checklist below which should help speed up the review process:
+
+- [ ] Have you signed the [contributor license agreement](https://github.com/snowplow/snowplow/wiki/CLA)?
+- [ ] Have you read the [contributing guide](https://github.com/snowplow-incubator/snowplow-postgres-loader/blob/master/CONTRIBUTING.md)?
+- [ ] Have you added the appropriate unit tests?
+- [ ] Have you run the tests through `sbt test`?
+- [ ] Is your pull request against the `master` branch?
+-->
+

.github/check_tag.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+tag=$1
+
+project_version=$(sbt version -Dsbt.log.noformat=true | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n')
+
+if [[ "${tag}" = "${project_version}" ]]; then
+  echo "Tag version (${tag}) matches project version (${project_version}). Deploying!"
+else
+  echo "Tag version (${tag}) doesn't match version in scala project (${project_version}). Aborting!"
+  exit 1
+fi

.github/schemas/me.chuwy/pg-test/jsonschema/1-0-0

@@ -0,0 +1,22 @@
+{
+  "self": {
+    "vendor": "me.chuwy",
+    "name": "pg-test",
+    "format": "jsonschema",
+    "version": "1-0-0"
+  },
+  "properties": {
+    "requiredString": { "type": "string" },
+    "requiredUnion": { "type": ["string", "boolean"] },
+    "nested": {
+      "properties": {
+        "a": { "type": "number" },
+        "b": {}
+      },
+      "required": ["a"]
+    },
+    "someArray": { "type": "array" },
+    "id": { "type": "string", "format": "uuid" }
+  },
+  "required": ["requiredString", "requiredUnion"]
+}

.github/schemas/me.chuwy/pg-test/jsonschema/1-0-1

@@ -0,0 +1,24 @@
+{
+  "self": {
+    "vendor": "me.chuwy",
+    "name": "pg-test",
+    "format": "jsonschema",
+    "version": "1-0-1"
+  },
+  "properties": {
+    "requiredString": { "type": "string" },
+    "requiredUnion": { "type": ["string", "boolean"] },
+    "nested": {
+      "properties": {
+        "a": { "type": "number" },
+        "b": {},
+        "c": { "type": ["integer", "null"] }
+      },
+      "required": ["a"]
+    },
+    "someArray": { "type": "array" },
+    "id": { "type": "string", "format": "uuid" },
+    "someDate": { "type": "string", "format": "date-time" }
+  },
+  "required": ["requiredString", "requiredUnion"]
+}

.github/schemas/me.chuwy/pg-test/jsonschema/1-0-2

@@ -0,0 +1,25 @@
+{
+  "self": {
+    "vendor": "me.chuwy",
+    "name": "pg-test",
+    "format": "jsonschema",
+    "version": "1-0-2"
+  },
+  "properties": {
+    "requiredString": { "type": "string" },
+    "requiredUnion": { "type": ["string", "boolean"] },
+    "nested": {
+      "properties": {
+        "a": { "type": "number" },
+        "b": {},
+        "c": { "type": ["integer", "null"] }
+      },
+      "required": ["a"]
+    },
+    "someArray": { "type": "array" },
+    "id": { "type": "string", "format": "uuid" },
+    "someDate": { "type": "string", "format": "date-time" },
+    "bigInt": { "type": "integer", "maximum": 100000000000000 }
+  },
+  "required": ["requiredString", "requiredUnion"]
+}

.github/server.conf

@@ -0,0 +1,16 @@
+# Dummy Iglu Server configuration to assist in testing
+
+repo-server {
+  interface = "0.0.0.0"
+  port = 8080
+  idleTimeout = 5
+  threadPool = {
+    type = "global"
+  }
+}
+
+database {
+  type = "dummy"
+}
+
+debug = true

.github/start_environment.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+
+set -e
+
+if [ -z ${GITHUB_WORKSPACE+x} ]; then
+    echo "GITHUB_WORKSPACE is unset";
+    exit 1
+fi
+
+IGLUCTL_ZIP="igluctl_0.7.2_rc1.zip"
+IGLUCTL_URI="http://dl.bintray.com/snowplow/snowplow-generic/$IGLUCTL_ZIP"
+IGLUCENTRAL_PATH="$GITHUB_WORKSPACE/iglu-central"
+SCHEMAS_PATH="$IGLUCENTRAL_PATH/schemas/" 
+TEST_SCHEMAS="$GITHUB_WORKSPACE/.github/schemas/"
+POSTGRES_PASSWORD=mysecretpassword
+
+git clone https://github.com/snowplow/iglu-central.git $IGLUCENTRAL_PATH
+
+docker run \
+    -p 8080:8080 \
+    -v $GITHUB_WORKSPACE/.github:/iglu \
+    --rm -d \
+    snowplow-docker-registry.bintray.io/snowplow/iglu-server:0.6.1 \
+    --config /iglu/server.conf
+
+echo "Waiting for Iglu Server..."
+sleep 5
+
+wget $IGLUCTL_URI
+unzip -j $IGLUCTL_ZIP
+
+./igluctl static push \
+    $SCHEMAS_PATH \
+    http://localhost:8080/ \
+    48b267d7-cd2b-4f22-bae4-0f002008b5ad \
+    --public
+
+./igluctl static push \
+    $TEST_SCHEMAS \
+    http://localhost:8080/ \
+    48b267d7-cd2b-4f22-bae4-0f002008b5ad \
+    --public

.github/workflows/snyk.yml

@@ -0,0 +1,20 @@
+name: Snyk
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Run Snyk to check for vulnerabilities
+      uses: snyk/actions/scala@master
+      with:
+        command: monitor
+        args: --project-name=snowplow-postgres-loader
+      env:
+        SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,56 @@
+name: Test
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres
+        ports:
+          - 5432:5432
+        env:
+          # See src/test/scala/com/snowplowanalytics/snowplow/postgres/loader/Database.scala
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: mysecretpassword
+          POSTGRES_DB: snowplow
+          POSTGRES_PORT: 5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Prepare test environment
+      run: $GITHUB_WORKSPACE/.github/start_environment.sh
+    - name: Run tests
+      run: sbt clean coverage test
+    - name: Submit coveralls data
+      if: ${{ always() }}
+      run: sbt coverageReport coveralls
+      env:
+        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+
+  deploy:
+    needs: test
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Compare SBT version with git tag
+      run: .github/check_tag.sh ${GITHUB_REF##*/}
+    - name: Docker login
+      run: docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Build and publish Docker image
+      run: sbt docker:publish

.gitignore

@@ -1,93 +1,4 @@
-# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
-# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
-
-# .idea/
-
-# User-specific stuff
-.idea/**/workspace.xml
-.idea/**/tasks.xml
-.idea/**/usage.statistics.xml
-.idea/**/dictionaries
-.idea/**/shelf
-
-# Generated files
-.idea/**/contentModel.xml
-
-# Sensitive or high-churn files
-.idea/**/dataSources/
-.idea/**/dataSources.ids
-.idea/**/dataSources.local.xml
-.idea/**/sqlDataSources.xml
-.idea/**/dynamic.xml
-.idea/**/uiDesigner.xml
-.idea/**/dbnavigator.xml
-
-# Gradle
-.idea/**/gradle.xml
-.idea/**/libraries
-
-# Gradle and Maven with auto-import
-# When using Gradle or Maven with auto-import, you should exclude module files,
-# since they will be recreated, and may cause churn.  Uncomment if using
-# auto-import.
-# .idea/artifacts
-# .idea/compiler.xml
-# .idea/jarRepositories.xml
-# .idea/modules.xml
-# .idea/*.iml
-# .idea/modules
-# *.iml
-# *.ipr
-
-# CMake
-cmake-build-*/
-
-# Mongo Explorer plugin
-.idea/**/mongoSettings.xml
-
-# File-based project format
-*.iws
-
-# IntelliJ
-out/
-
-# mpeltonen/sbt-idea plugin
-.idea_modules/
-
-# JIRA plugin
-atlassian-ide-plugin.xml
-
-# Cursive Clojure plugin
-.idea/replstate.xml
-
-# Crashlytics plugin (for Android Studio and IntelliJ)
-com_crashlytics_export_strings.xml
-crashlytics.properties
-crashlytics-build.properties
-fabric.properties
-
-# Editor-based Rest Client
-.idea/httpRequests
-
-# Android studio 3.1+ serialized cache file
-.idea/caches/build_file_checksums.ser
-
-# Misc
-*.class
-*.log
 target/
 project/target/
-project/boot/
-dist/
-boot/
-logs/
-tmp/
-projectFilesBackup/
-.history
-.DS_STORE
-.cache
-.settings
-.project
-.classpath
-version.properties
-RUNNING_PID
+
+.travis/igluctl*

.scalafix.conf

@@ -0,0 +1,17 @@
+rules = [
+  DisableSyntax
+]
+DisableSyntax.noUniversalEquality = true
+DisableSyntax.noVars = true
+DisableSyntax.noNulls = true
+DisableSyntax.noReturns = true
+DisableSyntax.noWhileLoops = true
+DisableSyntax.noAsInstanceOf = true
+DisableSyntax.noIsInstanceOf = true
+DisableSyntax.noXml = true
+DisableSyntax.noDefaultArgs = true
+DisableSyntax.noFinalVal = true
+DisableSyntax.noFinalize = true
+DisableSyntax.noValPatterns = true
+DisableSyntax.noUniversalEquality = true
+// DisableSyntax.noThrows = true