Initial commit

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,23 @@
+<!--
+
+** Please read the guidelines below. **
+
+GitHub issues are reserved for bug reports.
+The best place to ask general questions or ask for community support is
+[on our Discourse](https://discourse.snowplowanalytics.com/).
+
+-->
+
+**Version**:
+
+**Expected behavior**:
+
+**Actual behavior**:
+
+**Steps to reproduce**:
+
+<!-- Please try to be as detailed as possible so that we can reproduce and fix the issue
+as quickly as possible. -->
+ 1.
+ 2.
+

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,12 @@
+<!--
+Thank you for contributing to Snowplow Postgres Loader!
+
+You'll find a small checklist below which should help speed up the review process:
+
+- [ ] Have you signed the [contributor license agreement](https://github.com/snowplow/snowplow/wiki/CLA)?
+- [ ] Have you read the [contributing guide](https://github.com/snowplow-incubator/snowplow-postgres-loader/blob/master/CONTRIBUTING.md)?
+- [ ] Have you added the appropriate unit tests?
+- [ ] Have you run the tests through `sbt test`?
+- [ ] Is your pull request against the `master` branch?
+-->
+

.github/check_tag.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+tag=$1
+
+project_version=$(sbt version -Dsbt.log.noformat=true | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n')
+
+if [[ "${tag}" = "${project_version}" ]]; then
+  echo "Tag version (${tag}) matches project version (${project_version}). Deploying!"
+else
+  echo "Tag version (${tag}) doesn't match version in scala project (${project_version}). Aborting!"
+  exit 1
+fi

.github/deploy_common.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -e
+
+tag=$1
+
+mkdir ~/.bintray/
+FILE=$HOME/.bintray/.credentials
+cat <<EOF >$FILE
+realm = Bintray API Realm
+host = api.bintray.com
+user = $BINTRAY_SNOWPLOW_MAVEN_USER
+password = $BINTRAY_SNOWPLOW_MAVEN_API_KEY
+EOF
+
+sbt "project common" +publish
+echo "Snowplow Postgres: published to Bintray Maven"
+sbt "project common" +bintraySyncMavenCentral
+echo "Snowplow Postgres: synced to Maven Central"

.github/schemas/me.chuwy/pg-test/jsonschema/1-0-0

@@ -0,0 +1,22 @@
+{
+  "self": {
+    "vendor": "me.chuwy",
+    "name": "pg-test",
+    "format": "jsonschema",
+    "version": "1-0-0"
+  },
+  "properties": {
+    "requiredString": { "type": "string" },
+    "requiredUnion": { "type": ["string", "boolean"] },
+    "nested": {
+      "properties": {
+        "a": { "type": "number" },
+        "b": {}
+      },
+      "required": ["a"]
+    },
+    "someArray": { "type": "array" },
+    "id": { "type": "string", "format": "uuid" }
+  },
+  "required": ["requiredString", "requiredUnion"]
+}

.github/schemas/me.chuwy/pg-test/jsonschema/1-0-1

@@ -0,0 +1,24 @@
+{
+  "self": {
+    "vendor": "me.chuwy",
+    "name": "pg-test",
+    "format": "jsonschema",
+    "version": "1-0-1"
+  },
+  "properties": {
+    "requiredString": { "type": "string" },
+    "requiredUnion": { "type": ["string", "boolean"] },
+    "nested": {
+      "properties": {
+        "a": { "type": "number" },
+        "b": {},
+        "c": { "type": ["integer", "null"] }
+      },
+      "required": ["a"]
+    },
+    "someArray": { "type": "array" },
+    "id": { "type": "string", "format": "uuid" },
+    "someDate": { "type": "string", "format": "date-time" }
+  },
+  "required": ["requiredString", "requiredUnion"]
+}

.github/schemas/me.chuwy/pg-test/jsonschema/1-0-2

@@ -0,0 +1,25 @@
+{
+  "self": {
+    "vendor": "me.chuwy",
+    "name": "pg-test",
+    "format": "jsonschema",
+    "version": "1-0-2"
+  },
+  "properties": {
+    "requiredString": { "type": "string" },
+    "requiredUnion": { "type": ["string", "boolean"] },
+    "nested": {
+      "properties": {
+        "a": { "type": "number" },
+        "b": {},
+        "c": { "type": ["integer", "null"] }
+      },
+      "required": ["a"]
+    },
+    "someArray": { "type": "array" },
+    "id": { "type": "string", "format": "uuid" },
+    "someDate": { "type": "string", "format": "date-time" },
+    "bigInt": { "type": "integer", "maximum": 100000000000000 }
+  },
+  "required": ["requiredString", "requiredUnion"]
+}

.github/server.conf

@@ -0,0 +1,16 @@
+# Dummy Iglu Server configuration to assist in testing
+
+repo-server {
+  interface = "0.0.0.0"
+  port = 8080
+  idleTimeout = 5
+  threadPool = {
+    type = "global"
+  }
+}
+
+database {
+  type = "dummy"
+}
+
+debug = true

.github/start_environment.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+
+set -e
+
+if [ -z ${GITHUB_WORKSPACE+x} ]; then
+    echo "GITHUB_WORKSPACE is unset";
+    exit 1
+fi
+
+IGLUCTL_ZIP="igluctl_0.7.2_rc1.zip"
+IGLUCTL_URI="http://dl.bintray.com/snowplow/snowplow-generic/$IGLUCTL_ZIP"
+IGLUCENTRAL_PATH="$GITHUB_WORKSPACE/iglu-central"
+SCHEMAS_PATH="$IGLUCENTRAL_PATH/schemas/" 
+TEST_SCHEMAS="$GITHUB_WORKSPACE/.github/schemas/"
+POSTGRES_PASSWORD=mysecretpassword
+
+git clone https://github.com/snowplow/iglu-central.git $IGLUCENTRAL_PATH
+
+docker run \
+    -p 8080:8080 \
+    -v $GITHUB_WORKSPACE/.github:/iglu \
+    --rm -d \
+    snowplow-docker-registry.bintray.io/snowplow/iglu-server:0.6.1 \
+    --config /iglu/server.conf
+
+echo "Waiting for Iglu Server..."
+sleep 5
+
+wget $IGLUCTL_URI
+unzip -j $IGLUCTL_ZIP
+
+./igluctl static push \
+    $SCHEMAS_PATH \
+    http://localhost:8080/ \
+    48b267d7-cd2b-4f22-bae4-0f002008b5ad \
+    --public
+
+./igluctl static push \
+    $TEST_SCHEMAS \
+    http://localhost:8080/ \
+    48b267d7-cd2b-4f22-bae4-0f002008b5ad \
+    --public

.github/workflows/snyk.yml

@@ -0,0 +1,20 @@
+name: Snyk
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  security:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Run Snyk to check for vulnerabilities
+      uses: snyk/actions/scala@master
+      with:
+        command: monitor
+        args: --project-name=snowplow-postgres-loader
+      env:
+        SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,79 @@
+name: Test
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres
+        ports:
+          - 5432:5432
+        env:
+          # See src/test/scala/com/snowplowanalytics/snowplow/postgres/loader/Database.scala
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: mysecretpassword
+          POSTGRES_DB: snowplow
+          POSTGRES_PORT: 5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Prepare test environment
+      run: $GITHUB_WORKSPACE/.github/start_environment.sh
+    - name: Run tests
+      run: sbt clean coverage test
+    - name: Aggregate coverage data
+      if: ${{ always() }}
+      run: sbt coverageAggregate
+    - name: Submit coveralls data
+      if: ${{ always() }}
+      run: sbt coverageReport coveralls
+      env:
+        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+
+  deploy:
+    needs: test
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Compare SBT version with git tag
+      run: .github/check_tag.sh ${GITHUB_REF##*/}
+    - name: Docker login
+      run: docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Build and publish Docker image
+      run: sbt "project loader" docker:publish
+
+  deploy_common:
+    needs: test
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 1.8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: Compare SBT version with git tag
+      run: .github/check_tag.sh ${GITHUB_REF##*/}
+    - name: Deploy SCE on Bintray Maven and Maven Central
+      run: .github/deploy_common.sh
+      env:
+        SONA_USER: snowplow
+        SONA_PASS: ${{ secrets.SONA_PASS }}
+        BINTRAY_SNOWPLOW_MAVEN_USER: ${{ secrets.BINTRAY_SNOWPLOW_MAVEN_USER }}
+        BINTRAY_SNOWPLOW_MAVEN_API_KEY: ${{ secrets.BINTRAY_SNOWPLOW_MAVEN_API_KEY }}

.gitignore

@@ -0,0 +1,2 @@
+target/
+project/target/

.scalafix.conf

@@ -0,0 +1,17 @@
+rules = [
+  DisableSyntax
+]
+DisableSyntax.noUniversalEquality = true
+DisableSyntax.noVars = true
+DisableSyntax.noNulls = true
+DisableSyntax.noReturns = true
+DisableSyntax.noWhileLoops = true
+DisableSyntax.noAsInstanceOf = true
+DisableSyntax.noIsInstanceOf = true
+DisableSyntax.noXml = true
+DisableSyntax.noDefaultArgs = true
+DisableSyntax.noFinalVal = true
+DisableSyntax.noFinalize = true
+DisableSyntax.noValPatterns = true
+DisableSyntax.noUniversalEquality = true
+// DisableSyntax.noThrows = true

.scalafmt.conf

@@ -0,0 +1,22 @@
+version = 2.5.2
+style = default
+maxColumn = 140
+optIn.breakChainOnFirstMethodDot = false
+assumeStandardLibraryStripMargin = true
+align = none
+align.openParenCallSite = true
+align.openParenDefnSite = true
+danglingParentheses = true
+verticalMultiline.newlineAfterOpenParen = true
+newlines.afterCurlyLambda = preserve
+continuationIndent.defnSite = 2
+rewrite.rules = [
+  AsciiSortImports,
+  AvoidInfix,
+  PreferCurlyFors,
+  RedundantBraces,
+  RedundantParens,
+  SortModifiers
+]
+project.git = true
+includeNoParensInSelectChains = true

.travis.yml

@@ -0,0 +1,31 @@
+language: scala
+dist: trusty
+scala:
+  - 2.13.2
+cache:
+  directories:
+    - $HOME/.ivy2
+    - $HOME/.coursier
+    - $HOME/.sbt
+jdk:
+  - oraclejdk8
+services:
+  - docker
+script:
+  - sbt test
+before_install:
+  - bash ./.travis/checkTag.sh $TRAVIS_TAG
+  - docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
+  - sbt publishLocal
+deploy:
+  provider: script
+  script: ./.travis/deploy.sh $TRAVIS_TAG
+  skip_cleanup: true
+  on:
+    tags: true
+env:
+  global:
+    # DOCKER_USERNAME
+    - secure: encrypted_username
+    # DOCKER_PASSWORD
+    - secure: encrypted_password

CHANGELOG

@@ -0,0 +1,3 @@
+Version 0.1.0 (2020-10-05)
+--------------------------
+Initial release