Skip to content

Commit

Permalink
Merge 79e4da6 into 1c977c4
Browse files Browse the repository at this point in the history
  • Loading branch information
chuwy committed Nov 30, 2020
2 parents 1c977c4 + 79e4da6 commit c95f29f
Show file tree
Hide file tree
Showing 118 changed files with 2,315 additions and 4,523 deletions.
19 changes: 19 additions & 0 deletions .github/deploy.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/bin/bash

set -e

project=$1
tag=$2

export RDB_LOADER_VERSION=${tag##*/}

./.github/is_release_tag.sh $project $tag

echo "Launching release manager for $project"

release-manager \
--config "./.github/release_${project}.yml" \
--check-version \
--make-version \
--make-artifact \
--upload-artifact
9 changes: 8 additions & 1 deletion .travis/is_release_tag.sh → .github/is_release_tag.sh
Original file line number Diff line number Diff line change
@@ -1,20 +1,27 @@
#!/bin/bash

REFS_TAGS="refs/tags/"

project=$1
tag=$2

refs_tags_len=${#REFS_TAGS}

slashed="${project}/"
slashed_len=${#slashed}

cicd=${tag:0:${slashed_len}}
cicd=${tag:$refs_tags_len:${slashed_len}}
release=${tag:${slashed_len}}

echo "Comparing project name from git tag ${cicd} (from ${tag}) and specified ${slashed}"

if [ "${cicd}" == "${slashed}" ]; then
if [ "${release}" == "" ]; then
echo "Warning! No release specified! Ignoring."
exit 2
fi
exit 0
else
echo "${cicd} is not equal to ${slashed}"
exit 1
fi
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
---
# --- Variables --- #

local:
root_dir : <%= ENV['TRAVIS_BUILD_DIR'] %>

# --- Release Manager Config --- #
root_dir : <%= ENV['GITHUB_WORKSPACE'] %>

# Required: deployment targets
targets:
Expand Down Expand Up @@ -61,19 +57,19 @@ packages:
publish : true
override : false
continue_on_conflict : false
version : <%= FUNC['sbt_version(.)'] %>
version : <%= CMD['sbt "project loader" version -Dsbt.log.noformat=true | awk "END{print}" | sed -r "s/\[info\]\s(.*)/\1/" | xargs echo -n'] %>

# Required IF '--check-version' is passed: will assert that both versions are the same
build_version : <%= ENV['TRAVIS_BUILD_RELEASE_TAG'] %>
build_version : <%= ENV['RDB_LOADER_VERSION'] %>

# Optional: Build commands
build_commands:
- sbt assembly
- sbt "project loader" assembly

# Required: Artifact. Version will be inserted between prefix and suffix
artifacts:
- type : "asis"
prefix : "snowplow-rdb-loader-"
suffix : ".jar"
binary_paths:
- "target/scala-2.11/snowplow-rdb-loader-{{ packages.0.build_version }}.jar"
- "modules/loader/target/scala-2.12/snowplow-rdb-loader-{{ packages.0.build_version }}.jar"
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
---
# --- Variables --- #

local:
root_dir : <%= ENV['TRAVIS_BUILD_DIR'] %>

# --- Release Manager Config --- #
root_dir : <%= ENV['GITHUB_WORKSPACE'] %>

# Required: deployment targets
targets:
Expand Down Expand Up @@ -61,10 +57,10 @@ packages:
publish : true
override : false
continue_on_conflict : false
version : <%= CMD['sbt "project shredder" version -Dsbt.log.noformat=true | awk "END{print}" | sed -r "s/\[info\]\s(.*)/\1/" | xargs echo -n'] %>
version : <%= CMD['sbt "project shredder" version -Dsbt.log.noformat=true | awk "END{print}" | sed -r "s/\[info\]\s(.*)/\1/" | xargs echo -n'] %>

# Required IF '--check-version' is passed: will assert that both versions are the same
build_version : <%= ENV['TRAVIS_BUILD_RELEASE_TAG'] %>
build_version : <%= ENV['RDB_LOADER_VERSION'] %>

# Optional: Build commands
build_commands:
Expand All @@ -76,4 +72,4 @@ packages:
prefix : "snowplow-rdb-shredder-"
suffix : ".jar"
binary_paths:
- "shredder/target/scala-2.11/snowplow-rdb-shredder-{{ packages.0.build_version }}.jar"
- "modules/shredder/target/scala-2.12/snowplow-rdb-shredder-{{ packages.0.build_version }}.jar"
File renamed without changes.
4 changes: 2 additions & 2 deletions .travis/start_iglu.sh → .github/start_iglu.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ set -e

IGLUCTL_ZIP="igluctl_0.6.0.zip"
IGLUCTL_URI="http://dl.bintray.com/snowplow/snowplow-generic/$IGLUCTL_ZIP"
SCHEMAS_PATH="$TRAVIS_BUILD_DIR/iglu-central/schemas/"
SCHEMAS_PATH="$pwd/iglu-central/schemas/"

git clone https://github.com/snowplow/iglu-central.git

docker run \
-p 8080:8080 \
-v $TRAVIS_BUILD_DIR/.travis:/iglu \
-v $pwd/.github:/iglu \
--rm -d \
snowplow-docker-registry.bintray.io/snowplow/iglu-server:0.6.0-rc16 \
--config /iglu/server.conf
Expand Down
26 changes: 26 additions & 0 deletions .github/workflows/deploy_loader.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
name: Deploy Loader

on: [push, pull_request]

jobs:
deploy:
if: startsWith(github.ref, 'refs/tags/rdb_loader')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '2.x'
- uses: BSFishy/pip-action@v1
with:
packages: |
release-manager
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
java-version: 1.8
- name: Build and publish Loader
run: .github/deploy.sh rdb_loader $GITHUB_REF
env:
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
26 changes: 26 additions & 0 deletions .github/workflows/deploy_shredder.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
name: Deploy Shredder

on: [push, pull_request]

jobs:
deploy:
if: startsWith(github.ref, 'refs/tags/rdb_shredder')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: '2.x'
- uses: BSFishy/pip-action@v1
with:
packages: |
release-manager
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
java-version: 1.8
- name: Build and publish Shredder
run: .github/deploy.sh rdb_shredder $GITHUB_REF
env:
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
22 changes: 22 additions & 0 deletions .github/workflows/snyk.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
name: Snyk

on:
push:
branches: [ master ]
create:
tags: [ rdb* ]

jobs:
security:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2

- name: Run Snyk to check for vulnerabilities
uses: snyk/actions/scala@master
with:
command: monitor
args: --project-name=snowplow-rdb-loader
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
24 changes: 24 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
name: Test

on: [push, pull_request]

jobs:
test:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
java-version: 1.8
- name: Run tests
run: sbt coverage test coverageReport
- name: Aggregate coverage data
if: ${{ always() }}
run: sbt coverageAggregate
- name: Submit coveralls data
if: ${{ always() }}
run: sbt coveralls
env:
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
2 changes: 2 additions & 0 deletions .scala-steward.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
updates.includeScala = true
commits.message = "Bump ${artifactName} from ${currentVersion} to ${nextVersion}"
28 changes: 0 additions & 28 deletions .travis.yml

This file was deleted.

32 changes: 0 additions & 32 deletions .travis/deploy.sh

This file was deleted.

26 changes: 26 additions & 0 deletions CHANGELOG
Original file line number Diff line number Diff line change
@@ -1,3 +1,29 @@
Release 33 (2020-12-01)
-----------------------
Common: integrate coveralls (#220)
Common: integrate Snyk (#188)
Common: integrate Scala Steward (#206)
Common: remove Processing Manifest (#186)
Common: disable test in assembly (#202)
Common: fix ambigrous empty string (#171)
Common: create a separate modules root (#195)
Common: switch to Github Actions (#189)
Common: bump snowplow-scala-analytics-sdk to 2.1.0 (#201)
Common: bump snowplow-badrows to 2.1.0 (#199)
Common: bump specs2-core to 4.10.5 (#197)
Common: bump schema-ddl to 0.12.0 (#192)
Common: bump sbt to 1.4.4 (#196)
Common: bump Scala to 2.12 (#127)
RDB Shredder: optimize DAG (#181)
RDB Shredder: use single etl_tstamp for cross-batch deduplication (#180)
RDB Shredder: bump Spark to 3.0.1 (#101)
RDB Loader: remove Postgres support (#191)
RDB Loader: bump jsch to 0.1.55 (#177)
Loader: mitigate exhausted input error (#203)
Loader: drop legacy S3 paths (#198)
Loader: switch from Free monad to cats-effect IO (#184)
Loader: fix ignored schema in metadata queries (#178)

Release 32 (2020-03-06)
-----------------------
Common: remove VERSION file (#164)
Expand Down
32 changes: 20 additions & 12 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
# Relational Database Loader

[![Build Status][travis-image]][travis]
[![Build Status][build-image]][build]
[![Release][release-image]][releases]
[![License][license-image]][license]
[![Scala Steward][scala-steward-image]][scala-steward]
[![Coverage Status][coveralls-image]][coveralls]

## Introduction

Expand All @@ -18,7 +20,7 @@ RDB Shredder is a [Spark][spark] job which:
4. Adds metadata to these JSONs to track their origins
5. Writes these JSONs out to nested folders dependent on their schema

It is designed to be run by the [EmrEtlRunner][emr-etl-runner] immediately after the [Spark Enrich][spark-enrich] job.
It is designed to be run by the [EmrEtlRunner][emr-etl-runner] immediately after the [Enrich][enrich] job.

### RDB Loader

Expand Down Expand Up @@ -49,21 +51,27 @@ limitations under the License.
[techdocs-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/techdocs.png
[setup-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/setup.png
[roadmap-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/roadmap.png
[setup]: https://github.com/snowplow/snowplow/wiki/6-Configuring-shredding
[techdocs]: https://github.com/snowplow/snowplow/wiki/Relational-Database-Loader
[setup]: https://docs.snowplowanalytics.com/docs/getting-started-on-snowplow-open-source/setup-snowplow-on-aws/setup-destinations/setup-redshift/
[techdocs]: https://docs.snowplowanalytics.com/docs/pipeline-components-and-applications/loaders-storage-targets/snowplow-rdb-loader/

[spark]: http://spark.apache.org/
[emr-etl-runner]: https://github.com/snowplow/snowplow/tree/master/3-enrich/emr-etl-runner
[spark-enrich]: https://github.com/snowplow/snowplow/tree/master/3-enrich/spark-enrich
[emr-etl-runner]: https://github.com/snowplow/emr-etl-runner
[enrich]: https://github.com/snowplow/snowplow/enrich

[targets]: https://github.com/snowplow/snowplow/wiki/Configuring-storage-targets
[shred]: https://github.com/snowplow/snowplow/wiki/Scala-Hadoop-Shred
[shred]: https://docs.snowplowanalytics.com/docs/pipeline-components-and-applications/loaders-storage-targets/snowplow-rdb-loader/rdb-shredder/

[travis-image]: https://travis-ci.org/snowplow/snowplow-rdb-loader.png?branch=master
[travis]: http://travis-ci.org/snowplow/snowplow-rdb-loader
[build-image]: https://github.com/snowplow/snowplow-rdb-loader/workflows/Test%20and%20deploy/badge.svg
[build]: https://github.com/snowplow/snowplow-rdb-loader/actions?query=workflow%3A%22Test%22

[release-image]: http://img.shields.io/badge/release-r32-blue.svg?style=flat
[release-image]: https://img.shields.io/badge/release-r33-blue.svg?style=flat
[releases]: https://github.com/snowplow/snowplow-rdb-loader/releases

[license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat
[license]: http://www.apache.org/licenses/LICENSE-2.0
[license-image]: https://img.shields.io/badge/license-Apache--2-blue.svg?style=flat
[license]: https://www.apache.org/licenses/LICENSE-2.0

[scala-steward-image]: https://img.shields.io/badge/Scala_Steward-helping-blue.svg?style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA4AAAAQCAMAAAARSr4IAAAAVFBMVEUAAACHjojlOy5NWlrKzcYRKjGFjIbp293YycuLa3pYY2LSqql4f3pCUFTgSjNodYRmcXUsPD/NTTbjRS+2jomhgnzNc223cGvZS0HaSD0XLjbaSjElhIr+AAAAAXRSTlMAQObYZgAAAHlJREFUCNdNyosOwyAIhWHAQS1Vt7a77/3fcxxdmv0xwmckutAR1nkm4ggbyEcg/wWmlGLDAA3oL50xi6fk5ffZ3E2E3QfZDCcCN2YtbEWZt+Drc6u6rlqv7Uk0LdKqqr5rk2UCRXOk0vmQKGfc94nOJyQjouF9H/wCc9gECEYfONoAAAAASUVORK5CYII=
[scala-steward]: https://scala-steward.org

[coveralls]: https://coveralls.io/github/snowplow/snowplow-rdb-loader?branch=master
[coveralls-image]: https://coveralls.io/repos/github/snowplow/snowplow-rdb-loader/badge.svg?branch=master
Loading

0 comments on commit c95f29f

Please sign in to comment.