From 32e4638d3df5b0e94cb873c62132138add973cb0 Mon Sep 17 00:00:00 2001 From: spenes Date: Thu, 23 Nov 2023 01:31:38 +0300 Subject: [PATCH 1/5] Scan Docker images in Snyk Github action (close #285) --- .github/workflows/ci.yml | 123 +++++++++++++++++++++++++ .github/workflows/snyk.yml | 20 ---- .github/workflows/test_and_publish.yml | 72 --------------- 3 files changed, 123 insertions(+), 92 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/snyk.yml delete mode 100644 .github/workflows/test_and_publish.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8c8fcde --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,123 @@ +name: CI + +on: + push: + tags: + - '*' + branches: + - master + - develop + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: 11 + distribution: adopt + + - name: Install LZO + run: sudo apt-get install -y lzop liblzo2-dev + + - name: Run tests + run: | + sbt "project main" test + sbt "project lzo" test + + - name: Check formatting + run: sbt scalafmtCheck + + publish_docker: + needs: test + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + strategy: + matrix: + app: + - main + - lzo + - distroless + include: + - suffix: "" + - app: lzo + run_snyk: ${{ !contains(github.ref, 'rc') }} + - app: distroless + run_snyk: ${{ !contains(github.ref, 'rc') }} + + steps: + - uses: actions/checkout@v2 + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: 11 + distribution: adopt + + - name: Install LZO + run: sudo apt-get install -y lzop liblzo2-dev + + - name: Login to Docker Hub + run: docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + - name: Publish to Docker Hub + run: sbt "project ${{ matrix.app }}" docker:publish + + - name: Build local image, which is needed to run Snyk + if: matrix.run_snyk + run: sbt "project ${{ matrix.app }}" docker:publishLocal + - name: Run Snyk to check for vulnerabilities + uses: snyk/actions/docker@master + if: matrix.run_snyk + with: + image: "snowplow/snowplow-s3-loader:${{ github.ref_name }}-${{ matrix.app }}" + args: "--app-vulns --org=data-processing-new" + command: monitor + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + + create_release: + needs: test + if: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc') }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: 11 + distribution: adopt + + - name: Install LZO + run: sudo apt-get install -y lzop liblzo2-dev + + - name: Build artifacts + run: | + sbt assembly + - name: Get current version + id: ver + run: | + export PROJECT_VERSION=$(sbt version -Dsbt.log.noformat=true | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n') + echo "::set-output name=project_version::$PROJECT_VERSION" + - name: Create GitHub release and attach artifacts + uses: softprops/action-gh-release@v1 + with: + draft: true + prerelease: true + name: Version ${{ steps.ver.outputs.project_version }} + tag_name: ${{ steps.ver.outputs.project_version }} + files: | + modules/main/target/scala-2.13/snowplow-s3-loader-${{ steps.ver.outputs.project_version }}.jar + modules/lzo/target/scala-2.13/snowplow-s3-loader-lzo-${{ steps.ver.outputs.project_version }}.jar + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/snyk.yml b/.github/workflows/snyk.yml deleted file mode 100644 index ef82d4b..0000000 --- a/.github/workflows/snyk.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Snyk - -on: - push: - branches: [ master ] - -jobs: - check-vulnerabilities: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Snyk monitor - Check for vulnerabilities - uses: snyk/actions/scala@master - with: - command: monitor - args: --project-name=s3-loader - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} diff --git a/.github/workflows/test_and_publish.yml b/.github/workflows/test_and_publish.yml deleted file mode 100644 index 35d6c7c..0000000 --- a/.github/workflows/test_and_publish.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Test and publish - -on: push - -jobs: - test_and_publish: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Set up JDK 11 - uses: actions/setup-java@v2 - with: - java-version: 11 - distribution: adopt - - - name: Install LZO - run: sudo apt-get install -y lzop liblzo2-dev - - - name: Run tests - run: sbt coverage test - - - name: Check formatting - run: sbt scalafmtCheck - - - name: Login to Docker Hub - if: startsWith(github.ref, 'refs/tags/') - run: docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - - - name: Publish to Docker Hub - if: startsWith(github.ref, 'refs/tags/') - run: sbt 'project main' docker:publish - - - name: Publish to Docker Hub lzo - if: startsWith(github.ref, 'refs/tags/') - run: sbt 'project lzo' docker:publish - - - name: Publish to Docker Hub distroless - if: startsWith(github.ref, 'refs/tags/') - run: sbt 'project distroless' docker:publish - - - name: Build artifacts - run: | - sbt assembly - - name: Get current version - id: ver - run: | - export PROJECT_VERSION=$(sbt version -Dsbt.log.noformat=true | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n') - echo "::set-output name=project_version::$PROJECT_VERSION" - - name: Create GitHub release and attach artifacts - uses: softprops/action-gh-release@v1 - with: - draft: true - prerelease: true - name: Version ${{ steps.ver.outputs.project_version }} - tag_name: ${{ steps.ver.outputs.project_version }} - files: | - modules/main/target/scala-2.13/snowplow-s3-loader-${{ steps.ver.outputs.project_version }}.jar - modules/lzo/target/scala-2.13/snowplow-s3-loader-lzo-${{ steps.ver.outputs.project_version }}.jar - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Submit coveralls data - run: | - sbt coverageAggregate - sbt coveralls - env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} From 81b0359a61df4012324c5b5d1df2d102aa737512 Mon Sep 17 00:00:00 2001 From: spenes Date: Thu, 23 Nov 2023 12:09:42 +0300 Subject: [PATCH 2/5] Bump pureconfig to 0.15.0 --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6e4c686..d7a9236 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -42,7 +42,7 @@ object Dependencies { val circe = "0.13.0" val snowplowTracker = "0.7.0" val snowplowBadrows = "2.1.0" - val pureconfig = "0.14.1" + val pureconfig = "0.15.0" val igluCore = "1.0.0" // Scala (test only) val specs2 = "4.10.5" From c842de6d2056970f5c28150a18e7010e0b9968c1 Mon Sep 17 00:00:00 2001 From: spenes Date: Thu, 23 Nov 2023 15:35:57 +0300 Subject: [PATCH 3/5] Bump reload4j to 1.2.22 --- project/Dependencies.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d7a9236..febf7c8 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -34,6 +34,7 @@ object Dependencies { val collections = "3.2.2" // Address vulnerability val jaxbApi = "2.3.1" val protobuf = "3.21.12" + val reload4j = "1.2.22" // Address vulnerability // Thrift (test only) val collectorPayload = "0.0.0" val thrift = "0.15.0" // Address vulnerabilities @@ -61,6 +62,7 @@ object Dependencies { val hadoopMapReduce = "org.apache.hadoop" % "hadoop-mapreduce-client-core" % V.hadoop val hadoop = "org.apache.hadoop" % "hadoop-common" % V.hadoop val protobuf = "com.google.protobuf" % "protobuf-java" % V.protobuf + val reload4j = "ch.qos.reload4j" % "reload4j" % V.reload4j val collections = "commons-collections" % "commons-collections" % V.collections val jaxbApi = "javax.xml.bind" % "jaxb-api" % V.jaxbApi % Runtime @@ -93,6 +95,7 @@ object Dependencies { Libraries.sentry, Libraries.jaxbApi, Libraries.protobuf, + Libraries.reload4j, // Scala Libraries.decline, Libraries.circe, From 7a45343ae7845b1160ce39a126da3ce0f8e9591c Mon Sep 17 00:00:00 2001 From: spenes Date: Thu, 23 Nov 2023 15:39:30 +0300 Subject: [PATCH 4/5] Bump snappy-java to 1.1.10.4 --- project/Dependencies.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index febf7c8..b4ce72d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -35,6 +35,7 @@ object Dependencies { val jaxbApi = "2.3.1" val protobuf = "3.21.12" val reload4j = "1.2.22" // Address vulnerability + val snappyJava = "1.1.10.4" // Address vulnerability // Thrift (test only) val collectorPayload = "0.0.0" val thrift = "0.15.0" // Address vulnerabilities @@ -63,6 +64,7 @@ object Dependencies { val hadoop = "org.apache.hadoop" % "hadoop-common" % V.hadoop val protobuf = "com.google.protobuf" % "protobuf-java" % V.protobuf val reload4j = "ch.qos.reload4j" % "reload4j" % V.reload4j + val snappyJava = "org.xerial.snappy" % "snappy-java" % V.snappyJava val collections = "commons-collections" % "commons-collections" % V.collections val jaxbApi = "javax.xml.bind" % "jaxb-api" % V.jaxbApi % Runtime @@ -96,6 +98,7 @@ object Dependencies { Libraries.jaxbApi, Libraries.protobuf, Libraries.reload4j, + Libraries.snappyJava, // Scala Libraries.decline, Libraries.circe, From 6eb910d751ca0818dea45df2a317fb7ee8a31965 Mon Sep 17 00:00:00 2001 From: spenes Date: Thu, 23 Nov 2023 15:56:56 +0300 Subject: [PATCH 5/5] Prepare for 2.2.8 release --- CHANGELOG | 7 +++++++ README.md | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 8a8ac85..e56dd35 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +Version 2.2.8 (2023-11-24) +-------------------------- +Scan Docker images in Snyk Github action (#285) +Bump pureconfig to 0.15.0 (#286) +Bump reload4j to 1.2.22 (#286) +Bump snappy-java to 1.1.10.4 (#286) + Version 2.2.7 (2023-04-14) -------------------------- Bump sbt-snowplow-release to 0.3.1 (#282) diff --git a/README.md b/README.md index 6849942..c87e686 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ limitations under the License. [travis-image]: https://travis-ci.org/snowplow/snowplow-s3-loader.png?branch=master [travis]: http://travis-ci.org/snowplow/snowplow-s3-loader -[release-image]: http://img.shields.io/badge/release-2.2.7-blue.svg?style=flat +[release-image]: http://img.shields.io/badge/release-2.2.8-blue.svg?style=flat [releases]: https://github.com/snowplow/snowplow-s3-loader/releases [license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat