Add github workflows (flyteorg#32)

* Add github workflows * Update boilerplate * Remove endtoend tests for now * nudge * Add go releaser and temp change to test master workflow * typo * Deprecate Travis
eapolinario · Dec 1, 2020 · 8060e0e · 8060e0e
1 parent a70696b
commit 8060e0e
Show file tree

Hide file tree

Showing 10 changed files with 529 additions and 24 deletions.
diff --git a/datacalog/.github/workflows/master.yml b/datacalog/.github/workflows/master.yml
@@ -0,0 +1,117 @@
+name: Master
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  # Duplicated from pull request workflow because sharing is not yet supported
+  build-docker:
+    name: Build Docker Image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v1
+        with:
+          path: /tmp/tmp/docker-images
+          key: /tmp/docker-images-${{ github.event.after }}
+          restore-keys: |
+            /tmp/docker-images-${{ github.event.before }}
+            /tmp/docker-images-${{ github.event.pull_request.base.sha }}
+      - name: Prime docker cache
+        run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true)
+      - name: Build dockerfile
+        run: |
+          docker build -t lyft/${{ github.event.repository.name }}:builder --target builder --cache-from=lyft/${{ github.event.repository.name }}:builder .
+          docker build -t lyft/${{ github.event.repository.name }}:latest --cache-from=lyft/${{ github.event.repository.name }}:builder .
+
+      - name: Tag and cache docker image
+        run: mkdir -p /tmp/tmp/docker-images && docker save lyft/${{ github.event.repository.name }}:builder -o /tmp/tmp/docker-images/snapshot-builder.tar && docker save lyft/${{ github.event.repository.name }}:latest -o /tmp/tmp/docker-images/snapshot.tar
+
+  bump-version:
+    name: Bump Version
+    if: github.actor != 'goreleaserbot'
+    runs-on: ubuntu-latest
+    needs: build-docker # Only to ensure it can successfully build
+    outputs:
+      version: ${{ steps.bump-version.outputs.tag }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: '0'
+      - name: Bump version and push tag
+        id: bump-version
+        uses: anothrNick/github-tag-action@1.17.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WITH_V: true
+          DEFAULT_BUMP: patch
+
+  goreleaser:
+    name: Goreleaser
+    runs-on: ubuntu-latest
+    needs: [bump-version]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: '0'
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.14
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          version: latest
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }}
+
+  push-github:
+    name: Push to Github Registry
+    runs-on: ubuntu-latest
+    needs: bump-version
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: '0'
+      - name: Push Docker Image to Github Registry
+        uses: whoan/docker-build-with-cache-action@v5
+        with:
+          username: "${{ secrets.PUBLISH_CONTAINER_ACTOR }}"
+          password: "${{ secrets.PUBLISH_CONTAINER }}"
+          image_name: ${{ github.repository }}
+          image_tag: latest,${{ github.sha }},${{ needs.bump-version.outputs.version }}
+          push_git_tag: true
+          registry: ghcr.io
+          build_extra_args: "--compress=true"
+
+  tests-lint:
+    name: Run tests and lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: '0'
+      - name: Unit Tests
+        uses: cedrickring/golang-action@1.5.2
+        env:
+          GO111MODULE: "on"
+        with:
+          args: make install && make test_unit_codecov
+      - name: Push CodeCov
+        uses: codecov/codecov-action@v1
+        with:
+          file: coverage.txt
+          flags: unittests
+          fail_ci_if_error: true
+      - name: Lint
+        uses: cedrickring/golang-action@1.5.2
+        env:
+          GO111MODULE: "on"
+        with:
+          args: make install && make lint
diff --git a/datacalog/.github/workflows/pull_request.yml b/datacalog/.github/workflows/pull_request.yml
@@ -0,0 +1,55 @@
+name: Pull Request
+
+on:
+  pull_request
+
+jobs:
+  build-docker:
+    name: Build Docker Image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - id: load-docker-cache
+        name: Load Docker Cache
+        uses: actions/cache@v1
+        with:
+          path: /tmp/tmp/docker-images
+          key: /tmp/docker-images-${{ github.event.after }}
+          restore-keys: |
+            /tmp/docker-images-${{ github.event.before }}
+            /tmp/docker-images-${{ github.event.pull_request.base.sha }}
+      - name: Prime docker cache
+        run: (docker load -i /tmp/tmp/docker-images/snapshot-builder.tar || true) && (docker load -i /tmp/tmp/docker-images/snapshot.tar || true)
+      - name: Build dockerfile
+        run: |
+          docker build -t lyft/${{ github.event.repository.name }}:builder --target builder --cache-from=lyft/${{ github.event.repository.name }}:builder .
+          docker build -t lyft/${{ github.event.repository.name }}:latest --cache-from=lyft/${{ github.event.repository.name }}:builder .
+
+      - name: Tag and cache docker image
+        run: mkdir -p /tmp/tmp/docker-images && docker save lyft/${{ github.event.repository.name }}:builder -o /tmp/tmp/docker-images/snapshot-builder.tar && docker save lyft/${{ github.event.repository.name }}:latest -o /tmp/tmp/docker-images/snapshot.tar
+
+  tests-lint:
+    name: Run tests and lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Unit Tests
+        uses: cedrickring/golang-action@1.5.2
+        env:
+          GO111MODULE: "on"
+        with:
+          args: make install && make test_unit_codecov
+      - name: Push CodeCov
+        uses: codecov/codecov-action@v1
+        with:
+          file: coverage.txt
+          flags: unittests
+          fail_ci_if_error: true
+      - name: Lint
+        uses: cedrickring/golang-action@1.5.2
+        env:
+          GO111MODULE: "on"
+        with:
+          args: make install && make lint
diff --git a/datacalog/.goreleaser.yml b/datacalog/.goreleaser.yml
@@ -0,0 +1,66 @@
+project_name: datacatalog
+before:
+  hooks:
+    - go mod download
+builds:
+  - id: datacatalog
+    env:
+      - CGO_ENABLED=0
+    main: ./cmd/main.go
+    binary: datacatalog
+    goos:
+      - linux
+      - windows
+      - darwin
+archives:
+  - id: datacatalog-archive
+    name_template: |-
+      datacatalog_{{ .Tag }}_{{ .Os }}_{{ .Arch -}}
+      {{- with .Arm -}}
+      {{- if (eq . "6") -}}hf
+      {{- else -}}v{{- . -}}
+      {{- end -}}
+      {{- end -}}
+    builds:
+      - datacatalog
+    replacements:
+      386: i386
+      amd64: x86_64
+    format_overrides:
+      - goos: windows
+        format: zip
+
+checksum:
+  name_template: 'checksums.txt'
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+scoop:
+  # Default is "https://github.com/<repo_owner>/<repo_name>/releases/download/{{ .Tag }}/{{ .ArtifactName }}"
+  # url_template: "http://github.mycompany.com/foo/bar/releases/{{ .Tag }}/{{ .ArtifactName }}"
+
+  # Repository to push the app manifest to.
+  bucket:
+    owner: lyft
+    name: datacatalog
+
+  # Git author used to commit to the repository.
+  # Defaults are shown.
+  commit_author:
+    name: goreleaserbot
+    email: goreleaser@carlosbecker.com
+
+  # Your app's homepage.
+  # Default is empty.
+  homepage: "https://godoc.org/github.com/lyft/datacatalog"
+
+  # Your app's description.
+  # Default is empty.
+  description: "datacatalog is the a memoization & lineage tracking service."
+
+  # Your app's license
+  # Default is empty.
+  license: Apache-2.0
diff --git a/datacalog/.travis.yml b/datacalog/.travis.yml
diff --git a/datacalog/boilerplate/lyft/github_workflows/Readme.rst b/datacalog/boilerplate/lyft/github_workflows/Readme.rst
@@ -0,0 +1,20 @@
+Golang Github Actions
+~~~~~~~~~~~~~~~~~
+
+Provides a two github actions workflows.
+
+**To Enable:**
+
+Add ``lyft/github_workflows`` to your ``boilerplate/update.cfg`` file.
+
+Add a github secret ``flytegithub_repo`` with a the name of your fork (e.g. ``my_company/flytepropeller``).
+
+The actions will push to 2 repos:
+
+	1. ``docker.pkg.github.com/lyft/<repo>/operator``
+	2. ``docker.pkg.github.com/lyft/<repo>/operator-stages`` : this repo is used to cache build stages to speed up iterative builds after.
+
+There are two workflows that get deployed:
+
+	1. A workflow that runs on Pull Requests to build and push images to github registy tagged with the commit sha.
+	2. A workflow that runs on master merges that bump the patch version of release tag, builds and pushes images to github registry tagged with the version, commit sha as well as "latest"