Merge branch 'master' into kenny/type-metadata

flyteorg · Feb 8, 2022 · 83afa90 · 83afa90
2 parents 1cb7f4d + f6f4934
commit 83afa90
Show file tree

Hide file tree

Showing 52 changed files with 1,507 additions and 263 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '23 3 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -1,6 +1,7 @@
 name: Master
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - master
@@ -57,7 +58,10 @@ jobs:
         run: |
           flytectl sandbox exec -- docker load -i /root/snapshot.tar
       - name: Upgrade Helm charts
-        run: flytectl sandbox exec -- helm upgrade flyte -n flyte --kubeconfig=/etc/rancher/k3s/k3s.yaml /flyteorg/share/flyte -f /flyteorg/share/flyte/values-sandbox.yaml --set flytepropeller.image.repository=flyteorg/${{ github.event.repository.name }},flytepropeller.image.tag=latest
+        run: |
+          flytectl sandbox exec -- helm repo add flyteorg https://flyteorg.github.io/flyte
+          flytectl sandbox exec -- helm repo update
+          flytectl sandbox exec -- helm upgrade flyte -n flyte --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte -f /flyteorg/share/flyte/values.yaml --set flytepropeller.image.repository=flyteorg/${{ github.event.repository.name }},flytepropeller.image.tag=latest
       - name: End2End
         env:
           KUBECONFIG: /home/runner/.flyte/k3s/k3s.yaml
@@ -73,20 +77,24 @@ jobs:
     if: github.event.commits[0].author.name != 'goreleaserbot'
     runs-on: ubuntu-latest
     # Only to ensure it can successfully build
-    needs: build-docker
+    needs: [ endtoend, tests-lint ]
     outputs:
       version: ${{ steps.bump-version.outputs.tag }}
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: '0'
+      - name: Set branch name
+        id: extract_branch
+        run: echo "::set-output name=branch_name::$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}})"
       - name: Bump version and push tag
         id: bump-version
         uses: anothrNick/github-tag-action@1.17.2
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
           WITH_V: true
           DEFAULT_BUMP: patch
+          RELEASE_BRANCHES: ${{ steps.extract_branch.outputs.branch_name }}
 
   goreleaser:
     name: Goreleaser

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -53,7 +53,10 @@ jobs:
         run: |
           flytectl sandbox exec -- docker load -i /root/snapshot.tar
       - name: Upgrade Helm charts
-        run:  flytectl sandbox exec -- helm upgrade flyte -n flyte --kubeconfig=/etc/rancher/k3s/k3s.yaml /flyteorg/share/flyte -f /flyteorg/share/flyte/values-sandbox.yaml --set flytepropeller.image.repository=flyteorg/${{ github.event.repository.name }},flytepropeller.image.tag=latest
+        run:  |
+          flytectl sandbox exec -- helm repo add flyteorg https://flyteorg.github.io/flyte
+          flytectl sandbox exec -- helm repo update
+          flytectl sandbox exec -- helm upgrade flyte -n flyte --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte -f /flyteorg/share/flyte/values.yaml --set flytepropeller.image.repository=flyteorg/${{ github.event.repository.name }},flytepropeller.image.tag=latest
       - name: End2End
         env:
           KUBECONFIG: /home/runner/.flyte/k3s/k3s.yaml
@@ -87,7 +90,10 @@ jobs:
       - name: Prime docker cache
         run: flytectl sandbox exec -- docker load -i /root/snapshot.tar
       - name: Upgrade Helm charts
-        run:  flytectl sandbox exec -- helm upgrade flyte -n flyte --kubeconfig=/etc/rancher/k3s/k3s.yaml /flyteorg/share/flyte -f /flyteorg/share/flyte/values-sandbox.yaml --set flytepropeller.image.repository=flyteorg/${{ github.event.repository.name }},flytepropeller.image.tag=builder
+        run:  |
+          flytectl sandbox exec -- helm repo add flyteorg https://flyteorg.github.io/flyte
+          flytectl sandbox exec -- helm repo update
+          flytectl sandbox exec -- helm upgrade flyte -n flyte --kubeconfig=/etc/rancher/k3s/k3s.yaml flyteorg/flyte -f /flyteorg/share/flyte/values.yaml --set flytepropeller.image.repository=flyteorg/${{ github.event.repository.name }},flytepropeller.image.tag=builder
       - name: Integration
         env:
           KUBECONFIG: /home/runner/.flyte/k3s/k3s.yaml
@@ -128,3 +134,13 @@ jobs:
           GO111MODULE: "on"
         with:
           args: make install && make lint
+
+  generate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+      - name: Go generate and diff
+        run: DELTA_CHECK=true make generate
diff --git a/Makefile b/Makefile
@@ -54,6 +54,3 @@ golden:
 	go test ./cmd/kubectl-flyte/cmd -update
 	go test ./pkg/compiler/test -update
 
-.PHONY: generate
-generate: download_tooling
-	@go generate ./...
diff --git a/README.md b/README.md
@@ -114,15 +114,15 @@ To delete a specific workflow
 ```
 
 To delete all completed workflows - they have to be either success/failed with a special isCompleted label set on them.
-The Label is set `here <https://github.com/flyteorg/flytepropeller/blob/master/pkg/controller/controller.go#L247>`
+The Label is set [here](https://github.com/flyteorg/flytepropeller/blob/master/pkg/controller/controller.go#L247)
 
 ```
    $ kubectl-flyte delete --namespace flytekit-development --all-completed
 ```
 
 Running propeller locally
 -------------------------
-use the config.yaml in root found `here <https://github.com/flyteorg/flytepropeller/blob/master/config.yaml>`. Cd into
+use the config.yaml in root found [here](https://github.com/flyteorg/flytepropeller/blob/master/config.yaml). Cd into
 this folder and then run
 
 ```

diff --git a/boilerplate/flyte/end2end/end2end.sh b/boilerplate/flyte/end2end/end2end.sh
@@ -12,6 +12,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 OUT="${DIR}/tmp"
 rm -rf ${OUT}
 git clone https://github.com/flyteorg/flyte.git "${OUT}"
+
 pushd ${OUT}
 
 make end2end_execute

diff --git a/boilerplate/flyte/golang_support_tools/go.mod b/boilerplate/flyte/golang_support_tools/go.mod
@@ -1,15 +1,193 @@
 module github.com/flyteorg/boilerplate
 
-go 1.16
+go 1.17
 
 require (
 	github.com/alvaroloes/enumer v1.1.2
-	github.com/flyteorg/flytestdlib v0.3.22
+	github.com/flyteorg/flytestdlib v0.4.7
 	github.com/golangci/golangci-lint v1.38.0
-	github.com/pseudomuto/protoc-gen-doc v1.4.1 // indirect
+	github.com/pseudomuto/protoc-gen-doc v1.4.1
 	github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5
 )
 
+require (
+	4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a // indirect
+	cloud.google.com/go v0.75.0 // indirect
+	cloud.google.com/go/storage v1.12.0 // indirect
+	github.com/Azure/azure-sdk-for-go v51.0.0+incompatible // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.17 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.10 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.0 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/Masterminds/sprig v2.15.0+incompatible // indirect
+	github.com/OpenPeeDeeP/depguard v1.0.1 // indirect
+	github.com/alexkohler/prealloc v1.0.0 // indirect
+	github.com/aokoli/goutils v1.0.1 // indirect
+	github.com/ashanbrown/forbidigo v1.1.0 // indirect
+	github.com/ashanbrown/makezero v0.0.0-20201205152432-7b7cdbb3025a // indirect
+	github.com/aws/aws-sdk-go v1.37.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bkielbasa/cyclop v1.2.0 // indirect
+	github.com/bombsimon/wsl/v3 v3.2.0 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/charithe/durationcheck v0.0.6 // indirect
+	github.com/coocood/freecache v1.1.1 // indirect
+	github.com/daixiang0/gci v0.2.8 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/denis-tingajkin/go-header v0.4.2 // indirect
+	github.com/envoyproxy/protoc-gen-validate v0.3.0-java // indirect
+	github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607 // indirect
+	github.com/esimonov/ifshort v1.0.1 // indirect
+	github.com/fatih/color v1.10.0 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/fzipp/gocyclo v0.3.1 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-critic/go-critic v0.5.4 // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-toolsmith/astcast v1.0.0 // indirect
+	github.com/go-toolsmith/astcopy v1.0.0 // indirect
+	github.com/go-toolsmith/astequal v1.0.0 // indirect
+	github.com/go-toolsmith/astfmt v1.0.0 // indirect
+	github.com/go-toolsmith/astp v1.0.0 // indirect
+	github.com/go-toolsmith/strparse v1.0.0 // indirect
+	github.com/go-toolsmith/typep v1.0.2 // indirect
+	github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gofrs/flock v0.8.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
+	github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
+	github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 // indirect
+	github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a // indirect
+	github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 // indirect
+	github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca // indirect
+	github.com/golangci/misspell v0.3.5 // indirect
+	github.com/golangci/revgrep v0.0.0-20210208091834-cd28932614b5 // indirect
+	github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 // indirect
+	github.com/google/go-cmp v0.5.4 // indirect
+	github.com/google/uuid v1.1.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/gordonklaus/ineffassign v0.0.0-20210225214923-2e10b2664254 // indirect
+	github.com/gostaticanalysis/analysisutil v0.4.1 // indirect
+	github.com/gostaticanalysis/comment v1.4.1 // indirect
+	github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5 // indirect
+	github.com/gostaticanalysis/nilerr v0.1.1 // indirect
+	github.com/graymeta/stow v0.2.7 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/huandu/xstrings v1.0.0 // indirect
+	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jgautheron/goconst v1.4.0 // indirect
+	github.com/jingyugao/rowserrcheck v0.0.0-20210130005344-c6a0c12dd98d // indirect
+	github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jstemmer/go-junit-report v0.9.1 // indirect
+	github.com/julz/importas v0.0.0-20210226073942-60b4fa260dd0 // indirect
+	github.com/kisielk/errcheck v1.6.0 // indirect
+	github.com/kisielk/gotool v1.0.0 // indirect
+	github.com/kulti/thelper v0.4.0 // indirect
+	github.com/kunwardeep/paralleltest v1.0.2 // indirect
+	github.com/kyoh86/exportloopref v0.1.8 // indirect
+	github.com/magefile/mage v1.10.0 // indirect
+	github.com/magiconair/properties v1.8.4 // indirect
+	github.com/maratori/testpackage v1.0.1 // indirect
+	github.com/matoous/godox v0.0.0-20210227103229-6504466cf951 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-runewidth v0.0.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mbilski/exhaustivestruct v1.2.0 // indirect
+	github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 // indirect
+	github.com/mgechev/revive v1.0.3 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/moricho/tparallel v0.2.1 // indirect
+	github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 // indirect
+	github.com/nakabonne/nestif v0.3.0 // indirect
+	github.com/nbutton23/zxcvbn-go v0.0.0-20201221231540-e56b841a3c88 // indirect
+	github.com/ncw/swift v1.0.53 // indirect
+	github.com/nishanths/exhaustive v0.1.0 // indirect
+	github.com/nishanths/predeclared v0.2.1 // indirect
+	github.com/olekukonko/tablewriter v0.0.4 // indirect
+	github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 // indirect
+	github.com/pelletier/go-toml v1.8.1 // indirect
+	github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/polyfloyd/go-errorlint v0.0.0-20201127212506-19bd8db6546f // indirect
+	github.com/prometheus/client_golang v1.9.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.15.0 // indirect
+	github.com/prometheus/procfs v0.3.0 // indirect
+	github.com/pseudomuto/protokit v0.2.0 // indirect
+	github.com/quasilyte/go-ruleguard v0.3.0 // indirect
+	github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect
+	github.com/ryancurrah/gomodguard v1.2.0 // indirect
+	github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect
+	github.com/sanposhiho/wastedassign v0.1.3 // indirect
+	github.com/satori/go.uuid v1.2.0 // indirect
+	github.com/securego/gosec/v2 v2.6.1 // indirect
+	github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
+	github.com/sirupsen/logrus v1.8.0 // indirect
+	github.com/sonatard/noctx v0.0.1 // indirect
+	github.com/sourcegraph/go-diff v0.6.1 // indirect
+	github.com/spf13/afero v1.5.1 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/cobra v1.1.3 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.7.1 // indirect
+	github.com/ssgreg/nlreturn/v2 v2.1.0 // indirect
+	github.com/stretchr/objx v0.3.0 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b // indirect
+	github.com/tetafro/godot v1.4.4 // indirect
+	github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 // indirect
+	github.com/tomarrell/wrapcheck v0.0.0-20201130113247-1683564d9756 // indirect
+	github.com/tommy-muehle/go-mnd/v2 v2.3.1 // indirect
+	github.com/ultraware/funlen v0.0.3 // indirect
+	github.com/ultraware/whitespace v0.0.4 // indirect
+	github.com/uudashr/gocognit v1.0.1 // indirect
+	go.opencensus.io v0.22.6 // indirect
+	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
+	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect
+	golang.org/x/mod v0.4.1 // indirect
+	golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect
+	golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 // indirect
+	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
+	golang.org/x/text v0.3.5 // indirect
+	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+	golang.org/x/tools v0.1.0 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/api v0.38.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20210126160654-44e461bb6506 // indirect
+	google.golang.org/grpc v1.35.0 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	honnef.co/go/tools v0.1.2 // indirect
+	k8s.io/apimachinery v0.20.2 // indirect
+	k8s.io/client-go v0.0.0-20210217172142-7279fc64d847 // indirect
+	k8s.io/klog/v2 v2.5.0 // indirect
+	mvdan.cc/gofumpt v0.1.0 // indirect
+	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
+	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
+	mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7 // indirect
+)
+
 replace github.com/vektra/mockery => github.com/enghabu/mockery v0.0.0-20191009061720-9d0c8670c2f0
 
 replace github.com/pseudomuto/protoc-gen-doc => github.com/flyteorg/protoc-gen-doc v1.4.2