Merge branch 'master' into dependabot/go_modules/github.com/stretchr/…

…testify-1.8.4

.github/workflows/ci.yml

@@ -2,10 +2,10 @@ name: CI
 
 on:
   push:
+    branches:
+      - master
     tags:
       - v*
-    branches-ignore:
-      - gh-pages
   pull_request:
     branches-ignore:
       - gh-pages

.github/workflows/test.yml

@@ -214,7 +214,7 @@ jobs:
 
           echo -e "### Git status" >> $GITHUB_STEP_SUMMARY
           if [[ "$changed" -gt 0 ]]; then
-            echo -e "Generated proto files are out of date. Please run 'make proto' and commit the changes." >> $GITHUB_STEP_SUMMARY
+            echo -e "Generated proto files are out of date. Please run 'mage proto' and commit the changes." >> $GITHUB_STEP_SUMMARY
 
             git status -s -uno >> $GITHUB_STEP_SUMMARY
 

CONTRIBUTING.md

@@ -53,6 +53,42 @@ Note the names of the branch must follow proper docker names:
 
 >A tag name must be valid ASCII and may contain lowercase and uppercase letters, digits, underscores, periods and dashes. A tag name may not start with a period or a dash and may contain a maximum of 128 characters.
 
+#### Signing Off Commits
+
+To enhance the integrity of contributions to the Armada repository, we've adopted the use of the DCO (Developer Certificate of Origin) plug-in. This means that for every commit you contribute via Pull Requests, you'll need to sign off your commits to certify that you have the right to submit it under the open source license used by this project.
+
+**Every commit in your PRs must have a "Signed-Off" attribute.**
+
+When committing to the repository, ensure you use the `--signoff` option with `git commit`. This will append a sign-off message at the end of the commit log to indicate that the commit has your signature.
+
+You sign-off by adding the following to your commit messages:
+
+```
+Author: Your Name <your.name@example.com>
+Date:   Thu Feb 2 11:41:15 2018 -0800
+
+    This is my commit message
+
+    Signed-off-by: Your Name <your.name@example.com>
+```
+
+Notice the `Author` and `Signed-off-by` lines match. If they don't, the PR will
+be rejected by the automated DCO check.
+
+Git has a `-s` command line option to do this automatically:
+
+    git commit -s -m 'This is my commit message'
+
+If you forgot to do this and have not yet pushed your changes to the remote
+repository, you can amend your commit with the sign-off by running 
+
+    git commit --amend -s
+
+This command will modify the latest commit and add the required sign-off.    
+
+For more details checkout [DCO](https://github.com/apps/dco)
+
+
 ## Chat & Discussions
 
 Sometimes, it's good to hash things out in real time.

client/python/CONTRIBUTING.md

@@ -26,7 +26,7 @@ workflow for contributing. First time contributors can follow the guide below to
 Unlike most python projects, the Armada python client contains a large quantity of generated code. This code must be
 generated in order to compile and develop against the client.
 
-From the root of the repository, run `make python`. This will generate python code needed to build
+From the root of the repository, run `mage buildPython`. This will generate python code needed to build
 and use the client. This command needs to be re-run anytime an API change is committed (e.g. a change to a `*.proto`
 file).
 

client/python/README.md

@@ -26,5 +26,5 @@ Before beginning, ensure you have:
 - Network access to fetch docker images and go dependencies.
 
 To generate all needed code, and install the python client:
-1) From the root of the repository, run `make python`
+1) From the root of the repository, run `mage buildPython`
 2) Install the client using `pip install client/python`. It's strongly recommended you do this inside a virtualenv.

client/python/docs/README.md

@@ -9,13 +9,13 @@ Usage
 
 Easy way:
 - Ensure all protobufs files needed for the client are generated by running
-  `make python` from the repository root.
+  `mage buildPython` from the repository root.
 - `tox -e docs` will create a valid virtual environment and use it to generate
   documentation. The generated files will be placed under `build/jekyll/*.md`.
 
 Manual way:
 - Ensure all protobufs files needed for the client are generated by running
-  `make python` from the repository root.
+  `mage buildPython` from the repository root.
 - Create a virtual environment containing all the deps listed in `tox.ini`
   under `[testenv:docs]`.
 - Run `poetry install -v` from inside `client/python` to install the client

cmd/armada/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"fmt"
 	"net/http"
 	_ "net/http/pprof"
@@ -13,11 +12,11 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
-	"golang.org/x/sync/errgroup"
 
 	"github.com/armadaproject/armada/internal/armada"
 	"github.com/armadaproject/armada/internal/armada/configuration"
 	"github.com/armadaproject/armada/internal/common"
+	"github.com/armadaproject/armada/internal/common/armadacontext"
 	gateway "github.com/armadaproject/armada/internal/common/grpc"
 	"github.com/armadaproject/armada/internal/common/health"
 	"github.com/armadaproject/armada/internal/common/logging"
@@ -67,7 +66,7 @@ func main() {
 	}
 
 	// Run services within an errgroup to propagate errors between services.
-	g, ctx := errgroup.WithContext(context.Background())
+	g, ctx := armadacontext.ErrGroup(armadacontext.Background())
 
 	// Cancel the errgroup context on SIGINT and SIGTERM,
 	// which shuts everything down gracefully.

cmd/eventsprinter/logic/logic.go

@@ -1,14 +1,14 @@
 package logic
 
 import (
-	"context"
 	"fmt"
 	"time"
 
 	"github.com/apache/pulsar-client-go/pulsar"
 	"github.com/gogo/protobuf/proto"
 	v1 "k8s.io/api/core/v1"
 
+	"github.com/armadaproject/armada/internal/common/armadacontext"
 	"github.com/armadaproject/armada/internal/common/util"
 	"github.com/armadaproject/armada/pkg/armadaevents"
 )
@@ -18,7 +18,7 @@ func PrintEvents(url, topic, subscription string, verbose bool) error {
 	fmt.Println("URL:", url)
 	fmt.Println("Topic:", topic)
 	fmt.Println("Subscription", subscription)
-	return withSetup(url, topic, subscription, func(ctx context.Context, producer pulsar.Producer, consumer pulsar.Consumer) error {
+	return withSetup(url, topic, subscription, func(ctx *armadacontext.Context, producer pulsar.Producer, consumer pulsar.Consumer) error {
 		// Number of active jobs.
 		numJobs := 0
 
@@ -199,7 +199,7 @@ func stripPodSpec(spec *v1.PodSpec) *v1.PodSpec {
 }
 
 // Run action with an Armada submit client and a Pulsar producer and consumer.
-func withSetup(url, topic, subscription string, action func(ctx context.Context, producer pulsar.Producer, consumer pulsar.Consumer) error) error {
+func withSetup(url, topic, subscription string, action func(ctx *armadacontext.Context, producer pulsar.Producer, consumer pulsar.Consumer) error) error {
 	pulsarClient, err := pulsar.NewClient(pulsar.ClientOptions{
 		URL: url,
 	})
@@ -225,5 +225,5 @@ func withSetup(url, topic, subscription string, action func(ctx context.Context,
 	}
 	defer consumer.Close()
 
-	return action(context.Background(), producer, consumer)
+	return action(armadacontext.Background(), producer, consumer)
 }

cmd/executor/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"net/http"
 	"os"
 	"os/signal"
@@ -13,6 +12,7 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/armadaproject/armada/internal/common"
+	"github.com/armadaproject/armada/internal/common/armadacontext"
 	"github.com/armadaproject/armada/internal/common/health"
 	"github.com/armadaproject/armada/internal/executor"
 	"github.com/armadaproject/armada/internal/executor/configuration"
@@ -55,7 +55,7 @@ func main() {
 	)
 	defer shutdownMetricServer()
 
-	shutdown, wg := executor.StartUp(context.Background(), logrus.NewEntry(logrus.New()), config)
+	shutdown, wg := executor.StartUp(armadacontext.Background(), logrus.NewEntry(logrus.New()), config)
 	go func() {
 		<-shutdownChannel
 		shutdown()

cmd/lookoutv2/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"os"
 	"os/signal"
 	"syscall"
@@ -12,6 +11,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/clock"
 
 	"github.com/armadaproject/armada/internal/common"
+	"github.com/armadaproject/armada/internal/common/armadacontext"
 	"github.com/armadaproject/armada/internal/common/database"
 	"github.com/armadaproject/armada/internal/lookoutv2"
 	"github.com/armadaproject/armada/internal/lookoutv2/configuration"
@@ -36,9 +36,9 @@ func init() {
 	pflag.Parse()
 }
 
-func makeContext() (context.Context, func()) {
-	ctx := context.Background()
-	ctx, cancel := context.WithCancel(ctx)
+func makeContext() (*armadacontext.Context, func()) {
+	ctx := armadacontext.Background()
+	ctx, cancel := armadacontext.WithCancel(ctx)
 
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
@@ -57,7 +57,7 @@ func makeContext() (context.Context, func()) {
 	}
 }
 
-func migrate(ctx context.Context, config configuration.LookoutV2Configuration) {
+func migrate(ctx *armadacontext.Context, config configuration.LookoutV2Configuration) {
 	db, err := database.OpenPgxPool(config.Postgres)
 	if err != nil {
 		panic(err)
@@ -74,7 +74,7 @@ func migrate(ctx context.Context, config configuration.LookoutV2Configuration) {
 	}
 }
 
-func prune(ctx context.Context, config configuration.LookoutV2Configuration) {
+func prune(ctx *armadacontext.Context, config configuration.LookoutV2Configuration) {
 	db, err := database.OpenPgxConn(config.Postgres)
 	if err != nil {
 		panic(err)
@@ -92,7 +92,7 @@ func prune(ctx context.Context, config configuration.LookoutV2Configuration) {
 	log.Infof("expireAfter: %v, batchSize: %v, timeout: %v",
 		config.PrunerConfig.ExpireAfter, config.PrunerConfig.BatchSize, config.PrunerConfig.Timeout)
 
-	ctxTimeout, cancel := context.WithTimeout(ctx, config.PrunerConfig.Timeout)
+	ctxTimeout, cancel := armadacontext.WithTimeout(ctx, config.PrunerConfig.Timeout)
 	defer cancel()
 	err = pruner.PruneDb(ctxTimeout, db, config.PrunerConfig.ExpireAfter, config.PrunerConfig.BatchSize, clock.RealClock{})
 	if err != nil {

cmd/scheduler/cmd/migrate_database.go

@@ -1,13 +1,13 @@
 package cmd
 
 import (
-	"context"
 	"time"
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/armadaproject/armada/internal/common/armadacontext"
 	"github.com/armadaproject/armada/internal/common/database"
 	schedulerdb "github.com/armadaproject/armada/internal/scheduler/database"
 )
@@ -43,7 +43,7 @@ func migrateDatabase(cmd *cobra.Command, _ []string) error {
 		return errors.WithMessagef(err, "Failed to connect to database")
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), timeout)
 	defer cancel()
 	return schedulerdb.Migrate(ctx, db)
 }

cmd/scheduler/cmd/prune_database.go

@@ -1,13 +1,13 @@
 package cmd
 
 import (
-	"context"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"k8s.io/apimachinery/pkg/util/clock"
 
+	"github.com/armadaproject/armada/internal/common/armadacontext"
 	"github.com/armadaproject/armada/internal/common/database"
 	schedulerdb "github.com/armadaproject/armada/internal/scheduler/database"
 )
@@ -57,7 +57,7 @@ func pruneDatabase(cmd *cobra.Command, _ []string) error {
 		return errors.WithMessagef(err, "Failed to connect to database")
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), timeout)
 	defer cancel()
 	return schedulerdb.PruneDb(ctx, db, batchSize, expireAfter, clock.RealClock{})
 }

docs/developer/manual-localdev.md

@@ -28,7 +28,7 @@ mage BootstrapTools
 # Compile .pb.go files from .proto files
 # (only necessary after changing a .proto file).
 mage proto
-make dotnet
+mage dotnet
 
 # Build the Docker images containing all Armada components.
 # Only the main "bundle" is needed for quickly testing Armada.