OCPBUGS-17157: scripts: add a Go-based bumper, sync upstream

Makefile

@@ -139,15 +139,18 @@ vendor:
 manifests: ## Generate manifests
 	OLM_VERSION=$(OLM_VERSION) ./scripts/generate_crds_manifests.sh
 
+.PHONY: generate-manifests
+generate-manifests: OLM_VERSION=0.0.1-snapshot
+generate-manifests: manifests
+
 .PHONY: diff
 diff:
 	git diff --stat HEAD --ignore-submodules --exit-code
 
 verify-vendor: vendor
 	$(MAKE) diff
 
-verify-manifests: OLM_VERSION=0.0.1-snapshot
-verify-manifests: manifests
+verify-manifests: generate-manifests
 	$(MAKE) diff
 
 verify-nested-vendor:

scripts/README.md

@@ -86,9 +86,12 @@ or amended to the last commit of the branch.
 
 Once `make -k verify` is resolved, create a PR from this working sync branch.
 
-## TODO
+# Long-lived Carry Commits
 
-* Add `make verify` to the `sync_pop_candidate.sh` and/or `sync.sh` scripts.
+It is required at times to write commits that will live in the `vendor/` directory
+on top of upstream code and for those commits to be carried on top for the forseeable
+future. In these cases, prefix your commit message with `[CARRY]` to pass the commit
+verification routines.
 
 ## References
 1. [Downstream to operator-framework-olm](https://spaces.redhat.com/display/OOLM/Downstream+to+operator-framework-olm)

scripts/bumper/go.mod

@@ -0,0 +1,7 @@
+module github.com/openshift/operator-framework-olm/scripts/bumper
+
+go 1.20
+
+require github.com/sirupsen/logrus v1.9.3
+
+require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect

scripts/bumper/go.sum

@@ -0,0 +1,15 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

scripts/bumper/main.go

@@ -0,0 +1,320 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/fs"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+type mode string
+
+const (
+	summarize   mode = "summarize"
+	synchronize mode = "synchronize"
+)
+
+type options struct {
+	stagingDir       string
+	commitFileOutput string
+	commitFileInput  string
+	mode             string
+	logLevel         string
+}
+
+func (o *options) Bind(fs *flag.FlagSet) {
+	fs.StringVar(&o.stagingDir, "staging-dir", "staging/", "Directory for staging repositories.")
+	fs.StringVar(&o.mode, "mode", string(summarize), "Operation mode.")
+	fs.StringVar(&o.commitFileOutput, "commits-output", "", "File to write commits data to after resolving what needs to be synced.")
+	fs.StringVar(&o.commitFileInput, "commits-input", "", "File to read commits data from in order to drive sync process.")
+	fs.StringVar(&o.logLevel, "log-level", logrus.InfoLevel.String(), "Logging level.")
+}
+
+func (o *options) Validate() error {
+	switch mode(o.mode) {
+	case summarize, synchronize:
+	default:
+		return fmt.Errorf("--mode must be one of %v", []mode{summarize, synchronize})
+	}
+
+	if _, err := logrus.ParseLevel(o.logLevel); err != nil {
+		return fmt.Errorf("--log-level invalid: %w", err)
+	}
+	return nil
+}
+
+func main() {
+	logger := logrus.New()
+	opts := options{}
+	opts.Bind(flag.CommandLine)
+	flag.Parse()
+
+	if err := opts.Validate(); err != nil {
+		logger.WithError(err).Fatal("invalid options")
+	}
+
+	logLevel, _ := logrus.ParseLevel(opts.logLevel)
+	logger.SetLevel(logLevel)
+
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer cancel()
+
+	var commits []commit
+	var err error
+	if opts.commitFileInput != "" {
+		rawCommits, err := os.ReadFile(opts.commitFileInput)
+		if err != nil {
+			logrus.WithError(err).Fatal("could not read input file")
+		}
+		if err := json.Unmarshal(rawCommits, &commits); err != nil {
+			logrus.WithError(err).Fatal("could not unmarshal input commits")
+		}
+	} else {
+		commits, err = detectNewCommits(ctx, logger.WithField("phase", "detect"), opts.stagingDir)
+		if err != nil {
+			logger.WithError(err).Fatal("failed to detect commits")
+		}
+	}
+
+	if opts.commitFileOutput != "" {
+		commitsJson, err := json.Marshal(commits)
+		if err != nil {
+			logrus.WithError(err).Fatal("could not marshal commits")
+		}
+		if err := os.WriteFile(opts.commitFileOutput, commitsJson, 0666); err != nil {
+			logrus.WithError(err).Fatal("could not write commits")
+		}
+	}
+
+	switch mode(opts.mode) {
+	case summarize:
+		writer := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0)
+		for _, commit := range commits {
+			if _, err := fmt.Fprintln(writer, commit.Date.Format(time.DateTime)+"\t"+"operator-framework/"+commit.Repo+"\t", commit.Hash+"\t"+commit.Author+"\t"+commit.Message); err != nil {
+				logger.WithError(err).Error("failed to write output")
+			}
+		}
+		if err := writer.Flush(); err != nil {
+			logger.WithError(err).Error("failed to flush output")
+		}
+	case synchronize:
+		for i, commit := range commits {
+			commitLogger := logger.WithField("commit", commit.Hash)
+			commitLogger.Infof("cherry-picking commit %d/%d", i+1, len(commits))
+			if err := cherryPick(ctx, commitLogger, commit); err != nil {
+				logger.WithError(err).Error("failed to cherry-pick commit")
+				break
+			}
+		}
+	}
+}
+
+type commit struct {
+	Date    time.Time `json:"date"`
+	Hash    string    `json:"hash,omitempty"`
+	Author  string    `json:"author,omitempty"`
+	Message string    `json:"message,omitempty"`
+	Repo    string    `json:"repo,omitempty"`
+}
+
+var repoRegex = regexp.MustCompile(`Upstream-repository: ([^ ]+)\n`)
+var commitRegex = regexp.MustCompile(`Upstream-commit: ([a-f0-9]+)\n`)
+
+func detectNewCommits(ctx context.Context, logger *logrus.Entry, stagingDir string) ([]commit, error) {
+	lastCommits := map[string]string{}
+	if err := fs.WalkDir(os.DirFS(stagingDir), ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d == nil || !d.IsDir() {
+			return nil
+		}
+
+		if path == "." {
+			return nil
+		}
+		logger = logger.WithField("repo", path)
+		logger.Debug("detecting commits")
+		output, err := runCommand(logger, exec.CommandContext(ctx,
+			"git", "log",
+			"-n", "1",
+			"--grep", "Upstream-repository: "+path,
+			"--grep", "Upstream-commit",
+			"--all-match",
+			"--pretty=%B",
+			"--",
+			filepath.Join(stagingDir, path),
+		))
+		if err != nil {
+			return err
+		}
+		var lastCommit string
+		commitMatches := commitRegex.FindStringSubmatch(output)
+		if len(commitMatches) > 0 {
+			if len(commitMatches[0]) > 1 {
+				lastCommit = string(commitMatches[1])
+			}
+		}
+		if lastCommit != "" {
+			logger.WithField("commit", lastCommit).Debug("found last commit synchronized with staging")
+			lastCommits[path] = lastCommit
+		}
+
+		if path != "." {
+			return fs.SkipDir
+		}
+		return nil
+	}); err != nil {
+		return nil, fmt.Errorf("failed to walk %s: %w", stagingDir, err)
+	}
+
+	var commits []commit
+	for repo, lastCommit := range lastCommits {
+		if _, err := runCommand(logger, exec.CommandContext(ctx,
+			"git", "fetch",
+			"git@github.com:operator-framework/"+repo,
+			"master",
+		)); err != nil {
+			return nil, err
+		}
+
+		output, err := runCommand(logger, exec.CommandContext(ctx,
+			"git", "log",
+			"--pretty=%H",
+			lastCommit+"...FETCH_HEAD",
+		))
+		if err != nil {
+			return nil, err
+		}
+
+		for _, line := range strings.Split(output, "\n") {
+			line = strings.TrimSpace(line)
+			if line != "" {
+				infoCmd := exec.CommandContext(ctx,
+					"git", "show",
+					line,
+					"--pretty=format:%H\u00A0%cI\u00A0%an\u00A0%s",
+					"--quiet",
+				)
+				stdout, stderr := bytes.Buffer{}, bytes.Buffer{}
+				infoCmd.Stdout = &stdout
+				infoCmd.Stderr = &stderr
+				logger.WithField("command", infoCmd.String()).Debug("running command")
+				if err := infoCmd.Run(); err != nil {
+					return nil, fmt.Errorf("failed to run command: %s %s: %w", stdout.String(), stderr.String(), err)
+				}
+				parts := strings.Split(stdout.String(), "\u00A0")
+				if len(parts) != 4 {
+					return nil, fmt.Errorf("incorrect parts from git output: %v", stdout.String())
+				}
+				committedTime, err := time.Parse(time.RFC3339, parts[1])
+				if err != nil {
+					return nil, fmt.Errorf("invalid time %s: %w", parts[1], err)
+				}
+				commits = append(commits, commit{
+					Hash:    parts[0],
+					Date:    committedTime,
+					Author:  parts[2],
+					Message: parts[3],
+					Repo:    repo,
+				})
+			}
+		}
+	}
+	sort.Slice(commits, func(i, j int) bool {
+		return commits[i].Date.Before(commits[j].Date)
+	})
+	return commits, nil
+}
+
+func cherryPick(ctx context.Context, logger *logrus.Entry, c commit) error {
+	{
+		output, err := runCommand(logger, exec.CommandContext(ctx,
+			"git", "cherry-pick",
+			"--allow-empty", "--keep-redundant-commits",
+			"-Xsubtree=staging/"+c.Repo, c.Hash,
+		))
+		if err != nil {
+			if strings.Contains(output, "vendor/modules.txt deleted in HEAD and modified in") {
+				// we remove vendor directories for everything under staging/, but some of the upstream repos have them
+				if _, err := runCommand(logger, exec.CommandContext(ctx,
+					"git", "rm", "--cached", "-r", "--ignore-unmatch", "staging/"+c.Repo+"/vendor",
+				)); err != nil {
+					return err
+				}
+				if _, err := runCommand(logger, exec.CommandContext(ctx,
+					"git", "cherry-pick", "--continue",
+				)); err != nil {
+					return err
+				}
+			} else {
+				return err
+			}
+		}
+	}
+
+	for _, cmd := range []*exec.Cmd{
+		withEnv(exec.CommandContext(ctx,
+			"go", "mod", "tidy",
+		), os.Environ()...),
+		withEnv(exec.CommandContext(ctx,
+			"go", "mod", "vendor",
+		), os.Environ()...),
+		withEnv(exec.CommandContext(ctx,
+			"go", "mod", "verify",
+		), os.Environ()...),
+		withEnv(exec.CommandContext(ctx,
+			"make", "generate-manifests",
+		), os.Environ()...),
+		exec.CommandContext(ctx,
+			"git", "add",
+			"staging/"+c.Repo,
+			"vendor", "go.mod", "go.sum",
+			"manifests", "pkg/manifests",
+		),
+		exec.CommandContext(ctx,
+			"git", "commit",
+			"--amend", "--allow-empty", "--no-edit",
+			"--trailer", "Upstream-repository: "+c.Repo,
+			"--trailer", "Upstream-commit: "+c.Hash,
+			"staging/"+c.Repo,
+			"vendor", "go.mod", "go.sum",
+			"manifests", "pkg/manifests",
+		),
+	} {
+		if _, err := runCommand(logger, cmd); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func runCommand(logger *logrus.Entry, cmd *exec.Cmd) (string, error) {
+	output := bytes.Buffer{}
+	cmd.Stdout = &output
+	cmd.Stderr = &output
+	logger.WithField("command", cmd.String()).Debug("running command")
+	if err := cmd.Run(); err != nil {
+		return output.String(), fmt.Errorf("failed to run command: %s: %w", output.String(), err)
+	}
+	return output.String(), nil
+}
+
+func withEnv(command *exec.Cmd, env ...string) *exec.Cmd {
+	command.Env = append(command.Env, env...)
+	return command
+}

scripts/verify_commits.sh

@@ -37,9 +37,12 @@ function verify_downstream_only() {
     local inside_staging
     inside_staging="$(git show --name-only "${downstream_commit}" -- staging)"
     if [[ -n "${inside_staging}" ]]; then
-        err "downstream non-staging commit ${downstream_commit} changes staging"
+        if git log -n 1 "${downstream_commit}" --pretty=%s | grep -q '[CARRY]'; then
+          return 0
+        fi
+        err "downstream non-staging commit ${downstream_commit} changes staging and is not labeled [CARRY]"
         err "${inside_staging}"
-        err "only staging commits (i.e. from an upstream cherry-pick) may change staging"
+        err "only staging commits (i.e. from an upstream cherry-pick) or commits labeled as downstream carries with [CARRY] may change staging"
         return 1
     fi
 }