Skip to content
Permalink
Browse files
fix(build): virtual merge commits and inconsistent build cache
An inconsistency in the werf/actions or github-actions lead to invalid
--virtual-merge-into-commit param passed into werf. It leads to
creating inconsistent stage in the repo by the werf.

Solution: remove --virtual-merge-into-commit and
--virtual-merge-from-commit params and detect from/into commits by
itself reading virtual-merge commit metadata from the git.
  • Loading branch information
distorhead committed Jan 19, 2022
1 parent 4ec0151 commit 73729924905ef03a199a5d2fb26cccded5f6c69a
@@ -1,4 +1,4 @@
.PHONY: all werf buildah-test unit-test fmt lint clean
.PHONY: all werf buildah-test unit-test fmt lint docs clean

all: werf

@@ -20,6 +20,10 @@ lint:
golangci-lint run ./... --build-tags="dfrunmount dfssh containers_image_openpgp osusergo exclude_graphdriver_devicemapper netgo no_devmapper static_build"


docs:
./docs/regen.sh


clean:
rm -f $$GOPATH/bin/werf
rm -f $$GOPATH/buildah-test
@@ -107,8 +107,6 @@ If one or more IMAGE_NAME parameters specified, werf will build only these image

common.SetupAddCustomTag(&commonCmdData, cmd)
common.SetupVirtualMerge(&commonCmdData, cmd)
common.SetupVirtualMergeFromCommit(&commonCmdData, cmd)
common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd)

common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultBuildParallelTasksLimit)
common.SetupFollow(&commonCmdData, cmd)
@@ -7,7 +7,7 @@ import (
"time"

"github.com/spf13/cobra"
"helm.sh/helm/v3/cmd/helm"
helm_v3 "helm.sh/helm/v3/cmd/helm"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli/values"
@@ -107,8 +107,6 @@ func NewCmd() *cobra.Command {

common.SetupUseCustomTag(&commonCmdData, cmd)
common.SetupVirtualMerge(&commonCmdData, cmd)
common.SetupVirtualMergeFromCommit(&commonCmdData, cmd)
common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd)

common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultBuildParallelTasksLimit)

@@ -10,7 +10,7 @@ import (
"github.com/Masterminds/semver"
uuid "github.com/satori/go.uuid"
"github.com/spf13/cobra"
"helm.sh/helm/v3/cmd/helm"
helm_v3 "helm.sh/helm/v3/cmd/helm"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli/values"
@@ -117,8 +117,6 @@ Published into container registry bundle can be rolled out by the "werf bundle"

common.SetupUseCustomTag(&commonCmdData, cmd)
common.SetupVirtualMerge(&commonCmdData, cmd)
common.SetupVirtualMergeFromCommit(&commonCmdData, cmd)
common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd)

common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultBuildParallelTasksLimit)

@@ -111,9 +111,7 @@ type CmdData struct {
ReportPath *string
ReportFormat *string

VirtualMerge *bool
VirtualMergeFromCommit *string
VirtualMergeIntoCommit *string
VirtualMerge *bool

ScanContextNamespaceOnly *bool

@@ -1504,16 +1502,6 @@ func SetupVirtualMerge(cmdData *CmdData, cmd *cobra.Command) {
cmd.Flags().BoolVarP(cmdData.VirtualMerge, "virtual-merge", "", GetBoolEnvironmentDefaultFalse("WERF_VIRTUAL_MERGE"), "Enable virtual/ephemeral merge commit mode when building current application state ($WERF_VIRTUAL_MERGE by default)")
}

func SetupVirtualMergeFromCommit(cmdData *CmdData, cmd *cobra.Command) {
cmdData.VirtualMergeFromCommit = new(string)
cmd.Flags().StringVarP(cmdData.VirtualMergeFromCommit, "virtual-merge-from-commit", "", os.Getenv("WERF_VIRTUAL_MERGE_FROM_COMMIT"), "Commit hash for virtual/ephemeral merge commit with new changes introduced in the pull request ($WERF_VIRTUAL_MERGE_FROM_COMMIT by default)")
}

func SetupVirtualMergeIntoCommit(cmdData *CmdData, cmd *cobra.Command) {
cmdData.VirtualMergeIntoCommit = new(string)
cmd.Flags().StringVarP(cmdData.VirtualMergeIntoCommit, "virtual-merge-into-commit", "", os.Getenv("WERF_VIRTUAL_MERGE_INTO_COMMIT"), "Commit hash for virtual/ephemeral merge commit which is base for changes introduced in the pull request ($WERF_VIRTUAL_MERGE_INTO_COMMIT by default)")
}

func SetupPlatform(cmdData *CmdData, cmd *cobra.Command) {
cmdData.Platform = new(string)

@@ -18,9 +18,7 @@ import (
func GetConveyorOptions(commonCmdData *CmdData) build.ConveyorOptions {
return build.ConveyorOptions{
LocalGitRepoVirtualMergeOptions: stage.VirtualMergeOptions{
VirtualMerge: *commonCmdData.VirtualMerge,
VirtualMergeFromCommit: *commonCmdData.VirtualMergeFromCommit,
VirtualMergeIntoCommit: *commonCmdData.VirtualMergeIntoCommit,
VirtualMerge: *commonCmdData.VirtualMerge,
},
}
}
@@ -236,8 +236,6 @@ services:
common.SetupDryRun(&commonCmdData, cmd)

common.SetupVirtualMerge(&commonCmdData, cmd)
common.SetupVirtualMergeFromCommit(&commonCmdData, cmd)
common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd)

common.SetupDisableAutoHostCleanup(&commonCmdData, cmd)
common.SetupAllowedDockerStorageVolumeUsage(&commonCmdData, cmd)
@@ -139,8 +139,6 @@ werf converge --repo registry.mydomain.com/web --env production`,

common.SetupUseCustomTag(&commonCmdData, cmd)
common.SetupVirtualMerge(&commonCmdData, cmd)
common.SetupVirtualMergeFromCommit(&commonCmdData, cmd)
common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd)

common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultBuildParallelTasksLimit)
common.SetupSkipBuild(&commonCmdData, cmd)
@@ -97,8 +97,6 @@ All meta-information related to werf is removed from the exported images, and th
common.SetupDryRun(&commonCmdData, cmd)

common.SetupVirtualMerge(&commonCmdData, cmd)
common.SetupVirtualMergeFromCommit(&commonCmdData, cmd)
common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd)

common.SetupPlatform(&commonCmdData, cmd)

@@ -72,8 +72,6 @@ These values includes project name, docker images ids and other`),

common.SetupUseCustomTag(&getAutogeneratedValuedCmdData, cmd)
common.SetupVirtualMerge(&getAutogeneratedValuedCmdData, cmd)
common.SetupVirtualMergeFromCommit(&getAutogeneratedValuedCmdData, cmd)
common.SetupVirtualMergeIntoCommit(&getAutogeneratedValuedCmdData, cmd)

common.SetupNamespace(&getAutogeneratedValuedCmdData, cmd)

@@ -8,7 +8,7 @@ import (
"path/filepath"

"github.com/spf13/cobra"
"helm.sh/helm/v3/cmd/helm"
helm_v3 "helm.sh/helm/v3/cmd/helm"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
@@ -116,8 +116,6 @@ func NewCmd() *cobra.Command {

common.SetupUseCustomTag(&commonCmdData, cmd)
common.SetupVirtualMerge(&commonCmdData, cmd)
common.SetupVirtualMergeFromCommit(&commonCmdData, cmd)
common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd)

common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultBuildParallelTasksLimit)

@@ -147,8 +147,6 @@ func NewCmd() *cobra.Command {
common.SetupDryRun(&commonCmdData, cmd)

common.SetupVirtualMerge(&commonCmdData, cmd)
common.SetupVirtualMergeFromCommit(&commonCmdData, cmd)
common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd)

common.SetupPlatform(&commonCmdData, cmd)

@@ -83,8 +83,6 @@ func NewCmd() *cobra.Command {
common.SetupKubeContext(&commonCmdData, cmd)

common.SetupVirtualMerge(&commonCmdData, cmd)
common.SetupVirtualMergeFromCommit(&commonCmdData, cmd)
common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd)

common.SetupPlatform(&commonCmdData, cmd)

3 go.mod
@@ -55,7 +55,8 @@ require (
github.com/mvdan/xurls v1.1.0 // indirect
github.com/oleiade/reflections v1.0.1 // indirect
github.com/onsi/ginkgo v1.16.4
github.com/onsi/gomega v1.16.0
github.com/onsi/ginkgo/v2 v2.0.0 // indirect
github.com/onsi/gomega v1.17.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84
github.com/opencontainers/runc v1.0.3 // indirect
5 go.sum
@@ -979,6 +979,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
@@ -1494,6 +1495,8 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ=
github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -1506,6 +1509,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c=
github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -19,6 +19,7 @@ import (
"github.com/werf/logboek/pkg/types"
"github.com/werf/werf/pkg/build/stage"
"github.com/werf/werf/pkg/container_runtime"
"github.com/werf/werf/pkg/git_repo"
imagePkg "github.com/werf/werf/pkg/image"
"github.com/werf/werf/pkg/stapel"
"github.com/werf/werf/pkg/storage"
@@ -272,7 +273,13 @@ func (phase *BuildPhase) publishImageMetadata(ctx context.Context, img *Image) e
commits = append(commits, headCommit)

if phase.Conveyor.GetLocalGitRepoVirtualMergeOptions().VirtualMerge {
fromCommit := phase.Conveyor.GetLocalGitRepoVirtualMergeOptions().VirtualMergeFromCommit
phase.Conveyor.giterminismManager.LocalGitRepo().GetMergeCommitParents(ctx, headCommit)

fromCommit, _, err := git_repo.GetVirtualMergeParents(ctx, phase.Conveyor.giterminismManager.LocalGitRepo(), headCommit)
if err != nil {
return fmt.Errorf("unable to get virtual merge commit %q parents: %s", headCommit, err)
}

commits = append(commits, fromCommit)
}

@@ -1077,7 +1077,7 @@ func gitRemoteArtifactInit(ctx context.Context, remoteGitMappingConfig *config.G
gitMapping.Branch = remoteGitMappingConfig.Branch

gitMapping.Name = remoteGitMappingConfig.Name
gitMapping.RemoteGitRepo = remoteGitRepo
gitMapping.SetGitRepo(remoteGitRepo)

gitMappingTo, err := makeGitMappingTo(ctx, gitMapping, remoteGitMappingConfig.GitLocalExport.GitMappingTo(), c)
if err != nil {
@@ -1092,7 +1092,7 @@ func gitLocalPathInit(ctx context.Context, localGitMappingConfig *config.GitLoca
gitMapping := baseGitMappingInit(localGitMappingConfig.GitLocalExport, imageName, c)

gitMapping.Name = "own"
gitMapping.LocalGitRepo = c.giterminismManager.LocalGitRepo()
gitMapping.SetGitRepo(c.giterminismManager.LocalGitRepo())

gitMappingTo, err := makeGitMappingTo(ctx, gitMapping, localGitMappingConfig.GitLocalExport.GitMappingTo(), c)
if err != nil {
@@ -29,7 +29,5 @@ type Conveyor interface {
}

type VirtualMergeOptions struct {
VirtualMerge bool
VirtualMergeFromCommit string
VirtualMergeIntoCommit string
VirtualMerge bool
}
@@ -50,7 +50,7 @@ func (s *GitArchiveStage) SelectSuitableStage(ctx context.Context, c Conveyor, s
func (s *GitArchiveStage) GetDependencies(ctx context.Context, c Conveyor, _, _ container_runtime.LegacyImageInterface) (string, error) {
var args []string
for _, gitMapping := range s.gitMappings {
if gitMapping.LocalGitRepo != nil {
if gitMapping.IsLocal() {
if err := c.GiterminismManager().Inspector().InspectBuildContextFiles(ctx, gitMapping.getPathMatcher()); err != nil {
return "", err
}
@@ -22,9 +22,6 @@ import (
)

type GitMapping struct {
LocalGitRepo *git_repo.Local
RemoteGitRepo *git_repo.Remote

Name string
As string
Branch string
@@ -45,6 +42,7 @@ type GitMapping struct {

BaseCommitByPrevBuiltImageName map[string]string

gitRepo git_repo.GitRepo
mutexes map[string]*sync.Mutex
mutex sync.Mutex
}
@@ -89,14 +87,12 @@ func (gm *GitMapping) getMutex(key string) *sync.Mutex {
return m
}

func (gm *GitMapping) GitRepo() git_repo.GitRepo {
if gm.LocalGitRepo != nil {
return gm.LocalGitRepo
} else if gm.RemoteGitRepo != nil {
return gm.RemoteGitRepo
}
func (gm *GitMapping) SetGitRepo(gitRepo git_repo.GitRepo) {
gm.gitRepo = gitRepo
}

panic("GitRepo not initialized")
func (gm *GitMapping) GitRepo() git_repo.GitRepo {
return gm.gitRepo
}

func (gm *GitMapping) makeArchiveOptions(ctx context.Context, commit string) (*git_repo.ArchiveOptions, error) {
@@ -184,11 +180,7 @@ func (gm *GitMapping) getFileRenames(ctx context.Context, commit string) (map[st
}

func (gm *GitMapping) IsLocal() bool {
if gm.LocalGitRepo != nil {
return true
} else {
return false
}
return gm.GitRepo().IsLocal()
}

func (gm *GitMapping) getLatestCommit(ctx context.Context) (string, error) {
@@ -280,31 +272,25 @@ func (gm *GitMapping) GetLatestCommitInfo(ctx context.Context, c Conveyor) (Imag
res.Commit = commit
}

if _, isLocal := gm.GitRepo().(*git_repo.Local); isLocal && c.GetLocalGitRepoVirtualMergeOptions().VirtualMerge {
if gm.GitRepo().IsLocal() && c.GetLocalGitRepoVirtualMergeOptions().VirtualMerge {
res.VirtualMerge = true
res.VirtualMergeFromCommit = c.GetLocalGitRepoVirtualMergeOptions().VirtualMergeFromCommit
res.VirtualMergeIntoCommit = c.GetLocalGitRepoVirtualMergeOptions().VirtualMergeIntoCommit

if res.VirtualMergeFromCommit == "" || res.VirtualMergeIntoCommit == "" {
if parents, err := gm.GitRepo().GetMergeCommitParents(ctx, res.Commit); err != nil {
return ImageCommitInfo{}, fmt.Errorf("unable to get virtual merge commit %s parents for git repo %s: %s", res.Commit, gm.GitRepo().GetName(), err)
} else if len(parents) == 2 {
if res.VirtualMergeIntoCommit == "" {
res.VirtualMergeIntoCommit = parents[0]
logboek.Context(ctx).Debug().LogF("Got virtual-merge-into-commit from parents of %s => %s\n", res.Commit, res.VirtualMergeIntoCommit)
}
if res.VirtualMergeFromCommit == "" {
res.VirtualMergeFromCommit = parents[1]
logboek.Context(ctx).Debug().LogF("Got virtual-merge-from-commit from parents of %s => %s\n", res.Commit, res.VirtualMergeFromCommit)
}
}

fromCommit, intoCommit, err := git_repo.GetVirtualMergeParents(ctx, gm.GitRepo(), res.Commit)
if err != nil {
return ImageCommitInfo{}, fmt.Errorf("unable to get virtual merge commit %q parents: %s", res.Commit, err)
}

res.VirtualMergeFromCommit = fromCommit
logboek.Context(ctx).Debug().LogF("Got virtual-merge-from-commit from parents of %s => %s\n", res.Commit, res.VirtualMergeFromCommit)

res.VirtualMergeIntoCommit = intoCommit
logboek.Context(ctx).Debug().LogF("Got virtual-merge-into-commit from parents of %s => %s\n", res.Commit, res.VirtualMergeIntoCommit)

if res.VirtualMergeFromCommit == "" {
return ImageCommitInfo{}, fmt.Errorf("unable to detect --virtual-merge-from-commit for virtual merge commit %s", res.Commit)
return ImageCommitInfo{}, fmt.Errorf("unable to detect virtual-merge-from-commit for virtual merge commit %q", res.Commit)
}
if res.VirtualMergeIntoCommit == "" {
return ImageCommitInfo{}, fmt.Errorf("unable to detect --virtual-merge-into-commit for virtual merge commit %s", res.Commit)
return ImageCommitInfo{}, fmt.Errorf("unable to detect virtual-merge-into-commit for virtual merge commit %q", res.Commit)
}
}

@@ -346,7 +332,7 @@ func (gm *GitMapping) GetBaseCommitForPrevBuiltImage(ctx context.Context, c Conv
if prevBuiltImageCommitInfo.VirtualMerge {
if latestCommit, err := gm.getLatestCommit(ctx); err != nil {
return "", err
} else if _, isLocal := gm.GitRepo().(*git_repo.Local); isLocal && c.GetLocalGitRepoVirtualMergeOptions().VirtualMerge && latestCommit == prevBuiltImageCommitInfo.Commit {
} else if gm.GitRepo().IsLocal() && c.GetLocalGitRepoVirtualMergeOptions().VirtualMerge && latestCommit == prevBuiltImageCommitInfo.Commit {
baseCommit = prevBuiltImageCommitInfo.Commit
} else {
if detachedMergeCommit, err := gm.GitRepo().CreateDetachedMergeCommit(ctx, prevBuiltImageCommitInfo.VirtualMergeFromCommit, prevBuiltImageCommitInfo.VirtualMergeIntoCommit); err != nil {
@@ -365,8 +351,10 @@ func (gm *GitMapping) GetBaseCommitForPrevBuiltImage(ctx context.Context, c Conv
}

type ImageCommitInfo struct {
Commit string
VirtualMergeOptions
Commit string
VirtualMerge bool
VirtualMergeFromCommit string
VirtualMergeIntoCommit string
}

func makeInvalidImageError(label string) error {

0 comments on commit 7372992

Please sign in to comment.