Skip to content
Permalink
Browse files
feat(stapel-to-buildah): support git patches related stages
* Refactor container-backend paths removal method.
* Added git patch archive creation using in-memory buffer when passing patch-stage data.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Apr 13, 2022
1 parent 367beb8 commit 79f71c1890aba92ea9130e55afb5e5df87539815
@@ -46,8 +46,10 @@ type BuildOptions struct {
CustomTagFuncList []CustomTagFunc
}

type CustomTagFunc func(string, string) string
type ExportTagFunc func(string, string) string
type (
CustomTagFunc func(string, string) string
ExportTagFunc func(string, string) string
)

type IntrospectOptions struct {
Targets []IntrospectTarget
@@ -339,7 +339,7 @@ func (s *BaseStage) addServiceMountsVolumes(mountpointsByType map[string][]strin
} else {
stageImage.Builder.StapelStageBuilder().AddBuildVolumes(volume)
if cleanupMountpoints {
stageImage.Builder.StapelStageBuilder().AddPathsToRemove(absoluteMountpoint)
stageImage.Builder.StapelStageBuilder().RemoveData(container_backend.RemoveInsidePath, []string{absoluteMountpoint}, nil)
}
}
}
@@ -439,7 +439,7 @@ func (s *BaseStage) addCustomMountVolumes(mountpointsByFrom map[string][]string,
} else {
stageImage.Builder.StapelStageBuilder().AddBuildVolumes(volume)
if cleanupMountpoints {
stageImage.Builder.StapelStageBuilder().AddPathsToRemove(absoluteMountpoint)
stageImage.Builder.StapelStageBuilder().RemoveData(container_backend.RemoveInsidePath, []string{absoluteMountpoint}, nil)
}
}
}
@@ -1,6 +1,7 @@
package stage

import (
"archive/tar"
"context"
"crypto/sha256"
"fmt"
@@ -13,6 +14,9 @@ import (
"strings"
"sync"

"github.com/djherbis/buffer"
"github.com/djherbis/nio/v3"

"github.com/werf/logboek"
"github.com/werf/werf/pkg/container_backend"
"github.com/werf/werf/pkg/git_repo"
@@ -238,31 +242,6 @@ func (gm *GitMapping) applyPatchCommand(patchFile *ContainerFileDescriptor, arch
return commands, nil
}

func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
fromCommit, err := gm.GetBaseCommitForPrevBuiltImage(ctx, c, prevBuiltImage)
if err != nil {
return fmt.Errorf("unable to get base commit from built image: %w", err)
}

toCommitInfo, err := gm.GetLatestCommitInfo(ctx, c)
if err != nil {
return fmt.Errorf("unable to get latest commit info: %w", err)
}

commands, err := gm.baseApplyPatchCommand(ctx, fromCommit, toCommitInfo.Commit, prevBuiltImage)
if err != nil {
return err
}

if err := gm.applyScript(stageImage, commands); err != nil {
return err
}

gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, toCommitInfo)

return nil
}

func (gm *GitMapping) GetLatestCommitInfo(ctx context.Context, c Conveyor) (ImageCommitInfo, error) {
res := ImageCommitInfo{}

@@ -571,7 +550,111 @@ func (gm *GitMapping) applyArchiveCommand(archiveFile *ContainerFileDescriptor,
return commands, nil
}

func (gm *GitMapping) PreparePatchForImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
fromCommit, err := gm.GetBaseCommitForPrevBuiltImage(ctx, c, prevBuiltImage)
if err != nil {
return fmt.Errorf("unable to get base commit from built image: %w", err)
}
toCommitInfo, err := gm.GetLatestCommitInfo(ctx, c)
if err != nil {
return fmt.Errorf("unable to get latest commit info: %w", err)
}

if c.UseLegacyStapelBuilder(cb) {
commands, err := gm.baseApplyPatchCommand(ctx, fromCommit, toCommitInfo.Commit, prevBuiltImage)
if err != nil {
return err
}

if err := gm.applyScript(stageImage, commands); err != nil {
return err
}
} else {
patchOpts, err := gm.makePatchOptions(ctx, fromCommit, toCommitInfo.Commit, false, false)
if err != nil {
return fmt.Errorf("unable to make patch options: %w", err)
}
logboek.Context(ctx).Debug().LogF("Creating patch from %s to %s\n", fromCommit, toCommitInfo.Commit)
patch, err := gm.GitRepo().GetOrCreatePatch(ctx, *patchOpts)
if err != nil {
return fmt.Errorf("unable to create patch: %w", err)
}
if patch.IsEmpty() {
return nil
}

archiveOpts, err := gm.makeArchiveOptions(ctx, toCommitInfo.Commit)
if err != nil {
return err
}
logboek.Context(ctx).Debug().LogF("Creating archive for commit %s\n", toCommitInfo.Commit)
archive, err := gm.GitRepo().GetOrCreateArchive(ctx, *archiveOpts)
if err != nil {
return fmt.Errorf("unable to create git archive for commit %s with path scope %s: %w", archiveOpts.Commit, archiveOpts.PathScope, err)
}
var archiveType container_backend.ArchiveType
gitArchiveType, err := gm.getArchiveType(ctx, toCommitInfo.Commit)
if err != nil {
return fmt.Errorf("unable to determine git archive type: %w", err)
}
switch gitArchiveType {
case git_repo.FileArchive:
archiveType = container_backend.FileArchive
case git_repo.DirectoryArchive:
archiveType = container_backend.DirectoryArchive
}

tarBuf := buffer.New(64 * 1024 * 1024)
patchArchiveReader, patchArchiveWriter := nio.Pipe(tarBuf)
f, err := os.Open(archive.GetFilePath())
if err != nil {
return fmt.Errorf("unable to open archive file %q: %w", archive.GetFilePath(), err)
}

var includePaths []string
for _, path := range patch.GetPaths() {
if util.IsStringsContainValue(patch.GetPathsToRemove(), path) {
continue
}
includePaths = append(includePaths, path)
}

go func() {
logboek.Context(ctx).Debug().LogF("Starting archive %q filtering process, includePaths: %v\n", archive.GetFilePath(), includePaths)
tw := tar.NewWriter(patchArchiveWriter)
defer func() {
if err := tw.Close(); err != nil {
logboek.Context(ctx).Error().LogF("ERROR: %s\n", err)
panic("tar writer close failed")
}
}()

if err := util.CopyTar(ctx, f, tw, util.CopyTarOptions{IncludePaths: includePaths}); err != nil {
logboek.Context(ctx).Error().LogF("ERROR: %s\n", err)
panic("tar copy failed")
}
}()

logboek.Context(ctx).Debug().LogF("Adding git patch data archive with included paths: %v\n", includePaths)
stageImage.Builder.StapelStageBuilder().AddDataArchive(patchArchiveReader, archiveType, gm.To)

logboek.Context(ctx).Debug().LogF("Adding git paths to remove: %v\n", patch.GetPathsToRemove())
var pathsToRemove []string
for _, path := range patch.GetPathsToRemove() {
pathsToRemove = append(pathsToRemove, filepath.Join(gm.To, path))
}
stageImage.Builder.StapelStageBuilder().RemoveData(container_backend.RemoveExactPathWithEmptyParentDirs, pathsToRemove, []string{gm.To})
}

gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, toCommitInfo)

return nil
}

func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, stageImage *StageImage) error {
// FIXME: legacy stapel
// FIXME: file-archive type

commitInfo, err := gm.GetLatestCommitInfo(ctx, c)
if err != nil {
return fmt.Errorf("unable to get latest commit info: %w", err)
@@ -591,12 +674,10 @@ func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb
if err != nil {
return err
}

archive, err := gm.GitRepo().GetOrCreateArchive(ctx, *archiveOpts)
if err != nil {
return fmt.Errorf("unable to create git archive for commit %s with path scope %s: %w", archiveOpts.Commit, archiveOpts.PathScope, err)
}

var archiveType container_backend.ArchiveType

gitArchiveType, err := gm.getArchiveType(ctx, commitInfo.Commit)
@@ -618,11 +699,7 @@ func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb
return fmt.Errorf("unable to open archive file %q: %w", archive.GetFilePath(), err)
}

stageImage.Builder.StapelStageBuilder().AddDataArchives(container_backend.DataArchive{
Data: f,
Type: archiveType,
To: gm.To,
})
stageImage.Builder.StapelStageBuilder().AddDataArchive(f, archiveType, gm.To)
}

gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, commitInfo)
@@ -78,20 +78,17 @@ func (s *GitPatchStage) PrepareImage(ctx context.Context, c Conveyor, cr contain
}

func (s *GitPatchStage) prepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
if c.UseLegacyStapelBuilder(cr) {
for _, gitMapping := range s.gitMappings {
if err := gitMapping.ApplyPatchCommand(ctx, c, cr, prevBuiltImage, stageImage); err != nil {
return err
}
for _, gitMapping := range s.gitMappings {
if err := gitMapping.PreparePatchForImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil {
return err
}
}

if c.UseLegacyStapelBuilder(cr) {
stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", git_repo.CommonGitDataManager.GetPatchesCacheDir(), s.ContainerPatchesDir))
stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", git_repo.CommonGitDataManager.GetArchivesCacheDir(), s.ContainerArchivesDir))
stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", s.ScriptsDir, s.ContainerScriptsDir))

return nil
} else {
// TODO(stapel-to-buildah)
panic("not implemented")
}

return nil
}
@@ -21,8 +21,8 @@ type BuildStapelStageOptionsInterface interface {
AddBuildVolumes(volumes ...string) BuildStapelStageOptionsInterface
AddCommands(commands ...string) BuildStapelStageOptionsInterface

AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface
AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface
AddDataArchive(archive io.ReadCloser, archiveType ArchiveType, to string) BuildStapelStageOptionsInterface
RemoveData(removeType RemoveType, paths []string, keepParentDirs []string) BuildStapelStageOptionsInterface
AddDependencyImport(imageName, fromPath, toPath string, includePaths, excludePaths []string, owner, group string) BuildStapelStageOptionsInterface
}

@@ -42,9 +42,9 @@ type BuildStapelStageOptions struct {
BuildVolumes []string
Commands []string

DataArchives []DataArchive
PathsToRemove []string
DependenciesImports []DependencyImport
DataArchiveSpecs []DataArchiveSpec
RemoveDataSpecs []RemoveDataSpec
DependencyImportSpecs []DependencyImportSpec
}

type ArchiveType int
@@ -55,13 +55,28 @@ const (
DirectoryArchive
)

type DataArchive struct {
Data io.ReadCloser
Type ArchiveType
To string
type DataArchiveSpec struct {
Archive io.ReadCloser
Type ArchiveType
To string
}

type DependencyImport struct {
type RemoveType int

//go:generate stringer -type=RemoveType
const (
RemoveExactPath RemoveType = iota
RemoveExactPathWithEmptyParentDirs
RemoveInsidePath
)

type RemoveDataSpec struct {
Type RemoveType
Paths []string
KeepParentDirs []string
}

type DependencyImportSpec struct {
ImageName string
FromPath string
ToPath string
@@ -140,18 +155,26 @@ func (opts *BuildStapelStageOptions) AddCommands(commands ...string) BuildStapel
return opts
}

func (opts *BuildStapelStageOptions) AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface {
opts.DataArchives = append(opts.DataArchives, archives...)
func (opts *BuildStapelStageOptions) AddDataArchive(archive io.ReadCloser, archiveType ArchiveType, to string) BuildStapelStageOptionsInterface {
opts.DataArchiveSpecs = append(opts.DataArchiveSpecs, DataArchiveSpec{
Archive: archive,
Type: archiveType,
To: to,
})
return opts
}

func (opts *BuildStapelStageOptions) AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface {
opts.PathsToRemove = append(opts.PathsToRemove, paths...)
func (opts *BuildStapelStageOptions) RemoveData(removeType RemoveType, paths []string, keepParentDirs []string) BuildStapelStageOptionsInterface {
opts.RemoveDataSpecs = append(opts.RemoveDataSpecs, RemoveDataSpec{
Type: removeType,
Paths: paths,
KeepParentDirs: keepParentDirs,
})
return opts
}

func (opts *BuildStapelStageOptions) AddDependencyImport(imageName, fromPath, toPath string, includePaths, excludePaths []string, owner, group string) BuildStapelStageOptionsInterface {
opts.DependenciesImports = append(opts.DependenciesImports, DependencyImport{
opts.DependencyImportSpecs = append(opts.DependencyImportSpecs, DependencyImportSpec{
ImageName: imageName,
FromPath: fromPath,
ToPath: toPath,

0 comments on commit 79f71c1

Please sign in to comment.