Skip to content
Permalink
Browse files
feat(stapel-to-buildah): git archive stage implementation
Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Apr 8, 2022
1 parent 258ed14 commit 328b0338f586636ba6eee6a361aab58c787d083b
@@ -30,14 +30,14 @@ func (s *GitStage) PrepareImage(ctx context.Context, c Conveyor, cr container_ba
return err
}

if c.UseLegacyStapelBuilder(cr) {
if c.GiterminismManager().Dev() {
stageImage.Builder.LegacyStapelStageBuilder().BuilderContainer().AddLabel(map[string]string{imagePkg.WerfDevLabel: "true"})
if c.GiterminismManager().Dev() {
addLabels := map[string]string{imagePkg.WerfDevLabel: "true"}
if c.UseLegacyStapelBuilder(cr) {
stageImage.Builder.LegacyStapelStageBuilder().BuilderContainer().AddLabel(addLabels)
} else {
stageImage.Builder.StapelStageBuilder().AddLabels(addLabels)
}

return nil
} else {
// TODO(stapel-to-buildah)
panic("not implemented")
}

return nil
}
@@ -69,25 +69,26 @@ func (s *GitArchiveStage) GetNextStageDependencies(ctx context.Context, c Convey
}

func (s *GitArchiveStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
if !c.UseLegacyStapelBuilder(cr) {
stageImage.Builder.StapelStageBuilder().SetStageType(container_backend.DataArchivesStage)
}

if err := s.GitStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil {
return err
}

if c.UseLegacyStapelBuilder(cr) {
for _, gitMapping := range s.gitMappings {
if err := gitMapping.ApplyArchiveCommand(ctx, c, stageImage); err != nil {
return err
}
for _, gitMapping := range s.gitMappings {
if err := gitMapping.PrepareArchiveForImage(ctx, c, cr, stageImage); err != nil {
return fmt.Errorf("unable to prepare git mapping %s for image stage: %w", gitMapping.Name, err)
}
}

if c.UseLegacyStapelBuilder(cr) {
stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", git_repo.CommonGitDataManager.GetArchivesCacheDir(), s.ContainerArchivesDir))
stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", s.ScriptsDir, s.ContainerScriptsDir))

return nil
} else {
// TODO(stapel-to-buildah)
panic("not implemented")
}

return nil
}

func (s *GitArchiveStage) IsEmpty(ctx context.Context, c Conveyor, stageImage *StageImage) (bool, error) {
@@ -14,6 +14,7 @@ import (
"sync"

"github.com/werf/logboek"
"github.com/werf/werf/pkg/container_backend"
"github.com/werf/werf/pkg/git_repo"
"github.com/werf/werf/pkg/path_matcher"
"github.com/werf/werf/pkg/stapel"
@@ -237,7 +238,7 @@ func (gm *GitMapping) applyPatchCommand(patchFile *ContainerFileDescriptor, arch
return commands, nil
}

func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, prevBuiltImage, stageImage *StageImage) error {
func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
fromCommit, err := gm.GetBaseCommitForPrevBuiltImage(ctx, c, prevBuiltImage)
if err != nil {
return fmt.Errorf("unable to get base commit from built image: %w", err)
@@ -257,7 +258,7 @@ func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, prevBui
return err
}

gm.AddGitCommitToImageLabels(stageImage, toCommitInfo)
gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, toCommitInfo)

return nil
}
@@ -296,21 +297,24 @@ func (gm *GitMapping) GetLatestCommitInfo(ctx context.Context, c Conveyor) (Imag
return res, nil
}

func (gm *GitMapping) AddGitCommitToImageLabels(stageImage *StageImage, commitInfo ImageCommitInfo) {
stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(map[string]string{
gm.ImageGitCommitLabel(): commitInfo.Commit,
})
func (gm *GitMapping) AddGitCommitToImageLabels(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, stageImage *StageImage, commitInfo ImageCommitInfo) {
addLabels := make(map[string]string)
addLabels[gm.ImageGitCommitLabel()] = commitInfo.Commit

if commitInfo.VirtualMerge {
stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(map[string]string{
gm.VirtualMergeLabel(): "true",
gm.VirtualMergeFromCommitLabel(): commitInfo.VirtualMergeFromCommit,
gm.VirtualMergeIntoCommitLabel(): commitInfo.VirtualMergeIntoCommit,
})
addLabels[gm.VirtualMergeLabel()] = "true"
addLabels[gm.VirtualMergeFromCommitLabel()] = commitInfo.VirtualMergeFromCommit
addLabels[gm.VirtualMergeIntoCommitLabel()] = commitInfo.VirtualMergeIntoCommit
} else {
stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(map[string]string{
gm.VirtualMergeLabel(): "false",
})
addLabels[gm.VirtualMergeLabel()] = "false"
}

if len(addLabels) > 0 {
if c.UseLegacyStapelBuilder(cb) {
stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(addLabels)
} else {
stageImage.Builder.StapelStageBuilder().AddLabels(addLabels)
}
}
}

@@ -567,22 +571,61 @@ func (gm *GitMapping) applyArchiveCommand(archiveFile *ContainerFileDescriptor,
return commands, nil
}

func (gm *GitMapping) ApplyArchiveCommand(ctx context.Context, c Conveyor, stageImage *StageImage) error {
func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, stageImage *StageImage) error {
commitInfo, err := gm.GetLatestCommitInfo(ctx, c)
if err != nil {
return fmt.Errorf("unable to get latest commit info: %w", err)
}

commands, err := gm.baseApplyArchiveCommand(ctx, commitInfo.Commit, stageImage)
if err != nil {
return err
}
if c.UseLegacyStapelBuilder(cb) {
commands, err := gm.baseApplyArchiveCommand(ctx, commitInfo.Commit, stageImage)
if err != nil {
return err
}

if err := gm.applyScript(stageImage, commands); err != nil {
return err
if err := gm.applyScript(stageImage, commands); err != nil {
return err
}
} else {
archiveOpts, err := gm.makeArchiveOptions(ctx, commitInfo.Commit)
if err != nil {
return err
}

archive, err := gm.GitRepo().GetOrCreateArchive(ctx, *archiveOpts)
if err != nil {
return fmt.Errorf("unable to create git archive for commit %s with path scope %s: %w", archiveOpts.Commit, archiveOpts.PathScope, err)
}

var archiveType container_backend.ArchiveType

gitArchiveType, err := gm.getArchiveType(ctx, commitInfo.Commit)
if err != nil {
return fmt.Errorf("unable to determine git archive type: %w", err)
}

stageImage.Builder.StapelStageBuilder().AddLabels(map[string]string{gm.getArchiveTypeLabelName(): string(gitArchiveType)})

switch gitArchiveType {
case git_repo.FileArchive:
archiveType = container_backend.FileArchive
case git_repo.DirectoryArchive:
archiveType = container_backend.DirectoryArchive
}

f, err := os.Open(archive.GetFilePath())
if err != nil {
return fmt.Errorf("unable to open archive file %q: %w", archive.GetFilePath(), err)
}

stageImage.Builder.StapelStageBuilder().AddDataArchives(container_backend.DataArchive{
Data: f,
Type: archiveType,
To: gm.To,
})
}

gm.AddGitCommitToImageLabels(stageImage, commitInfo)
gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, commitInfo)

return nil
}
@@ -80,7 +80,7 @@ func (s *GitPatchStage) PrepareImage(ctx context.Context, c Conveyor, cr contain
func (s *GitPatchStage) prepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
if c.UseLegacyStapelBuilder(cr) {
for _, gitMapping := range s.gitMappings {
if err := gitMapping.ApplyPatchCommand(ctx, c, prevBuiltImage, stageImage); err != nil {
if err := gitMapping.ApplyPatchCommand(ctx, c, cr, prevBuiltImage, stageImage); err != nil {
return err
}
}
@@ -14,6 +14,7 @@ import (
"github.com/werf/logboek"
"github.com/werf/werf/pkg/buildah"
"github.com/werf/werf/pkg/image"
"github.com/werf/werf/pkg/util"
)

type BuildahBackend struct {
@@ -148,6 +149,64 @@ type dependencyContainer struct {
Import DependencyImport
}

func (runtime *BuildahBackend) applyDataArchives(ctx context.Context, container *containerDesc, opts BuildStapelStageOptions) error {
logboek.Context(ctx).Debug().LogF("Mounting container %q\n", container.Name)
if err := runtime.mountContainers(ctx, []*containerDesc{container}); err != nil {
return fmt.Errorf("unable to mount containers: %w", err)
}
defer func() {
logboek.Context(ctx).Debug().LogF("Unmounting container %q\n", container.Name)
if err := runtime.unmountContainers(ctx, []*containerDesc{container}); err != nil {
logboek.Context(ctx).Error().LogF("ERROR: unable to unmount containers: %s\n", err)
}
}()

for _, archive := range opts.DataArchives {
destPath := filepath.Join(container.RootMount, archive.To)

var extractDestPath string
switch archive.Type {
case DirectoryArchive:
extractDestPath = destPath
case FileArchive:
extractDestPath = filepath.Dir(destPath)
default:
return fmt.Errorf("unknown archive type %q", archive.Type)
}

_, err := os.Stat(destPath)
switch {
case os.IsNotExist(err):
case err != nil:
return fmt.Errorf("unable to access container path %q: %w", destPath, err)
default:
logboek.Context(ctx).Debug().LogF("Removing archive destination path %s\n", archive.To)
if err := os.RemoveAll(destPath); err != nil {
return fmt.Errorf("unable to cleanup archive destination path %s: %w", archive.To, err)
}
}

logboek.Context(ctx).Debug().LogF("Extracting archive into container path %s\n", archive.To)
if err := util.ExtractTar(archive.Data, extractDestPath); err != nil {
return fmt.Errorf("unable to extract data archive into %s: %w", archive.To, err)
}
if err := archive.Data.Close(); err != nil {
return fmt.Errorf("error closing archive data stream: %w", err)
}
}

for _, path := range opts.PathsToRemove {
destPath := filepath.Join(container.RootMount, path)

logboek.Context(ctx).Debug().LogF("Removing container path %s\n", path)
if err := os.RemoveAll(destPath); err != nil {
return fmt.Errorf("unable to remove path %s: %w", path, err)
}
}

return nil
}

func (runtime *BuildahBackend) applyDependencies(ctx context.Context, container *containerDesc, opts BuildStapelStageOptions) error {
var dependencies []*dependencyContainer

@@ -181,12 +240,12 @@ func (runtime *BuildahBackend) applyDependencies(ctx context.Context, container
}
}

logboek.Context(ctx).Debug().LogF("Mounting dependencies containers %v\n", dependenciesContainers)
logboek.Context(ctx).Debug().LogF("Mounting containers %v\n", append(dependenciesContainers, container))
if err := runtime.mountContainers(ctx, append(dependenciesContainers, container)); err != nil {
return fmt.Errorf("unable to mount containers: %w", err)
}
defer func() {
logboek.Context(ctx).Debug().LogF("Unmounting dependencies containers %v\n", dependenciesContainers)
logboek.Context(ctx).Debug().LogF("Unmounting containers %v\n", append(dependenciesContainers, container))
if err := runtime.unmountContainers(ctx, append(dependenciesContainers, container)); err != nil {
logboek.Context(ctx).Error().LogF("ERROR: unable to unmount containers: %s\n", err)
}
@@ -236,6 +295,10 @@ func (runtime *BuildahBackend) BuildStapelStage(ctx context.Context, stageType S
if err := runtime.applyDependencies(ctx, container, opts); err != nil {
return "", err
}
case DataArchivesStage:
if err := runtime.applyDataArchives(ctx, container, opts); err != nil {
return "", err
}
default:
return "", fmt.Errorf("unsupported stage type %q", stageType.String())
}
@@ -1,6 +1,9 @@
package container_backend

import "fmt"
import (
"fmt"
"io"
)

type StapelStageType int

@@ -10,6 +13,7 @@ const (
UserCommandsStage
DockerInstructionsStage
DependenciesStage
DataArchivesStage
)

type BuildStapelStageOptionsInterface interface {
@@ -25,6 +29,9 @@ type BuildStapelStageOptionsInterface interface {
SetWorkdir(workdir string) BuildStapelStageOptionsInterface
SetHealthcheck(healthcheck string) BuildStapelStageOptionsInterface

AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface
AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface

UserCommandsStage() UserCommandsStageOptionsInterface
DependenciesStage() DependenciesStageOptionsInterface
}
@@ -50,10 +57,27 @@ type BuildStapelStageOptions struct {
Workdir string
Healthcheck string

DataArchives []DataArchive
PathsToRemove []string

UserCommandsStageOptions
DependenciesStageOptions
}

type ArchiveType int

//go:generate stringer -type=ArchiveType
const (
FileArchive ArchiveType = iota
DirectoryArchive
)

type DataArchive struct {
Data io.ReadCloser
Type ArchiveType
To string
}

func (opts *BuildStapelStageOptions) SetBaseImage(baseImage string) BuildStapelStageOptionsInterface {
opts.BaseImage = baseImage
return opts
@@ -118,6 +142,16 @@ func (opts *BuildStapelStageOptions) AddBuildVolumes(volumes ...string) BuildSta
return opts
}

func (opts *BuildStapelStageOptions) AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface {
opts.DataArchives = append(opts.DataArchives, archives...)
return opts
}

func (opts *BuildStapelStageOptions) AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface {
opts.PathsToRemove = append(opts.PathsToRemove, paths...)
return opts
}

func (opts *BuildStapelStageOptions) UserCommandsStage() UserCommandsStageOptionsInterface {
return &opts.UserCommandsStageOptions
}

0 comments on commit 328b033

Please sign in to comment.