Skip to content

Commit 328b033

Browse files
distorheaddistorhead
committed
feat(stapel-to-buildah): git archive stage implementation
Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
1 parent 258ed14 commit 328b033

File tree

6 files changed

+185
-44
lines changed

6 files changed

+185
-44
lines changed

‎pkg/build/stage/git.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,14 @@ func (s *GitStage) PrepareImage(ctx context.Context, c Conveyor, cr container_ba
3030
return err
3131
}
3232

33-
if c.UseLegacyStapelBuilder(cr) {
34-
if c.GiterminismManager().Dev() {
35-
stageImage.Builder.LegacyStapelStageBuilder().BuilderContainer().AddLabel(map[string]string{imagePkg.WerfDevLabel: "true"})
33+
if c.GiterminismManager().Dev() {
34+
addLabels := map[string]string{imagePkg.WerfDevLabel: "true"}
35+
if c.UseLegacyStapelBuilder(cr) {
36+
stageImage.Builder.LegacyStapelStageBuilder().BuilderContainer().AddLabel(addLabels)
37+
} else {
38+
stageImage.Builder.StapelStageBuilder().AddLabels(addLabels)
3639
}
37-
38-
return nil
39-
} else {
40-
// TODO(stapel-to-buildah)
41-
panic("not implemented")
4240
}
41+
42+
return nil
4343
}

‎pkg/build/stage/git_archive.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -69,25 +69,26 @@ func (s *GitArchiveStage) GetNextStageDependencies(ctx context.Context, c Convey
6969
}
7070

7171
func (s *GitArchiveStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
72+
if !c.UseLegacyStapelBuilder(cr) {
73+
stageImage.Builder.StapelStageBuilder().SetStageType(container_backend.DataArchivesStage)
74+
}
75+
7276
if err := s.GitStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil {
7377
return err
7478
}
7579

76-
if c.UseLegacyStapelBuilder(cr) {
77-
for _, gitMapping := range s.gitMappings {
78-
if err := gitMapping.ApplyArchiveCommand(ctx, c, stageImage); err != nil {
79-
return err
80-
}
80+
for _, gitMapping := range s.gitMappings {
81+
if err := gitMapping.PrepareArchiveForImage(ctx, c, cr, stageImage); err != nil {
82+
return fmt.Errorf("unable to prepare git mapping %s for image stage: %w", gitMapping.Name, err)
8183
}
84+
}
8285

86+
if c.UseLegacyStapelBuilder(cr) {
8387
stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", git_repo.CommonGitDataManager.GetArchivesCacheDir(), s.ContainerArchivesDir))
8488
stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", s.ScriptsDir, s.ContainerScriptsDir))
85-
86-
return nil
87-
} else {
88-
// TODO(stapel-to-buildah)
89-
panic("not implemented")
9089
}
90+
91+
return nil
9192
}
9293

9394
func (s *GitArchiveStage) IsEmpty(ctx context.Context, c Conveyor, stageImage *StageImage) (bool, error) {

‎pkg/build/stage/git_mapping.go‎

Lines changed: 65 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"sync"
1515

1616
"github.com/werf/logboek"
17+
"github.com/werf/werf/pkg/container_backend"
1718
"github.com/werf/werf/pkg/git_repo"
1819
"github.com/werf/werf/pkg/path_matcher"
1920
"github.com/werf/werf/pkg/stapel"
@@ -237,7 +238,7 @@ func (gm *GitMapping) applyPatchCommand(patchFile *ContainerFileDescriptor, arch
237238
return commands, nil
238239
}
239240

240-
func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, prevBuiltImage, stageImage *StageImage) error {
241+
func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
241242
fromCommit, err := gm.GetBaseCommitForPrevBuiltImage(ctx, c, prevBuiltImage)
242243
if err != nil {
243244
return fmt.Errorf("unable to get base commit from built image: %w", err)
@@ -257,7 +258,7 @@ func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, prevBui
257258
return err
258259
}
259260

260-
gm.AddGitCommitToImageLabels(stageImage, toCommitInfo)
261+
gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, toCommitInfo)
261262

262263
return nil
263264
}
@@ -296,21 +297,24 @@ func (gm *GitMapping) GetLatestCommitInfo(ctx context.Context, c Conveyor) (Imag
296297
return res, nil
297298
}
298299

299-
func (gm *GitMapping) AddGitCommitToImageLabels(stageImage *StageImage, commitInfo ImageCommitInfo) {
300-
stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(map[string]string{
301-
gm.ImageGitCommitLabel(): commitInfo.Commit,
302-
})
300+
func (gm *GitMapping) AddGitCommitToImageLabels(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, stageImage *StageImage, commitInfo ImageCommitInfo) {
301+
addLabels := make(map[string]string)
302+
addLabels[gm.ImageGitCommitLabel()] = commitInfo.Commit
303303

304304
if commitInfo.VirtualMerge {
305-
stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(map[string]string{
306-
gm.VirtualMergeLabel(): "true",
307-
gm.VirtualMergeFromCommitLabel(): commitInfo.VirtualMergeFromCommit,
308-
gm.VirtualMergeIntoCommitLabel(): commitInfo.VirtualMergeIntoCommit,
309-
})
305+
addLabels[gm.VirtualMergeLabel()] = "true"
306+
addLabels[gm.VirtualMergeFromCommitLabel()] = commitInfo.VirtualMergeFromCommit
307+
addLabels[gm.VirtualMergeIntoCommitLabel()] = commitInfo.VirtualMergeIntoCommit
310308
} else {
311-
stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(map[string]string{
312-
gm.VirtualMergeLabel(): "false",
313-
})
309+
addLabels[gm.VirtualMergeLabel()] = "false"
310+
}
311+
312+
if len(addLabels) > 0 {
313+
if c.UseLegacyStapelBuilder(cb) {
314+
stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(addLabels)
315+
} else {
316+
stageImage.Builder.StapelStageBuilder().AddLabels(addLabels)
317+
}
314318
}
315319
}
316320

@@ -567,22 +571,61 @@ func (gm *GitMapping) applyArchiveCommand(archiveFile *ContainerFileDescriptor,
567571
return commands, nil
568572
}
569573

570-
func (gm *GitMapping) ApplyArchiveCommand(ctx context.Context, c Conveyor, stageImage *StageImage) error {
574+
func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, stageImage *StageImage) error {
571575
commitInfo, err := gm.GetLatestCommitInfo(ctx, c)
572576
if err != nil {
573577
return fmt.Errorf("unable to get latest commit info: %w", err)
574578
}
575579

576-
commands, err := gm.baseApplyArchiveCommand(ctx, commitInfo.Commit, stageImage)
577-
if err != nil {
578-
return err
579-
}
580+
if c.UseLegacyStapelBuilder(cb) {
581+
commands, err := gm.baseApplyArchiveCommand(ctx, commitInfo.Commit, stageImage)
582+
if err != nil {
583+
return err
584+
}
580585

581-
if err := gm.applyScript(stageImage, commands); err != nil {
582-
return err
586+
if err := gm.applyScript(stageImage, commands); err != nil {
587+
return err
588+
}
589+
} else {
590+
archiveOpts, err := gm.makeArchiveOptions(ctx, commitInfo.Commit)
591+
if err != nil {
592+
return err
593+
}
594+
595+
archive, err := gm.GitRepo().GetOrCreateArchive(ctx, *archiveOpts)
596+
if err != nil {
597+
return fmt.Errorf("unable to create git archive for commit %s with path scope %s: %w", archiveOpts.Commit, archiveOpts.PathScope, err)
598+
}
599+
600+
var archiveType container_backend.ArchiveType
601+
602+
gitArchiveType, err := gm.getArchiveType(ctx, commitInfo.Commit)
603+
if err != nil {
604+
return fmt.Errorf("unable to determine git archive type: %w", err)
605+
}
606+
607+
stageImage.Builder.StapelStageBuilder().AddLabels(map[string]string{gm.getArchiveTypeLabelName(): string(gitArchiveType)})
608+
609+
switch gitArchiveType {
610+
case git_repo.FileArchive:
611+
archiveType = container_backend.FileArchive
612+
case git_repo.DirectoryArchive:
613+
archiveType = container_backend.DirectoryArchive
614+
}
615+
616+
f, err := os.Open(archive.GetFilePath())
617+
if err != nil {
618+
return fmt.Errorf("unable to open archive file %q: %w", archive.GetFilePath(), err)
619+
}
620+
621+
stageImage.Builder.StapelStageBuilder().AddDataArchives(container_backend.DataArchive{
622+
Data: f,
623+
Type: archiveType,
624+
To: gm.To,
625+
})
583626
}
584627

585-
gm.AddGitCommitToImageLabels(stageImage, commitInfo)
628+
gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, commitInfo)
586629

587630
return nil
588631
}

‎pkg/build/stage/git_patch.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ func (s *GitPatchStage) PrepareImage(ctx context.Context, c Conveyor, cr contain
8080
func (s *GitPatchStage) prepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
8181
if c.UseLegacyStapelBuilder(cr) {
8282
for _, gitMapping := range s.gitMappings {
83-
if err := gitMapping.ApplyPatchCommand(ctx, c, prevBuiltImage, stageImage); err != nil {
83+
if err := gitMapping.ApplyPatchCommand(ctx, c, cr, prevBuiltImage, stageImage); err != nil {
8484
return err
8585
}
8686
}

‎pkg/container_backend/buildah_backend.go‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/werf/logboek"
1515
"github.com/werf/werf/pkg/buildah"
1616
"github.com/werf/werf/pkg/image"
17+
"github.com/werf/werf/pkg/util"
1718
)
1819

1920
type BuildahBackend struct {
@@ -148,6 +149,64 @@ type dependencyContainer struct {
148149
Import DependencyImport
149150
}
150151

152+
func (runtime *BuildahBackend) applyDataArchives(ctx context.Context, container *containerDesc, opts BuildStapelStageOptions) error {
153+
logboek.Context(ctx).Debug().LogF("Mounting container %q\n", container.Name)
154+
if err := runtime.mountContainers(ctx, []*containerDesc{container}); err != nil {
155+
return fmt.Errorf("unable to mount containers: %w", err)
156+
}
157+
defer func() {
158+
logboek.Context(ctx).Debug().LogF("Unmounting container %q\n", container.Name)
159+
if err := runtime.unmountContainers(ctx, []*containerDesc{container}); err != nil {
160+
logboek.Context(ctx).Error().LogF("ERROR: unable to unmount containers: %s\n", err)
161+
}
162+
}()
163+
164+
for _, archive := range opts.DataArchives {
165+
destPath := filepath.Join(container.RootMount, archive.To)
166+
167+
var extractDestPath string
168+
switch archive.Type {
169+
case DirectoryArchive:
170+
extractDestPath = destPath
171+
case FileArchive:
172+
extractDestPath = filepath.Dir(destPath)
173+
default:
174+
return fmt.Errorf("unknown archive type %q", archive.Type)
175+
}
176+
177+
_, err := os.Stat(destPath)
178+
switch {
179+
case os.IsNotExist(err):
180+
case err != nil:
181+
return fmt.Errorf("unable to access container path %q: %w", destPath, err)
182+
default:
183+
logboek.Context(ctx).Debug().LogF("Removing archive destination path %s\n", archive.To)
184+
if err := os.RemoveAll(destPath); err != nil {
185+
return fmt.Errorf("unable to cleanup archive destination path %s: %w", archive.To, err)
186+
}
187+
}
188+
189+
logboek.Context(ctx).Debug().LogF("Extracting archive into container path %s\n", archive.To)
190+
if err := util.ExtractTar(archive.Data, extractDestPath); err != nil {
191+
return fmt.Errorf("unable to extract data archive into %s: %w", archive.To, err)
192+
}
193+
if err := archive.Data.Close(); err != nil {
194+
return fmt.Errorf("error closing archive data stream: %w", err)
195+
}
196+
}
197+
198+
for _, path := range opts.PathsToRemove {
199+
destPath := filepath.Join(container.RootMount, path)
200+
201+
logboek.Context(ctx).Debug().LogF("Removing container path %s\n", path)
202+
if err := os.RemoveAll(destPath); err != nil {
203+
return fmt.Errorf("unable to remove path %s: %w", path, err)
204+
}
205+
}
206+
207+
return nil
208+
}
209+
151210
func (runtime *BuildahBackend) applyDependencies(ctx context.Context, container *containerDesc, opts BuildStapelStageOptions) error {
152211
var dependencies []*dependencyContainer
153212

@@ -181,12 +240,12 @@ func (runtime *BuildahBackend) applyDependencies(ctx context.Context, container
181240
}
182241
}
183242

184-
logboek.Context(ctx).Debug().LogF("Mounting dependencies containers %v\n", dependenciesContainers)
243+
logboek.Context(ctx).Debug().LogF("Mounting containers %v\n", append(dependenciesContainers, container))
185244
if err := runtime.mountContainers(ctx, append(dependenciesContainers, container)); err != nil {
186245
return fmt.Errorf("unable to mount containers: %w", err)
187246
}
188247
defer func() {
189-
logboek.Context(ctx).Debug().LogF("Unmounting dependencies containers %v\n", dependenciesContainers)
248+
logboek.Context(ctx).Debug().LogF("Unmounting containers %v\n", append(dependenciesContainers, container))
190249
if err := runtime.unmountContainers(ctx, append(dependenciesContainers, container)); err != nil {
191250
logboek.Context(ctx).Error().LogF("ERROR: unable to unmount containers: %s\n", err)
192251
}
@@ -236,6 +295,10 @@ func (runtime *BuildahBackend) BuildStapelStage(ctx context.Context, stageType S
236295
if err := runtime.applyDependencies(ctx, container, opts); err != nil {
237296
return "", err
238297
}
298+
case DataArchivesStage:
299+
if err := runtime.applyDataArchives(ctx, container, opts); err != nil {
300+
return "", err
301+
}
239302
default:
240303
return "", fmt.Errorf("unsupported stage type %q", stageType.String())
241304
}

‎pkg/container_backend/stapelstagetype.go‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package container_backend
22

3-
import "fmt"
3+
import (
4+
"fmt"
5+
"io"
6+
)
47

58
type StapelStageType int
69

@@ -10,6 +13,7 @@ const (
1013
UserCommandsStage
1114
DockerInstructionsStage
1215
DependenciesStage
16+
DataArchivesStage
1317
)
1418

1519
type BuildStapelStageOptionsInterface interface {
@@ -25,6 +29,9 @@ type BuildStapelStageOptionsInterface interface {
2529
SetWorkdir(workdir string) BuildStapelStageOptionsInterface
2630
SetHealthcheck(healthcheck string) BuildStapelStageOptionsInterface
2731

32+
AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface
33+
AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface
34+
2835
UserCommandsStage() UserCommandsStageOptionsInterface
2936
DependenciesStage() DependenciesStageOptionsInterface
3037
}
@@ -50,10 +57,27 @@ type BuildStapelStageOptions struct {
5057
Workdir string
5158
Healthcheck string
5259

60+
DataArchives []DataArchive
61+
PathsToRemove []string
62+
5363
UserCommandsStageOptions
5464
DependenciesStageOptions
5565
}
5666

67+
type ArchiveType int
68+
69+
//go:generate stringer -type=ArchiveType
70+
const (
71+
FileArchive ArchiveType = iota
72+
DirectoryArchive
73+
)
74+
75+
type DataArchive struct {
76+
Data io.ReadCloser
77+
Type ArchiveType
78+
To string
79+
}
80+
5781
func (opts *BuildStapelStageOptions) SetBaseImage(baseImage string) BuildStapelStageOptionsInterface {
5882
opts.BaseImage = baseImage
5983
return opts
@@ -118,6 +142,16 @@ func (opts *BuildStapelStageOptions) AddBuildVolumes(volumes ...string) BuildSta
118142
return opts
119143
}
120144

145+
func (opts *BuildStapelStageOptions) AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface {
146+
opts.DataArchives = append(opts.DataArchives, archives...)
147+
return opts
148+
}
149+
150+
func (opts *BuildStapelStageOptions) AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface {
151+
opts.PathsToRemove = append(opts.PathsToRemove, paths...)
152+
return opts
153+
}
154+
121155
func (opts *BuildStapelStageOptions) UserCommandsStage() UserCommandsStageOptionsInterface {
122156
return &opts.UserCommandsStageOptions
123157
}

0 commit comments

Comments
 (0)