Skip to content
Permalink
Browse files
feat(stapel-to-buildah): implemented dependencies checksum using buil…
…dah container backend

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Apr 15, 2022
1 parent b59afde commit 9596f6db61db4da88c3ff58ebafffacde871191d
@@ -567,7 +567,7 @@ func extractLegacyStageImage(stageImage *stage.StageImage) *container_backend.Le

func (phase *BuildPhase) calculateStage(ctx context.Context, img *Image, stg stage.Interface) (bool, func(), error) {
// FIXME(stapel-to-buildah): store StageImage-s everywhere in stage and build pkgs
stageDependencies, err := stg.GetDependencies(ctx, phase.Conveyor, phase.StagesIterator.GetPrevImage(img, stg), phase.StagesIterator.GetPrevBuiltImage(img, stg))
stageDependencies, err := stg.GetDependencies(ctx, phase.Conveyor, phase.Conveyor.ContainerBackend, phase.StagesIterator.GetPrevImage(img, stg), phase.StagesIterator.GetPrevBuiltImage(img, stg))
if err != nil {
return false, nil, err
}
@@ -126,7 +126,7 @@ func (s *BaseStage) FetchDependencies(_ context.Context, _ Conveyor, _ container
return nil
}

func (s *BaseStage) GetDependencies(_ context.Context, _ Conveyor, _, _ *StageImage) (string, error) {
func (s *BaseStage) GetDependencies(_ context.Context, _ Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
panic("method must be implemented!")
}

@@ -27,7 +27,7 @@ type BeforeInstallStage struct {
*UserStage
}

func (s *BeforeInstallStage) GetDependencies(ctx context.Context, _ Conveyor, _, _ *StageImage) (string, error) {
func (s *BeforeInstallStage) GetDependencies(ctx context.Context, _ Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
return s.builder.BeforeInstallChecksum(ctx), nil
}

@@ -28,7 +28,7 @@ type BeforeSetupStage struct {
*UserWithGitPatchStage
}

func (s *BeforeSetupStage) GetDependencies(ctx context.Context, c Conveyor, _, _ *StageImage) (string, error) {
func (s *BeforeSetupStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
stageDependenciesChecksum, err := s.getStageDependenciesChecksum(ctx, c, BeforeSetup)
if err != nil {
return "", err
@@ -30,7 +30,7 @@ type Conveyor interface {

GiterminismManager() giterminism_manager.Interface

UseLegacyStapelBuilder(cr container_backend.ContainerBackend) bool
UseLegacyStapelBuilder(cb container_backend.ContainerBackend) bool
}

type VirtualMergeOptions struct {
@@ -67,14 +67,14 @@ type DependenciesStage struct {
dependencies []*config.Dependency
}

func (s *DependenciesStage) GetDependencies(ctx context.Context, c Conveyor, _, _ *StageImage) (string, error) {
func (s *DependenciesStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
var args []string

for ind, elm := range s.imports {
var sourceChecksum string
var err error
if err := logboek.Context(ctx).Info().LogProcess("Getting import %d source checksum ...", ind).DoError(func() error {
sourceChecksum, err = s.getImportSourceChecksum(ctx, c, elm)
sourceChecksum, err = s.getImportSourceChecksum(ctx, c, cb, elm)
return err
}); err != nil {
return "", fmt.Errorf("unable to get import %d source checksum: %w", ind, err)
@@ -217,15 +217,15 @@ func (s *DependenciesStage) PrepareImage(ctx context.Context, c Conveyor, cr con
}
}

func (s *DependenciesStage) getImportSourceChecksum(ctx context.Context, c Conveyor, importElm *config.Import) (string, error) {
func (s *DependenciesStage) getImportSourceChecksum(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, importElm *config.Import) (string, error) {
importSourceID := getImportSourceID(c, importElm)
importMetadata, err := c.GetImportMetadata(ctx, s.projectName, importSourceID)
if err != nil {
return "", fmt.Errorf("unable to get import metadata: %w", err)
}

if importMetadata == nil {
checksum, err := s.generateImportChecksum(ctx, c, importElm)
checksum, err := s.generateImportChecksum(ctx, c, cb, importElm)
if err != nil {
return "", fmt.Errorf("unable to generate import source checksum: %w", err)
}
@@ -245,59 +245,83 @@ func (s *DependenciesStage) getImportSourceChecksum(ctx context.Context, c Conve
return importMetadata.Checksum, nil
}

func (s *DependenciesStage) generateImportChecksum(ctx context.Context, c Conveyor, importElm *config.Import) (string, error) {
func (s *DependenciesStage) generateImportChecksum(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, importElm *config.Import) (string, error) {
if err := fetchSourceImageDockerImage(ctx, c, importElm); err != nil {
return "", fmt.Errorf("unable to fetch source image: %w", err)
}

sourceImageDockerImageName := getSourceImageDockerImageName(c, importElm)
importSourceID := getImportSourceID(c, importElm)

stapelContainerName, err := stapel.GetOrCreateContainer(ctx)
if err != nil {
return "", err
}
if c.UseLegacyStapelBuilder(cb) {
importSourceID := getImportSourceID(c, importElm)

importHostTmpDir := filepath.Join(s.imageTmpDir, string(s.Name()), "imports", importSourceID)
importContainerDir := s.containerWerfDir
stapelContainerName, err := stapel.GetOrCreateContainer(ctx)
if err != nil {
return "", err
}

importScriptHostTmpPath := filepath.Join(importHostTmpDir, "script.sh")
resultChecksumHostTmpPath := filepath.Join(importHostTmpDir, "checksum")
importScriptContainerPath := path.Join(importContainerDir, "script.sh")
resultChecksumContainerPath := path.Join(importContainerDir, "checksum")
importHostTmpDir := filepath.Join(s.imageTmpDir, string(s.Name()), "imports", importSourceID)
importContainerDir := s.containerWerfDir

command := generateChecksumCommand(importElm.Add, importElm.IncludePaths, importElm.ExcludePaths, resultChecksumContainerPath)
if err := stapel.CreateScript(importScriptHostTmpPath, []string{command}); err != nil {
return "", fmt.Errorf("unable to create script: %w", err)
}
importScriptHostTmpPath := filepath.Join(importHostTmpDir, "script.sh")
resultChecksumHostTmpPath := filepath.Join(importHostTmpDir, "checksum")
importScriptContainerPath := path.Join(importContainerDir, "script.sh")
resultChecksumContainerPath := path.Join(importContainerDir, "checksum")

runArgs := []string{
"--rm",
"--user=0:0",
"--workdir=/",
fmt.Sprintf("--volumes-from=%s", stapelContainerName),
fmt.Sprintf("--volume=%s:%s", importHostTmpDir, importContainerDir),
fmt.Sprintf("--entrypoint=%s", stapel.BashBinPath()),
sourceImageDockerImageName,
importScriptContainerPath,
}
command := generateChecksumCommand(importElm.Add, importElm.IncludePaths, importElm.ExcludePaths, resultChecksumContainerPath)
if err := stapel.CreateScript(importScriptHostTmpPath, []string{command}); err != nil {
return "", fmt.Errorf("unable to create script: %w", err)
}

if debugImportSourceChecksum() {
fmt.Println(runArgs)
}
runArgs := []string{
"--rm",
"--user=0:0",
"--workdir=/",
fmt.Sprintf("--volumes-from=%s", stapelContainerName),
fmt.Sprintf("--volume=%s:%s", importHostTmpDir, importContainerDir),
fmt.Sprintf("--entrypoint=%s", stapel.BashBinPath()),
sourceImageDockerImageName,
importScriptContainerPath,
}

if output, err := docker.CliRun_RecordedOutput(ctx, runArgs...); err != nil {
logboek.Context(ctx).Error().LogF("%s", output)
return "", err
}
if debugImportSourceChecksum() {
fmt.Println(runArgs)
}

data, err := ioutil.ReadFile(resultChecksumHostTmpPath)
if err != nil {
return "", fmt.Errorf("unable to read file with import source checksum: %w", err)
}
if output, err := docker.CliRun_RecordedOutput(ctx, runArgs...); err != nil {
logboek.Context(ctx).Error().LogF("%s", output)
return "", err
}

checksum := strings.TrimSpace(string(data))
return checksum, nil
data, err := ioutil.ReadFile(resultChecksumHostTmpPath)
if err != nil {
return "", fmt.Errorf("unable to read file with import source checksum: %w", err)
}

checksum := strings.TrimSpace(string(data))

return checksum, nil
} else {
var checksum string
var err error

logboek.Context(ctx).Debug().LogProcess("Calculating dependency import checksum").Do(func() {
checksum, err = cb.CalculateDependencyImportChecksum(ctx, container_backend.DependencyImportSpec{
ImageName: sourceImageDockerImageName,
FromPath: importElm.Add,
ToPath: importElm.To,
IncludePaths: importElm.IncludePaths,
ExcludePaths: importElm.ExcludePaths,
Owner: importElm.Owner,
Group: importElm.Group,
})
})

if err != nil {
return "", fmt.Errorf("unable to calculate dependency import checksum in %s: %w", sourceImageDockerImageName, err)
}
return checksum, nil
}
}

func generateChecksumCommand(from string, includePaths, excludePaths []string, resultChecksumPath string) string {
@@ -30,7 +30,7 @@ var _ = Describe("DependenciesStage", func() {
Builder: stageBuilder,
}

digest, err := stage.GetDependencies(ctx, conveyor, nil, stageImage)
digest, err := stage.GetDependencies(ctx, conveyor, containerBackend, nil, stageImage)
Expect(err).To(Succeed())
Expect(digest).To(Equal(data.ExpectedDigest))

@@ -30,7 +30,7 @@ type DockerInstructionsStage struct {
instructions *config.Docker
}

func (s *DockerInstructionsStage) GetDependencies(_ context.Context, _ Conveyor, _, _ *StageImage) (string, error) {
func (s *DockerInstructionsStage) GetDependencies(_ context.Context, _ Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
var args []string

args = append(args, s.instructions.Volume...)
@@ -411,7 +411,7 @@ func isUnsupportedMediaTypeError(err error) bool {

var errImageNotExistLocally = errors.New("IMAGE_NOT_EXIST_LOCALLY")

func (s *DockerfileStage) GetDependencies(ctx context.Context, c Conveyor, _, _ *StageImage) (string, error) {
func (s *DockerfileStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
resolvedDependenciesArgsHash := resolveDependenciesArgsHash(s.dependencies, c)

resolvedDockerMetaArgsHash, err := s.resolveDockerMetaArgs(resolvedDependenciesArgsHash)
@@ -78,7 +78,7 @@ var _ = Describe("DockerfileStage", func() {
Builder: stageBuilder,
}

digest, err := stage.GetDependencies(ctx, conveyor, nil, stageImage)
digest, err := stage.GetDependencies(ctx, conveyor, containerBackend, nil, stageImage)
Expect(err).To(Succeed())
Expect(digest).To(Equal(data.TestDependencies.ExpectedDigest))

@@ -47,7 +47,7 @@ type FromStage struct {
cacheVersion string
}

func (s *FromStage) GetDependencies(_ context.Context, c Conveyor, prevImage, _ *StageImage) (string, error) {
func (s *FromStage) GetDependencies(_ context.Context, c Conveyor, _ container_backend.ContainerBackend, prevImage, _ *StageImage) (string, error) {
var args []string

if s.cacheVersion != "" {
@@ -47,7 +47,7 @@ func (s *GitArchiveStage) SelectSuitableStage(ctx context.Context, c Conveyor, s
}

// TODO: 1.3 add git mapping type (dir, file, ...) to gitArchive stage digest
func (s *GitArchiveStage) GetDependencies(ctx context.Context, c Conveyor, _, _ *StageImage) (string, error) {
func (s *GitArchiveStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
var args []string
for _, gitMapping := range s.gitMappings {
if gitMapping.IsLocal() {
@@ -4,6 +4,7 @@ import (
"context"
"fmt"

"github.com/werf/werf/pkg/container_backend"
"github.com/werf/werf/pkg/image"
"github.com/werf/werf/pkg/util"
)
@@ -45,7 +46,7 @@ func (s *GitCacheStage) IsEmpty(ctx context.Context, c Conveyor, prevBuiltImage
return isEmpty, nil
}

func (s *GitCacheStage) GetDependencies(ctx context.Context, c Conveyor, _, prevBuiltImage *StageImage) (string, error) {
func (s *GitCacheStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, prevBuiltImage *StageImage) (string, error) {
patchSize, err := s.gitMappingsPatchSize(ctx, c, prevBuiltImage)
if err != nil {
return "", err
@@ -4,6 +4,7 @@ import (
"context"
"fmt"

"github.com/werf/werf/pkg/container_backend"
"github.com/werf/werf/pkg/image"
"github.com/werf/werf/pkg/util"
)
@@ -49,7 +50,7 @@ func (s *GitLatestPatchStage) IsEmpty(ctx context.Context, c Conveyor, prevBuilt
return isEmpty, nil
}

func (s *GitLatestPatchStage) GetDependencies(ctx context.Context, c Conveyor, _, prevBuiltImage *StageImage) (string, error) {
func (s *GitLatestPatchStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, prevBuiltImage *StageImage) (string, error) {
var args []string

for _, gitMapping := range s.gitMappings {
@@ -28,7 +28,7 @@ type InstallStage struct {
*UserWithGitPatchStage
}

func (s *InstallStage) GetDependencies(ctx context.Context, c Conveyor, _, _ *StageImage) (string, error) {
func (s *InstallStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
stageDependenciesChecksum, err := s.getStageDependenciesChecksum(ctx, c, Install)
if err != nil {
return "", err
@@ -14,11 +14,11 @@ type Interface interface {

IsEmpty(ctx context.Context, c Conveyor, prevBuiltImage *StageImage) (bool, error)

FetchDependencies(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, dockerRegistry docker_registry.ApiInterface) error
GetDependencies(ctx context.Context, c Conveyor, prevImage *StageImage, prevBuiltImage *StageImage) (string, error)
FetchDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, dockerRegistry docker_registry.ApiInterface) error
GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage *StageImage, prevBuiltImage *StageImage) (string, error)
GetNextStageDependencies(ctx context.Context, c Conveyor) (string, error)

PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error
PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error

PreRunHook(context.Context, Conveyor) error

@@ -28,7 +28,7 @@ type SetupStage struct {
*UserWithGitPatchStage
}

func (s *SetupStage) GetDependencies(ctx context.Context, c Conveyor, _, _ *StageImage) (string, error) {
func (s *SetupStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
stageDependenciesChecksum, err := s.getStageDependenciesChecksum(ctx, c, Setup)
if err != nil {
return "", err

0 comments on commit 9596f6d

Please sign in to comment.