Skip to content
Permalink
Browse files
feat(staged-dockerfile): Dockerfile and DockerfileStage primitives re…
…worked

* Initialize DockerfileStage with instructions.
* Build dependencies tree between different DockerfileStage-s based on FROM, COPY and RUN instructions usage.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Oct 14, 2022
1 parent 186f563 commit 78e29110ae3cb2e2addabaaa84e9b24408e7d86e
Show file tree
Hide file tree
Showing 6 changed files with 199 additions and 111 deletions.
@@ -16,6 +16,7 @@ import (
"github.com/werf/werf/pkg/config"
backend_instruction "github.com/werf/werf/pkg/container_backend/instruction"
"github.com/werf/werf/pkg/dockerfile"
"github.com/werf/werf/pkg/dockerfile/frontend"
dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction"
"github.com/werf/werf/pkg/path_matcher"
"github.com/werf/werf/pkg/util"
@@ -29,7 +30,7 @@ func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig
return nil, fmt.Errorf("unable to read dockerfile %s: %w", relDockerfilePath, err)
}

d, err := dockerfile.ParseDockerfile(dockerfileData, dockerfile.DockerfileOptions{
d, err := frontend.ParseDockerfileWithBuildkit(dockerfileData, dockerfile.DockerfileOptions{
Target: dockerfileImageConfig.Target,
BuildArgs: util.MapStringInterfaceToMapStringString(dockerfileImageConfig.Args),
AddHost: dockerfileImageConfig.AddHost,
@@ -178,9 +179,9 @@ func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *conf
return nil, err
}

dockerfile.ResolveDockerStagesFromValue(dockerStages)
frontend.ResolveDockerStagesFromValue(dockerStages)

dockerTargetIndex, err := dockerfile.GetDockerTargetStageIndex(dockerStages, dockerfileImageConfig.Target)
dockerTargetIndex, err := frontend.GetDockerTargetStageIndex(dockerStages, dockerfileImageConfig.Target)
if err != nil {
return nil, err
}
@@ -10,7 +10,7 @@ import (
. "github.com/onsi/gomega"

"github.com/werf/werf/pkg/container_backend/stage_builder"
"github.com/werf/werf/pkg/dockerfile"
"github.com/werf/werf/pkg/dockerfile/frontend"
"github.com/werf/werf/pkg/util"
)

@@ -21,13 +21,13 @@ func testDockerfileToDockerStages(dockerfileData []byte) ([]instructions.Stage,
dockerStages, dockerMetaArgs, err := instructions.Parse(p.AST)
Expect(err).To(Succeed())

dockerfile.ResolveDockerStagesFromValue(dockerStages)
frontend.ResolveDockerStagesFromValue(dockerStages)

return dockerStages, dockerMetaArgs
}

func newTestFullDockerfileStage(dockerfileData []byte, target string, buildArgs map[string]interface{}, dockerStages []instructions.Stage, dockerMetaArgs []instructions.ArgCommand, dependencies []*TestDependency) *FullDockerfileStage {
dockerTargetIndex, err := dockerfile.GetDockerTargetStageIndex(dockerStages, target)
dockerTargetIndex, err := frontend.GetDockerTargetStageIndex(dockerStages, target)
Expect(err).To(Succeed())

ds := NewDockerStages(
@@ -1,36 +1,9 @@
package dockerfile

import (
"bytes"
"context"
"fmt"

"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/moby/buildkit/frontend/dockerfile/parser"
)

func ParseDockerfile(dockerfile []byte, opts DockerfileOptions) (*Dockerfile, error) {
p, err := parser.Parse(bytes.NewReader(dockerfile))
if err != nil {
return nil, fmt.Errorf("parsing dockerfile data: %w", err)
}

dockerStages, dockerMetaArgs, err := instructions.Parse(p.AST)
if err != nil {
return nil, fmt.Errorf("parsing instructions tree: %w", err)
}

// FIXME(staged-dockerfile): is this needed?
ResolveDockerStagesFromValue(dockerStages)

dockerTargetIndex, err := GetDockerTargetStageIndex(dockerStages, opts.Target)
if err != nil {
return nil, fmt.Errorf("determine target stage: %w", err)
}

return newDockerfile(dockerStages, dockerMetaArgs, dockerTargetIndex, opts), nil
}

type DockerfileOptions struct {
Target string
BuildArgs map[string]string
@@ -39,26 +12,37 @@ type DockerfileOptions struct {
SSH string
}

func newDockerfile(dockerStages []instructions.Stage, dockerMetaArgs []instructions.ArgCommand, dockerTargetStageIndex int, opts DockerfileOptions) *Dockerfile {
func NewDockerfile(stages []*DockerfileStage, opts DockerfileOptions) *Dockerfile {
return &Dockerfile{
DockerfileOptions: opts,

dockerStages: dockerStages,
dockerMetaArgs: dockerMetaArgs,
dockerTargetStageIndex: dockerTargetStageIndex,
nameToIndex: GetDockerStagesNameToIndexMap(dockerStages),
Stages: stages,
}
}

type Dockerfile struct {
DockerfileOptions

dockerStages []instructions.Stage
dockerMetaArgs []instructions.ArgCommand
dockerTargetStageIndex int
nameToIndex map[string]string
Stages []*DockerfileStage
}

func (dockerfile *Dockerfile) GroupStagesByIndependentSets(ctx context.Context) ([][]*DockerfileStage, error) {
func (df *Dockerfile) GroupStagesByIndependentSets(ctx context.Context) ([][]*DockerfileStage, error) {
// FIXME(staged-dockerfile): build real dependencies tree

// var res [][]*DockerfileStage
// var curLevel []*DockerfileStage

// stagesQueue

// res = append(res, curLevel)

// for _, stg := range df.Stages {
// stg.Dependencies
// }

// var res [][]*DockerfileStage
// for _, stg := range df.Stages {
// res = append(res, []*DockerfileStage{stg})
// }
// return res, nil
return nil, nil
}
@@ -1,10 +1,85 @@
package dockerfile

func NewDockerfileStage(dockerfile *Dockerfile, instructions []InstructionInterface) *DockerfileStage {
return &DockerfileStage{Dockerfile: dockerfile, Instructions: instructions}
import (
"fmt"
"strconv"
"strings"

dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction"
)

func NewDockerfileStage(index int, baseName, stageName string, instructions []InstructionInterface, platform string) *DockerfileStage {
return &DockerfileStage{BaseName: baseName, StageName: stageName, Instructions: instructions, Platform: platform}
}

type DockerfileStage struct {
Dockerfile *Dockerfile
Dependencies []*DockerfileStage

BaseName string
Index int
StageName string
Platform string
Instructions []InstructionInterface
}

func (stage DockerfileStage) LogName() string {
if stage.HasStageName() {
return stage.StageName
} else {
return fmt.Sprintf("<%d>", stage.Index)
}
}

func (stage DockerfileStage) HasStageName() bool {
return stage.StageName != ""
}

func SetupDockerfileStagesDependencies(stages []*DockerfileStage) error {
stageByName := make(map[string]*DockerfileStage)
for _, stage := range stages {
if stage.HasStageName() {
stageByName[strings.ToLower(stage.StageName)] = stage
}
}

for _, stage := range stages {
// Base image dependency
if dependency, hasKey := stageByName[strings.ToLower(stage.BaseName)]; hasKey {
stage.Dependencies = append(stage.Dependencies, dependency)
}

for _, instr := range stage.Instructions {
switch typedInstr := instr.(type) {
case *dockerfile_instruction.Copy:
if dep := findStageByNameOrIndex(typedInstr.From, stages, stageByName); dep != nil {
stage.Dependencies = append(stage.Dependencies, dep)
} else {
return fmt.Errorf("unable to resolve stage %q instruction %s --from=%q: no such stage", stage.LogName(), instr.Name(), typedInstr.From)
}

case *dockerfile_instruction.Run:
for _, mount := range typedInstr.Mounts {
if mount.From != "" {
if dep := findStageByNameOrIndex(mount.From, stages, stageByName); dep != nil {
stage.Dependencies = append(stage.Dependencies, dep)
} else {
return fmt.Errorf("unable to resolve stage %q instruction %s --mount=from=%s: no such stage", stage.LogName(), instr.Name(), mount.From)
}
}
}
}
}
}

return nil
}

func findStageByNameOrIndex(ref string, stages []*DockerfileStage, stageByName map[string]*DockerfileStage) *DockerfileStage {
if stg, found := stageByName[strings.ToLower(ref)]; found {
return stg
} else if ind, err := strconv.Atoi(ref); err == nil && ind >= 0 && ind < len(stages) {
return stages[ind]
}
return nil
}
@@ -1,15 +1,53 @@
package frontend

import (
"bytes"
"fmt"
"strconv"
"strings"

"github.com/moby/buildkit/frontend/dockerfile/instructions"
"github.com/moby/buildkit/frontend/dockerfile/parser"

"github.com/werf/werf/pkg/dockerfile"
dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction"
)

func DockerfileStageFromBuildkitStage(d *dockerfile.Dockerfile, stage instructions.Stage) (*dockerfile.DockerfileStage, error) {
func ParseDockerfileWithBuildkit(dockerfileBytes []byte, opts dockerfile.DockerfileOptions) (*dockerfile.Dockerfile, error) {
p, err := parser.Parse(bytes.NewReader(dockerfileBytes))
if err != nil {
return nil, fmt.Errorf("parsing dockerfile data: %w", err)
}

dockerStages, dockerMetaArgs, err := instructions.Parse(p.AST)
if err != nil {
return nil, fmt.Errorf("parsing instructions tree: %w", err)
}

dockerTargetIndex, err := GetDockerTargetStageIndex(dockerStages, opts.Target)
if err != nil {
return nil, fmt.Errorf("determine target stage: %w", err)
}

var stages []*dockerfile.DockerfileStage
for i, dockerStage := range dockerStages {
stages = append(stages, DockerfileStageFromBuildkitStage(i, dockerStage))
}

// TODO(staged-dockerfile): convert meta-args and initialize into Dockerfile obj
_ = dockerMetaArgs
_ = dockerTargetIndex

dockerfile.SetupDockerfileStagesDependencies(stages)

d := dockerfile.NewDockerfile(stages, opts)
for _, stage := range d.Stages {
stage.Dockerfile = d
}
return d, nil
}

func DockerfileStageFromBuildkitStage(index int, stage instructions.Stage) *dockerfile.DockerfileStage {
var i []dockerfile.InstructionInterface

for _, cmd := range stage.Commands {
@@ -56,7 +94,7 @@ func DockerfileStageFromBuildkitStage(d *dockerfile.Dockerfile, stage instructio
}
}

return dockerfile.NewDockerfileStage(d, i), nil
return dockerfile.NewDockerfileStage(index, stage.BaseName, stage.Name, i, stage.Platform)
}

func extractSrcAndDst(sourcesAndDest instructions.SourcesAndDest) ([]string, string) {
@@ -75,3 +113,56 @@ func extractKeyValuePairsAsMap(pairs instructions.KeyValuePairs) (res map[string
}
return
}

func GetDockerStagesNameToIndexMap(stages []instructions.Stage) map[string]int {
nameToIndex := make(map[string]int)
for i, s := range stages {
name := strings.ToLower(s.Name)
if name != strconv.Itoa(i) {
nameToIndex[name] = i
}
}
return nameToIndex
}

func ResolveDockerStagesFromValue(stages []instructions.Stage) {
nameToIndex := GetDockerStagesNameToIndexMap(stages)

for _, s := range stages {
for _, cmd := range s.Commands {
switch typedCmd := cmd.(type) {
case *instructions.CopyCommand:
if typedCmd.From != "" {
from := strings.ToLower(typedCmd.From)
if val, ok := nameToIndex[from]; ok {
typedCmd.From = strconv.Itoa(val)
}
}

case *instructions.RunCommand:
for _, mount := range instructions.GetMounts(typedCmd) {
if mount.From != "" {
from := strings.ToLower(mount.From)
if val, ok := nameToIndex[from]; ok {
mount.From = strconv.Itoa(val)
}
}
}
}
}
}
}

func GetDockerTargetStageIndex(dockerStages []instructions.Stage, dockerTargetStage string) (int, error) {
if dockerTargetStage == "" {
return len(dockerStages) - 1, nil
}

for i, s := range dockerStages {
if s.Name == dockerTargetStage {
return i, nil
}
}

return -1, fmt.Errorf("%s is not a valid target build stage", dockerTargetStage)
}

0 comments on commit 78e2911

Please sign in to comment.