Skip to content

Commit 2a34fbc

Browse files
nervghalexey-igrychev
nervgh
and
alexey-igrychev
authored
feat(build): add build.cacheVersion and <image>.cacheVersion directives (#6643)
Signed-off-by: Alexandr Zaytsev <alexandr.zaytsev@flant.com> Signed-off-by: Aleksei Igrychev <aleksei.igrychev@palark.com> Co-authored-by: Aleksei Igrychev <aleksei.igrychev@palark.com>
1 parent 97fcf0f commit 2a34fbc

33 files changed

+1291
-159
lines changed

‎docs/_data/werf_yaml.yml

+8
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@ sections:
2323
collapsible: true
2424
isCollapsedByDefault: false
2525
directives:
26+
- &meta-section-build-cache-version
27+
name: cacheVersion
28+
description:
29+
en: "Cache version"
30+
ru: "Версия кеша"
31+
value: "string"
2632
- name: platform
2733
description:
2834
en: Common list of target platforms for all images (for example ['linux/amd64', 'linux/arm64', 'linux/arm/v8'])
@@ -268,6 +274,7 @@ sections:
268274
description:
269275
en: "Enable layer-by-layer caching of Dockerfile instructions in container registry"
270276
ru: "Включить послойное кеширование Dockerfile-инструкций в container registry"
277+
- << : *meta-section-build-cache-version
271278
- name: context
272279
value: "string"
273280
description:
@@ -505,6 +512,7 @@ sections:
505512
detailsArticle:
506513
all: "/usage/build/stapel/imports.html"
507514
- << : *dockerfile-image-section-final
515+
- << : *meta-section-build-cache-version
508516
- name: platform
509517
description:
510518
en: List of target platforms for this image (for example ['linux/amd64', 'linux/arm64', 'linux/arm/v8'])

‎docs/pages_en/usage/build/process.md

+26
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,32 @@ There are several generations of the staged dockerfile builder. You can switch b
152152

153153
Stapel images are cached layer-by-layer in the container registry by default and do not require any configuration.
154154

155+
### Cache versioning
156+
157+
You can use the global directive `build.cacheVersion` or its local alternative `<image>.cacheVersion` to explicitly manage the cache version of images through configuration and ensure reproducibility of all previous builds. If both directives are specified, the local one takes precedence.
158+
159+
**Usage example:**
160+
161+
```yaml
162+
project: test
163+
configVersion: 1
164+
build:
165+
cacheVersion: global-cache-version
166+
---
167+
image: backend
168+
cacheVersion: user-cache-version
169+
dockerfile: Dockerfile
170+
---
171+
image: frontend
172+
cacheVersion: frontend-cache-version
173+
dockerfile: Dockerfile
174+
staged: true
175+
---
176+
image: user
177+
cacheVersion: user-cache-version
178+
from: alpine:3.14
179+
```
180+
155181
## Parallelism and image assembly order
156182

157183
<!-- reference: https://werf.io/docs/v2/internals/build_process.html#parallel-build -->

‎docs/pages_ru/usage/build/process.md

+27
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,33 @@ staged: true
150150

151151
Образы stapel кешируются в режиме послойного кеширования в container registry по умолчанию без дополнительной конфигурации.
152152

153+
### Версионирование кеша
154+
155+
Вы можете использовать глобальную директиву `build.cacheVersion` или её локальный аналог `<image>.cacheVersion`,
156+
чтобы явно управлять версией кеша образов через конфигурацию и сохранять воспроизводимость всех сборок. Если обе директивы указаны, локальная имеет приоритет.
157+
158+
**Пример использования:**
159+
160+
```yaml
161+
project: test
162+
configVersion: 1
163+
build:
164+
cacheVersion: global-cache-version
165+
---
166+
image: backend
167+
cacheVersion: user-cache-version
168+
dockerfile: Dockerfile
169+
---
170+
image: frontend
171+
cacheVersion: frontend-cache-version
172+
dockerfile: Dockerfile
173+
staged: true
174+
---
175+
image: user
176+
cacheVersion: user-cache-version
177+
from: alpine:3.14
178+
```
179+
153180
## Параллельность и порядок сборки образов
154181

155182
<!-- прим. для перевода: на основе https://werf.io/docs/v2/internals/build_process.html#parallel-build -->

‎pkg/build/build_phase.go

+2-13
Original file line numberDiff line numberDiff line change
@@ -927,8 +927,6 @@ func (phase *BuildPhase) calculateStage(ctx context.Context, img *image.Image, s
927927
}
928928

929929
var opts calculateDigestOptions
930-
// TODO: common cache version / per image cache version / fromCacheVersion goes into this
931-
opts.CacheVersionParts = nil
932930
opts.TargetPlatform = img.TargetPlatform
933931

934932
if img.IsDockerfileImage && img.DockerfileImageConfig.Staged {
@@ -1269,9 +1267,8 @@ func introspectStage(ctx context.Context, s stage.Interface) error {
12691267
}
12701268

12711269
type calculateDigestOptions struct {
1272-
TargetPlatform string
1273-
CacheVersionParts []string
1274-
BaseImage string // TODO(staged-dockerfile): legacy compatibility field
1270+
TargetPlatform string
1271+
BaseImage string // TODO(staged-dockerfile): legacy compatibility field
12751272
}
12761273

12771274
func calculateDigest(ctx context.Context, stageName, stageDependencies string, prevNonEmptyStage stage.Interface, conveyor *Conveyor, opts calculateDigestOptions) (string, error) {
@@ -1304,14 +1301,6 @@ func calculateDigest(ctx context.Context, stageName, stageDependencies string, p
13041301
)
13051302
}
13061303

1307-
if len(opts.CacheVersionParts) > 0 {
1308-
for i, cacheVersion := range opts.CacheVersionParts {
1309-
name := fmt.Sprintf("CacheVersion%d", i)
1310-
checksumArgsNames = append(checksumArgsNames, name)
1311-
checksumArgs = append(checksumArgs, name, cacheVersion)
1312-
}
1313-
}
1314-
13151304
if opts.BaseImage != "" {
13161305
checksumArgs = append(checksumArgs, opts.BaseImage)
13171306
checksumArgsNames = append(checksumArgsNames, "BaseImage")

‎pkg/build/image/dockerfile.go

+26-30
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,11 @@ import (
2020
"github.com/werf/werf/v2/pkg/dockerfile/frontend"
2121
"github.com/werf/werf/v2/pkg/giterminism_manager"
2222
"github.com/werf/werf/v2/pkg/path_matcher"
23+
"github.com/werf/werf/v2/pkg/util/option"
2324
"github.com/werf/werf/v2/pkg/werf"
2425
)
2526

26-
func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig *config.ImageFromDockerfile, targetPlatform string, opts CommonImageOptions) (ImagesSets, error) {
27+
func MapDockerfileConfigToImagesSets(ctx context.Context, metaConfig *config.Meta, dockerfileImageConfig *config.ImageFromDockerfile, targetPlatform string, opts CommonImageOptions) (ImagesSets, error) {
2728
if dockerfileImageConfig.Staged {
2829
relDockerfilePath := filepath.Join(dockerfileImageConfig.Context, dockerfileImageConfig.Dockerfile)
2930
dockerfileData, err := opts.GiterminismManager.FileReader().ReadDockerfile(ctx, relDockerfilePath)
@@ -46,22 +47,20 @@ func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig
4647
return nil, fmt.Errorf("unable to parse dockerfile %s: %w", relDockerfilePath, err)
4748
}
4849

49-
return mapDockerfileToImagesSets(ctx, d, dockerfileImageConfig, targetPlatform, opts)
50+
return mapDockerfileToImagesSets(ctx, d, metaConfig, dockerfileImageConfig, targetPlatform, opts)
5051
}
5152

52-
img, err := mapLegacyDockerfileToImage(ctx, dockerfileImageConfig, targetPlatform, opts)
53+
img, err := mapLegacyDockerfileToImage(ctx, metaConfig, dockerfileImageConfig, targetPlatform, opts)
5354
if err != nil {
5455
return nil, err
5556
}
5657

57-
var ret ImagesSets
58-
59-
ret = append(ret, []*Image{img})
58+
ret := ImagesSets{[]*Image{img}}
6059

6160
return ret, nil
6261
}
6362

64-
func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile, dockerfileImageConfig *config.ImageFromDockerfile, targetPlatform string, opts CommonImageOptions) (ImagesSets, error) {
63+
func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile, metaConfig *config.Meta, dockerfileImageConfig *config.ImageFromDockerfile, targetPlatform string, opts CommonImageOptions) (ImagesSets, error) {
6564
var ret ImagesSets
6665

6766
targetStage, err := cfg.GetTargetStage()
@@ -167,10 +166,11 @@ func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile,
167166
if werf.GetStagedDockerfileVersion() == werf.StagedDockerfileV2 {
168167
baseStageOptions := *commonBaseStageOptions
169168
baseStageOptions.LogName = "FROM1"
170-
img.stages = append(img.stages, stage_instruction.NewFrom(
171-
img.GetBaseImageReference(), img.GetBaseImageRepoDigest(),
172-
&baseStageOptions,
173-
))
169+
170+
imageCacheVersion := option.ValueOrDefault(dockerfileImageConfig.CacheVersion(), metaConfig.Build.CacheVersion)
171+
fromStage := stage_instruction.NewFrom(img.GetBaseImageReference(), img.GetBaseImageRepoDigest(), imageCacheVersion, &baseStageOptions)
172+
173+
img.stages = append(img.stages, fromStage)
174174
instrNum = 1
175175
} else {
176176
instrNum = 0
@@ -255,7 +255,7 @@ func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile,
255255
return ret, nil
256256
}
257257

258-
func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *config.ImageFromDockerfile, targetPlatform string, opts CommonImageOptions) (*Image, error) {
258+
func mapLegacyDockerfileToImage(ctx context.Context, metaConfig *config.Meta, dockerfileImageConfig *config.ImageFromDockerfile, targetPlatform string, opts CommonImageOptions) (*Image, error) {
259259
img, err := NewImage(ctx, targetPlatform, dockerfileImageConfig.Name, NoBaseImage, ImageOptions{
260260
CommonImageOptions: opts,
261261
IsFinal: dockerfileImageConfig.IsFinal(),
@@ -320,24 +320,20 @@ func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *conf
320320
ProjectName: opts.ProjectName,
321321
}
322322

323-
dockerfileStage := stage.GenerateFullDockerfileStage(
324-
stage.NewDockerRunArgs(
325-
dockerfileData,
326-
dockerfileImageConfig.Dockerfile,
327-
dockerfileImageConfig.Target,
328-
dockerfileImageConfig.Context,
329-
dockerfileImageConfig.ContextAddFiles,
330-
dockerfileImageConfig.Args,
331-
dockerfileImageConfig.AddHost,
332-
dockerfileImageConfig.Network,
333-
dockerfileImageConfig.SSH,
334-
dockerfileImageConfig.Secrets,
335-
),
336-
ds,
337-
stage.NewContextChecksum(dockerIgnorePathMatcher),
338-
baseStageOptions,
339-
dockerfileImageConfig.Dependencies,
340-
)
323+
imageCacheVersion := option.ValueOrDefault(dockerfileImageConfig.CacheVersion(), metaConfig.Build.CacheVersion)
324+
325+
dockerfileStage := stage.GenerateFullDockerfileStage(stage.NewDockerRunArgs(
326+
dockerfileData,
327+
dockerfileImageConfig.Dockerfile,
328+
dockerfileImageConfig.Target,
329+
dockerfileImageConfig.Context,
330+
dockerfileImageConfig.ContextAddFiles,
331+
dockerfileImageConfig.Args,
332+
dockerfileImageConfig.AddHost,
333+
dockerfileImageConfig.Network,
334+
dockerfileImageConfig.SSH,
335+
dockerfileImageConfig.Secrets,
336+
), ds, stage.NewContextChecksum(dockerIgnorePathMatcher), baseStageOptions, dockerfileImageConfig.Dependencies, imageCacheVersion)
341337

342338
img.stages = append(img.stages, dockerfileStage)
343339

‎pkg/build/image/image_tree.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ func (tree *ImagesTree) Calculate(ctx context.Context) error {
104104
return fmt.Errorf("unable to map stapel config to images sets: %w", err)
105105
}
106106
case *config.ImageFromDockerfile:
107-
newImagesSets, err = MapDockerfileConfigToImagesSets(ctx, imageConfig, targetPlatform, commonImageOpts)
107+
newImagesSets, err = MapDockerfileConfigToImagesSets(ctx, tree.werfConfig.Meta, imageConfig, targetPlatform, commonImageOpts)
108108
if err != nil {
109109
return fmt.Errorf("unable to map dockerfile to images sets: %w", err)
110110
}

‎pkg/build/image/stapel.go

+5-4
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/werf/werf/v2/pkg/build/stage"
1111
"github.com/werf/werf/v2/pkg/config"
1212
"github.com/werf/werf/v2/pkg/git_repo"
13+
"github.com/werf/werf/v2/pkg/util/option"
1314
)
1415

1516
func MapStapelConfigToImagesSets(ctx context.Context, metaConfig *config.Meta, stapelImageConfig config.StapelImageInterface, targetPlatform string, opts CommonImageOptions) (ImagesSets, error) {
@@ -18,9 +19,7 @@ func MapStapelConfigToImagesSets(ctx context.Context, metaConfig *config.Meta, s
1819
return nil, err
1920
}
2021

21-
var ret ImagesSets
22-
23-
ret = append(ret, []*Image{img})
22+
ret := ImagesSets{[]*Image{img}}
2423

2524
return ret, nil
2625
}
@@ -92,7 +91,9 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap
9291

9392
gitMappingsExist := len(gitMappings) != 0
9493

95-
stages = appendIfExist(ctx, stages, stage.GenerateFromStage(imageBaseConfig, image.baseImageRepoId, baseStageOptions))
94+
imageCacheVersion := option.ValueOrDefault(stapelImageConfig.CacheVersion(), metaConfig.Build.CacheVersion)
95+
96+
stages = appendIfExist(ctx, stages, stage.GenerateFromStage(imageBaseConfig, image.baseImageRepoId, imageCacheVersion, baseStageOptions))
9697
stages = appendIfExist(ctx, stages, stage.GenerateBeforeInstallStage(ctx, imageBaseConfig, baseStageOptions))
9798
stages = appendIfExist(ctx, stages, stage.GenerateDependenciesBeforeInstallStage(imageBaseConfig, baseStageOptions))
9899

‎pkg/build/stage/base.go

+8-7
Original file line numberDiff line numberDiff line change
@@ -76,13 +76,14 @@ var AllStages = []StageName{
7676
}
7777

7878
type BaseStageOptions struct {
79-
LogName string
80-
TargetPlatform string
81-
ImageName string
82-
ConfigMounts []*config.Mount
83-
ImageTmpDir string
84-
ContainerWerfDir string
85-
ProjectName string
79+
LogName string
80+
TargetPlatform string
81+
ImageName string
82+
ImageCacheVersion string
83+
ConfigMounts []*config.Mount
84+
ImageTmpDir string
85+
ContainerWerfDir string
86+
ProjectName string
8687
}
8788

8889
func NewBaseStage(name StageName, options *BaseStageOptions) *BaseStage {

‎pkg/build/stage/data_test.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,9 @@ import (
1111
)
1212

1313
type TestDependencies struct {
14-
Dependencies []*TestDependency
15-
ExpectedDigest string
14+
Dependencies []*TestDependency
15+
ImageCacheVersion string
16+
ExpectedDigest string
1617
}
1718

1819
type TestDependency struct {

‎pkg/build/stage/from.go

+17-14
Original file line numberDiff line numberDiff line change
@@ -12,39 +12,38 @@ import (
1212
"github.com/werf/werf/v2/pkg/container_backend"
1313
imagePkg "github.com/werf/werf/v2/pkg/image"
1414
"github.com/werf/werf/v2/pkg/stapel"
15+
"github.com/werf/werf/v2/pkg/util/option"
1516
)
1617

17-
func GenerateFromStage(imageBaseConfig *config.StapelImageBase, baseImageRepoId string, baseStageOptions *BaseStageOptions) *FromStage {
18+
func GenerateFromStage(imageBaseConfig *config.StapelImageBase, baseImageRepoId, imageCacheVersion string, baseStageOptions *BaseStageOptions) *FromStage {
1819
var baseImageRepoIdOrNone string
1920
if imageBaseConfig.FromLatest {
2021
baseImageRepoIdOrNone = baseImageRepoId
2122
}
2223

23-
var fromImageOrArtifactImageName string
24-
if imageBaseConfig.FromImageName != "" {
25-
fromImageOrArtifactImageName = imageBaseConfig.FromImageName
26-
} else if imageBaseConfig.FromArtifactName != "" {
27-
fromImageOrArtifactImageName = imageBaseConfig.FromArtifactName
28-
}
24+
fromImageOrArtifactImageName := option.ValueOrDefault(imageBaseConfig.FromImageName, imageBaseConfig.FromArtifactName)
2925

30-
return newFromStage(fromImageOrArtifactImageName, baseImageRepoIdOrNone, imageBaseConfig.FromCacheVersion, baseStageOptions)
26+
return newFromStage(fromImageOrArtifactImageName, baseImageRepoIdOrNone, imageBaseConfig.FromCacheVersion, imageCacheVersion, baseStageOptions)
3127
}
3228

33-
func newFromStage(fromImageOrArtifactImageName, baseImageRepoIdOrNone, cacheVersion string, baseStageOptions *BaseStageOptions) *FromStage {
29+
func newFromStage(fromImageOrArtifactImageName, baseImageRepoIdOrNone, fromCacheVersion, imageCacheVersion string, baseStageOptions *BaseStageOptions) *FromStage {
3430
s := &FromStage{}
35-
s.cacheVersion = cacheVersion
31+
s.fromCacheVersion = fromCacheVersion
3632
s.fromImageOrArtifactImageName = fromImageOrArtifactImageName
3733
s.baseImageRepoIdOrNone = baseImageRepoIdOrNone
3834
s.BaseStage = NewBaseStage(From, baseStageOptions)
35+
s.imageCacheVersion = imageCacheVersion
3936
return s
4037
}
4138

4239
type FromStage struct {
4340
*BaseStage
4441

45-
fromImageOrArtifactImageName string
4642
baseImageRepoIdOrNone string
47-
cacheVersion string
43+
fromCacheVersion string
44+
fromImageOrArtifactImageName string
45+
46+
imageCacheVersion string
4847
}
4948

5049
func (s *FromStage) HasPrevStage() bool {
@@ -54,8 +53,12 @@ func (s *FromStage) HasPrevStage() bool {
5453
func (s *FromStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
5554
var args []string
5655

57-
if s.cacheVersion != "" {
58-
args = append(args, s.cacheVersion)
56+
if s.imageCacheVersion != "" {
57+
args = append(args, s.imageCacheVersion)
58+
}
59+
60+
if s.fromCacheVersion != "" {
61+
args = append(args, s.fromCacheVersion)
5962
}
6063

6164
if s.baseImageRepoIdOrNone != "" {

0 commit comments

Comments
 (0)