Skip to content

Commit

Permalink
Use long running session in builder
Browse files Browse the repository at this point in the history 
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

Add incremental context send support

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
  • Loading branch information
@tonistiigi
tonistiigi committed Jun 26, 2017
1 parent ae8d049 commit b95638a
Show file tree
Hide file tree
Showing 6 changed files with 309 additions and 18 deletions.
50 changes: 45 additions & 5 deletions cli/command/formatter/disk_usage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,12 @@ const (
// DiskUsageContext contains disk usage specific information required by the formatter, encapsulate a Context struct.
type DiskUsageContext struct {
Context
Verbose bool
LayersSize int64
Images []*types.ImageSummary
Containers []*types.Container
Volumes []*types.Volume
Verbose bool
LayersSize int64
Images []*types.ImageSummary
Containers []*types.Container
Volumes []*types.Volume
BuilderSize int64
}

func (ctx *DiskUsageContext) startSubsection(format string) (*template.Template, error) {
Expand Down Expand Up @@ -97,6 +98,13 @@ func (ctx *DiskUsageContext) Write() (err error) {
return err
}

err = ctx.contextFormat(tmpl, &diskUsageBuilderContext{
builderSize: ctx.BuilderSize,
})
if err != nil {
return err
}

diskUsageContainersCtx := diskUsageContainersContext{containers: []*types.Container{}}
diskUsageContainersCtx.header = map[string]string{
"Type": typeHeader,
Expand Down Expand Up @@ -179,6 +187,9 @@ func (ctx *DiskUsageContext) verboseWrite() (err error) {
}
}
ctx.postFormat(tmpl, newVolumeContext())

// And build cache
fmt.Fprintf(ctx.Output, "\nBuild cache usage: %s\n\n", units.HumanSize(float64(ctx.BuilderSize)))
return
}

Expand Down Expand Up @@ -357,3 +368,32 @@ func (c *diskUsageVolumesContext) Reclaimable() string {

return fmt.Sprintf("%s", units.HumanSize(float64(reclaimable)))
}

type diskUsageBuilderContext struct {
HeaderContext
builderSize int64
}

func (c *diskUsageBuilderContext) MarshalJSON() ([]byte, error) {
return marshalJSON(c)
}

func (c *diskUsageBuilderContext) Type() string {
return "Build Cache"
}

func (c *diskUsageBuilderContext) TotalCount() string {
return ""
}

func (c *diskUsageBuilderContext) Active() string {
return ""
}

func (c *diskUsageBuilderContext) Size() string {
return units.HumanSize(float64(c.builderSize))
}

func (c *diskUsageBuilderContext) Reclaimable() string {
return c.Size()
}
12 changes: 12 additions & 0 deletions cli/command/formatter/disk_usage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ func TestDiskUsageContextFormatWrite(t *testing.T) {
Images 0 0 0B 0B
Containers 0 0 0B 0B
Local Volumes 0 0 0B 0B
Build Cache 0B 0B
`,
},
{
Expand All @@ -38,6 +39,9 @@ CONTAINER ID IMAGE COMMAND LOCAL VOLUMES
Local Volumes space usage:
VOLUME NAME LINKS SIZE
Build cache usage: 0B
`,
},
// Errors
Expand Down Expand Up @@ -70,6 +74,7 @@ VOLUME NAME LINKS SIZE
Images 0 0 0B 0B
Containers 0 0 0B 0B
Local Volumes 0 0 0B 0B
Build Cache 0B 0B
`,
},
{
Expand All @@ -82,6 +87,7 @@ Local Volumes 0 0 0B
Images 0
Containers 0
Local Volumes 0
Build Cache
`,
},
// Raw Format
Expand Down Expand Up @@ -109,6 +115,12 @@ active: 0
size: 0B
reclaimable: 0B
type: Build Cache
total:
active:
size: 0B
reclaimable: 0B
`,
},
}
Expand Down
95 changes: 87 additions & 8 deletions cli/command/image/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"regexp"
"runtime"

"github.com/Sirupsen/logrus"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
"github.com/docker/cli/cli/command/image/build"
Expand Down Expand Up @@ -61,6 +62,7 @@ type buildOptions struct {
squash bool
target string
imageIDFile string
stream bool
}

// dockerfileFromStdin returns true when the user specified that the Dockerfile
Expand Down Expand Up @@ -133,6 +135,10 @@ func NewBuildCommand(dockerCli *command.DockerCli) *cobra.Command {
flags.SetAnnotation("squash", "experimental", nil)
flags.SetAnnotation("squash", "version", []string{"1.25"})

flags.BoolVar(&options.stream, "stream", false, "Stream attaches to server to negotiate build context")
flags.SetAnnotation("stream", "experimental", nil)
flags.SetAnnotation("stream", "version", []string{"1.31"})

return cmd
}

Expand Down Expand Up @@ -163,6 +169,7 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
relDockerfile string
progBuff io.Writer
buildBuff io.Writer
remote string
)

if options.dockerfileFromStdin() {
Expand All @@ -188,6 +195,7 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {

switch {
case options.contextFromStdin():
// buildCtx is tar archive. if stdin was dockerfile then it is wrapped
buildCtx, relDockerfile, err = build.GetContextFromReader(dockerCli.In(), options.dockerfileName)
case isLocalDir(specifiedContext):
contextDir, relDockerfile, err = build.GetContextFromLocalDir(specifiedContext, options.dockerfileName)
Expand All @@ -211,7 +219,8 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
contextDir = tempDir
}

if buildCtx == nil {
// read from a directory into tar archive
if buildCtx == nil && !options.stream {
excludes, err := build.ReadDockerignore(contextDir)
if err != nil {
return err
Expand Down Expand Up @@ -242,24 +251,45 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
}
}

// replace Dockerfile if added dynamically
if dockerfileCtx != nil {
// replace Dockerfile if it was added from stdin and there is archive context
if dockerfileCtx != nil && buildCtx != nil {
buildCtx, relDockerfile, err = build.AddDockerfileToBuildContext(dockerfileCtx, buildCtx)
if err != nil {
return err
}
}

ctx := context.Background()
// if streaming and dockerfile was not from stdin then read from file
// to the same reader that is usually stdin
if options.stream && dockerfileCtx == nil {
dockerfileCtx, err = os.Open(relDockerfile)
if err != nil {
return errors.Wrapf(err, "failed to open %s", relDockerfile)
}
defer dockerfileCtx.Close()
}

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

var resolvedTags []*resolvedTag
if command.IsTrusted() {
translator := func(ctx context.Context, ref reference.NamedTagged) (reference.Canonical, error) {
return TrustedReference(ctx, dockerCli, ref, nil)
}
// Wrap the tar archive to replace the Dockerfile entry with the rewritten
// Dockerfile which uses trusted pulls.
buildCtx = replaceDockerfileTarWrapper(ctx, buildCtx, relDockerfile, translator, &resolvedTags)
// if there is a tar wrapper, the dockerfile needs to be replaced inside it
if buildCtx != nil {
// Wrap the tar archive to replace the Dockerfile entry with the rewritten
// Dockerfile which uses trusted pulls.
buildCtx = replaceDockerfileTarWrapper(ctx, buildCtx, relDockerfile, translator, &resolvedTags)
} else if dockerfileCtx != nil {
// if there was not archive context still do the possible replacements in Dockerfile
newDockerfile, _, err := rewriteDockerfileFrom(ctx, dockerfileCtx, translator)
if err != nil {
return err
}
dockerfileCtx = ioutil.NopCloser(bytes.NewBuffer(newDockerfile))
}
}

// Setup an upload progress bar
Expand All @@ -268,7 +298,43 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
progressOutput = &lastProgressOutput{output: progressOutput}
}

var body io.Reader = progress.NewProgressReader(buildCtx, progressOutput, 0, "", "Sending build context to Docker daemon")
// if up to this point nothing has set the context then we must have have
// another way for sending it(streaming) and set the context to the Dockerfile
if dockerfileCtx != nil && buildCtx == nil {
buildCtx = dockerfileCtx
}

s, err := trySession(dockerCli, contextDir)
if err != nil {
return err
}

var body io.Reader
if buildCtx != nil && !options.stream {
body = progress.NewProgressReader(buildCtx, progressOutput, 0, "", "Sending build context to Docker daemon")
}

// add context stream to the session
if options.stream && s != nil {
syncDone := make(chan error) // used to signal first progress reporting completed.
// progress would also send errors but don't need it here as errors
// are handled by session.Run() and ImageBuild()
if err := addDirToSession(s, contextDir, progressOutput, syncDone); err != nil {
return err
}

buf := newBufferedWriter(syncDone, buildBuff)
defer func() {
select {
case <-buf.flushed:
case <-ctx.Done():
}
}()
buildBuff = buf

remote = clientSessionRemote
body = buildCtx
}

authConfigs, _ := dockerCli.GetAllCredentials()
buildOptions := types.ImageBuildOptions{
Expand Down Expand Up @@ -299,13 +365,26 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
Squash: options.squash,
ExtraHosts: options.extraHosts.GetAll(),
Target: options.target,
RemoteContext: remote,
}

if s != nil {
go func() {
logrus.Debugf("running session: %v", s.UUID())
if err := s.Run(ctx, dockerCli.Client().DialSession); err != nil {
logrus.Error(err)
cancel() // cancel progress context
}
}()
buildOptions.SessionID = s.UUID()
}

response, err := dockerCli.Client().ImageBuild(ctx, body, buildOptions)
if err != nil {
if options.quiet {
fmt.Fprintf(dockerCli.Err(), "%s", progBuff)
}
cancel()
return err
}
defer response.Body.Close()
Expand Down
Loading

0 comments on commit b95638a

Please sign in to comment.