Merge e70e55f into 30f3fe0

.travis.yml

@@ -7,6 +7,7 @@ matrix:
 go:
 - 1.2.2
 - 1.3.3
+- 1.4
 - tip
 install:
 - make build

Deps

@@ -1,6 +1,6 @@
 {
 	"ImportPath": "github.com/winchman/builder-core",
-	"GoVersion": "go1.3.3",
+	"GoVersion": "go1.4",
 	"Deps": [
 		{
 			"ImportPath": "github.com/BurntSushi/toml",
@@ -9,52 +9,52 @@
 		},
 		{
 			"ImportPath": "github.com/Sirupsen/logrus",
-			"Comment": "v0.6.2-10-g51fe59a",
-			"Rev": "51fe59aca108dc5680109e7b2051cbdcfa5a253c"
+			"Comment": "v0.6.1-10-g6c28e16",
+			"Rev": "6c28e16200e55c2549705ac24ea58976c07f18ac"
 		},
 		{
 			"ImportPath": "github.com/docker/docker/pkg/archive",
-			"Comment": "v1.4.1-115-g502a1a4",
-			"Rev": "502a1a4e1a36cfcdd88139b0b35723e86d6198cf"
+			"Comment": "v1.4.0",
+			"Rev": "4595d4fb03093acf87b905bebc5ba4964d7c0707"
 		},
 		{
 			"ImportPath": "github.com/docker/docker/pkg/fileutils",
-			"Comment": "v1.4.1-115-g502a1a4",
-			"Rev": "502a1a4e1a36cfcdd88139b0b35723e86d6198cf"
+			"Comment": "v1.4.0",
+			"Rev": "4595d4fb03093acf87b905bebc5ba4964d7c0707"
 		},
 		{
 			"ImportPath": "github.com/docker/docker/pkg/ioutils",
-			"Comment": "v1.4.1-115-g502a1a4",
-			"Rev": "502a1a4e1a36cfcdd88139b0b35723e86d6198cf"
+			"Comment": "v1.4.0",
+			"Rev": "4595d4fb03093acf87b905bebc5ba4964d7c0707"
 		},
 		{
 			"ImportPath": "github.com/docker/docker/pkg/pools",
-			"Comment": "v1.4.1-115-g502a1a4",
-			"Rev": "502a1a4e1a36cfcdd88139b0b35723e86d6198cf"
+			"Comment": "v1.4.0",
+			"Rev": "4595d4fb03093acf87b905bebc5ba4964d7c0707"
 		},
 		{
 			"ImportPath": "github.com/docker/docker/pkg/promise",
-			"Comment": "v1.4.1-115-g502a1a4",
-			"Rev": "502a1a4e1a36cfcdd88139b0b35723e86d6198cf"
+			"Comment": "v1.4.0",
+			"Rev": "4595d4fb03093acf87b905bebc5ba4964d7c0707"
 		},
 		{
 			"ImportPath": "github.com/docker/docker/pkg/system",
-			"Comment": "v1.4.1-115-g502a1a4",
-			"Rev": "502a1a4e1a36cfcdd88139b0b35723e86d6198cf"
+			"Comment": "v1.4.0",
+			"Rev": "4595d4fb03093acf87b905bebc5ba4964d7c0707"
 		},
 		{
 			"ImportPath": "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar",
-			"Comment": "v1.4.1-115-g502a1a4",
-			"Rev": "502a1a4e1a36cfcdd88139b0b35723e86d6198cf"
+			"Comment": "v1.4.0",
+			"Rev": "4595d4fb03093acf87b905bebc5ba4964d7c0707"
 		},
 		{
 			"ImportPath": "github.com/fsouza/go-dockerclient",
-			"Comment": "0.2.1-336-g77d44de",
-			"Rev": "77d44de354d4cb20ad5b03a9e118dc203e59095b"
+			"Comment": "0.2.1-321-g335b6a8",
+			"Rev": "335b6a888db6a8e33550c5faeab78017d420abcd"
 		},
 		{
 			"ImportPath": "github.com/modcloth/go-fileutils",
-			"Rev": "f2bf9a2a685355797adbdd7f1d9191ad32f76115"
+			"Rev": "a7a54f3b6086bcd6c70ae0f84cdb3991b015f6b0"
 		},
 		{
 			"ImportPath": "github.com/nu7hatch/gouuid",
@@ -66,7 +66,7 @@
 		},
 		{
 			"ImportPath": "github.com/rafecolton/go-dockerclient-quick",
-			"Rev": "ebab26ac4bc48eaffb7a57f64bf723ff960c2d96"
+			"Rev": "713f8731d921ca4009b67d83de5350d8b38c66e1"
 		},
 		{
 			"ImportPath": "github.com/rafecolton/go-dockerclient-sort",
@@ -76,9 +76,14 @@
 			"ImportPath": "github.com/rafecolton/go-gitutils",
 			"Rev": "98169906211372ca2fd02e4e8e6c01eec7eed21c"
 		},
+		{
+			"ImportPath": "github.com/winchman/libsquash",
+			"Comment": "v0.1.1-34-g786fb10",
+			"Rev": "786fb10eb6af5ff7467485f9463ca85487193e8d"
+		},
 		{
 			"ImportPath": "gopkg.in/yaml.v2",
-			"Rev": "d466437aa4adc35830964cffc5b5f262c63ddcb4"
+			"Rev": "eca94c41d994ae2215d455ce578ae6e2dc6ee516"
 		}
 	]
 }

Makefile

@@ -39,7 +39,6 @@ $(GOPATH)/bin/deppy:
 
 .PHONY: get
 get: $(GOPATH)/bin/deppy
-	go get -t ./...
 	$(GOPATH)/bin/deppy restore
 
 $(PWD)/coverage:

TODO

@@ -0,0 +1,4 @@
+TODO:
+
+- document how to write a Bobfile to optimize squashing (see docker-builder)
+- add test for squashing

builder/builder.go

@@ -121,6 +121,7 @@ func (bob *Builder) BuildCommandSequence(commandSequence *parser.CommandSequence
 				Stdout:       bob.Stdout,
 				Workdir:      bob.workdir,
 				Reporter:     bob.reporter,
+				Squash:       seq.Metadata.Squash,
 			}
 			cmd = cmd.WithOpts(opts)
 
@@ -134,14 +135,6 @@ func (bob *Builder) BuildCommandSequence(commandSequence *parser.CommandSequence
 					return err
 				}
 			}
-
-			bob.reporter.Log(
-				l.WithFields(l.Fields{
-					"command":  cmd.Message(),
-					"image_id": imageID,
-				}),
-				"finished running docker command",
-			)
 		}
 
 		if !bob.KeepTemporaryTag {
@@ -163,7 +156,7 @@ func (bob *Builder) attemptToDeleteTemporaryUUIDTag(uuid string) {
 	image, err := bob.dockerClient.LatestImageByRegex(regex)
 	if err != nil {
 		bob.reporter.LogLevel(
-			l.WithField("err", err),
+			l.WithField("error", err),
 			"error getting repo taggged with temporary tag",
 			l.WarnLevel,
 		)
@@ -186,7 +179,7 @@ func (bob *Builder) attemptToDeleteTemporaryUUIDTag(uuid string) {
 
 			if err = bob.dockerClient.Client().RemoveImage(tag); err != nil {
 				bob.reporter.LogLevel(
-					l.WithField("err", err),
+					l.WithField("error", err),
 					"error deleting temporary tag",
 					l.WarnLevel,
 				)

communication/event.go

@@ -1,7 +1,7 @@
 package comm
 
 // EventType is a type for constants that indicate the type of event reported
-type EventType uint8
+type EventType uint8 //go:generate stringer -type=EventType
 
 // Event is the type that will be sent over the event channel
 type Event interface {
@@ -16,6 +16,24 @@ const (
 	// BuildEvent is for when a `docker build` command starts
 	BuildEvent
 
+	// BuildEventSquashStartSave - start exporting tar for squashing
+	BuildEventSquashStartSave
+
+	// BuildEventSquashFinishSave - finish exporting tar for squashing
+	BuildEventSquashFinishSave
+
+	// BuildEventSquashStartSquash - start squashing
+	BuildEventSquashStartSquash
+
+	// BuildEventSquashFinishSquash - finish squashing
+	BuildEventSquashFinishSquash
+
+	// BuildEventSquashStartLoad - start importing squashed image
+	BuildEventSquashStartLoad
+
+	// BuildEventSquashFinishLoad - finish importing squashed image
+	BuildEventSquashFinishLoad
+
 	// BuildCompletedEvent is for when a `docker build` command completes
 	BuildCompletedEvent
 
@@ -35,29 +53,6 @@ const (
 	CompletedEvent
 )
 
-func (t EventType) String() string {
-	switch t {
-	case RequestedEvent:
-		return "RequestedEvent"
-	case CompletedEvent:
-		return "CompletedEvent"
-	case BuildEvent:
-		return "BuildEvent"
-	case BuildCompletedEvent:
-		return "BuildCompletedEvent"
-	case TagEvent:
-		return "TagEvent"
-	case TagCompletedEvent:
-		return "TagCompletedEvent"
-	case PushEvent:
-		return "PushEvent"
-	case PushCompletedEvent:
-		return "PushCompletedEvent"
-	default:
-		return ""
-	}
-}
-
 type event struct {
 	eventType EventType
 	data      map[string]interface{}

communication/eventtype_string.go

@@ -0,0 +1,21 @@
+// generated by stringer -type=EventType; DO NOT EDIT
+
+package comm
+
+import "fmt"
+
+const _EventTypeName = "RequestedEventBuildEventBuildEventSquashStartSaveBuildEventSquashFinishSaveBuildEventSquashStartSquashBuildEventSquashFinishSquashBuildEventSquashStartLoadBuildEventSquashFinishLoadBuildCompletedEventTagEventTagCompletedEventPushEventPushCompletedEventCompletedEvent"
+
+var _EventTypeIndex = [...]uint16{14, 24, 49, 75, 102, 130, 155, 181, 200, 208, 225, 234, 252, 266}
+
+func (i EventType) String() string {
+	if i >= EventType(len(_EventTypeIndex)) {
+		return fmt.Sprintf("EventType(%d)", i)
+	}
+	hi := _EventTypeIndex[i]
+	lo := uint16(0)
+	if i > 0 {
+		lo = _EventTypeIndex[i-1]
+	}
+	return _EventTypeName[lo:hi]
+}

parser/docker_cmd.go

@@ -1,9 +1,11 @@
 package parser
 
 import (
+	"bytes"
 	"io"
 	"strings"
 
+	log "github.com/Sirupsen/logrus"
 	"github.com/fsouza/go-dockerclient"
 	"github.com/rafecolton/go-dockerclient-quick"
 	"github.com/winchman/builder-core/communication"
@@ -20,6 +22,7 @@ type DockerCmdOpts struct {
 	Stdout       io.Writer
 	Stderr       io.Writer
 	SkipPush     bool
+	Squash       bool
 	ImageUUID    string
 	Reporter     *comm.Reporter
 }
@@ -72,7 +75,7 @@ func (err NilClientError) Error() string {
 //Run is the command that actually calls docker build shell command.  Determine
 //the image ID for the resulting image and return that as well.
 func (b *BuildCmd) Run() (string, error) {
-	var opts = b.opts
+	opts := b.opts
 	if b.test {
 		return opts.ImageUUID, NilClientError{}
 	}
@@ -106,6 +109,7 @@ func (b *BuildCmd) Run() (string, error) {
 		return "", err
 	}
 
+	// build successful, handle squashig
 	b.reporter.Event(comm.EventOptions{
 		EventType: comm.BuildCompletedEvent,
 		Data: map[string]interface{}{
@@ -114,7 +118,46 @@ func (b *BuildCmd) Run() (string, error) {
 			"error":    nil,
 		},
 	})
-	return image.ID, nil
+	retID := image.ID
+
+	if opts.Squash {
+		imageReader, pipeWriter := io.Pipe()
+		squashedImageReader, squashedImageWriter := io.Pipe()
+		retIDBuffer := new(bytes.Buffer)
+
+		defer func() {
+			imageReader.Close()
+			squashedImageReader.Close()
+			pipeWriter.Close()
+			squashedImageWriter.Close()
+		}()
+
+		// exporting async
+		go b.exportImage(image, pipeWriter, opts)
+
+		// squash async
+		go b.squash(imageReader, squashedImageWriter, retIDBuffer)
+
+		// import and wait for export, squash, and import to finish
+		if err := b.loadImage(squashedImageReader, opts); err != nil {
+			return "", err
+		}
+
+		// if all steps are successful, retID will contain the id of the
+		// imported, squashed imaged
+		retID = retIDBuffer.String()
+
+		b.reporter.Event(comm.EventOptions{
+			EventType: comm.BuildEventSquashFinishLoad,
+			Data:      map[string]interface{}{"squashed_image_id": retID},
+		})
+
+		b.reporter.Log(
+			log.WithFields(log.Fields{"squashed_image_id": retID}),
+			"finished import of squashed image",
+		)
+	}
+	return retID, nil
 }
 
 //Message returns the shell command that gets run for docker build commands

parser/logic.go

@@ -67,6 +67,7 @@ func mergeGlobals(container, globals *unitconfig.ContainerSection) *unitconfig.C
 	}
 
 	container.SkipPush = container.SkipPush || globals.SkipPush
+	container.Squash = container.Squash || globals.Squash
 
 	return container
 }
@@ -181,6 +182,7 @@ func (parser *Parser) step2(is *InstructionSet) *CommandSequence {
 				Name:       v.Name,
 				Dockerfile: v.Dockerfile,
 				UUID:       uuid.String(),
+				Squash:     v.Squash,
 			},
 			SubCommand: containerCommands,
 		})