Merge pull request #98 from soltysh/card416

Merged by openshift-bot

cmd/sti/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/spf13/pflag"
 
 	"github.com/openshift/source-to-image/pkg/sti"
+	"github.com/openshift/source-to-image/pkg/sti/api"
 	"github.com/openshift/source-to-image/pkg/sti/errors"
 	"github.com/openshift/source-to-image/pkg/sti/version"
 )
@@ -52,7 +53,7 @@ func newCmdVersion() *cobra.Command {
 	}
 }
 
-func newCmdBuild(req *sti.Request) *cobra.Command {
+func newCmdBuild(req *api.Request) *cobra.Command {
 	buildCmd := &cobra.Command{
 		Use:   "build <source> <image> <tag>",
 		Short: "Build a new image",
@@ -85,12 +86,13 @@ func newCmdBuild(req *sti.Request) *cobra.Command {
 	buildCmd.Flags().StringVarP(&(req.Ref), "ref", "r", "", "Specify a ref to check-out")
 	buildCmd.Flags().StringVar(&(req.CallbackURL), "callbackURL", "", "Specify a URL to invoke via HTTP POST upon build completion")
 	buildCmd.Flags().StringVarP(&(req.ScriptsURL), "scripts", "s", "", "Specify a URL for the assemble and run scripts")
+	buildCmd.Flags().StringVarP(&(req.Location), "location", "l", "", "Specify a destination location for untar operation")
 	buildCmd.Flags().BoolVar(&(req.ForcePull), "forcePull", true, "Always pull the builder image even if it is present locally")
 	buildCmd.Flags().BoolVar(&(req.PreserveWorkingDir), "saveTempDir", false, "Save the temporary directory used by STI instead of deleting it")
 	return buildCmd
 }
 
-func newCmdUsage(req *sti.Request) *cobra.Command {
+func newCmdUsage(req *api.Request) *cobra.Command {
 	usageCmd := &cobra.Command{
 		Use:   "usage <image>",
 		Short: "Print usage of the assemble script associated with the image",
@@ -116,6 +118,7 @@ func newCmdUsage(req *sti.Request) *cobra.Command {
 	usageCmd.Flags().StringVarP(&(req.ScriptsURL), "scripts", "s", "", "Specify a URL for the assemble and run scripts")
 	usageCmd.Flags().BoolVar(&(req.ForcePull), "forcePull", true, "Always pull the builder image even if it is present locally")
 	usageCmd.Flags().BoolVar(&(req.PreserveWorkingDir), "saveTempDir", false, "Save the temporary directory used by STI instead of deleting it")
+	usageCmd.Flags().StringVarP(&(req.Location), "location", "l", "", "Specify a destination location for untar operation")
 	return usageCmd
 }
 
@@ -151,7 +154,7 @@ func checkErr(err error) {
 }
 
 func main() {
-	req := &sti.Request{}
+	req := &api.Request{}
 	stiCmd := &cobra.Command{
 		Use: "sti",
 		Long: "Source-to-image (STI) is a tool for building repeatable docker images.\n\n" +

hack/build-test-images.sh

@@ -25,3 +25,4 @@ buildimage sti_test/sti-fake test/integration/images/sti-fake
 buildimage sti_test/sti-fake-user test/integration/images/sti-fake-user
 buildimage sti_test/sti-fake-scripts test/integration/images/sti-fake-scripts
 buildimage sti_test/sti-fake-scripts-no-save-artifacts test/integration/images/sti-fake-scripts-no-save-artifacts
+buildimage sti_test/sti-fake-no-tar test/integration/images/sti-fake-no-tar

hack/test-integration.sh

@@ -17,7 +17,7 @@ set +e
 img_count=$(docker images | grep -c sti_test/sti-fake)
 set -e
 
-if [ "${img_count}" != "4" ]; then
+if [ "${img_count}" != "5" ]; then
     echo "You do not have necessary test images, be sure to run 'hack/build-test-images.sh' beforehand."
     exit 1
 fi

pkg/sti/api/doc.go

@@ -0,0 +1,2 @@
+// Provides types used for processing sti requests.
+package api

pkg/sti/api/script.go

@@ -0,0 +1,20 @@
+package api
+
+// Script defines script names used by STI process.
+type Script string
+
+const (
+	// Assemble is the name of the script responsible for build process of the resulting image.
+	Assemble Script = "assemble"
+	// Run is the name of the script responsible for running the final application.
+	Run Script = "run"
+	// SaveArtifacts is the name of the script responsible for storing dependencies etc. between builds.
+	SaveArtifacts Script = "save-artifacts"
+	// Usage i the name of the script responsible for printing the builder image's short info.
+	Usage Script = "usage"
+)
+
+// String returns name of the script.
+func (s Script) String() string {
+	return string(s)
+}

pkg/sti/types.go → pkg/sti/api/types.go

@@ -1,4 +1,4 @@
-package sti
+package api
 
 // Request contains essential fields for any request.
 type Request struct {
@@ -37,20 +37,26 @@ type Request struct {
 	// ScriptsURL is a URL describing the localization of STI scripts used during build process.
 	ScriptsURL string
 
+	// Location specifies a location where the untar operation will place its artifacts.
+	Location string
+
 	// ForcePull describes if the builder should pull the images from registry prior to building.
 	ForcePull bool
 
-	// incremental describes incremental status of current build
-	incremental bool
+	// Incremental describes incremental status of current build
+	Incremental bool
+
+	// WorkingDir describes temporary directory used for downloading sources, scripts and tar operations.
+	WorkingDir string
 
-	// workingDir describes temporary directory used for downloading sources, scripts and tar operations.
-	workingDir string
+	// ExternalRequiredScripts describes if required scripts are from external URL.
+	ExternalRequiredScripts bool
 
-	// externalRequiredScripts describes if required scripts are from external URL.
-	externalRequiredScripts bool
+	// ExternalOptionalScripts describes if optional scripts are from external URL.
+	ExternalOptionalScripts bool
 
-	// externalOptionalScripts describes if optional scripts are from external URL.
-	externalOptionalScripts bool
+	// LayeredBuild describes if this is build which layered scripts and sources on top of BaseImage.
+	LayeredBuild bool
 }
 
 // Result structure contains information from build process.

pkg/sti/build.go

@@ -3,9 +3,11 @@ package sti
 import (
 	"io"
 	"path/filepath"
+	"regexp"
 
 	"github.com/golang/glog"
 
+	"github.com/openshift/source-to-image/pkg/sti/api"
 	"github.com/openshift/source-to-image/pkg/sti/docker"
 	"github.com/openshift/source-to-image/pkg/sti/errors"
 	"github.com/openshift/source-to-image/pkg/sti/util"
@@ -18,13 +20,15 @@ type Builder struct {
 
 type buildHandlerInterface interface {
 	cleanup()
-	setup(requiredScripts, optionalScripts []string) error
+	setup(required []api.Script, optional []api.Script) error
 	determineIncremental() error
-	Request() *Request
-	Result() *Result
+	Request() *api.Request
+	Result() *api.Result
 	saveArtifacts() error
 	fetchSource() error
-	execute(command string) error
+	execute(command api.Script) error
+	wasExpectedError(text string) bool
+	build() error
 }
 
 type buildHandler struct {
@@ -33,7 +37,7 @@ type buildHandler struct {
 }
 
 // NewBuilder returns a new Builder
-func NewBuilder(req *Request) (*Builder, error) {
+func NewBuilder(req *api.Request) (*Builder, error) {
 	handler, err := newBuildHandler(req)
 	if err != nil {
 		return nil, err
@@ -43,7 +47,7 @@ func NewBuilder(req *Request) (*Builder, error) {
 	}, nil
 }
 
-func newBuildHandler(req *Request) (*buildHandler, error) {
+func newBuildHandler(req *api.Request) (*buildHandler, error) {
 	rh, err := newRequestHandler(req)
 	if err != nil {
 		return nil, err
@@ -53,18 +57,19 @@ func newBuildHandler(req *Request) (*buildHandler, error) {
 		callbackInvoker: util.NewCallbackInvoker(),
 	}
 	rh.postExecutor = bh
+	rh.errorChecker = bh
 	return bh, nil
 }
 
-// Build processes a Request and returns a *Result and an error.
+// Build processes a Request and returns a *api.Result and an error.
 // An error represents a failure performing the build rather than a failure
 // of the build itself.  Callers should check the Success field of the result
 // to determine whether a build succeeded or not.
-func (b *Builder) Build() (*Result, error) {
+func (b *Builder) Build() (*api.Result, error) {
 	bh := b.handler
 	defer bh.cleanup()
 
-	err := bh.setup([]string{"assemble", "run"}, []string{"save-artifacts"})
+	err := bh.setup([]api.Script{api.Assemble, api.Run}, []api.Script{api.SaveArtifacts})
 	if err != nil {
 		return nil, err
 	}
@@ -73,40 +78,61 @@ func (b *Builder) Build() (*Result, error) {
 	if err != nil {
 		return nil, err
 	}
-	if bh.Request().incremental {
+	if bh.Request().Incremental {
 		glog.V(1).Infof("Existing image for tag %s detected for incremental build.", bh.Request().Tag)
 	} else {
 		glog.V(1).Infof("Clean build will be performed")
 	}
 
 	glog.V(2).Infof("Performing source build from %s", bh.Request().Source)
-	if bh.Request().incremental {
+	if bh.Request().Incremental {
 		if err = bh.saveArtifacts(); err != nil {
 			glog.Warning("Error saving previous build artifacts: %v", err)
 			glog.Warning("Clean build will be performed!")
 		}
 	}
 
-	if err = bh.execute("assemble"); err != nil {
+	glog.V(1).Infof("Building %s", bh.Request().Tag)
+	err = bh.execute(api.Assemble)
+	if e, ok := err.(errors.ContainerError); ok && bh.wasExpectedError(e.ExpectedError) {
+		glog.Warningf("Image %s does not have tar! Performing additional build to add the scripts and sources.",
+			bh.Request().BaseImage)
+		if err := bh.build(); err != nil {
+			return nil, err
+		}
+		glog.V(2).Infof("Building %s using sti-enabled image", bh.Request().Tag)
+		if err := bh.execute(api.Assemble); err != nil {
+			return nil, err
+		}
+	} else if err != nil {
 		return nil, err
 	}
 
 	return bh.Result(), nil
 }
 
-func (h *buildHandler) PostExecute(containerID string, cmd []string) error {
+// wasExpectedError is used for determining whether the error that appeared
+// authorizes us to do the additional build injecting the scripts and sources.
+func (h *buildHandler) wasExpectedError(text string) bool {
+	tar, _ := regexp.MatchString(`.*tar.*not found`, text)
+	sh, _ := regexp.MatchString(`.*/bin/sh.*no such file or directory`, text)
+	return tar || sh
+}
+
+func (h *buildHandler) PostExecute(containerID string, location string) error {
 	var (
 		err             error
 		previousImageID string
 	)
-	if h.request.incremental && h.request.RemovePreviousImage {
+	if h.request.Incremental && h.request.RemovePreviousImage {
 		if previousImageID, err = h.docker.GetImageID(h.request.Tag); err != nil {
 			glog.Errorf("Error retrieving previous image's metadata: %v", err)
 		}
 	}
 
+	cmd := []string{}
 	opts := docker.CommitContainerOptions{
-		Command:     append(cmd, "run"),
+		Command:     append(cmd, filepath.Join(location, string(api.Run))),
 		Env:         h.generateConfigEnv(),
 		ContainerID: containerID,
 		Repository:  h.request.Tag,
@@ -119,7 +145,7 @@ func (h *buildHandler) PostExecute(containerID string, cmd []string) error {
 	h.result.ImageID = imageID
 	glog.V(1).Infof("Tagged %s as %s", imageID, h.request.Tag)
 
-	if h.request.incremental && h.request.RemovePreviousImage && previousImageID != "" {
+	if h.request.Incremental && h.request.RemovePreviousImage && previousImageID != "" {
 		glog.V(1).Infof("Removing previously-tagged image %s", previousImageID)
 		if err = h.docker.RemoveImage(previousImageID); err != nil {
 			glog.Errorf("Unable to remove previous image: %v", err)
@@ -136,7 +162,7 @@ func (h *buildHandler) PostExecute(containerID string, cmd []string) error {
 }
 
 func (h *buildHandler) determineIncremental() (err error) {
-	h.request.incremental = false
+	h.request.Incremental = false
 	if h.request.Clean {
 		return
 	}
@@ -150,14 +176,14 @@ func (h *buildHandler) determineIncremental() (err error) {
 	// we're assuming save-artifacts to exists for embedded scripts (if not we'll
 	// warn a user upon container failure and proceed with clean build)
 	// for external save-artifacts - check its existence
-	saveArtifactsExists := !h.request.externalOptionalScripts ||
-		h.fs.Exists(filepath.Join(h.request.workingDir, "upload", "scripts", "save-artifacts"))
-	h.request.incremental = previousImageExists && saveArtifactsExists
+	saveArtifactsExists := !h.request.ExternalOptionalScripts ||
+		h.fs.Exists(filepath.Join(h.request.WorkingDir, "upload", "scripts", string(api.SaveArtifacts)))
+	h.request.Incremental = previousImageExists && saveArtifactsExists
 	return nil
 }
 
 func (h *buildHandler) saveArtifacts() (err error) {
-	artifactTmpDir := filepath.Join(h.request.workingDir, "upload", "artifacts")
+	artifactTmpDir := filepath.Join(h.request.WorkingDir, "upload", "artifacts")
 	if err = h.fs.Mkdir(artifactTmpDir); err != nil {
 		return err
 	}
@@ -171,11 +197,13 @@ func (h *buildHandler) saveArtifacts() (err error) {
 	}
 
 	opts := docker.RunContainerOptions{
-		Image:        image,
-		OverwriteCmd: true,
-		Command:      "save-artifacts",
-		Stdout:       writer,
-		OnStart:      extractFunc,
+		Image:           image,
+		ExternalScripts: h.request.ExternalRequiredScripts,
+		ScriptsURL:      h.request.ScriptsURL,
+		Location:        h.request.Location,
+		Command:         api.SaveArtifacts,
+		Stdout:          writer,
+		OnStart:         extractFunc,
 	}
 	err = h.docker.RunContainer(opts)
 	writer.Close()
@@ -185,10 +213,10 @@ func (h *buildHandler) saveArtifacts() (err error) {
 	return err
 }
 
-func (h *buildHandler) Request() *Request {
+func (h *buildHandler) Request() *api.Request {
 	return h.request
 }
 
-func (h *buildHandler) Result() *Result {
+func (h *buildHandler) Result() *api.Result {
 	return h.result
 }