FAB-16599 Add optional release to externalbuilders

Per the design, the external builders are to be given the opportunity to
supply a 'release' binary, which will be invoked after the build, and
give the chaincode an opportunity to supply assets such as connection
information for a remote chaincode, or chaincode db indicates which must
be installed into the statedb for the chaincode to function properly.

This CR adds rudimentary support for release.

Signed-off-by: Jason Yellick <jyellick@us.ibm.com>
Change-Id: I743174c825eae7fd62eaca03bb2905ddfa7cdc1b

Author: Jason Yellick

core/container/externalbuilders/externalbuilders.go

@@ -141,6 +141,10 @@ func (d *Detector) Build(ccid string, md *persistence.ChaincodePackageMetadata,
 		return nil, errors.WithMessage(err, "external builder failed to build")
 	}
 
+	if err := builder.Release(buildContext); err != nil {
+		return nil, errors.WithMessage(err, "external builder failed to release")
+	}
+
 	durablePath := filepath.Join(d.DurablePath, ccid)
 
 	err = os.Mkdir(durablePath, 0700)
@@ -162,9 +166,16 @@ func (d *Detector) Build(ccid string, md *persistence.ChaincodePackageMetadata,
 		return nil, errors.WithMessage(err, "could not write build-info.json")
 	}
 
+	err = os.Rename(buildContext.ReleaseDir, filepath.Join(durablePath, "release"))
+	if err != nil {
+		os.RemoveAll(durablePath)
+		return nil, errors.WithMessagef(err, "could not move build context release to persistent location '%s'", durablePath)
+	}
+
 	durableBldDir := filepath.Join(durablePath, "bld")
 	err = os.Rename(buildContext.BldDir, durableBldDir)
 	if err != nil {
+		os.RemoveAll(durablePath)
 		return nil, errors.WithMessagef(err, "could not move build context bld to persistent location '%s'", durablePath)
 	}
 
@@ -180,48 +191,52 @@ type BuildContext struct {
 	Metadata    *persistence.ChaincodePackageMetadata
 	ScratchDir  string
 	SourceDir   string
+	ReleaseDir  string
 	MetadataDir string
 	BldDir      string
 }
 
 var pkgIDreg = regexp.MustCompile("[^a-zA-Z0-9]+")
 
-func NewBuildContext(ccid string, md *persistence.ChaincodePackageMetadata, codePackage io.Reader) (*BuildContext, error) {
+func NewBuildContext(ccid string, md *persistence.ChaincodePackageMetadata, codePackage io.Reader) (bc *BuildContext, err error) {
 	scratchDir, err := ioutil.TempDir("", "fabric-"+pkgIDreg.ReplaceAllString(ccid, "-"))
 	if err != nil {
 		return nil, errors.WithMessage(err, "could not create temp dir")
 	}
 
+	defer func() {
+		if err != nil {
+			os.RemoveAll(scratchDir)
+		}
+	}()
+
 	sourceDir := filepath.Join(scratchDir, "src")
-	err = os.Mkdir(sourceDir, 0700)
-	if err != nil {
-		os.RemoveAll(scratchDir)
+	if err = os.Mkdir(sourceDir, 0700); err != nil {
 		return nil, errors.WithMessage(err, "could not create source dir")
 	}
 
 	metadataDir := filepath.Join(scratchDir, "metadata")
-	err = os.Mkdir(metadataDir, 0700)
-	if err != nil {
-		os.RemoveAll(scratchDir)
+	if err = os.Mkdir(metadataDir, 0700); err != nil {
 		return nil, errors.WithMessage(err, "could not create metadata dir")
 	}
 
 	outputDir := filepath.Join(scratchDir, "bld")
-	err = os.Mkdir(outputDir, 0700)
-	if err != nil {
-		os.RemoveAll(scratchDir)
+	if err = os.Mkdir(outputDir, 0700); err != nil {
 		return nil, errors.WithMessage(err, "could not create build dir")
 	}
 
+	releaseDir := filepath.Join(scratchDir, "release")
+	if err = os.Mkdir(releaseDir, 0700); err != nil {
+		return nil, errors.WithMessage(err, "could not create release dir")
+	}
+
 	err = Untar(codePackage, sourceDir)
 	if err != nil {
-		os.RemoveAll(scratchDir)
 		return nil, errors.WithMessage(err, "could not untar source package")
 	}
 
 	err = writeMetadataFile(ccid, md, metadataDir)
 	if err != nil {
-		os.RemoveAll(scratchDir)
 		return nil, errors.WithMessage(err, "could not write metadata file")
 	}
 
@@ -230,6 +245,7 @@ func NewBuildContext(ccid string, md *persistence.ChaincodePackageMetadata, code
 		SourceDir:   sourceDir,
 		MetadataDir: metadataDir,
 		BldDir:      outputDir,
+		ReleaseDir:  releaseDir,
 		Metadata:    md,
 		CCID:        ccid,
 	}, nil
@@ -317,6 +333,33 @@ func (b *Builder) Build(buildContext *BuildContext) error {
 	return nil
 }
 
+func (b *Builder) Release(buildContext *BuildContext) error {
+	release := filepath.Join(b.Location, "bin", "release")
+
+	_, err := os.Stat(release)
+	if os.IsNotExist(err) {
+		b.Logger.Debugf("Skipping release step for '%s' as no release binary found", buildContext.CCID)
+		return nil
+	}
+
+	if err != nil {
+		return errors.WithMessagef(err, "could not stat release binary '%s'", release)
+	}
+
+	cmd := NewCommand(
+		release,
+		b.EnvWhitelist,
+		buildContext.BldDir,
+		buildContext.ReleaseDir,
+	)
+
+	if err := RunCommand(b.Logger, cmd); err != nil {
+		return errors.Wrapf(err, "builder '%s' release failed", b.Name)
+	}
+
+	return nil
+}
+
 // LaunchConfig is serialized to disk when launching
 type LaunchConfig struct {
 	CCID        string `json:"chaincode_id"`

core/container/externalbuilders/externalbuilders_test.go

@@ -63,6 +63,14 @@ var _ = Describe("Externalbuilders", func() {
 			_, err = os.Stat(buildContext.SourceDir)
 			Expect(err).NotTo(HaveOccurred())
 
+			Expect(buildContext.ReleaseDir).NotTo(BeEmpty())
+			_, err = os.Stat(buildContext.SourceDir)
+			Expect(err).NotTo(HaveOccurred())
+
+			Expect(buildContext.BldDir).NotTo(BeEmpty())
+			_, err = os.Stat(buildContext.SourceDir)
+			Expect(err).NotTo(HaveOccurred())
+
 			_, err = os.Stat(filepath.Join(buildContext.SourceDir, "a/test.file"))
 			Expect(err).NotTo(HaveOccurred())
 		})
@@ -104,8 +112,8 @@ var _ = Describe("Externalbuilders", func() {
 						Name: "bad1",
 					},
 					{
-						Path: "testdata",
-						Name: "testdata",
+						Path: "testdata/goodbuilder",
+						Name: "goodbuilder",
 					},
 					{
 						Path: "bad2",
@@ -127,7 +135,7 @@ var _ = Describe("Externalbuilders", func() {
 			It("iterates over all detectors and chooses the one that matches", func() {
 				instance, err := detector.Build("fake-package-id", md, codePackage)
 				Expect(err).NotTo(HaveOccurred())
-				Expect(instance.Builder.Name).To(Equal("testdata"))
+				Expect(instance.Builder.Name).To(Equal("goodbuilder"))
 			})
 
 			Context("when no builder can be found", func() {
@@ -148,6 +156,9 @@ var _ = Describe("Externalbuilders", func() {
 				_, err = os.Stat(filepath.Join(durablePath, "fake-package-id", "bld"))
 				Expect(err).NotTo(HaveOccurred())
 
+				_, err = os.Stat(filepath.Join(durablePath, "fake-package-id", "release"))
+				Expect(err).NotTo(HaveOccurred())
+
 				_, err = os.Stat(filepath.Join(durablePath, "fake-package-id", "build-info.json"))
 				Expect(err).NotTo(HaveOccurred())
 			})
@@ -215,7 +226,7 @@ var _ = Describe("Externalbuilders", func() {
 
 				It("returns an error", func() {
 					_, err := detector.Build("fake-package-id", md, codePackage)
-					Expect(err).To(MatchError("existing build could not be restored: chaincode 'fake-package-id' was already built with builder 'testdata', but that builder is no longer available"))
+					Expect(err).To(MatchError("existing build could not be restored: chaincode 'fake-package-id' was already built with builder 'goodbuilder', but that builder is no longer available"))
 				})
 			})
 		})
@@ -229,8 +240,9 @@ var _ = Describe("Externalbuilders", func() {
 
 		BeforeEach(func() {
 			builder = &externalbuilders.Builder{
-				Location: "testdata",
-				Name:     "testdata",
+				Location: "testdata/goodbuilder",
+				Name:     "goodbuilder",
+				Logger:   flogging.MustGetLogger("builder.test"),
 			}
 
 			var err error
@@ -250,13 +262,7 @@ var _ = Describe("Externalbuilders", func() {
 
 			Context("when the detector exits with a non-zero status", func() {
 				BeforeEach(func() {
-					md.Type = "foo"
-
-					var err error
-					codePackage, err = os.Open("testdata/normal_archive.tar.gz")
-					Expect(err).NotTo(HaveOccurred())
-					buildContext, err = externalbuilders.NewBuildContext("fake-package-id", md, codePackage)
-					Expect(err).NotTo(HaveOccurred())
+					builder.Location = "testdata/failbuilder"
 				})
 
 				It("returns false", func() {
@@ -274,18 +280,43 @@ var _ = Describe("Externalbuilders", func() {
 
 			Context("when the builder exits with a non-zero status", func() {
 				BeforeEach(func() {
-					md.Path = "bar"
+					builder.Location = "testdata/failbuilder"
+					builder.Name = "failbuilder"
+				})
 
-					var err error
-					codePackage, err = os.Open("testdata/normal_archive.tar.gz")
-					Expect(err).NotTo(HaveOccurred())
-					buildContext, err = externalbuilders.NewBuildContext("fake-package-id", md, codePackage)
+				It("returns an error", func() {
+					err := builder.Build(buildContext)
+					Expect(err).To(MatchError("builder 'failbuilder' build failed: exit status 1"))
+				})
+			})
+		})
+
+		Describe("Release", func() {
+			It("releases the package by invoking external builder", func() {
+				err := builder.Release(buildContext)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			When("the release binary is not in the builder", func() {
+				BeforeEach(func() {
+					builder.Location = "bad-builder-location"
+				})
+
+				It("returns no error as release is optional", func() {
+					err := builder.Release(buildContext)
 					Expect(err).NotTo(HaveOccurred())
 				})
+			})
+
+			When("the builder exits with a non-zero status", func() {
+				BeforeEach(func() {
+					builder.Location = "testdata/failbuilder"
+					builder.Name = "failbuilder"
+				})
 
 				It("returns an error", func() {
-					err := builder.Build(buildContext)
-					Expect(err).To(MatchError("builder 'testdata' build failed: exit status 1"))
+					err := builder.Release(buildContext)
+					Expect(err).To(MatchError("builder 'failbuilder' release failed: exit status 1"))
 				})
 			})
 		})
@@ -325,12 +356,13 @@ var _ = Describe("Externalbuilders", func() {
 
 			Context("when the launch exits with a non-zero status", func() {
 				BeforeEach(func() {
-					fakeConnection.Address = "different-fake-peer-address"
+					builder.Location = "testdata/failbuilder"
+					builder.Name = "failbuilder"
 				})
 
 				It("returns an error", func() {
 					err := builder.Launch("test-ccid", bldDir, fakeConnection)
-					Expect(err).To(MatchError("builder 'testdata' launch failed: exit status 1"))
+					Expect(err).To(MatchError("builder 'failbuilder' launch failed: exit status 1"))
 				})
 			})
 		})

core/container/externalbuilders/testdata/failbuilder/bin/build

@@ -0,0 +1,2 @@
+#!/bin/bash
+exit 1

core/container/externalbuilders/testdata/failbuilder/bin/detect

@@ -0,0 +1,2 @@
+#!/bin/bash
+exit 1

core/container/externalbuilders/testdata/failbuilder/bin/launch

@@ -0,0 +1,2 @@
+#!/bin/bash
+exit 1

core/container/externalbuilders/testdata/failbuilder/bin/release

@@ -0,0 +1,2 @@
+#!/bin/bash
+exit 1

core/container/externalbuilders/testdata/bin/build renamed to core/container/externalbuilders/testdata/goodbuilder/bin/build

@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ $# -ne 2 ] ; then
+  exit 2
+fi
+
+exit 0