Run custom commands at build time

This commit adds the ability to run custom commands at build time.

This PR adds the ability to run custom commands at build time.

A package specifies custom commands in its `pkg.yml` file.  There are
three types of commands:
1. pre_build_cmds (run before the build)
2. pre_link_cmds (run after compilation, before linking)
3. post_build_cmds (run after the build)

Example (apps/blinky/pkg.yml):
```
pkg.pre_build_cmds:
    scripts/pre_build1.sh: 100
    scripts/pre_build2.sh: 200

pkg.pre_link_cmds:
    scripts/pre_link.sh: 500

pkg.post_build_cmds:
    scripts/post_build.sh: 100
```

For each command, the string on the left specifies the command to run.
The number on the right indicates the command's relative ordering.

When newt builds this example, it performs the following sequence:

* scripts/pre_build1.sh
* scripts/pre_build2.sh
* [compile]
* scripts/pre_link.sh
* [link]
* scripts/post_build.sh

If other packages specify custom commands, those commands would also be
executed during the above sequence.  For example, if another package
specifies a pre command with an ordering of 150, that command would run
immediately after `pre_build1.sh`.  In the case of a tie, the commands
are run in lexicographic order.

All commands are run from the project's base directory.  In the above
example, the `scripts` directory is a sibling of `targets`.

See <#335> for details.

go.mod

@@ -5,6 +5,7 @@ go 1.12
 require (
 	github.com/NickBall/go-aes-key-wrap v0.0.0-20170929221519-1c3aa3e4dfc5
 	github.com/apache/mynewt-artifact v0.0.3
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cast v1.3.0
 	github.com/spf13/cobra v0.0.5

go.sum

@@ -16,6 +16,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=

newt/builder/build.go

@@ -83,10 +83,16 @@ func NewBuilder(
 		}
 	}
 
-	// Create a pseudo build package for the generated sysinit code.
+	// Create the pseudo build packages.
 	if _, err := b.addSysinitBpkg(); err != nil {
 		return nil, err
 	}
+	if _, err := b.addUserPreBuildBpkg(); err != nil {
+		return nil, err
+	}
+	if _, err := b.addUserPreLinkBpkg(); err != nil {
+		return nil, err
+	}
 
 	for api, rpkg := range apiMap {
 		bpkg := b.PkgMap[rpkg]
@@ -504,6 +510,10 @@ func (b *Builder) PrepBuild() error {
 	baseCi.Includes = append(baseCi.Includes,
 		GeneratedIncludeDir(b.targetPkg.rpkg.Lpkg.FullName()))
 
+	// All packages have access to the user generated header directory.
+	baseCi.Includes = append(baseCi.Includes,
+		UserPreBuildIncludeDir(b.targetPkg.rpkg.Lpkg.Name()))
+
 	// Let multiplatform libraries know that a Mynewt binary is being build.
 	baseCi.Cflags = append(baseCi.Cflags, "-DMYNEWT=1")
 
@@ -518,17 +528,40 @@ func (b *Builder) AddCompilerInfo(info *toolchain.CompilerInfo) {
 	b.compilerInfo.AddCompilerInfo(info)
 }
 
-func (b *Builder) addSysinitBpkg() (*BuildPackage, error) {
-	lpkg := pkg.NewLocalPackage(b.targetPkg.rpkg.Lpkg.Repo().(*repo.Repo),
-		GeneratedBaseDir(b.targetPkg.rpkg.Lpkg.FullName()))
-	lpkg.SetName(pkg.ShortName(b.targetPkg.rpkg.Lpkg) + "-sysinit-" +
-		b.buildName)
+// addPseudoBpkg creates a dynamic build package and adds it to the builder.
+func (b *Builder) addPseudoBpkg(name string,
+	dir string) (*BuildPackage, error) {
+
+	lpkg := pkg.NewLocalPackage(b.targetPkg.rpkg.Lpkg.Repo().(*repo.Repo), dir)
+	lpkg.SetName(fmt.Sprintf(
+		"%s-%s", pkg.ShortName(b.targetPkg.rpkg.Lpkg), name))
 	lpkg.SetType(pkg.PACKAGE_TYPE_GENERATED)
 
 	rpkg := resolve.NewResolvePkg(lpkg)
 	return b.addPackage(rpkg)
 }
 
+// addSysinitBpkg adds the pseudo sysinit build package to the builder.
+func (b *Builder) addSysinitBpkg() (*BuildPackage, error) {
+	name := fmt.Sprintf("%s-%s", "sysinit", b.buildName)
+	dir := GeneratedBaseDir(b.targetPkg.rpkg.Lpkg.FullName())
+	return b.addPseudoBpkg(name, dir)
+}
+
+// addUserPreBuildBpkg adds the pseudo user build package to the builder.  The
+// user build package contains inputs emitted by external scripts.
+func (b *Builder) addUserPreBuildBpkg() (*BuildPackage, error) {
+	return b.addPseudoBpkg("user-pre-build",
+		UserPreBuildDir(b.targetPkg.rpkg.Lpkg.FullName()))
+}
+
+// addUserPreLinkBpkg adds the pseudo user build package to the builder.  The
+// user build package contains inputs emitted by external scripts.
+func (b *Builder) addUserPreLinkBpkg() (*BuildPackage, error) {
+	return b.addPseudoBpkg("user-pre-link",
+		UserPreLinkDir(b.targetPkg.rpkg.Lpkg.FullName()))
+}
+
 // Runs build jobs while any remain.  On failure, signals the other workers to
 // stop via the stop channel.  On error, the error object is signaled via the
 // results channel.  On successful completion, nil is signaled via the results

newt/builder/buildutil.go

@@ -21,6 +21,7 @@ package builder
 
 import (
 	"bytes"
+	"fmt"
 	"sort"
 	"strconv"
 	"strings"
@@ -217,6 +218,42 @@ func SlotEnvVars(bspPkg *pkg.BspPackage,
 	return env, nil
 }
 
+type UserEnvParams struct {
+	Lpkg       *pkg.LocalPackage
+	TargetName string // Short name
+	AppName    string
+	BuildName  string // "app" or "loader"
+	UserSrcDir string // "" if none
+	UserIncDir string // "" if none
+	WorkDir    string
+}
+
+// UserEnvVars calculates the set of environment variables required by external
+// user scripts.
+func UserEnvVars(params UserEnvParams) map[string]string {
+	m := map[string]string{}
+
+	m["MYNEWT_APP_BIN_DIR"] = FileBinDir(
+		params.TargetName, params.BuildName, params.AppName)
+	m["MYNEWT_PKG_BIN_ARCHIVE"] = ArchivePath(
+		params.TargetName, params.BuildName, params.Lpkg.FullName(),
+		params.Lpkg.Type())
+	m["MYNEWT_PKG_BIN_DIR"] = PkgBinDir(
+		params.TargetName, params.BuildName, params.Lpkg.FullName(),
+		params.Lpkg.Type())
+	m["MYNEWT_PKG_NAME"] = params.Lpkg.FullName()
+	m["MYNEWT_USER_WORK_DIR"] = params.WorkDir
+
+	if params.UserSrcDir != "" {
+		m["MYNEWT_USER_SRC_DIR"] = params.UserSrcDir
+	}
+	if params.UserIncDir != "" {
+		m["MYNEWT_USER_INCLUDE_DIR"] = params.UserIncDir
+	}
+
+	return m
+}
+
 // EnvVars calculates the full set of environment variables passed to external
 // scripts.
 func (b *Builder) EnvVars(imageSlot int) (map[string]string, error) {
@@ -238,7 +275,7 @@ func (b *Builder) EnvVars(imageSlot int) (map[string]string, error) {
 		imageSlot = -1
 	}
 
-	slotEnv, err := SlotEnvVars(bspPkg, imageSlot, settings)
+	slotEnv, err := SlotEnvVars(bspPkg, imageSlot)
 	if err != nil {
 		return nil, err
 	}
@@ -252,3 +289,21 @@ func (b *Builder) EnvVars(imageSlot int) (map[string]string, error) {
 
 	return env, nil
 }
+
+// EnvVarsToSlice converts an environment variable map into a slice of strings
+// suitable for "shell command" functions defined in `util` (e.g.,
+// util.ShellCommand).
+func EnvVarsToSlice(env map[string]string) []string {
+	keys := make([]string, 0, len(env))
+	for k, _ := range env {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	slice := make([]string, 0, len(env))
+	for _, key := range keys {
+		slice = append(slice, fmt.Sprintf("%s=%s", key, env[key]))
+	}
+
+	return slice
+}