Run custom commands at build time

go.mod

@@ -5,6 +5,7 @@ go 1.12
 require (
 	github.com/NickBall/go-aes-key-wrap v0.0.0-20170929221519-1c3aa3e4dfc5
 	github.com/apache/mynewt-artifact v0.0.3
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/cast v1.3.0
 	github.com/spf13/cobra v0.0.5

go.sum

@@ -16,6 +16,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=

newt/builder/build.go

@@ -83,10 +83,16 @@ func NewBuilder(
 		}
 	}
 
-	// Create a pseudo build package for the generated sysinit code.
+	// Create the pseudo build packages.
 	if _, err := b.addSysinitBpkg(); err != nil {
 		return nil, err
 	}
+	if _, err := b.addUserPreBuildBpkg(); err != nil {
+		return nil, err
+	}
+	if _, err := b.addUserPreLinkBpkg(); err != nil {
+		return nil, err
+	}
 
 	for api, rpkg := range apiMap {
 		bpkg := b.PkgMap[rpkg]
@@ -504,6 +510,10 @@ func (b *Builder) PrepBuild() error {
 	baseCi.Includes = append(baseCi.Includes,
 		GeneratedIncludeDir(b.targetPkg.rpkg.Lpkg.FullName()))
 
+	// All packages have access to the user generated header directory.
+	baseCi.Includes = append(baseCi.Includes,
+		UserPreBuildIncludeDir(b.targetPkg.rpkg.Lpkg.Name()))
+
 	// Let multiplatform libraries know that a Mynewt binary is being build.
 	baseCi.Cflags = append(baseCi.Cflags, "-DMYNEWT=1")
 
@@ -518,17 +528,40 @@ func (b *Builder) AddCompilerInfo(info *toolchain.CompilerInfo) {
 	b.compilerInfo.AddCompilerInfo(info)
 }
 
-func (b *Builder) addSysinitBpkg() (*BuildPackage, error) {
-	lpkg := pkg.NewLocalPackage(b.targetPkg.rpkg.Lpkg.Repo().(*repo.Repo),
-		GeneratedBaseDir(b.targetPkg.rpkg.Lpkg.FullName()))
-	lpkg.SetName(pkg.ShortName(b.targetPkg.rpkg.Lpkg) + "-sysinit-" +
-		b.buildName)
+// addPseudoBpkg creates a dynamic build package and adds it to the builder.
+func (b *Builder) addPseudoBpkg(name string,
+	dir string) (*BuildPackage, error) {
+
+	lpkg := pkg.NewLocalPackage(b.targetPkg.rpkg.Lpkg.Repo().(*repo.Repo), dir)
+	lpkg.SetName(fmt.Sprintf(
+		"%s-%s", pkg.ShortName(b.targetPkg.rpkg.Lpkg), name))
 	lpkg.SetType(pkg.PACKAGE_TYPE_GENERATED)
 
 	rpkg := resolve.NewResolvePkg(lpkg)
 	return b.addPackage(rpkg)
 }
 
+// addSysinitBpkg adds the pseudo sysinit build package to the builder.
+func (b *Builder) addSysinitBpkg() (*BuildPackage, error) {
+	name := fmt.Sprintf("%s-%s", "sysinit", b.buildName)
+	dir := GeneratedBaseDir(b.targetPkg.rpkg.Lpkg.FullName())
+	return b.addPseudoBpkg(name, dir)
+}
+
+// addUserPreBuildBpkg adds the pseudo user build package to the builder.  The
+// user build package contains inputs emitted by external scripts.
+func (b *Builder) addUserPreBuildBpkg() (*BuildPackage, error) {
+	return b.addPseudoBpkg("user-pre-build",
+		UserPreBuildDir(b.targetPkg.rpkg.Lpkg.FullName()))
+}
+
+// addUserPreLinkBpkg adds the pseudo user build package to the builder.  The
+// user build package contains inputs emitted by external scripts.
+func (b *Builder) addUserPreLinkBpkg() (*BuildPackage, error) {
+	return b.addPseudoBpkg("user-pre-link",
+		UserPreLinkDir(b.targetPkg.rpkg.Lpkg.FullName()))
+}
+
 // Runs build jobs while any remain.  On failure, signals the other workers to
 // stop via the stop channel.  On error, the error object is signaled via the
 // results channel.  On successful completion, nil is signaled via the results

newt/builder/buildutil.go

@@ -21,25 +21,31 @@ package builder
 
 import (
 	"bytes"
+	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
 
 	"mynewt.apache.org/newt/newt/parse"
+	"mynewt.apache.org/newt/newt/pkg"
+	"mynewt.apache.org/newt/newt/project"
 	"mynewt.apache.org/newt/newt/resolve"
+	"mynewt.apache.org/newt/util"
 )
 
 func TestTargetName(testPkgName string) string {
 	return strings.Replace(testPkgName, "/", "_", -1)
 }
 
-func (b *Builder) FeatureString() string {
+// FeatureString converts a syscfg map to a string.  The string is a
+// space-separate list of "enabled" settings.
+func FeatureString(settings map[string]string) string {
 	var buffer bytes.Buffer
 
-	settingMap := b.cfg.SettingValues()
-	featureSlice := make([]string, 0, len(settingMap))
-	for k, v := range settingMap {
+	featureSlice := make([]string, 0, len(settings))
+	for k, v := range settings {
 		if parse.ValueIsTrue(v) {
 			featureSlice = append(featureSlice, k)
 		}
@@ -53,6 +59,7 @@ func (b *Builder) FeatureString() string {
 
 		buffer.WriteString(feature)
 	}
+
 	return buffer.String()
 }
 
@@ -126,3 +133,177 @@ func logDepInfo(res *resolve.Resolution) {
 		log.Debugf("%s", RevdepGraphText(rdg))
 	}
 }
+
+// binBasePath calculates the relative path of the "application binary"
+// directory.  Examples:
+//     * bin/targets/my_blinky_sim/app
+//     * bin/targets/splitty-nrf52dk/loader
+func (b *Builder) binBasePath() (string, error) {
+	if b.appPkg == nil {
+		return "", util.NewNewtError("app package not specified")
+	}
+
+	// Convert the binary path from absolute to relative.  This is required for
+	// Windows compatibility.
+	return util.TryRelPath(b.AppBinBasePath()), nil
+}
+
+// BasicEnvVars calculates the basic set of environment variables passed to all
+// external scripts.  `binBase` is the result of calling `binBasePath()`.
+func BasicEnvVars(binBase string, bspPkg *pkg.BspPackage) map[string]string {
+	coreRepo := project.GetProject().FindRepo("apache-mynewt-core")
+	bspPath := bspPkg.BasePath()
+
+	return map[string]string{
+		"CORE_PATH":           coreRepo.Path(),
+		"BSP_PATH":            bspPath,
+		"BIN_BASENAME":        binBase,
+		"BIN_ROOT":            BinRoot(),
+		"MYNEWT_PROJECT_ROOT": ProjectRoot(),
+	}
+}
+
+// SettingsEnvVars calculates the syscfg set of environment variables required
+// by image loading scripts.
+func SettingsEnvVars(settings map[string]string) map[string]string {
+	env := map[string]string{}
+
+	// Add all syscfg settings to the environment with the MYNEWT_VAL_ prefix.
+	for k, v := range settings {
+		env["MYNEWT_VAL_"+k] = v
+	}
+
+	if parse.ValueIsTrue(settings["BOOT_LOADER"]) {
+		env["BOOT_LOADER"] = "1"
+	}
+
+	env["FEATURES"] = FeatureString(settings)
+
+	return env
+}
+
+// SlotEnvVars calculates the image-slot set of environment variables required
+// by image loading scripts.  Pass a negative `imageSlot` value if the target
+// is a boot loader.
+func SlotEnvVars(bspPkg *pkg.BspPackage,
+	imageSlot int) (map[string]string, error) {
+
+	env := map[string]string{}
+
+	var flashTargetArea string
+	if imageSlot < 0 {
+		flashTargetArea = "FLASH_AREA_BOOTLOADER"
+	} else {
+		env["IMAGE_SLOT"] = strconv.Itoa(imageSlot)
+		switch imageSlot {
+		case 0:
+			flashTargetArea = "FLASH_AREA_IMAGE_0"
+		case 1:
+			flashTargetArea = "FLASH_AREA_IMAGE_1"
+		default:
+			return nil, util.FmtNewtError(
+				"invalid image slot: have=%d want=0or1", imageSlot)
+		}
+	}
+
+	tgtArea := bspPkg.FlashMap.Areas[flashTargetArea]
+	if tgtArea.Name == "" {
+		return nil, util.FmtNewtError(
+			"No flash target area %s", flashTargetArea)
+	}
+
+	env["FLASH_OFFSET"] = "0x" + strconv.FormatInt(int64(tgtArea.Offset), 16)
+	env["FLASH_AREA_SIZE"] = "0x" + strconv.FormatInt(int64(tgtArea.Size), 16)
+
+	return env, nil
+}
+
+type UserEnvParams struct {
+	Lpkg       *pkg.LocalPackage
+	TargetName string // Short name
+	AppName    string
+	BuildName  string // "app" or "loader"
+	UserSrcDir string // "" if none
+	UserIncDir string // "" if none
+	WorkDir    string
+}
+
+// UserEnvVars calculates the set of environment variables required by external
+// user scripts.
+func UserEnvVars(params UserEnvParams) map[string]string {
+	m := map[string]string{}
+
+	m["MYNEWT_APP_BIN_DIR"] = FileBinDir(
+		params.TargetName, params.BuildName, params.AppName)
+	m["MYNEWT_PKG_BIN_ARCHIVE"] = ArchivePath(
+		params.TargetName, params.BuildName, params.Lpkg.FullName(),
+		params.Lpkg.Type())
+	m["MYNEWT_PKG_BIN_DIR"] = PkgBinDir(
+		params.TargetName, params.BuildName, params.Lpkg.FullName(),
+		params.Lpkg.Type())
+	m["MYNEWT_PKG_NAME"] = params.Lpkg.FullName()
+	m["MYNEWT_USER_WORK_DIR"] = params.WorkDir
+
+	if params.UserSrcDir != "" {
+		m["MYNEWT_USER_SRC_DIR"] = params.UserSrcDir
+	}
+	if params.UserIncDir != "" {
+		m["MYNEWT_USER_INCLUDE_DIR"] = params.UserIncDir
+	}
+
+	return m
+}
+
+// EnvVars calculates the full set of environment variables passed to external
+// scripts.
+func (b *Builder) EnvVars(imageSlot int) (map[string]string, error) {
+	bspPkg := b.targetBuilder.bspPkg
+	settings := b.cfg.SettingValues()
+
+	binBasePath, err := b.binBasePath()
+	if err != nil {
+		return nil, err
+	}
+
+	// Calculate all three sets of environment variables: basic, settings, and
+	// slot.  Then merge the three sets into one.
+
+	env := BasicEnvVars(binBasePath, bspPkg)
+	setEnv := SettingsEnvVars(settings)
+
+	if parse.ValueIsTrue(settings["BOOT_LOADER"]) {
+		imageSlot = -1
+	}
+
+	slotEnv, err := SlotEnvVars(bspPkg, imageSlot)
+	if err != nil {
+		return nil, err
+	}
+
+	for k, v := range setEnv {
+		env[k] = v
+	}
+	for k, v := range slotEnv {
+		env[k] = v
+	}
+
+	return env, nil
+}
+
+// EnvVarsToSlice converts an environment variable map into a slice of strings
+// suitable for "shell command" functions defined in `util` (e.g.,
+// util.ShellCommand).
+func EnvVarsToSlice(env map[string]string) []string {
+	keys := make([]string, 0, len(env))
+	for k, _ := range env {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	slice := make([]string, 0, len(env))
+	for _, key := range keys {
+		slice = append(slice, fmt.Sprintf("%s=%s", key, env[key]))
+	}
+
+	return slice
+}

newt/builder/cmake.go

@@ -312,6 +312,14 @@ func (t *TargetBuilder) CMakeTargetBuilderWrite(w io.Writer, targetCompiler *too
 		return err
 	}
 
+	if len(t.res.PreBuildCmdCfg.StageFuncs) > 0 ||
+		len(t.res.PreLinkCmdCfg.StageFuncs) > 0 ||
+		len(t.res.PostBuildCmdCfg.StageFuncs) > 0 {
+
+		util.OneTimeWarning(
+			"custom commands not included in cmake output (unsupported)")
+	}
+
 	/* Build the Apps */
 	project.ResetDeps(t.AppList)