add .g10k-deploy.json deploy file and detect Puppetfile changes, fixes …

…#138

g10k.go

@@ -184,6 +184,16 @@ type ExecResult struct {
 	output     string
 }
 
+// DeployResult contains information about the Puppet environment which was deployed by g10k and tries to emulate the .r10k-deploy.json
+type DeployResult struct {
+	Name               string    `json:"name"`
+	Signature          string    `json:"signature"`
+	StartedAt          time.Time `json:"started_at"`
+	FinishedAt         time.Time `json:"finished_at"`
+	DeploySuccess      bool      `json:"deploy_success"`
+	PuppetfileChecksum string    `json:"puppetfile_checksum"`
+}
+
 func init() {
 	// initialize global maps
 	needSyncEnvs = make(map[string]struct{})

g10k_test.go

@@ -18,6 +18,20 @@ import (
 	"github.com/davecgh/go-spew/spew"
 )
 
+func removeTimestampsFromDeployfile(file string) {
+	if fileExists(file) {
+		dr := readDeployResultFile(file)
+		newDr := DeployResult{DeploySuccess: dr.DeploySuccess,
+			Name:               dr.Name,
+			Signature:          dr.Signature,
+			PuppetfileChecksum: dr.PuppetfileChecksum,
+		}
+
+		writeStructJSONFile(file, newDr)
+
+	}
+}
+
 func TestForgeChecksum(t *testing.T) {
 	expectedFmm := ForgeModule{md5sum: "8a8c741978e578921e489774f05e9a65", fileSize: 57358}
 	fmm := getMetadataForgeModule(ForgeModule{version: "2.2.0", name: "apt",
@@ -166,7 +180,7 @@ func TestConfigDeploy(t *testing.T) {
 	}
 }
 
-func TestResolvConfigAddWarning(t *testing.T) {
+func TestResolveConfigAddWarning(t *testing.T) {
 	funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1]
 	config = readConfigfile("tests/TestConfigAddWarning.yaml")
 	if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" {
@@ -191,21 +205,23 @@ func TestResolvConfigAddWarning(t *testing.T) {
 	}
 }
 
-func TestResolvStatic(t *testing.T) {
-
+func TestResolveStatic(t *testing.T) {
 	path, err := exec.LookPath("hashdeep")
 	if err != nil {
 		t.Skip("Skipping full Puppet environment resolv test, because package hashdeep is missing")
 	}
 
 	quiet = true
-	purgeDir("./cache/", "TestResolvStatic()")
-	purgeDir("./example/", "TestResolvStatic()")
+	purgeDir("./cache/", "TestResolveStatic()")
+	purgeDir("./example/", "TestResolveStatic()")
 	config = readConfigfile("tests/TestConfigStatic.yaml")
 	// increase maxworker to finish the test quicker
 	config.Maxworker = 500
 	resolvePuppetEnvironment("static", false, "")
 
+	// remove timestamps from .g10k-deploy.json otherwise hash sum would always differ
+	removeTimestampsFromDeployfile("example/example_static/.g10k-deploy.json")
+
 	cmd := exec.Command(path, "-vvv", "-l", "-r", "./example", "-a", "-k", "tests/hashdeep_example_static.hashdeep")
 	out, err := cmd.CombinedOutput()
 	exitCode := 0
@@ -220,9 +236,9 @@ func TestResolvStatic(t *testing.T) {
 	}
 	Debugf("hashdeep output:" + string(out))
 
-	purgeDir("example/example_static/external_modules/stdlib/spec/unit/facter/util", "TestResolvStatic()")
+	purgeDir("example/example_static/external_modules/stdlib/spec/unit/facter/util", "TestResolveStatic()")
 
-	cmd = exec.Command("hashdeep", "-r", "./example/", "-a", "-k", "tests/hashdeep_example_static.hashdeep")
+	cmd = exec.Command("hashdeep", "-l", "-r", "./example/", "-a", "-k", "tests/hashdeep_example_static.hashdeep")
 	out, err = cmd.CombinedOutput()
 	exitCode = 0
 	if msg, ok := err.(*exec.ExitError); ok { // there is error code
@@ -456,10 +472,10 @@ func TestModuleDirOverride(t *testing.T) {
 	moduleDirParam = ""
 }
 
-func TestResolvConfigExitIfUnreachable(t *testing.T) {
+func TestResolveConfigExitIfUnreachable(t *testing.T) {
 	funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1]
 	config = readConfigfile("tests/TestConfigExitIfUnreachable.yaml")
-	purgeDir(config.CacheDir, "TestResolvConfigExitIfUnreachable()")
+	purgeDir(config.CacheDir, "TestResolveConfigExitIfUnreachable()")
 	if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" {
 		resolvePuppetEnvironment("single", false, "")
 		return
@@ -483,10 +499,10 @@ func TestResolvConfigExitIfUnreachable(t *testing.T) {
 	}
 }
 
-func TestResolvConfigExitIfUnreachableFalse(t *testing.T) {
+func TestResolveConfigExitIfUnreachableFalse(t *testing.T) {
 	funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1]
 	config = readConfigfile("tests/TestConfigExitIfUnreachableFalse.yaml")
-	purgeDir(config.CacheDir, "TestResolvConfigExitIfUnreachableFalse()")
+	purgeDir(config.CacheDir, "TestResolveConfigExitIfUnreachableFalse()")
 	if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" {
 		resolvePuppetEnvironment("single", false, "")
 		return
@@ -2329,3 +2345,58 @@ func TestEnvironmentParameter(t *testing.T) {
 	purgeDir(cacheDir, funcName)
 	purgeDir("/tmp/out", funcName)
 }
+
+func TestDetectPuppetfileChanges(t *testing.T) {
+	quiet = true
+	funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1]
+	config = readConfigfile("tests/TestConfigUseCacheFallback.yaml")
+	if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" {
+		info = true
+		resolvePuppetEnvironment("single", false, "")
+		return
+	}
+
+	purgeDir("/tmp/example", funcName)
+
+	cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$")
+	cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1")
+	out, err := cmd.CombinedOutput()
+
+	exitCode := 0
+	if msg, ok := err.(*exec.ExitError); ok { // there is error code
+		exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus()
+	}
+
+	expectedExitCode := 0
+	if expectedExitCode != exitCode {
+		t.Errorf("terminated with %v, but we expected exit status %v", exitCode, expectedExitCode)
+	}
+	//fmt.Println(string(out))
+
+	expectedLines := []string{
+		"Need to sync /tmp/example/single/",
+		"Need to sync /tmp/example/single/external_modules/inifile/",
+	}
+
+	for _, expectedLine := range expectedLines {
+		if !strings.Contains(string(out), expectedLine) {
+			t.Errorf("Could not find expected line '" + expectedLine + "' in debug output")
+		}
+	}
+
+	// do it again, now g10k should detect that it's the same Puppetfile
+	cmdCached := exec.Command(os.Args[0], "-test.run="+funcName+"$")
+	cmdCached.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1")
+	outCached, err := cmdCached.CombinedOutput()
+	//fmt.Println(string(outCached))
+
+	expectedLines = []string{
+		"Skipping Puppetfile sync of branch example_single because /tmp/example/single/Puppetfile did not change",
+	}
+
+	for _, expectedLine := range expectedLines {
+		if !strings.Contains(string(outCached), expectedLine) {
+			t.Errorf("Could not find expected line '" + expectedLine + "' in debug output")
+		}
+	}
+}

git.go

@@ -128,6 +128,7 @@ func doMirrorOrUpdate(url string, workDir string, sshPrivateKey string, allowFai
 }
 
 func syncToModuleDir(srcDir string, targetDir string, tree string, allowFail bool, ignoreUnreachable bool, correspondingPuppetEnvironment string, onlyDelta bool) bool {
+	startedAt := time.Now()
 	mutex.Lock()
 	syncGitCount++
 	mutex.Unlock()
@@ -144,7 +145,8 @@ func syncToModuleDir(srcDir string, targetDir string, tree string, allowFail boo
 	}
 
 	er := executeCommand(logCmd, config.Timeout, allowFail)
-	hashFile := targetDir + ".latest_commit"
+	hashFile := filepath.Join(targetDir, ".latest_commit")
+	deployFile := filepath.Join(targetDir, ".g10k-deploy.json")
 	needToSync := true
 	if er.returnCode != 0 {
 		if allowFail && ignoreUnreachable {
@@ -155,10 +157,17 @@ func syncToModuleDir(srcDir string, targetDir string, tree string, allowFail boo
 	}
 
 	if len(er.output) > 0 {
-		targetHash, _ := ioutil.ReadFile(hashFile)
-		if string(targetHash) == er.output {
-			needToSync = false
-			//Debugf("Skipping, because no diff found between " + srcDir + "(" + er.output + ") and " + targetDir + "(" + string(targetHash) + ")")
+		if strings.HasPrefix(srcDir, config.EnvCacheDir) && fileExists(deployFile) {
+			dr := readDeployResultFile(deployFile)
+			if dr.Signature == strings.TrimSuffix(er.output, "\n") {
+				needToSync = false
+			}
+		} else {
+			targetHash, _ := ioutil.ReadFile(hashFile)
+			if string(targetHash) == er.output {
+				needToSync = false
+				//Debugf("Skipping, because no diff found between " + srcDir + "(" + er.output + ") and " + targetDir + "(" + string(targetHash) + ")")
+			}
 		}
 
 	}
@@ -211,11 +220,23 @@ func syncToModuleDir(srcDir string, targetDir string, tree string, allowFail boo
 
 			er = executeCommand(logCmd, config.Timeout, false)
 			if len(er.output) > 0 {
-				Debugf("Writing hash " + er.output + " from command " + logCmd + " to " + hashFile)
-				f, _ := os.Create(hashFile)
-				defer f.Close()
-				f.WriteString(er.output)
-				f.Sync()
+				commitHash := strings.TrimSuffix(er.output, "\n")
+				if strings.HasPrefix(srcDir, config.EnvCacheDir) {
+					Debugf("Writing to deploy file " + deployFile)
+					dr := DeployResult{
+						Name:      tree,
+						Signature: commitHash,
+						StartedAt: startedAt,
+					}
+					writeStructJSONFile(deployFile, dr)
+				} else {
+					Debugf("Writing hash " + commitHash + " from command " + logCmd + " to " + hashFile)
+					f, _ := os.Create(hashFile)
+					defer f.Close()
+					f.WriteString(commitHash)
+					f.Sync()
+				}
+
 			}
 		}
 	}

helper.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -260,7 +263,7 @@ func getSha256sumFile(file string) string {
 		Fatalf("failed to calculate SHA256 sum of file " + file + " Error: " + err.Error())
 	}
 
-	return string(h.Sum(nil))
+	return hex.EncodeToString(h.Sum(nil))
 }
 
 // moveFile uses io.Copy to create a copy of the given file https://stackoverflow.com/a/50741908/682847
@@ -298,3 +301,37 @@ func stringSliceContains(slice []string, element string) bool {
 	}
 	return false
 }
+
+func writeStructJSONFile(file string, v interface{}) {
+	content, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		Warnf("Could not encode JSON file " + file + " " + err.Error())
+	}
+
+	err = ioutil.WriteFile(file, content, 0644)
+	if err != nil {
+		Warnf("Could not write JSON file " + file + " " + err.Error())
+	}
+
+}
+
+func readDeployResultFile(file string) DeployResult {
+	// Open our jsonFile
+	jsonFile, err := os.Open(file)
+	// if we os.Open returns an error then handle it
+	if err != nil {
+		Warnf("Could not open JSON file " + file + " " + err.Error())
+	}
+	defer jsonFile.Close()
+
+	byteValue, err := ioutil.ReadAll(jsonFile)
+	if err != nil {
+		Warnf("Could not read JSON file " + file + " " + err.Error())
+	}
+
+	var dr DeployResult
+	json.Unmarshal([]byte(byteValue), &dr)
+
+	return dr
+
+}

puppetfile.go

@@ -8,6 +8,7 @@ import (
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/remeh/sizedwaitgroup"
 	"github.com/xorpaul/uiprogress"
@@ -127,10 +128,21 @@ func resolvePuppetEnvironment(envBranch string, tags bool, outputNameTag string)
 
 							env := strings.Replace(strings.Replace(targetDir, sa.Basedir, "", 1), "/", "", -1)
 							syncToModuleDir(workDir, targetDir, branch, false, false, env, true)
-							if !fileExists(targetDir + "Puppetfile") {
+							pf := filepath.Join(targetDir, "Puppetfile")
+							deployFile := filepath.Join(targetDir, ".g10k-deploy.json")
+							if !fileExists(pf) {
 								Debugf("Skipping branch " + source + "_" + branch + " because " + targetDir + "Puppetfile does not exist")
 							} else {
-								puppetfile := readPuppetfile(targetDir+"Puppetfile", sa.PrivateKey, source, sa.ForceForgeVersions, false)
+								if fileExists(deployFile) {
+									pfHashSum := getSha256sumFile(pf)
+									dr := readDeployResultFile(deployFile)
+									if pfHashSum == dr.PuppetfileChecksum && dr.DeploySuccess {
+										Infof("Skipping Puppetfile sync of branch " + source + "_" + branch + " because " + targetDir + "Puppetfile did not change")
+										dr.FinishedAt = time.Now()
+										writeStructJSONFile(deployFile, dr)
+									}
+								}
+								puppetfile := readPuppetfile(pf, sa.PrivateKey, source, sa.ForceForgeVersions, false)
 								puppetfile.workDir = normalizeDir(targetDir)
 								puppetfile.controlRepoBranch = branch
 								mutex.Lock()
@@ -170,7 +182,6 @@ func resolvePuppetEnvironment(envBranch string, tags bool, outputNameTag string)
 	resolvePuppetfile(allPuppetfiles)
 	//fmt.Println(desiredContent)
 	purgeUnmanagedContent(envBranch, allBasedirs, allEnvironments)
-
 }
 
 func purgeUnmanagedContent(envBranch string, allBasedirs map[string]bool, allEnvironments map[string]bool) {
@@ -444,9 +455,9 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) {
 				}
 
 				// remove this module from the exisitingModuleDirs map
-				moduleDirectory := moduleDir + gitName
+				moduleDirectory := filepath.Join(moduleDir, gitName)
 				if len(gitModule.installPath) > 0 {
-					moduleDirectory = normalizeDir(basedir) + normalizeDir(gitModule.installPath) + gitName
+					moduleDirectory = filepath.Join(normalizeDir(basedir), normalizeDir(gitModule.installPath), gitName)
 				}
 				moduleDirectory = normalizeDir(moduleDirectory)
 				mutex.Lock()
@@ -465,7 +476,7 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) {
 		}
 		for forgeModuleName, fm := range pf.forgeModules {
 			wg.Add()
-			moduleDir := pf.workDir + fm.moduleDir
+			moduleDir := filepath.Join(pf.workDir, fm.moduleDir)
 			moduleDir = normalizeDir(moduleDir)
 			go func(forgeModuleName string, fm ForgeModule, moduleDir string, env string) {
 				defer wg.Done()
@@ -478,7 +489,6 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) {
 				mutex.Unlock()
 			}(forgeModuleName, fm, moduleDir, env)
 		}
-
 	}
 	wg.Wait()
 
@@ -498,4 +508,17 @@ func resolvePuppetfile(allPuppetfiles map[string]Puppetfile) {
 	if !debug && !verbose && !info && !quiet && terminal.IsTerminal(int(os.Stdout.Fd())) {
 		uiprogress.Stop()
 	}
+
+	for _, pf := range allPuppetfiles {
+		deployFile := filepath.Join(pf.workDir, ".g10k-deploy.json")
+		if fileExists(deployFile) {
+			Debugf("Finishing writing to deploy file " + deployFile)
+			dr := readDeployResultFile(deployFile)
+			dr.DeploySuccess = true
+			dr.FinishedAt = time.Now()
+			dr.PuppetfileChecksum = getSha256sumFile(filepath.Join(pf.workDir, "Puppetfile"))
+			writeStructJSONFile(deployFile, dr)
+		}
+	}
+
 }