Merge pull request GoogleContainerTools#3489 from GoogleContainerTool…

…s/jib-sync-parts-2

Add common jib sync code for processing syncmap

pkg/skaffold/build/jib/sync.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2020 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package jib
+
+import (
+	"encoding/json"
+	"os"
+	"os/exec"
+	"regexp"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
+)
+
+type SyncMap map[string]SyncEntry
+
+type SyncEntry struct {
+	Dest     []string
+	FileTime time.Time
+	IsDirect bool
+}
+
+type JSONSyncMap struct {
+	Direct    []JSONSyncEntry `json:"direct"`
+	Generated []JSONSyncEntry `json:"generated"`
+}
+
+type JSONSyncEntry struct {
+	Src  string `json:"src"`
+	Dest string `json:"dest"`
+}
+
+func getSyncMapFromSystem(cmd *exec.Cmd) (*SyncMap, error) {
+	jsm := JSONSyncMap{}
+	stdout, err := util.RunCmdOut(cmd)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get Jib sync map")
+	}
+
+	// To parse the output, search for "BEGIN JIB JSON", then unmarshal the next line into the pathMap struct.
+	// Syncmap is transitioning to "BEGIN JIB JSON: SYNCMAP/1" starting in jib 2.0.0
+	// perhaps this feature should only be included from 2.0.0 onwards? And we generally avoid this?
+	matches := regexp.MustCompile(`BEGIN JIB JSON(?:: SYNCMAP/1)?\r?\n({.*})`).FindSubmatch(stdout)
+	if len(matches) == 0 {
+		return nil, errors.New("failed to get Jib Sync data")
+	}
+
+	if err := json.Unmarshal(matches[1], &jsm); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	sm := make(SyncMap)
+	if err := sm.addEntries(jsm.Direct, true); err != nil {
+		return nil, errors.WithStack(err)
+	}
+	if err := sm.addEntries(jsm.Generated, false); err != nil {
+		return nil, errors.WithStack(err)
+	}
+	return &sm, nil
+}
+
+func (sm SyncMap) addEntries(entries []JSONSyncEntry, direct bool) error {
+	for _, entry := range entries {
+		info, err := os.Stat(entry.Src)
+		if err != nil {
+			return errors.Wrapf(err, "could not obtain file mod time for %s", entry.Src)
+		}
+		sm[entry.Src] = SyncEntry{
+			Dest:     []string{entry.Dest},
+			FileTime: info.ModTime(),
+			IsDirect: direct,
+		}
+	}
+	return nil
+}

pkg/skaffold/build/jib/sync_test.go

@@ -0,0 +1,140 @@
+/*
+Copyright 2020 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package jib
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
+	"github.com/GoogleContainerTools/skaffold/testutil"
+)
+
+func TestGetSyncMapFromSystem(t *testing.T) {
+	tmpDir, cleanup := testutil.NewTempDir(t)
+	defer cleanup()
+
+	tmpDir.Touch("dep1", "dir/dep2")
+	dep1 := tmpDir.Path("dep1")
+	dep2 := tmpDir.Path("dir/dep2")
+
+	dep1Time := getFileTime(dep1, t)
+	dep2Time := getFileTime(dep2, t)
+
+	dep1Target := "/target/dep1"
+	dep2Target := "/target/anotherDir/dep2"
+
+	tests := []struct {
+		description string
+		stdout      string
+		shouldErr   bool
+		expected    *SyncMap
+	}{
+		{
+			description: "empty",
+			stdout:      "",
+			shouldErr:   true,
+			expected:    nil,
+		},
+		{
+			description: "old style marker",
+			stdout:      "BEGIN JIB JSON\n{}",
+			shouldErr:   false,
+			expected:    &SyncMap{},
+		},
+		{
+			description: "bad marker",
+			stdout:      "BEGIN JIB JSON: BAD/1\n{}",
+			shouldErr:   true,
+			expected:    nil,
+		},
+		{
+			description: "direct only",
+			stdout: "BEGIN JIB JSON: SYNCMAP/1\n" +
+				fmt.Sprintf(`{"direct":[{"src":"%s","dest":"%s"}]}`, escapeBackslashes(dep1), dep1Target),
+			shouldErr: false,
+			expected: &SyncMap{
+				dep1: SyncEntry{
+					[]string{dep1Target},
+					dep1Time,
+					true,
+				},
+			},
+		},
+		{
+			description: "generated only",
+			stdout: "BEGIN JIB JSON: SYNCMAP/1\n" +
+				fmt.Sprintf(`{"generated":[{"src":"%s","dest":"%s"}]}`, escapeBackslashes(dep1), dep1Target),
+			shouldErr: false,
+			expected: &SyncMap{
+				dep1: SyncEntry{
+					[]string{dep1Target},
+					dep1Time,
+					false,
+				},
+			},
+		},
+		{
+			description: "generated and direct",
+			stdout: "BEGIN JIB JSON: SYNCMAP/1\n" +
+				fmt.Sprintf(`{"direct":[{"src":"%s","dest":"%s"}],"generated":[{"src":"%s","dest":"%s"}]}"`, escapeBackslashes(dep1), dep1Target, escapeBackslashes(dep2), dep2Target),
+			shouldErr: false,
+			expected: &SyncMap{
+				dep1: SyncEntry{
+					[]string{dep1Target},
+					dep1Time,
+					true,
+				},
+				dep2: SyncEntry{
+					Dest:     []string{dep2Target},
+					FileTime: dep2Time,
+					IsDirect: false,
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		testutil.Run(t, test.description, func(t *testutil.T) {
+			t.Override(&util.DefaultExecCommand, testutil.CmdRunOut(
+				"ignored",
+				test.stdout,
+			))
+
+			results, err := getSyncMapFromSystem(&exec.Cmd{Args: []string{"ignored"}})
+
+			t.CheckErrorAndDeepEqual(test.shouldErr, err, test.expected, results)
+		})
+	}
+}
+
+func getFileTime(file string, t *testing.T) time.Time {
+	info, err := os.Stat(file)
+	if err != nil {
+		t.Fatalf("Failed to stat %s", file)
+		return time.Time{}
+	}
+	return info.ModTime()
+}
+
+// for paths that contain "\", they must be escaped in json strings
+func escapeBackslashes(path string) string {
+	return strings.Replace(path, `\`, `\\`, -1)
+}