Best effort copying of run directory

NOTICE.txt

@@ -3902,12 +3902,12 @@ Contents of probable licence file $GOMODCACHE/github.com/oklog/ulid@v1.3.1/LICEN
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/otiai10/copy
-Version: v1.2.0
+Dependency : github.com/michalpristas/copy
+Version: v1.9.1
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/otiai10/copy@v1.2.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/michalpristas/copy@v1.9.1/LICENSE:
 
 The MIT License (MIT)
 
@@ -13667,11 +13667,11 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/image-sp
 
 --------------------------------------------------------------------------------
 Dependency : github.com/otiai10/mint
-Version: v1.3.1
+Version: v1.4.1
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/otiai10/mint@v1.3.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/otiai10/mint@v1.4.1/LICENSE:
 
 Copyright 2017 otiai10 (Hiromu OCHIAI)
 

go.mod

@@ -33,7 +33,7 @@ require (
 	github.com/mitchellh/hashstructure v0.0.0-20170116052023-ab25296c0f51
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/oklog/ulid v1.3.1
-	github.com/otiai10/copy v1.2.0
+	github.com/otiai10/copy v1.9.0
 	github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.27.0
@@ -157,6 +157,7 @@ replace (
 	github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20
 	github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6
 	github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v1.4.8-0.20211018144411-a81f2b630e7c
+	github.com/otiai10/copy v1.9.0 => github.com/michalpristas/copy v1.9.1
 	github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c
 )
 

go.sum

@@ -892,6 +892,8 @@ github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88J
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4=
+github.com/michalpristas/copy v1.9.1 h1:jYTonyfnuxrOjb/LKwho7U++S6/iBuW1MyeTNzG7wyc=
+github.com/michalpristas/copy v1.9.1/go.mod h1:DiseS1dNi8N2JrJVHtzng5BgHfIMMP6Y4DOP1hqJCvM=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
@@ -1010,13 +1012,11 @@ github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqi
 github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
-github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k=
-github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
-github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc=
-github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
+github.com/otiai10/mint v1.4.1 h1:HOVBfKP1oXIc0wWo9hZ8JLdZtyCPWqjvmFDuVZ0yv2Y=
+github.com/otiai10/mint v1.4.1/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
@@ -1615,6 +1615,7 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

internal/pkg/agent/application/upgrade/test/case2/README.md

@@ -0,0 +1 @@
+test

internal/pkg/agent/application/upgrade/test/case3/README.md

@@ -0,0 +1 @@
+test 2

internal/pkg/agent/application/upgrade/upgrade.go

@@ -263,7 +263,7 @@ func copyRunDirectory(log *logger.Logger, newHash string) error {
 		return errors.New(err, "failed to create run directory")
 	}
 
-	err := copyDir(oldRunPath, newRunPath)
+	err := copyDir(log, oldRunPath, newRunPath, true)
 	if os.IsNotExist(err) {
 		// nothing to copy, operation ok
 		log.Debugw("Run directory not present", "old_run_path", oldRunPath)
@@ -279,7 +279,7 @@ func copyRunDirectory(log *logger.Logger, newHash string) error {
 // shutdownCallback returns a callback function to be executing during shutdown once all processes are closed.
 // this goes through runtime directory of agent and copies all the state files created by processes to new versioned
 // home directory with updated process name to match new version.
-func shutdownCallback(_ *logger.Logger, homePath, prevVersion, newVersion, newHash string) reexec.ShutdownCallbackFn {
+func shutdownCallback(l *logger.Logger, homePath, prevVersion, newVersion, newHash string) reexec.ShutdownCallbackFn {
 	if release.Snapshot() {
 		// SNAPSHOT is part of newVersion
 		prevVersion += "-SNAPSHOT"
@@ -297,7 +297,7 @@ func shutdownCallback(_ *logger.Logger, homePath, prevVersion, newVersion, newHa
 		for _, processDir := range processDirs {
 			newDir := strings.ReplaceAll(processDir, prevVersion, newVersion)
 			newDir = strings.ReplaceAll(newDir, oldHome, newHome)
-			if err := copyDir(processDir, newDir); err != nil {
+			if err := copyDir(l, processDir, newDir, true); err != nil {
 				return err
 			}
 		}
@@ -343,11 +343,22 @@ func readDirs(dir string) ([]string, error) {
 	return dirs, nil
 }
 
-func copyDir(from, to string) error {
+func copyDir(l *logger.Logger, from, to string, ignoreErrs bool) error {
+	var onErr func(error) error
+
+	if ignoreErrs {
+		onErr = func(err error) error {
+			// ignore all errors, just log them
+			l.Debugf("ignoring error: failed to copy %q to %q: %s", from, to, err.Error())
+			return nil
+		}
+	}
+
 	return copy.Copy(from, to, copy.Options{
 		OnSymlink: func(_ string) copy.SymlinkAction {
 			return copy.Shallow
 		},
-		Sync: true,
+		Sync:  true,
+		OnErr: onErr,
 	})
 }

internal/pkg/agent/application/upgrade/upgrade_test.go

@@ -9,15 +9,93 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 
+	"github.com/gofrs/flock"
 	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/elastic-agent/internal/pkg/release"
 	"github.com/elastic/elastic-agent/pkg/core/logger"
 )
 
+func Test_CopyFile(t *testing.T) {
+	l, _ := logger.New("test", false)
+	tt := []struct {
+		Name        string
+		From        string
+		To          string
+		IgnoreErr   bool
+		KeepOpen    bool
+		ExpectedErr bool
+	}{
+		{
+			"Existing, no onerr",
+			filepath.Join(".", "test", "case1", "README.md"),
+			filepath.Join(".", "test", "case1", "copy", "README.md"),
+			false,
+			false,
+			false,
+		},
+		{
+			"Existing but open",
+			filepath.Join(".", "test", "case2", "README.md"),
+			filepath.Join(".", "test", "case2", "copy", "README.md"),
+			false,
+			true,
+			runtime.GOOS == "windows", // this fails only on,
+		},
+		{
+			"Existing but open, ignore errors",
+			filepath.Join(".", "test", "case3", "README.md"),
+			filepath.Join(".", "test", "case3", "copy", "README.md"),
+			true,
+			true,
+			false,
+		},
+		{
+			"Not existing, accept errors",
+			filepath.Join(".", "test", "case4", "README.md"),
+			filepath.Join(".", "test", "case4", "copy", "README.md"),
+			false,
+			false,
+			true,
+		},
+		{
+			"Not existing, ignore errors",
+			filepath.Join(".", "test", "case4", "README.md"),
+			filepath.Join(".", "test", "case4", "copy", "README.md"),
+			true,
+			false,
+			false,
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.Name, func(t *testing.T) {
+			defer func() {
+				// cleanup
+				_ = os.RemoveAll(filepath.Dir(tc.To))
+			}()
+
+			var fl *flock.Flock
+			if tc.KeepOpen {
+				// this uses syscalls to create inter-process lock
+				fl = flock.New(tc.From)
+				_, err := fl.TryLock()
+				require.NoError(t, err)
+			}
+
+			err := copyDir(l, tc.From, tc.To, tc.IgnoreErr)
+			require.Equal(t, tc.ExpectedErr, err != nil)
+			if fl != nil {
+				require.NoError(t, fl.Unlock())
+			}
+		})
+	}
+}
+
 func TestShutdownCallback(t *testing.T) {
 	l, _ := logger.New("test", false)
 	tmpDir, err := ioutil.TempDir("", "shutdown-test-")