txn_writer.go

// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dep

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/golang/dep/internal/fs"
	"github.com/golang/dep/internal/gps"
	"github.com/pelletier/go-toml"
	"github.com/pkg/errors"
)

// Example string to be written to the manifest file
// if no dependencies are found in the project
// during `dep init`
var exampleTOML = []byte(`
## Gopkg.toml example (these lines may be deleted)

## "metadata" defines metadata about the project that could be used by other independent
## systems. The metadata defined here will be ignored by dep.
# [metadata]
# key1 = "value that convey data to other systems"
# system1-data = "value that is used by a system"
# system2-data = "value that is used by another system"

## "required" lists a set of packages (not projects) that must be included in
## Gopkg.lock. This list is merged with the set of packages imported by the current
## project. Use it when your project needs a package it doesn't explicitly import -
## including "main" packages.
# required = ["github.com/user/thing/cmd/thing"]

## "ignored" lists a set of packages (not projects) that are ignored when
## dep statically analyzes source code. Ignored packages can be in this project,
## or in a dependency.
# ignored = ["github.com/user/project/badpkg"]

## Constraints are rules for how directly imported projects
## may be incorporated into the depgraph. They are respected by
## dep whether coming from the Gopkg.toml of the current project or a dependency.
# [[constraint]]
## Required: the root import path of the project being constrained.
# name = "github.com/user/project"
#
## Recommended: the version constraint to enforce for the project.
## Only one of "branch", "version" or "revision" can be specified.
# version = "1.0.0"
# branch = "master"
# revision = "abc123"
#
## Optional: an alternate location (URL or import path) for the project's source.
# source = "https://github.com/myfork/package.git"
#
## "metadata" defines metadata about the dependency or override that could be used
## by other independent systems. The metadata defined here will be ignored by dep.
# [metadata]
# key1 = "value that convey data to other systems"
# system1-data = "value that is used by a system"
# system2-data = "value that is used by another system"

## Overrides have the same structure as [[constraint]], but supersede all
## [[constraint]] declarations from all projects. Only [[override]] from
## the current project's are applied.
##
## Overrides are a sledgehammer. Use them only as a last resort.
# [[override]]
## Required: the root import path of the project being constrained.
# name = "github.com/user/project"
#
## Optional: specifying a version constraint override will cause all other
## constraints on this project to be ignored; only the overridden constraint
## need be satisfied.
## Again, only one of "branch", "version" or "revision" can be specified.
# version = "1.0.0"
# branch = "master"
# revision = "abc123"
#
## Optional: specifying an alternate source location as an override will
## enforce that the alternate location is used for that project, regardless of
## what source location any dependent projects specify.
# source = "https://github.com/myfork/package.git"


`)

// String added on top of lock file
var lockFileComment = []byte(`# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.

`)

// SafeWriter transactionalizes writes of manifest, lock, and vendor dir, both
// individually and in any combination, into a pseudo-atomic action with
// transactional rollback.
//
// It is not impervious to errors (writing to disk is hard), but it should
// guard against non-arcane failure conditions.
type SafeWriter struct {
	Manifest    *Manifest
	lock        *Lock
	lockDiff    *gps.LockDiff
	writeVendor bool
}

// NewSafeWriter sets up a SafeWriter to write a set of config yaml, lock and vendor tree.
//
// - If manifest is provided, it will be written to the standard manifest file
//   name beneath root.
// - If newLock is provided, it will be written to the standard lock file
//   name beneath root.
// - If vendor is VendorAlways, or is VendorOnChanged and the locks are different,
//   the vendor directory will be written beneath root based on newLock.
// - If oldLock is provided without newLock, error.
// - If vendor is VendorAlways without a newLock, error.
func NewSafeWriter(manifest *Manifest, oldLock, newLock *Lock, vendor VendorBehavior) (*SafeWriter, error) {
	sw := &SafeWriter{
		Manifest: manifest,
		lock:     newLock,
	}
	if oldLock != nil {
		if newLock == nil {
			return nil, errors.New("must provide newLock when oldLock is specified")
		}
		sw.lockDiff = gps.DiffLocks(oldLock, newLock)
	}

	switch vendor {
	case VendorAlways:
		sw.writeVendor = true
	case VendorOnChanged:
		if sw.lockDiff != nil || (newLock != nil && oldLock == nil) {
			sw.writeVendor = true
		}
	}

	if sw.writeVendor && newLock == nil {
		return nil, errors.New("must provide newLock in order to write out vendor")
	}

	return sw, nil
}

// HasLock checks if a Lock is present in the SafeWriter
func (sw *SafeWriter) HasLock() bool {
	return sw.lock != nil
}

// HasManifest checks if a Manifest is present in the SafeWriter
func (sw *SafeWriter) HasManifest() bool {
	return sw.Manifest != nil
}

type rawStringDiff struct {
	*gps.StringDiff
}

func (diff rawStringDiff) MarshalTOML() ([]byte, error) {
	return []byte(diff.String()), nil
}

type rawLockedProjectDiff struct {
	Name     gps.ProjectRoot `toml:"name"`
	Source   *rawStringDiff  `toml:"source,omitempty"`
	Version  *rawStringDiff  `toml:"version,omitempty"`
	Branch   *rawStringDiff  `toml:"branch,omitempty"`
	Revision *rawStringDiff  `toml:"revision,omitempty"`
	Packages []rawStringDiff `toml:"packages,omitempty"`
}

func toRawLockedProjectDiff(diff gps.LockedProjectDiff) rawLockedProjectDiff {
	// this is a shallow copy since we aren't modifying the raw diff
	raw := rawLockedProjectDiff{Name: diff.Name}
	if diff.Source != nil {
		raw.Source = &rawStringDiff{diff.Source}
	}
	if diff.Version != nil {
		raw.Version = &rawStringDiff{diff.Version}
	}
	if diff.Branch != nil {
		raw.Branch = &rawStringDiff{diff.Branch}
	}
	if diff.Revision != nil {
		raw.Revision = &rawStringDiff{diff.Revision}
	}
	raw.Packages = make([]rawStringDiff, len(diff.Packages))
	for i := 0; i < len(diff.Packages); i++ {
		raw.Packages[i] = rawStringDiff{&diff.Packages[i]}
	}
	return raw
}

type rawLockedProjectDiffs struct {
	Projects []rawLockedProjectDiff `toml:"projects"`
}

func toRawLockedProjectDiffs(diffs []gps.LockedProjectDiff) rawLockedProjectDiffs {
	raw := rawLockedProjectDiffs{
		Projects: make([]rawLockedProjectDiff, len(diffs)),
	}

	for i := 0; i < len(diffs); i++ {
		raw.Projects[i] = toRawLockedProjectDiff(diffs[i])
	}

	return raw
}

func formatLockDiff(diff gps.LockDiff) (string, error) {
	var buf bytes.Buffer

	if diff.HashDiff != nil {
		buf.WriteString(fmt.Sprintf("Memo: %s\n\n", diff.HashDiff))
	}

	writeDiffs := func(diffs []gps.LockedProjectDiff) error {
		raw := toRawLockedProjectDiffs(diffs)
		chunk, err := toml.Marshal(raw)
		if err != nil {
			return err
		}
		buf.Write(chunk)
		buf.WriteString("\n")
		return nil
	}

	if len(diff.Add) > 0 {
		buf.WriteString("Add:")
		err := writeDiffs(diff.Add)
		if err != nil {
			return "", errors.Wrap(err, "Unable to format LockDiff.Add")
		}
	}

	if len(diff.Remove) > 0 {
		buf.WriteString("Remove:")
		err := writeDiffs(diff.Remove)
		if err != nil {
			return "", errors.Wrap(err, "Unable to format LockDiff.Remove")
		}
	}

	if len(diff.Modify) > 0 {
		buf.WriteString("Modify:")
		err := writeDiffs(diff.Modify)
		if err != nil {
			return "", errors.Wrap(err, "Unable to format LockDiff.Modify")
		}
	}

	return buf.String(), nil
}

// VendorBehavior defines when the vendor directory should be written.
type VendorBehavior int

const (
	// VendorOnChanged indicates that the vendor directory should be written when the lock is new or changed.
	VendorOnChanged VendorBehavior = iota
	// VendorAlways forces the vendor directory to always be written.
	VendorAlways
	// VendorNever indicates the vendor directory should never be written.
	VendorNever
)

func (sw SafeWriter) validate(root string, sm gps.SourceManager) error {
	if root == "" {
		return errors.New("root path must be non-empty")
	}
	if is, err := fs.IsDir(root); !is {
		if err != nil {
			return err
		}
		return errors.Errorf("root path %q does not exist", root)
	}

	if sw.writeVendor && sm == nil {
		return errors.New("must provide a SourceManager if writing out a vendor dir")
	}

	return nil
}

// Write saves some combination of config yaml, lock, and a vendor tree.
// root is the absolute path of root dir in which to write.
// sm is only required if vendor is being written.
//
// It first writes to a temp dir, then moves them in place if and only if all the write
// operations succeeded. It also does its best to roll back if any moves fail.
// This mostly guarantees that dep cannot exit with a partial write that would
// leave an undefined state on disk.
func (sw *SafeWriter) Write(root string, sm gps.SourceManager, noExamples bool) error {
	err := sw.validate(root, sm)
	if err != nil {
		return err
	}

	if !sw.HasManifest() && !sw.HasLock() && !sw.writeVendor {
		// nothing to do
		return nil
	}

	mpath := filepath.Join(root, ManifestName)
	lpath := filepath.Join(root, LockName)
	vpath := filepath.Join(root, "vendor")

	td, err := ioutil.TempDir(os.TempDir(), "dep")
	if err != nil {
		return errors.Wrap(err, "error while creating temp dir for writing manifest/lock/vendor")
	}
	defer os.RemoveAll(td)

	if sw.HasManifest() {
		// Always write the example text to the bottom of the TOML file.
		tb, err := sw.Manifest.MarshalTOML()
		if err != nil {
			return errors.Wrap(err, "failed to marshal manifest to TOML")
		}

		var initOutput []byte

		// If examples are NOT disabled, use the example text
		if !noExamples {
			initOutput = exampleTOML
		}

		if err = ioutil.WriteFile(filepath.Join(td, ManifestName), append(initOutput, tb...), 0666); err != nil {
			return errors.Wrap(err, "failed to write manifest file to temp dir")
		}
	}

	if sw.HasLock() {
		l, err := sw.lock.MarshalTOML()
		if err != nil {
			return errors.Wrap(err, "failed to marshal lock to TOML")
		}

		if err = ioutil.WriteFile(filepath.Join(td, LockName), append(lockFileComment, l...), 0666); err != nil {
			return errors.Wrap(err, "failed to write lock file to temp dir")
		}
	}

	if sw.writeVendor {
		err = gps.WriteDepTree(filepath.Join(td, "vendor"), sw.lock, sm, true)
		if err != nil {
			return errors.Wrap(err, "error while writing out vendor tree")
		}
	}

	// Ensure vendor/.git is preserved if present
	if hasDotGit(vpath) {
		err = fs.RenameWithFallback(filepath.Join(vpath, ".git"), filepath.Join(td, "vendor/.git"))
		if _, ok := err.(*os.LinkError); ok {
			return errors.Wrap(err, "failed to preserve vendor/.git")
		}
	}

	// Move the existing files and dirs to the temp dir while we put the new
	// ones in, to provide insurance against errors for as long as possible.
	type pathpair struct {
		from, to string
	}
	var restore []pathpair
	var failerr error
	var vendorbak string

	if sw.HasManifest() {
		if _, err := os.Stat(mpath); err == nil {
			// Move out the old one.
			tmploc := filepath.Join(td, ManifestName+".orig")
			failerr = fs.RenameWithFallback(mpath, tmploc)
			if failerr != nil {
				goto fail
			}
			restore = append(restore, pathpair{from: tmploc, to: mpath})
		}

		// Move in the new one.
		failerr = fs.RenameWithFallback(filepath.Join(td, ManifestName), mpath)
		if failerr != nil {
			goto fail
		}
	}

	if sw.HasLock() {
		if _, err := os.Stat(lpath); err == nil {
			// Move out the old one.
			tmploc := filepath.Join(td, LockName+".orig")

			failerr = fs.RenameWithFallback(lpath, tmploc)
			if failerr != nil {
				goto fail
			}
			restore = append(restore, pathpair{from: tmploc, to: lpath})
		}

		// Move in the new one.
		failerr = fs.RenameWithFallback(filepath.Join(td, LockName), lpath)
		if failerr != nil {
			goto fail
		}
	}

	if sw.writeVendor {
		if _, err := os.Stat(vpath); err == nil {
			// Move out the old vendor dir. just do it into an adjacent dir, to
			// try to mitigate the possibility of a pointless cross-filesystem
			// move with a temp directory.
			vendorbak = vpath + ".orig"
			if _, err := os.Stat(vendorbak); err == nil {
				// If the adjacent dir already exists, bite the bullet and move
				// to a proper tempdir.
				vendorbak = filepath.Join(td, "vendor.orig")
			}

			failerr = fs.RenameWithFallback(vpath, vendorbak)
			if failerr != nil {
				goto fail
			}
			restore = append(restore, pathpair{from: vendorbak, to: vpath})
		}

		// Move in the new one.
		failerr = fs.RenameWithFallback(filepath.Join(td, "vendor"), vpath)
		if failerr != nil {
			goto fail
		}
	}

	// Renames all went smoothly. The deferred os.RemoveAll will get the temp
	// dir, but if we wrote vendor, we have to clean that up directly
	if sw.writeVendor {
		// Nothing we can really do about an error at this point, so ignore it
		os.RemoveAll(vendorbak)
	}

	return nil

fail:
	// If we failed at any point, move all the things back into place, then bail.
	for _, pair := range restore {
		// Nothing we can do on err here, as we're already in recovery mode.
		fs.RenameWithFallback(pair.from, pair.to)
	}
	return failerr
}

func (sw *SafeWriter) PrintPreparedActions(output *log.Logger) error {
	if sw.HasManifest() {
		output.Printf("Would have written the following %s:\n", ManifestName)
		m, err := sw.Manifest.MarshalTOML()
		if err != nil {
			return errors.Wrap(err, "ensure DryRun cannot serialize manifest")
		}
		output.Println(string(m))
	}

	if sw.HasLock() {
		if sw.lockDiff == nil {
			output.Printf("Would have written the following %s:\n", LockName)
			l, err := sw.lock.MarshalTOML()
			if err != nil {
				return errors.Wrap(err, "ensure DryRun cannot serialize lock")
			}
			output.Println(string(l))
		} else {
			output.Printf("Would have written the following changes to %s:\n", LockName)
			diff, err := formatLockDiff(*sw.lockDiff)
			if err != nil {
				return errors.Wrap(err, "ensure DryRun cannot serialize the lock diff")
			}
			output.Println(diff)
		}
	}

	if sw.writeVendor {
		output.Println("Would have written the following projects to the vendor directory:")
		for _, project := range sw.lock.Projects() {
			prj := project.Ident()
			rev, _, _ := gps.VersionComponentStrings(project.Version())
			if prj.Source == "" {
				output.Printf("%s@%s\n", prj.ProjectRoot, rev)
			} else {
				output.Printf("%s -> %s@%s\n", prj.ProjectRoot, prj.Source, rev)
			}
		}
	}

	return nil
}

func PruneProject(p *Project, sm gps.SourceManager, logger *log.Logger) error {
	td, err := ioutil.TempDir(os.TempDir(), "dep")
	if err != nil {
		return errors.Wrap(err, "error while creating temp dir for writing manifest/lock/vendor")
	}
	defer os.RemoveAll(td)

	if err := gps.WriteDepTree(td, p.Lock, sm, true); err != nil {
		return err
	}

	var toKeep []string
	for _, project := range p.Lock.Projects() {
		projectRoot := string(project.Ident().ProjectRoot)
		for _, pkg := range project.Packages() {
			toKeep = append(toKeep, filepath.Join(projectRoot, pkg))
		}
	}

	toDelete, err := calculatePrune(td, toKeep, logger)
	if err != nil {
		return err
	}

	if logger != nil {
		if len(toDelete) > 0 {
			logger.Println("Calculated the following directories to prune:")
			for _, d := range toDelete {
				logger.Printf("  %s\n", d)
			}
		} else {
			logger.Println("No directories found to prune")
		}
	}

	if err := deleteDirs(toDelete); err != nil {
		return err
	}

	vpath := filepath.Join(p.AbsRoot, "vendor")
	vendorbak := vpath + ".orig"
	var failerr error
	if _, err := os.Stat(vpath); err == nil {
		// Move out the old vendor dir. just do it into an adjacent dir, to
		// try to mitigate the possibility of a pointless cross-filesystem
		// move with a temp directory.
		if _, err := os.Stat(vendorbak); err == nil {
			// If the adjacent dir already exists, bite the bullet and move
			// to a proper tempdir.
			vendorbak = filepath.Join(td, "vendor.orig")
		}
		failerr = fs.RenameWithFallback(vpath, vendorbak)
		if failerr != nil {
			goto fail
		}
	}

	// Move in the new one.
	failerr = fs.RenameWithFallback(td, vpath)
	if failerr != nil {
		goto fail
	}

	os.RemoveAll(vendorbak)

	return nil

fail:
	fs.RenameWithFallback(vendorbak, vpath)
	return failerr
}

func calculatePrune(vendorDir string, keep []string, logger *log.Logger) ([]string, error) {
	if logger != nil {
		logger.Println("Calculating prune. Checking the following packages:")
	}
	sort.Strings(keep)
	toDelete := []string{}
	err := filepath.Walk(vendorDir, func(path string, info os.FileInfo, err error) error {
		if _, err := os.Lstat(path); err != nil {
			return nil
		}
		if !info.IsDir() {
			return nil
		}
		if path == vendorDir {
			return nil
		}

		name := strings.TrimPrefix(path, vendorDir+"/")
		if logger != nil {
			logger.Printf("  %s", name)
		}
		i := sort.Search(len(keep), func(i int) bool {
			return name <= keep[i]
		})
		if i >= len(keep) || !strings.HasPrefix(keep[i], name) {
			toDelete = append(toDelete, path)
		}
		return nil
	})
	return toDelete, err
}

func deleteDirs(toDelete []string) error {
	// sort by length so we delete sub dirs first
	sort.Sort(byLen(toDelete))
	for _, path := range toDelete {
		if err := os.RemoveAll(path); err != nil {
			return err
		}
	}
	return nil
}

// hasDotGit checks if a given path has .git file or directory in it.
func hasDotGit(path string) bool {
	gitfilepath := filepath.Join(path, ".git")
	_, err := os.Stat(gitfilepath)
	return err == nil
}

type byLen []string

func (a byLen) Len() int           { return len(a) }
func (a byLen) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byLen) Less(i, j int) bool { return len(a[i]) > len(a[j]) }