Skip to content
This repository has been archived by the owner on Sep 9, 2020. It is now read-only.

Commit

Permalink
dep: Introduce dep check subcommand
Browse files Browse the repository at this point in the history
dep check is a read-only command that verifies dep's sync invariants
are met. It's primarily intended for automated use (e.g. CI or git
pre-commit hooks). We may expand it later to include more comprehensive
checks.
  • Loading branch information
sdboyer committed Jul 12, 2018
1 parent 296fe9d commit 3201ef6
Show file tree
Hide file tree
Showing 7 changed files with 245 additions and 42 deletions.
177 changes: 177 additions & 0 deletions cmd/dep/check.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
package main

import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"

"github.com/golang/dep"
"github.com/golang/dep/gps"
"github.com/golang/dep/gps/verify"
"github.com/pkg/errors"
)

const checkShortHelp = `Check if imports, Gopkg.toml, and Gopkg.lock are in sync`
const checkLongHelp = `
Check determines if your project is in a good state. If problems are found, it
prints a description of each issue, then exits 1. Passing -q suppresses output.
Flags control which specific checks will be run. By default,dep check verifies
that Gopkg.lock is in sync with Gopkg.toml and the imports in your project's .go
files, and that the vendor directory is in sync with Gopkg.lock. These checks
can be disabled with -skip-lock and -skip-vendor, respectively.
(See https://golang.github.io/dep/docs/ensure-mechanics.html#staying-in-sync for
more information on what it means to be "in sync.")
`

type checkCommand struct {
quiet bool
skiplock, skipvendor bool
}

func (cmd *checkCommand) Name() string { return "check" }
func (cmd *checkCommand) Args() string {
return "[-q] [-skip-lock] [-skip-vendor]"
}
func (cmd *checkCommand) ShortHelp() string { return checkShortHelp }
func (cmd *checkCommand) LongHelp() string { return checkLongHelp }
func (cmd *checkCommand) Hidden() bool { return false }

func (cmd *checkCommand) Register(fs *flag.FlagSet) {
fs.BoolVar(&cmd.skiplock, "skip-lock", false, "Skip checking that imports and Gopkg.toml are in sync with Gopkg.lock")
fs.BoolVar(&cmd.skipvendor, "skip-vendor", false, "Skip checking that vendor is in sync with Gopkg.lock")
fs.BoolVar(&cmd.quiet, "q", false, "Suppress non-error output")
}

func (cmd *checkCommand) Run(ctx *dep.Ctx, args []string) error {
logger := ctx.Out
if cmd.quiet {
logger = log.New(ioutil.Discard, "", 0)
}

p, err := ctx.LoadProject()
if err != nil {
return err
}

sm, err := ctx.SourceManager()
if err != nil {
return err
}

sm.UseDefaultSignalHandling()
defer sm.Release()

var fail bool
if !cmd.skiplock {
if p.Lock == nil {
return errors.New("Gopkg.lock does not exist, cannot check it against imports and Gopkg.toml")
}

lsat := verify.LockSatisfiesInputs(p.Lock, p.Manifest, p.RootPackageTree)
delta := verify.DiffLocks(p.Lock, p.ChangedLock)
sat, changed := lsat.Satisfied(), delta.Changed(verify.PruneOptsChanged|verify.HashVersionChanged)

if changed || !sat {
fail = true
logger.Println("# Gopkg.lock is out of sync:")
if !sat {
logger.Printf("%s\n", sprintLockUnsat(lsat))
}
if changed {
for pr, lpd := range delta.ProjectDeltas {
// Only two possible changes right now are prune opts
// changing or a missing hash digest (for old Gopkg.lock
// files)
if lpd.PruneOptsChanged() {
// Override what's on the lockdiff with the extra info we have;
// this lets us excise PruneNestedVendorDirs and get the real
// value from the input param in place.
old := lpd.PruneOptsBefore & ^gps.PruneNestedVendorDirs
new := lpd.PruneOptsAfter & ^gps.PruneNestedVendorDirs
logger.Printf("%s: prune options changed (%s -> %s)\n", pr, old, new)
}
if lpd.HashVersionWasZero() {
logger.Printf("%s: no hash digest in lock\n", pr)
}
}
}
}
}

if !cmd.skipvendor {
if p.Lock == nil {
return errors.New("Gopkg.lock does not exist, cannot check vendor against it")
}

statuses, err := p.VerifyVendor()
if err != nil {
return errors.Wrap(err, "error while verifying vendor")
}

if fail {
logger.Println()
}
// One full pass through, to see if we need to print the header.
for _, status := range statuses {
if status != verify.NoMismatch {
fail = true
logger.Println("# vendor is out of sync:")
break
}
}

for pr, status := range statuses {
switch status {
case verify.NotInTree:
logger.Printf("%s: missing from vendor\n", pr)
case verify.NotInLock:
fi, err := os.Stat(filepath.Join(p.AbsRoot, "vendor", pr))
if err != nil {
return errors.Wrap(err, "could not stat file that VerifyVendor claimed existed")
}

if fi.IsDir() {
logger.Printf("%s: unused project\n", pr)
} else {
logger.Printf("%s: orphaned file\n", pr)
}
case verify.DigestMismatchInLock:
logger.Printf("%s: hash of vendored tree didn't match digest in Gopkg.lock\n", pr)
case verify.HashVersionMismatch:
// This will double-print if the hash version is zero, but
// that's a rare case that really only occurs before the first
// run with a version of dep >=0.5.0, so it's fine.
logger.Printf("%s: hash algorithm mismatch, want version %v\n", pr, verify.HashVersion)
}
}
}

if fail {
return silentfail{}
}
return nil
}

func sprintLockUnsat(lsat verify.LockSatisfaction) string {
var buf bytes.Buffer
for _, missing := range lsat.MissingImports {
fmt.Fprintf(&buf, "%s: missing from input-imports\n", missing)
}
for _, excess := range lsat.ExcessImports {
fmt.Fprintf(&buf, "%s: in input-imports, but not imported\n", excess)
}
for pr, unmatched := range lsat.UnmetOverrides {
fmt.Fprintf(&buf, "%s@%s: not allowed by override %s\n", pr, unmatched.V, unmatched.C)
}
for pr, unmatched := range lsat.UnmetConstraints {
fmt.Fprintf(&buf, "%s@%s: not allowed by constraint %s\n", pr, unmatched.V, unmatched.C)
}
return strings.TrimSpace(buf.String())
}
15 changes: 1 addition & 14 deletions cmd/dep/ensure.go
Original file line number Diff line number Diff line change
Expand Up @@ -261,20 +261,7 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
lsat := verify.LockSatisfiesInputs(p.Lock, p.Manifest, params.RootPackageTree)
if !lsat.Satisfied() {
if ctx.Verbose {
ctx.Out.Println("# Gopkg.lock is out of sync with Gopkg.toml and project code:")
for _, missing := range lsat.MissingImports {
ctx.Out.Printf("%s: missing from input-imports\n", missing)
}
for _, excess := range lsat.ExcessImports {
ctx.Out.Printf("%s: in input-imports, but isn't imported\n", excess)
}
for pr, unmatched := range lsat.UnmetOverrides {
ctx.Out.Printf("%s@%s: not allowed by override %s\n", pr, unmatched.V, unmatched.C)
}
for pr, unmatched := range lsat.UnmetConstraints {
ctx.Out.Printf("%s@%s: not allowed by constraint %s\n", pr, unmatched.V, unmatched.C)
}
ctx.Out.Println()
ctx.Out.Printf("# Gopkg.lock is out of sync with Gopkg.toml and project imports:\n%s\n\n", sprintLockUnsat(lsat))
}
solve = true
} else if cmd.noVendor {
Expand Down
22 changes: 19 additions & 3 deletions cmd/dep/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,14 @@ type command interface {
Run(*dep.Ctx, []string) error
}

// Helper type so that commands can fail without generating any additional
// ouptut.
type silentfail struct{}

func (silentfail) Error() string {
return ""
}

func main() {
p := &profile{}

Expand Down Expand Up @@ -139,7 +147,12 @@ func (c *Config) Run() int {
// Build flag set with global flags in there.
flags := flag.NewFlagSet(cmdName, flag.ContinueOnError)
flags.SetOutput(c.Stderr)
verbose := flags.Bool("v", false, "enable verbose logging")

var verbose bool
// No verbose for verify
if cmdName != "check" {
flags.BoolVar(&verbose, "v", false, "enable verbose logging")
}

// Register the subcommand flags in there, too.
cmd.Register(flags)
Expand Down Expand Up @@ -186,7 +199,7 @@ func (c *Config) Run() int {
ctx := &dep.Ctx{
Out: outLogger,
Err: errLogger,
Verbose: *verbose,
Verbose: verbose,
DisableLocking: getEnv(c.Env, "DEPNOLOCK") != "",
Cachedir: cachedir,
CacheAge: cacheAge,
Expand All @@ -197,7 +210,9 @@ func (c *Config) Run() int {

// Run the command with the post-flag-processing args.
if err := cmd.Run(ctx, flags.Args()); err != nil {
errLogger.Printf("%v\n", err)
if _, ok := err.(silentfail); !ok {
errLogger.Printf("%v\n", err)
}
return errorExitCode
}

Expand All @@ -222,6 +237,7 @@ func commandList() []command {
&ensureCommand{},
&pruneCommand{},
&versionCommand{},
&checkCommand{},
}
}

Expand Down
15 changes: 12 additions & 3 deletions gps/verify/digest.go
Original file line number Diff line number Diff line change
Expand Up @@ -386,11 +386,23 @@ func ParseVersionedDigest(input string) (VersionedDigest, error) {
func CheckDepTree(osDirname string, wantDigests map[string]VersionedDigest) (map[string]VendorStatus, error) {
osDirname = filepath.Clean(osDirname)

// Create associative array to store the results of calling this function.
slashStatus := make(map[string]VendorStatus)

// Ensure top level pathname is a directory
fi, err := os.Stat(osDirname)
if err != nil {
// If the dir doesn't exist at all, that's OK - just consider all the
// wanted paths absent.
if os.IsNotExist(err) {
for path := range wantDigests {
slashStatus[path] = NotInTree
}
return slashStatus, nil
}
return nil, errors.Wrap(err, "cannot Stat")
}

if !fi.IsDir() {
return nil, errors.Errorf("cannot verify non directory: %q", osDirname)
}
Expand Down Expand Up @@ -438,9 +450,6 @@ func CheckDepTree(osDirname string, wantDigests map[string]VersionedDigest) (map
// `NotInLock`.
nodes := []*fsnode{currentNode}

// Create associative array to store the results of calling this function.
slashStatus := make(map[string]VendorStatus)

// Mark directories of expected projects as required. When each respective
// project is later found while traversing the vendor root hierarchy, its
// status will be updated to reflect whether its digest is empty, or,
Expand Down
39 changes: 25 additions & 14 deletions gps/verify/lockdiff.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,13 @@ type LockedProjectDelta struct {
// properties of two LockedProjects. It can represent deltas for
// VerifiableProject properties, as well.
type LockedProjectPropertiesDelta struct {
PackagesAdded, PackagesRemoved []string
VersionBefore, VersionAfter gps.UnpairedVersion
RevisionBefore, RevisionAfter gps.Revision
SourceBefore, SourceAfter string
PruneOptsBefore, PruneOptsAfter gps.PruneOptions
HashChanged, HashVersionChanged bool
PackagesAdded, PackagesRemoved []string
VersionBefore, VersionAfter gps.UnpairedVersion
RevisionBefore, RevisionAfter gps.Revision
SourceBefore, SourceAfter string
PruneOptsBefore, PruneOptsAfter gps.PruneOptions
HashVersionBefore, HashVersionAfter int
HashChanged bool
}

// DiffLocks compares two locks and computes a semantically rich delta between
Expand Down Expand Up @@ -200,20 +201,18 @@ func DiffLockedProjectProperties(lp1, lp2 gps.LockedProject) LockedProjectProper

if ok1 && ok2 {
ld.PruneOptsBefore, ld.PruneOptsAfter = vp1.PruneOpts, vp2.PruneOpts
ld.HashVersionBefore, ld.HashVersionAfter = vp1.Digest.HashVersion, vp2.Digest.HashVersion

if vp1.Digest.HashVersion != vp2.Digest.HashVersion {
ld.HashVersionChanged = true
}
if !bytes.Equal(vp1.Digest.Digest, vp2.Digest.Digest) {
ld.HashChanged = true
}
} else if ok1 {
ld.PruneOptsBefore = vp1.PruneOpts
ld.HashVersionChanged = true
ld.HashVersionBefore = vp1.Digest.HashVersion
ld.HashChanged = true
} else if ok2 {
ld.PruneOptsAfter = vp2.PruneOpts
ld.HashVersionChanged = true
ld.HashVersionAfter = vp2.Digest.HashVersion
ld.HashChanged = true
}

Expand Down Expand Up @@ -322,7 +321,7 @@ func (ld LockedProjectPropertiesDelta) Changed(dims DeltaDimension) bool {
if dims&HashChanged != 0 && ld.HashChanged {
return true
}
if dims&HashVersionChanged != 0 && ld.HashVersionChanged {
if dims&HashVersionChanged != 0 && ld.HashVersionChanged() {
return true
}
if dims&VersionChanged != 0 && ld.VersionChanged() {
Expand Down Expand Up @@ -351,7 +350,7 @@ func (ld LockedProjectPropertiesDelta) Changes() DeltaDimension {
if ld.HashChanged {
dd |= HashChanged
}
if ld.HashVersionChanged {
if ld.HashVersionChanged() {
dd |= HashVersionChanged
}
if ld.VersionChanged() {
Expand Down Expand Up @@ -400,11 +399,23 @@ func (ld LockedProjectPropertiesDelta) PackagesChanged() bool {
}

// PruneOptsChanged returns true if the pruning flags for the project changed
// between teh first and second locks.
// between the first and second locks.
func (ld LockedProjectPropertiesDelta) PruneOptsChanged() bool {
return ld.PruneOptsBefore != ld.PruneOptsAfter
}

// HashVersionChanged returns true if the version of the hashing algorithm
// changed between the first and second locks.
func (ld LockedProjectPropertiesDelta) HashVersionChanged() bool {
return ld.HashVersionBefore != ld.HashVersionAfter
}

// HashVersionWasZero returns true if the first lock had a zero hash version,
// which can only mean it was uninitialized.
func (ld LockedProjectPropertiesDelta) HashVersionWasZero() bool {
return ld.HashVersionBefore == 0
}

// sortLockedProjects returns a sorted copy of lps, or itself if already sorted.
func sortLockedProjects(lps []gps.LockedProject) []gps.LockedProject {
if len(lps) <= 1 || sort.SliceIsSorted(lps, func(i, j int) bool {
Expand Down
6 changes: 0 additions & 6 deletions project.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,12 +140,6 @@ func (p *Project) VerifyVendor() (map[string]verify.VendorStatus, error) {
lps = p.Lock.Projects()
}

err := os.MkdirAll(vendorDir, os.FileMode(0777))
if err != nil {
p.CheckVendorErr = err
return
}

sums := make(map[string]verify.VersionedDigest)
for _, lp := range lps {
sums[string(lp.Ident().ProjectRoot)] = lp.(verify.VerifiableProject).Digest
Expand Down
Loading

0 comments on commit 3201ef6

Please sign in to comment.