cmd: add hashSUM file support

cmd/all/all.go

@@ -10,6 +10,7 @@ import (
 	_ "github.com/rclone/rclone/cmd/cachestats"
 	_ "github.com/rclone/rclone/cmd/cat"
 	_ "github.com/rclone/rclone/cmd/check"
+	_ "github.com/rclone/rclone/cmd/checksum"
 	_ "github.com/rclone/rclone/cmd/cleanup"
 	_ "github.com/rclone/rclone/cmd/cmount"
 	_ "github.com/rclone/rclone/cmd/config"

cmd/check/check.go

@@ -2,6 +2,7 @@ package check
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
 	"strings"
@@ -17,20 +18,22 @@ import (
 
 // Globals
 var (
-	download     = false
-	oneway       = false
-	combined     = ""
-	missingOnSrc = ""
-	missingOnDst = ""
-	match        = ""
-	differ       = ""
-	errFile      = ""
+	download          = false
+	oneway            = false
+	combined          = ""
+	missingOnSrc      = ""
+	missingOnDst      = ""
+	match             = ""
+	differ            = ""
+	errFile           = ""
+	checkFileHashType = ""
 )
 
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
 	flags.BoolVarP(cmdFlags, &download, "download", "", download, "Check by downloading rather than with hash.")
+	flags.StringVarP(cmdFlags, &checkFileHashType, "checkfile", "C", checkFileHashType, "Treat source:path as a SUM file with hashes of given type")
 	AddFlags(cmdFlags)
 }
 
@@ -126,7 +129,6 @@ func GetCheckOpt(fsrc, fdst fs.Fs) (opt *operations.CheckOpt, close func(), err
 	}
 
 	return opt, close, nil
-
 }
 
 var commandDefinition = &cobra.Command{
@@ -144,16 +146,39 @@ If you supply the |--download| flag, it will download the data from
 both remotes and check them against each other on the fly.  This can
 be useful for remotes that don't support hashes or if you really want
 to check all the data.
+
+If you supply the |--checkfile HASH| flag with a valid hash name,
+the |source:path| must point to a text file in the SUM format.
 `, "|", "`") + FlagsHelp,
-	Run: func(command *cobra.Command, args []string) {
+	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(2, 2, command, args)
-		fsrc, fdst := cmd.NewFsSrcDst(args)
+		var (
+			fsrc, fdst fs.Fs
+			hashType   hash.Type
+			fsum       fs.Fs
+			sumFile    string
+		)
+		if checkFileHashType != "" {
+			if err := hashType.Set(checkFileHashType); err != nil {
+				fmt.Println(hash.HelpString(0))
+				return err
+			}
+			fsum, sumFile, fsrc = cmd.NewFsSrcFileDst(args)
+		} else {
+			fsrc, fdst = cmd.NewFsSrcDst(args)
+		}
+
 		cmd.Run(false, true, command, func() error {
 			opt, close, err := GetCheckOpt(fsrc, fdst)
 			if err != nil {
 				return err
 			}
 			defer close()
+
+			if checkFileHashType != "" {
+				return operations.CheckSum(context.Background(), fsrc, fsum, sumFile, hashType, opt, download)
+			}
+
 			if download {
 				return operations.CheckDownload(context.Background(), opt)
 			}
@@ -165,5 +190,6 @@ to check all the data.
 			}
 			return operations.Check(context.Background(), opt)
 		})
+		return nil
 	},
 }

cmd/checksum/checksum.go

@@ -0,0 +1,57 @@
+package checksum
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/cmd/check" // for common flags
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/spf13/cobra"
+)
+
+var download = false
+
+func init() {
+	cmd.Root.AddCommand(commandDefinition)
+	cmdFlags := commandDefinition.Flags()
+	flags.BoolVarP(cmdFlags, &download, "download", "", download, "Check by hashing the contents.")
+	check.AddFlags(cmdFlags)
+}
+
+var commandDefinition = &cobra.Command{
+	Use:   "checksum <hash> sumfile src:path",
+	Short: `Checks the files in the source against a SUM file.`,
+	Long: strings.ReplaceAll(`
+Checks that hashsums of source files match the SUM file.
+It compares hashes (MD5, SHA1, etc) and logs a report of files which
+don't match.  It doesn't alter the file system.
+
+If you supply the |--download| flag, it will download the data from remote
+and calculate the contents hash on the fly.  This can be useful for remotes
+that don't support hashes or if you really want to check all the data.
+`, "|", "`") + check.FlagsHelp,
+	RunE: func(command *cobra.Command, args []string) error {
+		cmd.CheckArgs(3, 3, command, args)
+		var hashType hash.Type
+		if err := hashType.Set(args[0]); err != nil {
+			fmt.Println(hash.HelpString(0))
+			return err
+		}
+		fsum, sumFile, fsrc := cmd.NewFsSrcFileDst(args[1:])
+
+		cmd.Run(false, true, command, func() error {
+			opt, close, err := check.GetCheckOpt(nil, fsrc)
+			if err != nil {
+				return err
+			}
+			defer close()
+
+			return operations.CheckSum(context.Background(), fsrc, fsum, sumFile, hashType, opt, download)
+		})
+		return nil
+	},
+}

cmd/hashsum/hashsum.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/cmd"
@@ -21,6 +20,7 @@ var (
 	OutputBase64   = false
 	DownloadFlag   = false
 	HashsumOutfile = ""
+	ChecksumFile   = ""
 )
 
 func init() {
@@ -33,6 +33,7 @@ func init() {
 func AddHashFlags(cmdFlags *pflag.FlagSet) {
 	flags.BoolVarP(cmdFlags, &OutputBase64, "base64", "", OutputBase64, "Output base64 encoded hashsum")
 	flags.StringVarP(cmdFlags, &HashsumOutfile, "output-file", "", HashsumOutfile, "Output hashsums to a file rather than the terminal")
+	flags.StringVarP(cmdFlags, &ChecksumFile, "checkfile", "C", ChecksumFile, "Validate hashes against a given SUM file instead of printing them")
 	flags.BoolVarP(cmdFlags, &DownloadFlag, "download", "", DownloadFlag, "Download the file and hash it locally; if this flag is not specified, the hash is requested from the remote")
 }
 
@@ -70,7 +71,7 @@ hashed locally enabling any hash for any remote.
 Run without a hash to see the list of all supported hashes, e.g.
 
     $ rclone hashsum
-` + hashListHelp("    ") + `
+` + hash.HelpString(4) + `
 Then
 
     $ rclone hashsum MD5 remote:path
@@ -80,20 +81,24 @@ Note that hash names are case insensitive.
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(0, 2, command, args)
 		if len(args) == 0 {
-			fmt.Print(hashListHelp(""))
+			fmt.Print(hash.HelpString(0))
 			return nil
 		} else if len(args) == 1 {
 			return errors.New("need hash type and remote")
 		}
 		var ht hash.Type
 		err := ht.Set(args[0])
 		if err != nil {
-			fmt.Println(hashListHelp(""))
+			fmt.Println(hash.HelpString(0))
 			return err
 		}
 		fsrc := cmd.NewFsSrc(args[1:])
 
 		cmd.Run(false, false, command, func() error {
+			if ChecksumFile != "" {
+				fsum, sumFile := cmd.NewFsFile(ChecksumFile)
+				return operations.CheckSum(context.Background(), fsrc, fsum, sumFile, ht, nil, DownloadFlag)
+			}
 			if HashsumOutfile == "" {
 				return operations.HashLister(context.Background(), ht, OutputBase64, DownloadFlag, fsrc, nil)
 			}
@@ -107,14 +112,3 @@ Note that hash names are case insensitive.
 		return nil
 	},
 }
-
-func hashListHelp(indent string) string {
-	var help strings.Builder
-	help.WriteString(indent)
-	help.WriteString("Supported hashes are:\n")
-	for _, ht := range hash.Supported().Array() {
-		help.WriteString(indent)
-		fmt.Fprintf(&help, "  * %v\n", ht.String())
-	}
-	return help.String()
-}

cmd/md5sum/md5sum.go

@@ -32,6 +32,10 @@ hashed locally enabling MD5 for any remote.
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
 		cmd.Run(false, false, command, func() error {
+			if hashsum.ChecksumFile != "" {
+				fsum, sumFile := cmd.NewFsFile(hashsum.ChecksumFile)
+				return operations.CheckSum(context.Background(), fsrc, fsum, sumFile, hash.MD5, nil, hashsum.DownloadFlag)
+			}
 			if hashsum.HashsumOutfile == "" {
 				return operations.HashLister(context.Background(), hash.MD5, hashsum.OutputBase64, hashsum.DownloadFlag, fsrc, nil)
 			}

cmd/sha1sum/sha1sum.go

@@ -32,6 +32,10 @@ hashed locally enabling SHA-1 for any remote.
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
 		cmd.Run(false, false, command, func() error {
+			if hashsum.ChecksumFile != "" {
+				fsum, sumFile := cmd.NewFsFile(hashsum.ChecksumFile)
+				return operations.CheckSum(context.Background(), fsrc, fsum, sumFile, hash.SHA1, nil, hashsum.DownloadFlag)
+			}
 			if hashsum.HashsumOutfile == "" {
 				return operations.HashLister(context.Background(), hash.SHA1, hashsum.OutputBase64, hashsum.DownloadFlag, fsrc, nil)
 			}

docs/content/commands/rclone_check.md

@@ -24,6 +24,9 @@ both remotes and check them against each other on the fly.  This can
 be useful for remotes that don't support hashes or if you really want
 to check all the data.
 
+If you supply the `--checkfile HASH` flag with a valid hash name,
+the `source:path` must point to a text file in the SUM format.
+
 If you supply the `--one-way` flag, it will only check that files in
 the source match the files in the destination, not the other way
 around. This means that extra files in the destination that are not in
@@ -53,6 +56,7 @@ rclone check source:path dest:path [flags]
 ## Options
 
 ```
+  -C, --checkfile string        Treat source:path as a SUM file with hashes of given type
       --combined string         Make a combined report of changes to this file
       --differ string           Report all non-matching files to this file
       --download                Check by downloading rather than with hash.

docs/content/commands/rclone_checksum.md

@@ -0,0 +1,68 @@
+---
+title: "rclone checksum"
+description: "Checks the files in the source against a SUM file."
+slug: rclone_checksum
+url: /commands/rclone_checksum/
+# autogenerated - DO NOT EDIT, instead edit the source code in cmd/checksum/ and as part of making a release run "make commanddocs"
+---
+# rclone checksum
+
+Checks the files in the source against a SUM file.
+
+## Synopsis
+
+
+Checks that hashsums of source files match the SUM file.
+It compares hashes (MD5, SHA1, etc) and logs a report of files which
+don't match.  It doesn't alter the file system.
+
+If you supply the `--download` flag, it will download the data from remote
+and calculate the contents hash on the fly.  This can be useful for remotes
+that don't support hashes or if you really want to check all the data.
+
+If you supply the `--one-way` flag, it will only check that files in
+the source match the files in the destination, not the other way
+around. This means that extra files in the destination that are not in
+the source will not be detected.
+
+The `--differ`, `--missing-on-dst`, `--missing-on-src`, `--match`
+and `--error` flags write paths, one per line, to the file name (or
+stdout if it is `-`) supplied. What they write is described in the
+help below. For example `--differ` will write all paths which are
+present on both the source and destination but different.
+
+The `--combined` flag will write a file (or stdout) which contains all
+file paths with a symbol and then a space and then the path to tell
+you what happened to it. These are reminiscent of diff files.
+
+- `= path` means path was found in source and destination and was identical
+- `- path` means path was missing on the source, so only in the destination
+- `+ path` means path was missing on the destination, so only in the source
+- `* path` means path was present in source and destination but different.
+- `! path` means there was an error reading or hashing the source or dest.
+
+
+```
+rclone checksum <hash> sumfile src:path [flags]
+```
+
+## Options
+
+```
+      --combined string         Make a combined report of changes to this file
+      --differ string           Report all non-matching files to this file
+      --download                Check by hashing the contents.
+      --error string            Report all files with errors (hashing or reading) to this file
+  -h, --help                    help for checksum
+      --match string            Report all matching files to this file
+      --missing-on-dst string   Report all files missing from the destination to this file
+      --missing-on-src string   Report all files missing from the source to this file
+      --one-way                 Check one way only, source files must exist on remote
+```
+
+See the [global flags page](/flags/) for global options not listed here.
+
+## SEE ALSO
+
+* [rclone](/commands/rclone/)	 - Show help for rclone commands, flags and backends.
+

docs/content/commands/rclone_hashsum.md

@@ -48,6 +48,7 @@ rclone hashsum <hash> remote:path [flags]
 
 ```
       --base64               Output base64 encoded hashsum
+  -C, --checkfile string     Validate hashes against a given SUM file instead of printing them
       --download             Download the file and hash it locally; if this flag is not specified, the hash is requested from the remote
   -h, --help                 help for hashsum
       --output-file string   Output hashsums to a file rather than the terminal

docs/content/commands/rclone_md5sum.md

@@ -29,6 +29,7 @@ rclone md5sum remote:path [flags]
 
 ```
       --base64               Output base64 encoded hashsum
+  -C, --checkfile string     Validate hashes against a given SUM file instead of printing them
       --download             Download the file and hash it locally; if this flag is not specified, the hash is requested from the remote
   -h, --help                 help for md5sum
       --output-file string   Output hashsums to a file rather than the terminal

docs/content/commands/rclone_sha1sum.md

@@ -29,6 +29,7 @@ rclone sha1sum remote:path [flags]
 
 ```
       --base64               Output base64 encoded hashsum
+  -C, --checkfile string     Validate hashes against a given SUM file instead of printing them
       --download             Download the file and hash it locally; if this flag is not specified, the hash is requested from the remote
   -h, --help                 help for sha1sum
       --output-file string   Output hashsums to a file rather than the terminal

fs/filter/filter.go

@@ -373,8 +373,8 @@ func (f *Filter) InActive() bool {
 		len(f.Opt.ExcludeFile) == 0)
 }
 
-// includeRemote returns whether this remote passes the filter rules.
-func (f *Filter) includeRemote(remote string) bool {
+// IncludeRemote returns whether this remote passes the filter rules.
+func (f *Filter) IncludeRemote(remote string) bool {
 	for _, rule := range f.fileRules.rules {
 		if rule.Match(remote) {
 			return rule.Include
@@ -467,7 +467,7 @@ func (f *Filter) Include(remote string, size int64, modTime time.Time) bool {
 	if f.Opt.MaxSize >= 0 && size > int64(f.Opt.MaxSize) {
 		return false
 	}
-	return f.includeRemote(remote)
+	return f.IncludeRemote(remote)
 }
 
 // IncludeObject returns whether this object should be included into