Cryptography module, integrity checking commands and small fixes

.gitignore

@@ -1,7 +1,10 @@
+# App related
 build/
 testing.log
 *.out
 *.exe
 .cpackget*
 tmp/
 .vagrant/
+# IDEs
+.vscode/

cmd/commands/checksum.go

@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+/* Copyright Contributors to the cpackget project. */
+
+package commands
+
+import (
+	"github.com/open-cmsis-pack/cpackget/cmd/cryptography"
+	"github.com/spf13/cobra"
+)
+
+var checksumCreateCmdFlags struct {
+	// hashAlgorithm is the cryptographic hash function to be used
+	hashAlgorithm string
+
+	// outputDir is the target directory where the checksum file is written to
+	outputDir string
+}
+
+func init() {
+	ChecksumCreateCmd.Flags().StringVarP(&checksumCreateCmdFlags.hashAlgorithm, "hash-function", "a", cryptography.Hashes[0], "specifies the hash function to be used")
+	ChecksumCreateCmd.Flags().StringVarP(&checksumCreateCmdFlags.outputDir, "output-dir", "o", "", "specifies output directory for the checksum file")
+}
+
+var ChecksumCreateCmd = &cobra.Command{
+	Use:   "checksum-create [<local .path pack>]",
+	Short: "Generates a .checksum file containing the digests of a pack",
+	Long: `
+Creates a .checksum file of a local pack. This is file contains the digests
+of the contents of the pack. Example <Vendor.Pack.1.2.3.sha256.checksum> file:
+
+  "6f95628e4e0824b0ff4a9f49dad1c3eb073b27c2dd84de3b985f0ef3405ca9ca Vendor.Pack.1.2.3.pdsc
+  435fsdf..."
+
+  The referenced pack must be in its original/compressed form (.pack), and be present locally:
+
+  $ cpackget checksum-create Vendor.Pack.1.2.3.pack
+
+The default Cryptographic Hash Function used is "` + cryptography.Hashes[0] + `". In the future other hash functions
+might be supported. The used function will be prefixed to the ".checksum" extension.
+
+By default the checksum file will be created in the same directory as the provided pack.`,
+	Args:              cobra.ExactArgs(1),
+	PersistentPreRunE: configureInstaller,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return cryptography.GenerateChecksum(args[0], checksumCreateCmdFlags.outputDir, checksumCreateCmdFlags.hashAlgorithm)
+	},
+}
+
+var ChecksumVerifyCmd = &cobra.Command{
+	Use:   "checksum-verify [<local .path pack>] [<local .checksum path>]",
+	Short: "Verifies the integrity of a pack using its .checksum file",
+	Long: `
+Verifies the contents of a pack, checking its integrity against its .checksum file (created
+with "checksum-create"):
+
+  $ cpackget checksum-verify Vendor.Pack.1.2.3.pack Vendor.Pack.1.2.3.sha256.checksum
+
+The used hash function is inferred from the checksum filename, and if any of the digests
+computed doesn't match the one provided in the checksum file an error will be thrown.`,
+	Args:              cobra.ExactArgs(2),
+	PersistentPreRunE: configureInstaller,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return cryptography.VerifyChecksum(args[0], args[1])
+	},
+}

cmd/commands/index.go

@@ -4,6 +4,9 @@
 package commands
 
 import (
+	"os"
+	"runtime"
+
 	"github.com/open-cmsis-pack/cpackget/cmd/installer"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -13,11 +16,10 @@ import (
 var overwrite bool
 
 var IndexCmd = &cobra.Command{
-	Deprecated: "Consider running `cpackget update-index` instead",
-	Use:        "index <index url>",
-	Short:      "Updates public index",
-	Long: `Updates the public index in CMSIS_PACK_ROOT/.Web/index.pidx using the file specified by the given url.
-If there's already an index file, cpackget won't overwrite it. Use "-f" to do so.`,
+	Deprecated:        "Consider running `cpackget update-index` instead",
+	Use:               "index <index url>",
+	Short:             "Updates public index",
+	Long:              getLongIndexDescription(),
 	PersistentPreRunE: configureInstaller,
 	Args:              cobra.ExactArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -27,6 +29,18 @@ If there's already an index file, cpackget won't overwrite it. Use "-f" to do so
 	},
 }
 
+// getLongIndexDescription prints a "Windows friendly" long description,
+// using the correct path slashes
+func getLongIndexDescription() string {
+	if runtime.GOOS == "windows" {
+		return `Updates the public index in ` + os.Getenv("CMSIS_PACK_ROOT") + `\.Web\index.pidx using the file specified by the given url.
+If there's already an index file, cpackget won't overwrite it. Use "-f" to do so.`
+	} else {
+		return `Updates the public index in ` + os.Getenv("CMSIS_PACK_ROOT") + `/.Web/index.pidx using the file specified by the given url.
+If there's already an index file, cpackget won't overwrite it. Use "-f" to do so.`
+	}
+}
+
 func init() {
 	IndexCmd.Flags().BoolVarP(&overwrite, "force", "f", false, "forces cpackget to overwrite an existing public index file")
 }

cmd/commands/root.go

@@ -26,6 +26,8 @@ var AllCommands = []*cobra.Command{
 	RmCmd,
 	ListCmd,
 	UpdateIndexCmd,
+	ChecksumCreateCmd,
+	ChecksumVerifyCmd,
 }
 
 // createPackRoot is a flag that determines if the pack root should be created or not

cmd/commands/update_index.go

@@ -4,6 +4,9 @@
 package commands
 
 import (
+	"os"
+	"runtime"
+
 	"github.com/open-cmsis-pack/cpackget/cmd/installer"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -15,10 +18,9 @@ var updateIndexCmdFlags struct {
 }
 
 var UpdateIndexCmd = &cobra.Command{
-	Use:   "update-index",
-	Short: "Update the public index",
-	Long: `Update the public index in CMSIS_PACK_ROOT/.Web/index.pidx using the URL in <url> tag inside index.pidx.
-By default it will also check if all PDSC files under .Web/ need update as well. This can be disabled via the "--sparse" flag.`,
+	Use:               "update-index",
+	Short:             "Update the public index",
+	Long:              getLongUpdateDescription(),
 	PersistentPreRunE: configureInstaller,
 	Args:              cobra.ExactArgs(0),
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -27,6 +29,18 @@ By default it will also check if all PDSC files under .Web/ need update as well.
 	},
 }
 
+// getLongUpdateDescription prints a "Windows friendly" long description,
+// using the correct path slashes
+func getLongUpdateDescription() string {
+	if runtime.GOOS == "windows" {
+		return `Updates the public index in ` + os.Getenv("CMSIS_PACK_ROOT") + `\.Web\index.pidx using the URL in <url> tag inside index.pidx.
+By default it will also check if all PDSC files under .Web/ need update as well. This can be disabled via the "--sparse" flag.`
+	} else {
+		return `Updates the public index in ` + os.Getenv("CMSIS_PACK_ROOT") + `/.Web/index.pidx using the URL in <url> tag inside index.pidx.
+By default it will also check if all PDSC files under .Web/ need update as well. This can be disabled via the "--sparse" flag.`
+	}
+}
+
 func init() {
 	UpdateIndexCmd.Flags().BoolVarP(&updateIndexCmdFlags.sparse, "sparse", "s", false, "avoid updating the pdsc files within .Web/ folder")
 }

cmd/cryptography/checksum.go

@@ -0,0 +1,176 @@
+package cryptography
+
+import (
+	"archive/zip"
+	"bufio"
+	"crypto/sha256"
+	"fmt"
+	"hash"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	errs "github.com/open-cmsis-pack/cpackget/cmd/errors"
+	"github.com/open-cmsis-pack/cpackget/cmd/utils"
+	log "github.com/sirupsen/logrus"
+)
+
+// isValidHash returns whether a hash function is
+// supported or not.
+func isValidHash(hashFunction string) bool {
+	for _, h := range Hashes {
+		if h == hashFunction {
+			return true
+		}
+	}
+	return false
+}
+
+// getChecksumList computes the digests of a pack according
+// to the specified hash function.
+func getChecksumList(sourcePack, hashFunction string) (map[string]string, error) {
+	var h hash.Hash
+	switch hashFunction {
+	case "sha256":
+		h = sha256.New()
+	} // Default will always be "sha256" if nothing is passed
+
+	zipReader, err := zip.OpenReader(sourcePack)
+	if err != nil {
+		log.Errorf("can't decompress \"%s\": %s", sourcePack, err)
+		return nil, errs.ErrFailedDecompressingFile
+	}
+
+	digests := make(map[string]string)
+	for _, file := range zipReader.File {
+		reader, err := file.Open()
+		if err != nil {
+			return nil, err
+		}
+		// Avoid Potential DoS vulnerability via decompression bomb
+		for {
+			_, err = io.CopyN(h, reader, 1024)
+			if err != nil {
+				if err == io.EOF {
+					break
+				}
+				return nil, err
+			}
+		}
+		digests[file.Name] = fmt.Sprintf("%x", h.Sum(nil))
+	}
+	return digests, nil
+}
+
+// GenerateChecksum creates a .checksum file for a pack.
+func GenerateChecksum(sourcePack, destinationDir, hashFunction string) error {
+	if !isValidHash(hashFunction) {
+		return errs.ErrInvalidHashFunction
+	}
+	if !utils.FileExists(sourcePack) {
+		log.Errorf("\"%s\" does not exist", sourcePack)
+		return errs.ErrFileNotFound
+	}
+
+	// Checksum file path defaults to the .pack's location
+	base := ""
+	if destinationDir == "" {
+		base = filepath.Clean(strings.TrimSuffix(sourcePack, filepath.Ext(sourcePack)))
+	} else {
+		if !utils.DirExists(destinationDir) {
+			return errs.ErrDirectoryNotFound
+		}
+		base = filepath.Clean(destinationDir) + string(filepath.Separator) + strings.TrimSuffix(string(filepath.Base(sourcePack)), ".pack")
+	}
+	checksumFilename := base + "." + strings.Replace(hashFunction, "-", "", -1) + ".checksum"
+	if utils.FileExists(checksumFilename) {
+		log.Errorf("\"%s\" already exists, choose a diferent path", checksumFilename)
+		return errs.ErrPathAlreadyExists
+	}
+
+	digests, err := getChecksumList(sourcePack, hashFunction)
+	if err != nil {
+		return err
+	}
+
+	out, err := os.Create(checksumFilename)
+	if err != nil {
+		log.Error(err)
+		return errs.ErrFailedCreatingFile
+	}
+	defer out.Close()
+	for filename, digest := range digests {
+		_, err := out.Write([]byte(digest + " " + filename + "\n"))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// VerifyChecksum validates the contents of a pack
+// according to a provided .checksum file.
+func VerifyChecksum(sourcePack, sourceChecksum string) error {
+	if !utils.FileExists(sourcePack) {
+		log.Errorf("\"%s\" does not exist", sourcePack)
+		return errs.ErrFileNotFound
+	}
+	if !utils.FileExists(sourceChecksum) {
+		log.Errorf("\"%s\" does not exist", sourceChecksum)
+		return errs.ErrFileNotFound
+	}
+	hashFunction := filepath.Ext(strings.Split(sourceChecksum, ".checksum")[0])[1:]
+	if !isValidHash(hashFunction) {
+		log.Errorf("\"%s\" is not a valid .checksum file (correct format is [<pack>].[<hash-algorithm>].checksum). Please confirm if the algorithm is supported.", sourceChecksum)
+		return errs.ErrInvalidHashFunction
+	}
+
+	// Compute pack's digests
+	digests, err := getChecksumList(sourcePack, hashFunction)
+	if err != nil {
+		return err
+	}
+
+	// Check if pack and checksum file have the same number of files listed
+	b, err := os.ReadFile(sourceChecksum)
+	if err != nil {
+		return err
+	}
+	if strings.Count(string(b), "\n") != len(digests) {
+		log.Errorf("provided checksum file lists %d file(s), but pack contains %d file(s)", len(digests), strings.Count(string(b), "\n"))
+		return errs.ErrIntegrityCheckFailed
+	}
+
+	// Compare with target checksum file
+	checksumFile, err := os.Open(sourceChecksum)
+	if err != nil {
+		return err
+	}
+	defer checksumFile.Close()
+
+	failure := false
+	scanner := bufio.NewScanner(checksumFile)
+	for scanner.Scan() {
+		if scanner.Text() == "" {
+			continue
+		}
+		targetFile := strings.Split(scanner.Text(), " ")[1]
+		targetDigest := strings.Split(scanner.Text(), " ")[0]
+		if digests[targetFile] != targetDigest {
+			if digests[targetFile] == "" {
+				log.Errorf("\"%s\" does not exist in the provided pack but is listed in the checksum file", targetFile)
+				return errs.ErrIntegrityCheckFailed
+			}
+			log.Debugf("%s != %s", digests[targetFile], targetDigest)
+			log.Errorf("%s: computed checksum did NOT match", targetFile)
+			failure = true
+		}
+	}
+	if failure {
+		return errs.ErrBadPackIntegrity
+	}
+
+	log.Info("pack integrity verified, all checksums match.")
+	return nil
+}

cmd/cryptography/const.go

@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+/* Copyright Contributors to the cpackget project. */
+
+package cryptography
+
+// Hashes is the list of supported Cryptographic Hash Functions used for the checksum feature
+var Hashes = [1]string{"sha256"}

cmd/errors/errors.go

@@ -47,9 +47,15 @@ var (
 	ErrFailedCreatingDirectory   = errors.New("fail to create directory")
 	ErrFileNotFound              = errors.New("file not found")
 	ErrDirectoryNotFound         = errors.New("directory not found")
+	ErrPathAlreadyExists         = errors.New("path already exists")
 	ErrCopyingEqualPaths         = errors.New("failed copying files: source is the same as destination")
 	ErrMovingEqualPaths          = errors.New("failed moving files: source is the same as destination")
 
+	// Cryptography errors
+	ErrBadPackIntegrity     = errors.New("bad pack integrity")
+	ErrIntegrityCheckFailed = errors.New("checksum verification failed")
+	ErrInvalidHashFunction  = errors.New("provided hash function is not supported")
+
 	// Security errors
 	ErrInsecureZipFileName = errors.New("zip file contains insecure characters: ../")
 	ErrFileTooBig          = errors.New("files cannot be over 20G")

cmd/installer/pack.go

@@ -256,6 +256,7 @@ func (p *PackType) install(installation *PacksInstallationType, checkEula bool)
 			}
 
 			if !ok {
+				log.Info("User does not agree with the pack's license, not installing it")
 				return errs.ErrEula
 			}
 		} else {

cmd/installer/root.go

@@ -67,6 +67,10 @@ func AddPack(packPath string, checkEula, extractEula bool, forceReinstall bool)
 	ui.Extract = extractEula
 
 	if err = pack.install(Installation, checkEula || extractEula); err != nil {
+		// Just for internal purposes, is not presented as an error to the user
+		if err == errs.ErrEula {
+			return nil
+		}
 		if dropPreInstalled {
 			log.Error("Error installing pack, reverting temporary pack to original state")
 			// Make sure the original directory doesn't exist to avoid moving errors