From 7010543475331e538872630dfce6d97c3d63d190 Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Thu, 23 Nov 2023 13:42:03 +0000 Subject: [PATCH 1/9] calculate package checksum locally --- cloudsmith/data_source_package.go | 41 ++++++++++++++++++++++++++ cloudsmith/data_source_package_test.go | 16 ++++++++++ docs/data-sources/package.md | 8 ++--- 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/cloudsmith/data_source_package.go b/cloudsmith/data_source_package.go index a99f6b7..91d997d 100644 --- a/cloudsmith/data_source_package.go +++ b/cloudsmith/data_source_package.go @@ -1,6 +1,11 @@ package cloudsmith import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" "fmt" "io" "net/http" @@ -36,6 +41,7 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { d.Set("slug", pkg.GetSlug()) d.Set("slug_perm", pkg.GetSlugPerm()) d.Set("version", pkg.GetVersion()) + // Grab the checksum from API in case they don't want to download the file directly via terraform (when returning just the cdn_url) d.Set("checksum_md5", pkg.GetChecksumMd5()) d.Set("checksum_sha1", pkg.GetChecksumSha1()) d.Set("checksum_sha256", pkg.GetChecksumSha256()) @@ -50,6 +56,17 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { } d.Set("output_path", outputPath) d.Set("output_directory", downloadDir) + + // Calculate checksums for the downloaded file + md5Checksum, sha1Checksum, sha256Checksum, sha512Checksum, err := calculateChecksums(outputPath) + if err != nil { + return err + } + + d.Set("checksum_md5", md5Checksum) + d.Set("checksum_sha1", sha1Checksum) + d.Set("checksum_sha256", sha256Checksum) + d.Set("checksum_sha512", sha512Checksum) } else { d.Set("output_path", pkg.GetCdnUrl()) d.Set("output_directory", "") @@ -95,6 +112,30 @@ func downloadPackage(url string, downloadDir string, pc *providerConfig) (string return outputPath, nil } +func calculateChecksums(filePath string) (string, string, string, string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", "", "", "", err + } + defer file.Close() + + md5hash := md5.New() + sha1hash := sha1.New() + sha256hash := sha256.New() + sha512hash := sha512.New() + + if _, err := io.Copy(io.MultiWriter(md5hash, sha1hash, sha256hash, sha512hash), file); err != nil { + return "", "", "", "", err + } + + md5Checksum := hex.EncodeToString(md5hash.Sum(nil)) + sha1Checksum := hex.EncodeToString(sha1hash.Sum(nil)) + sha256Checksum := hex.EncodeToString(sha256hash.Sum(nil)) + sha512Checksum := hex.EncodeToString(sha512hash.Sum(nil)) + + return md5Checksum, sha1Checksum, sha256Checksum, sha512Checksum, nil +} + func dataSourcePackage() *schema.Resource { return &schema.Resource{ Read: dataSourcePackageRead, diff --git a/cloudsmith/data_source_package_test.go b/cloudsmith/data_source_package_test.go index 5125bfd..6481adf 100644 --- a/cloudsmith/data_source_package_test.go +++ b/cloudsmith/data_source_package_test.go @@ -196,5 +196,21 @@ func testAccPackageDataReadPackageDownload(namespace, repository string) string identifier = data.cloudsmith_package_list.test.packages[0].slug_perm download = true } + + output "md5_checksum" { + value = data.cloudsmith_package.test.checksum_md5 + } + + output "sha1_checksum" { + value = data.cloudsmith_package.test.checksum_sha1 + } + + output "sha256_checksum" { + value = data.cloudsmith_package.test.checksum_sha256 + } + + output "sha512_checksum" { + value = data.cloudsmith_package.test.checksum_sha512 + } `, repository, namespace, repository, namespace, repository, namespace) } diff --git a/docs/data-sources/package.md b/docs/data-sources/package.md index 71b743a..d844b9f 100644 --- a/docs/data-sources/package.md +++ b/docs/data-sources/package.md @@ -43,10 +43,10 @@ data "cloudsmith_package" "test" { ## Attribute Reference - `cdn_url`: The URL of the package to download. This attribute is computed and available only when the `download` argument is set to `false`. -- `checksum_md5`: MD5 hash of the package. -- `checksum_sha1`: SHA1 hash of the package. -- `checksum_sha256`: SHA256 hash of the package. -- `checksum_sha512`: SHA512 hash of the package. +- `checksum_md5`: MD5 hash of the downloaded package. If `download` is set to `false`, the checksum is returned from the package API instead. +- `checksum_sha1`: SHA1 hash of the downloaded package.If `download` is set to `false`, the checksum is returned from the package API instead. +- `checksum_sha256`: SHA256 hash of the downloaded package.If `download` is set to `false`, the checksum is returned from the package API instead. +- `checksum_sha512`: SHA512 hash of the downloaded package.If `download` is set to `false`, the checksum is returned from the package API instead. - `format`: The format of the package. - `is_sync_awaiting`: Indicates whether the package is awaiting synchronization. - `is_sync_completed`: Indicates whether the package synchronization has completed. From 2257b928f057afb333c1f4ad1960858250c48417 Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Thu, 23 Nov 2023 13:44:39 +0000 Subject: [PATCH 2/9] remove output from test --- cloudsmith/data_source_package_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/cloudsmith/data_source_package_test.go b/cloudsmith/data_source_package_test.go index 6481adf..5125bfd 100644 --- a/cloudsmith/data_source_package_test.go +++ b/cloudsmith/data_source_package_test.go @@ -196,21 +196,5 @@ func testAccPackageDataReadPackageDownload(namespace, repository string) string identifier = data.cloudsmith_package_list.test.packages[0].slug_perm download = true } - - output "md5_checksum" { - value = data.cloudsmith_package.test.checksum_md5 - } - - output "sha1_checksum" { - value = data.cloudsmith_package.test.checksum_sha1 - } - - output "sha256_checksum" { - value = data.cloudsmith_package.test.checksum_sha256 - } - - output "sha512_checksum" { - value = data.cloudsmith_package.test.checksum_sha512 - } `, repository, namespace, repository, namespace, repository, namespace) } From cd15bcb057e52efcb52704402082d86c300a2cf1 Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Fri, 24 Nov 2023 13:02:00 +0000 Subject: [PATCH 3/9] Feedback improvements to main and test file --- cloudsmith/data_source_package.go | 110 +++++++++++++++++++++---- cloudsmith/data_source_package_test.go | 72 +++++++++++++++- 2 files changed, 162 insertions(+), 20 deletions(-) diff --git a/cloudsmith/data_source_package.go b/cloudsmith/data_source_package.go index 91d997d..e3c76ab 100644 --- a/cloudsmith/data_source_package.go +++ b/cloudsmith/data_source_package.go @@ -9,13 +9,23 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "path" + "strconv" + "time" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" ) +type Checksums struct { + MD5 string + SHA1 string + SHA256 string + SHA512 string +} + func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { pc := m.(*providerConfig) namespace := requiredString(d, "namespace") @@ -50,7 +60,7 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { d.SetId(fmt.Sprintf("%s_%s_%s", namespace, repository, pkg.GetSlugPerm())) if download { - outputPath, err := downloadPackage(pkg.GetCdnUrl(), downloadDir, pc) + outputPath, err := downloadPackage(pkg.GetCdnUrl(), downloadDir, pc, false) if err != nil { return err } @@ -58,15 +68,63 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { d.Set("output_directory", downloadDir) // Calculate checksums for the downloaded file - md5Checksum, sha1Checksum, sha256Checksum, sha512Checksum, err := calculateChecksums(outputPath) + localChecksums, err := calculateChecksums(outputPath) if err != nil { return err } - d.Set("checksum_md5", md5Checksum) - d.Set("checksum_sha1", sha1Checksum) - d.Set("checksum_sha256", sha256Checksum) - d.Set("checksum_sha512", sha512Checksum) + localMD5 := localChecksums.MD5 + localSHA1 := localChecksums.SHA1 + localSHA256 := localChecksums.SHA256 + localSHA512 := localChecksums.SHA512 + + // Check against API checksums + if localMD5 != pkg.GetChecksumMd5() || localSHA1 != pkg.GetChecksumSha1() || localSHA256 != pkg.GetChecksumSha256() || localSHA512 != pkg.GetChecksumSha512() { + // Checksum doesn't match, try to download again with isCached set to true + outputPath, err := downloadPackage(pkg.GetCdnUrl(), downloadDir, pc, true) + if err != nil { + return err + } + + // Calculate checksums for the downloaded file again + localChecksums, err := calculateChecksums(outputPath) + if err != nil { + return err + } + + localMD5 = localChecksums.MD5 + localSHA1 = localChecksums.SHA1 + localSHA256 = localChecksums.SHA256 + localSHA512 = localChecksums.SHA512 + + // Check again after the retry + if localMD5 != pkg.GetChecksumMd5() || localSHA1 != pkg.GetChecksumSha1() || localSHA256 != pkg.GetChecksumSha256() || localSHA512 != pkg.GetChecksumSha512() { + // Checksum still doesn't match, set the flag, and provide a warning + d.Set("download_checksum_mismatch", true) + + // Set the content for each checksum to "Checksum mismatch: Local File: , Remote File: " + mismatchMessageMD5 := fmt.Sprintf("Checksum mismatch: Local File: %s, Remote File: %s", localMD5, pkg.GetChecksumMd5()) + d.Set("checksum_md5", mismatchMessageMD5) + fmt.Println("Warning:", mismatchMessageMD5) + + mismatchMessageSHA1 := fmt.Sprintf("Checksum mismatch: Local File: %s, Remote File: %s", localSHA1, pkg.GetChecksumSha1()) + d.Set("checksum_sha1", mismatchMessageSHA1) + fmt.Println("Warning:", mismatchMessageSHA1) + + mismatchMessageSHA256 := fmt.Sprintf("Checksum mismatch: Local File: %s, Remote File: %s", localSHA256, pkg.GetChecksumSha256()) + d.Set("checksum_sha256", mismatchMessageSHA256) + fmt.Println("Warning:", mismatchMessageSHA256) + + mismatchMessageSHA512 := fmt.Sprintf("Checksum mismatch: Local File: %s, Remote File: %s", localSHA512, pkg.GetChecksumSha512()) + d.Set("checksum_sha512", mismatchMessageSHA512) + fmt.Println("Warning:", mismatchMessageSHA512) + } + } + + d.Set("checksum_md5", localMD5) + d.Set("checksum_sha1", localSHA1) + d.Set("checksum_sha256", localSHA256) + d.Set("checksum_sha512", localSHA512) } else { d.Set("output_path", pkg.GetCdnUrl()) d.Set("output_directory", "") @@ -75,8 +133,8 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { return nil } -func downloadPackage(url string, downloadDir string, pc *providerConfig) (string, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func downloadPackage(urlStr string, downloadDir string, pc *providerConfig, isCached bool) (string, error) { + req, err := http.NewRequest(http.MethodGet, urlStr, nil) if err != nil { return "", err } @@ -84,6 +142,20 @@ func downloadPackage(url string, downloadDir string, pc *providerConfig) (string req.Header.Add("Authorization", fmt.Sprintf("Token %s", pc.GetAPIKey())) client := pc.APIClient.GetConfig().HTTPClient + if isCached { + timestamp := time.Now().Unix() + parsedURL, err := url.Parse(urlStr) + if err != nil { + return "", err + } + + queryValues := parsedURL.Query() + queryValues.Set("time", strconv.FormatInt(timestamp, 10)) + parsedURL.RawQuery = queryValues.Encode() + + req.URL = parsedURL + } + resp, err := client.Do(req) if err != nil { return "", err @@ -91,11 +163,11 @@ func downloadPackage(url string, downloadDir string, pc *providerConfig) (string defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("failed to download file: %s, status code: %d", url, resp.StatusCode) + return "", fmt.Errorf("failed to download file: %s, status code: %d", urlStr, resp.StatusCode) } // Extract filename from CDN URL - filename := path.Base(url) + filename := path.Base(urlStr) outputPath := path.Join(downloadDir, filename) outputFile, err := os.Create(outputPath) @@ -112,10 +184,12 @@ func downloadPackage(url string, downloadDir string, pc *providerConfig) (string return outputPath, nil } -func calculateChecksums(filePath string) (string, string, string, string, error) { +func calculateChecksums(filePath string) (Checksums, error) { + var checksums Checksums + file, err := os.Open(filePath) if err != nil { - return "", "", "", "", err + return checksums, err } defer file.Close() @@ -125,15 +199,15 @@ func calculateChecksums(filePath string) (string, string, string, string, error) sha512hash := sha512.New() if _, err := io.Copy(io.MultiWriter(md5hash, sha1hash, sha256hash, sha512hash), file); err != nil { - return "", "", "", "", err + return checksums, err } - md5Checksum := hex.EncodeToString(md5hash.Sum(nil)) - sha1Checksum := hex.EncodeToString(sha1hash.Sum(nil)) - sha256Checksum := hex.EncodeToString(sha256hash.Sum(nil)) - sha512Checksum := hex.EncodeToString(sha512hash.Sum(nil)) + checksums.MD5 = hex.EncodeToString(md5hash.Sum(nil)) + checksums.SHA1 = hex.EncodeToString(sha1hash.Sum(nil)) + checksums.SHA256 = hex.EncodeToString(sha256hash.Sum(nil)) + checksums.SHA512 = hex.EncodeToString(sha512hash.Sum(nil)) - return md5Checksum, sha1Checksum, sha256Checksum, sha512Checksum, nil + return checksums, nil } func dataSourcePackage() *schema.Resource { diff --git a/cloudsmith/data_source_package_test.go b/cloudsmith/data_source_package_test.go index 5125bfd..149bea6 100644 --- a/cloudsmith/data_source_package_test.go +++ b/cloudsmith/data_source_package_test.go @@ -36,7 +36,7 @@ func TestAccPackage_data(t *testing.T) { resource.TestCheckResourceAttr("cloudsmith_repository.test", "name", dsPackageTestRepository), // Custom TestCheckFunc to upload the package and wait for sync after repository creation func(s *terraform.State) error { - return uploadPackage(testAccProvider.Meta().(*providerConfig)) + return uploadPackage(testAccProvider.Meta().(*providerConfig), false) }, ), }, @@ -58,6 +58,33 @@ func TestAccPackage_data(t *testing.T) { if _, err := os.Stat(filePath); os.IsNotExist(err) { return fmt.Errorf("file does not exist at path: %s", filePath) } + expectedContent := "Hello world" + if err := checkFileContent(filePath, expectedContent); err != nil { + return fmt.Errorf("file content check failed: %w", err) + } + return nil + }, + ), + }, + { + Config: testAccPackageDataReadPackageDownloadRepublish(dsPackageTestNamespace, dsPackageTestRepository), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("data.cloudsmith_package.test", "namespace", dsPackageTestNamespace), + resource.TestCheckResourceAttr("data.cloudsmith_package.test", "repository", dsPackageTestRepository), + func(s *terraform.State) error { + return uploadPackage(testAccProvider.Meta().(*providerConfig), true) + }, + func(s *terraform.State) error { + filePath := filepath.Join("/Users/bblizniak/Desktop/terrafor_test/2", "hello.txt") + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return fmt.Errorf("file does not exist at path: %s", filePath) + } + + expectedContent := "Hello world updated content" + if err := checkFileContent(filePath, expectedContent); err != nil { + return fmt.Errorf("file content check failed: %w", err) + } + return nil }, ), @@ -65,9 +92,25 @@ func TestAccPackage_data(t *testing.T) { }, }) } +func checkFileContent(filePath string, expectedContent string) error { + content, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read file: %w", err) + } -func uploadPackage(pc *providerConfig) error { + if string(content) != expectedContent { + return fmt.Errorf("file content does not match expected. Got: %s, Expected: %s", content, expectedContent) + } + + return nil +} + +func uploadPackage(pc *providerConfig, republish bool) error { fileContent := []byte("Hello world") + if republish { + updatedContent := []byte(" updated content") + fileContent = append(fileContent, updatedContent...) + } initPayload := cloudsmith.PackageFileUploadRequest{ Filename: "hello.txt", @@ -154,6 +197,7 @@ func testAccPackageDataSetup(namespace, repository string) string { resource "cloudsmith_repository" "test" { name = "%s" namespace = "%s" + replace_packages_by_default = true } `, repository, namespace) } @@ -163,6 +207,7 @@ func testAccPackageDataReadPackage(namespace, repository string) string { resource "cloudsmith_repository" "test" { name = "%s" namespace = "%s" + replace_packages_by_default = true } data "cloudsmith_package_list" "test" { @@ -183,6 +228,29 @@ func testAccPackageDataReadPackageDownload(namespace, repository string) string resource "cloudsmith_repository" "test" { name = "%s" namespace = "%s" + replace_packages_by_default = true + } + + data "cloudsmith_package_list" "test" { + repository = "%s" + namespace = "%s" + } + + data "cloudsmith_package" "test" { + repository = "%s" + namespace = "%s" + identifier = data.cloudsmith_package_list.test.packages[0].slug_perm + download = true + } + `, repository, namespace, repository, namespace, repository, namespace) +} + +func testAccPackageDataReadPackageDownloadRepublish(namespace, repository string) string { + return fmt.Sprintf(` + resource "cloudsmith_repository" "test" { + name = "%s" + namespace = "%s" + replace_packages_by_default = true } data "cloudsmith_package_list" "test" { From 7b522a1535f408d188f3c78ce759a54d723d9721 Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Fri, 24 Nov 2023 13:11:22 +0000 Subject: [PATCH 4/9] fix test dir --- cloudsmith/data_source_package_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudsmith/data_source_package_test.go b/cloudsmith/data_source_package_test.go index 149bea6..171672c 100644 --- a/cloudsmith/data_source_package_test.go +++ b/cloudsmith/data_source_package_test.go @@ -75,7 +75,7 @@ func TestAccPackage_data(t *testing.T) { return uploadPackage(testAccProvider.Meta().(*providerConfig), true) }, func(s *terraform.State) error { - filePath := filepath.Join("/Users/bblizniak/Desktop/terrafor_test/2", "hello.txt") + filePath := filepath.Join(os.TempDir(), "hello.txt") if _, err := os.Stat(filePath); os.IsNotExist(err) { return fmt.Errorf("file does not exist at path: %s", filePath) } From 641e36ea4ad4d26921b68eb83102de1a30100d2d Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Fri, 24 Nov 2023 14:00:36 +0000 Subject: [PATCH 5/9] Adjust test logic --- cloudsmith/data_source_package_test.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/cloudsmith/data_source_package_test.go b/cloudsmith/data_source_package_test.go index 171672c..8645b3d 100644 --- a/cloudsmith/data_source_package_test.go +++ b/cloudsmith/data_source_package_test.go @@ -58,12 +58,21 @@ func TestAccPackage_data(t *testing.T) { if _, err := os.Stat(filePath); os.IsNotExist(err) { return fmt.Errorf("file does not exist at path: %s", filePath) } + defer func() { + // Remove the file after the check is done + if err := os.Remove(filePath); err != nil { + fmt.Printf("Error removing file: %s\n", err) + } + }() expectedContent := "Hello world" if err := checkFileContent(filePath, expectedContent); err != nil { return fmt.Errorf("file content check failed: %w", err) } return nil }, + func(s *terraform.State) error { + return uploadPackage(testAccProvider.Meta().(*providerConfig), true) + }, ), }, { @@ -71,9 +80,6 @@ func TestAccPackage_data(t *testing.T) { Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr("data.cloudsmith_package.test", "namespace", dsPackageTestNamespace), resource.TestCheckResourceAttr("data.cloudsmith_package.test", "repository", dsPackageTestRepository), - func(s *terraform.State) error { - return uploadPackage(testAccProvider.Meta().(*providerConfig), true) - }, func(s *terraform.State) error { filePath := filepath.Join(os.TempDir(), "hello.txt") if _, err := os.Stat(filePath); os.IsNotExist(err) { @@ -106,10 +112,15 @@ func checkFileContent(filePath string, expectedContent string) error { } func uploadPackage(pc *providerConfig, republish bool) error { - fileContent := []byte("Hello world") + + var ( + fileContent []byte + ) + if republish { - updatedContent := []byte(" updated content") - fileContent = append(fileContent, updatedContent...) + fileContent = []byte("Hello world updated content") + } else { + fileContent = []byte("Hello world") } initPayload := cloudsmith.PackageFileUploadRequest{ From b5021c88b11f3b08e5fadc9a58a7d7b5cd4de487 Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Fri, 24 Nov 2023 14:13:16 +0000 Subject: [PATCH 6/9] Add output to notify cache reset --- cloudsmith/data_source_package.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudsmith/data_source_package.go b/cloudsmith/data_source_package.go index e3c76ab..c01e3f2 100644 --- a/cloudsmith/data_source_package.go +++ b/cloudsmith/data_source_package.go @@ -85,6 +85,7 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { if err != nil { return err } + fmt.Println("Package pulled again due to checksum mismatch.") // Calculate checksums for the downloaded file again localChecksums, err := calculateChecksums(outputPath) From 9de7727eaf5a2fb00551e30419cb1d34fd80dbf8 Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Thu, 30 Nov 2023 10:39:27 +0000 Subject: [PATCH 7/9] Review changes --- cloudsmith/data_source_package.go | 114 +++++++++++++++++------------- 1 file changed, 66 insertions(+), 48 deletions(-) diff --git a/cloudsmith/data_source_package.go b/cloudsmith/data_source_package.go index c01e3f2..89d5a99 100644 --- a/cloudsmith/data_source_package.go +++ b/cloudsmith/data_source_package.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "crypto/sha512" "encoding/hex" + "errors" "fmt" "io" "net/http" @@ -26,6 +27,11 @@ type Checksums struct { SHA512 string } +func checksumMismatchError(localChecksum string, remoteChecksum string, checksumType string) string { + formatString := fmt.Sprintf("Checksum mismatch (%s): expected=%s, got=%s", localChecksum, remoteChecksum, checksumType) + return formatString +} + func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { pc := m.(*providerConfig) namespace := requiredString(d, "namespace") @@ -33,6 +39,7 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { identifier := requiredString(d, "identifier") download := requiredBool(d, "download") downloadDir := requiredString(d, "download_dir") + ignoreChecksum := requiredBool(d, "ignore_checksums") req := pc.APIClient.PackagesApi.PackagesRead(pc.Auth, namespace, repository, identifier) pkg, _, err := pc.APIClient.PackagesApi.PackagesReadExecute(req) @@ -80,12 +87,12 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { // Check against API checksums if localMD5 != pkg.GetChecksumMd5() || localSHA1 != pkg.GetChecksumSha1() || localSHA256 != pkg.GetChecksumSha256() || localSHA512 != pkg.GetChecksumSha512() { - // Checksum doesn't match, try to download again with isCached set to true + // Checksum doesn't match, try to download again with bustCache set to true outputPath, err := downloadPackage(pkg.GetCdnUrl(), downloadDir, pc, true) if err != nil { return err } - fmt.Println("Package pulled again due to checksum mismatch.") + fmt.Println("Package pulled again with bustCache due to checksum mismatch.") // Calculate checksums for the downloaded file again localChecksums, err := calculateChecksums(outputPath) @@ -100,25 +107,30 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { // Check again after the retry if localMD5 != pkg.GetChecksumMd5() || localSHA1 != pkg.GetChecksumSha1() || localSHA256 != pkg.GetChecksumSha256() || localSHA512 != pkg.GetChecksumSha512() { - // Checksum still doesn't match, set the flag, and provide a warning - d.Set("download_checksum_mismatch", true) - - // Set the content for each checksum to "Checksum mismatch: Local File: , Remote File: " - mismatchMessageMD5 := fmt.Sprintf("Checksum mismatch: Local File: %s, Remote File: %s", localMD5, pkg.GetChecksumMd5()) - d.Set("checksum_md5", mismatchMessageMD5) - fmt.Println("Warning:", mismatchMessageMD5) - - mismatchMessageSHA1 := fmt.Sprintf("Checksum mismatch: Local File: %s, Remote File: %s", localSHA1, pkg.GetChecksumSha1()) - d.Set("checksum_sha1", mismatchMessageSHA1) - fmt.Println("Warning:", mismatchMessageSHA1) - - mismatchMessageSHA256 := fmt.Sprintf("Checksum mismatch: Local File: %s, Remote File: %s", localSHA256, pkg.GetChecksumSha256()) - d.Set("checksum_sha256", mismatchMessageSHA256) - fmt.Println("Warning:", mismatchMessageSHA256) - - mismatchMessageSHA512 := fmt.Sprintf("Checksum mismatch: Local File: %s, Remote File: %s", localSHA512, pkg.GetChecksumSha512()) - d.Set("checksum_sha512", mismatchMessageSHA512) - fmt.Println("Warning:", mismatchMessageSHA512) + if ignoreChecksum { + fmt.Println("Warning: ignore_checksums set to true, downloading mismatched checksum file.") + d.Set("checksum_md5", localMD5) + d.Set("checksum_sha1", localSHA1) + d.Set("checksum_sha256", localSHA256) + d.Set("checksum_sha512", localSHA512) + } else { + if localMD5 != pkg.GetChecksumMd5() || localSHA1 != pkg.GetChecksumSha1() || localSHA256 != pkg.GetChecksumSha256() || localSHA512 != pkg.GetChecksumSha512() { + errMsg := "" + if localMD5 != pkg.GetChecksumMd5() { + errMsg += checksumMismatchError(localMD5, pkg.GetChecksumMd5(), "MD5") + "\n" + } + if localSHA1 != pkg.GetChecksumSha1() { + errMsg += checksumMismatchError(localSHA1, pkg.GetChecksumSha1(), "SHA1") + "\n" + } + if localSHA256 != pkg.GetChecksumSha256() { + errMsg += checksumMismatchError(localSHA256, pkg.GetChecksumSha256(), "SHA256") + "\n" + } + if localSHA512 != pkg.GetChecksumSha512() { + errMsg += checksumMismatchError(localSHA512, pkg.GetChecksumSha512(), "SHA512") + "\n" + } + return errors.New(errMsg) + } + } } } @@ -134,8 +146,8 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { return nil } -func downloadPackage(urlStr string, downloadDir string, pc *providerConfig, isCached bool) (string, error) { - req, err := http.NewRequest(http.MethodGet, urlStr, nil) +func downloadPackage(downloadUrl string, downloadDir string, pc *providerConfig, bustCache bool) (string, error) { + req, err := http.NewRequest(http.MethodGet, downloadUrl, nil) if err != nil { return "", err } @@ -143,9 +155,9 @@ func downloadPackage(urlStr string, downloadDir string, pc *providerConfig, isCa req.Header.Add("Authorization", fmt.Sprintf("Token %s", pc.GetAPIKey())) client := pc.APIClient.GetConfig().HTTPClient - if isCached { + if bustCache { timestamp := time.Now().Unix() - parsedURL, err := url.Parse(urlStr) + parsedURL, err := url.Parse(downloadUrl) if err != nil { return "", err } @@ -164,11 +176,11 @@ func downloadPackage(urlStr string, downloadDir string, pc *providerConfig, isCa defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("failed to download file: %s, status code: %d", urlStr, resp.StatusCode) + return "", fmt.Errorf("failed to download file: %s, status code: %d", downloadUrl, resp.StatusCode) } // Extract filename from CDN URL - filename := path.Base(urlStr) + filename := path.Base(downloadUrl) outputPath := path.Join(downloadDir, filename) outputFile, err := os.Create(outputPath) @@ -247,11 +259,23 @@ func dataSourcePackage() *schema.Resource { Optional: true, Default: false, }, + "download_dir": { + Type: schema.TypeString, + Description: "The directory where the file will be downloaded if download is set to true", + Optional: true, + Default: os.TempDir(), + }, "format": { Type: schema.TypeString, Description: "The format of the package", Computed: true, }, + "ignore_checksums": { + Type: schema.TypeBool, + Description: "Ignore checksums for the package", + Optional: true, + Default: false, + }, "identifier": { Type: schema.TypeString, Description: "The identifier for this repository.", @@ -260,27 +284,27 @@ func dataSourcePackage() *schema.Resource { }, "is_sync_awaiting": { Type: schema.TypeBool, - Description: "Is the package awaiting synchronisation", + Description: "Is the package awaiting synchronization", Computed: true, }, "is_sync_completed": { Type: schema.TypeBool, - Description: "Has the package synchronisation completed", + Description: "Has the package synchronization completed", Computed: true, }, "is_sync_failed": { Type: schema.TypeBool, - Description: "Has the package synchronisation failed", + Description: "Has the package synchronization failed", Computed: true, }, "is_sync_in_flight": { Type: schema.TypeBool, - Description: "Is the package synchronisation currently in-flight", + Description: "Is the package synchronization currently in-flight", Computed: true, }, "is_sync_in_progress": { Type: schema.TypeBool, - Description: "Is the package synchronisation currently in-progress", + Description: "Is the package synchronization currently in-progress", Computed: true, }, "name": { @@ -294,22 +318,22 @@ func dataSourcePackage() *schema.Resource { Required: true, ValidateFunc: validation.StringIsNotEmpty, }, - "output_path": { + "output_directory": { Type: schema.TypeString, - Description: "The location of the package", + Description: "The directory where the file is downloaded", Computed: true, }, - "download_dir": { - Type: schema.TypeString, - Description: "The directory where the file will be downloaded if download is set to true", - Optional: true, - Default: os.TempDir(), - }, - "output_directory": { + "output_path": { Type: schema.TypeString, - Description: "The directory where the file is downloaded", + Description: "The location of the package", Computed: true, }, + "repository": { + Type: schema.TypeString, + Description: "The repository of the package", + Required: true, + ValidateFunc: validation.StringIsNotEmpty, + }, "slug": { Type: schema.TypeString, Description: "The slug identifies the package in URIs.", @@ -321,12 +345,6 @@ func dataSourcePackage() *schema.Resource { "It will never change once a package has been created.", Computed: true, }, - "repository": { - Type: schema.TypeString, - Description: "The repository of the package", - Required: true, - ValidateFunc: validation.StringIsNotEmpty, - }, "version": { Type: schema.TypeString, Description: "The version of the package", From 90b5555cdecae467b712e225a8f8a19792c2b916 Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Thu, 30 Nov 2023 10:56:42 +0000 Subject: [PATCH 8/9] Update documentation --- docs/data-sources/package.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/data-sources/package.md b/docs/data-sources/package.md index d844b9f..8d0b6ba 100644 --- a/docs/data-sources/package.md +++ b/docs/data-sources/package.md @@ -39,6 +39,7 @@ data "cloudsmith_package" "test" { - `identifier` (Required): The identifier for the package. - `download` (Optional): If set to true, the package will be downloaded. Defaults to false. If set to false, the CDN URL will be available in the `output_path`. - `download_dir` (Optional): The directory where the file will be downloaded to. If not set and `download` is set to `true`, it will default to the operating system's default temporary directory and save the file there. +- `ignore_checksums` (Optional): If set to `true`, any mismatched checksum from our API and local check will be ignored and download the package if `download` is set to `true`. ## Attribute Reference From ceccab6d699c92e9b83017884bd107fa5bf19294 Mon Sep 17 00:00:00 2001 From: Bartosz Blizniak Date: Mon, 11 Dec 2023 17:11:11 +0000 Subject: [PATCH 9/9] comment review --- cloudsmith/data_source_package.go | 121 ++++++++++++++---------------- 1 file changed, 58 insertions(+), 63 deletions(-) diff --git a/cloudsmith/data_source_package.go b/cloudsmith/data_source_package.go index 89d5a99..237be1a 100644 --- a/cloudsmith/data_source_package.go +++ b/cloudsmith/data_source_package.go @@ -6,7 +6,6 @@ import ( "crypto/sha256" "crypto/sha512" "encoding/hex" - "errors" "fmt" "io" "net/http" @@ -16,6 +15,7 @@ import ( "strconv" "time" + cloudsmith_api "github.com/cloudsmith-io/cloudsmith-api-go" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" ) @@ -27,6 +27,30 @@ type Checksums struct { SHA512 string } +func (c Checksums) CompareWithPkg(pkg *cloudsmith_api.Package) error { + var errs []error + + if c.MD5 != pkg.GetChecksumMd5() { + errs = append(errs, fmt.Errorf(checksumMismatchError(c.MD5, pkg.GetChecksumMd5(), "MD5"))) + } + if c.SHA1 != pkg.GetChecksumSha1() { + errs = append(errs, fmt.Errorf(checksumMismatchError(c.SHA1, pkg.GetChecksumSha1(), "SHA1"))) + } + if c.SHA256 != pkg.GetChecksumSha256() { + errs = append(errs, fmt.Errorf(checksumMismatchError(c.SHA256, pkg.GetChecksumSha256(), "SHA256"))) + } + if c.SHA512 != pkg.GetChecksumSha512() { + errs = append(errs, fmt.Errorf(checksumMismatchError(c.SHA512, pkg.GetChecksumSha512(), "SHA512"))) + } + + var finalError error = nil + for _, err := range errs { + finalError = fmt.Errorf("%w\n", err) + } + + return finalError +} + func checksumMismatchError(localChecksum string, remoteChecksum string, checksumType string) string { formatString := fmt.Sprintf("Checksum mismatch (%s): expected=%s, got=%s", localChecksum, remoteChecksum, checksumType) return formatString @@ -66,83 +90,54 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { d.SetId(fmt.Sprintf("%s_%s_%s", namespace, repository, pkg.GetSlugPerm())) - if download { - outputPath, err := downloadPackage(pkg.GetCdnUrl(), downloadDir, pc, false) + if !download { + d.Set("output_path", pkg.GetCdnUrl()) + d.Set("output_directory", "") + return nil + } + + bustCache := false + retryTimes := 0 + var checksumError error = nil + var localChecksums Checksums + + for retryTimes < 2 { + outputPath, err := downloadPackage(pkg.GetCdnUrl(), downloadDir, pc, bustCache) if err != nil { return err } + d.Set("output_path", outputPath) d.Set("output_directory", downloadDir) // Calculate checksums for the downloaded file - localChecksums, err := calculateChecksums(outputPath) + localChecksums, err = calculateChecksums(outputPath) if err != nil { return err } - localMD5 := localChecksums.MD5 - localSHA1 := localChecksums.SHA1 - localSHA256 := localChecksums.SHA256 - localSHA512 := localChecksums.SHA512 - - // Check against API checksums - if localMD5 != pkg.GetChecksumMd5() || localSHA1 != pkg.GetChecksumSha1() || localSHA256 != pkg.GetChecksumSha256() || localSHA512 != pkg.GetChecksumSha512() { - // Checksum doesn't match, try to download again with bustCache set to true - outputPath, err := downloadPackage(pkg.GetCdnUrl(), downloadDir, pc, true) - if err != nil { - return err - } - fmt.Println("Package pulled again with bustCache due to checksum mismatch.") - - // Calculate checksums for the downloaded file again - localChecksums, err := calculateChecksums(outputPath) - if err != nil { - return err - } - - localMD5 = localChecksums.MD5 - localSHA1 = localChecksums.SHA1 - localSHA256 = localChecksums.SHA256 - localSHA512 = localChecksums.SHA512 - - // Check again after the retry - if localMD5 != pkg.GetChecksumMd5() || localSHA1 != pkg.GetChecksumSha1() || localSHA256 != pkg.GetChecksumSha256() || localSHA512 != pkg.GetChecksumSha512() { - if ignoreChecksum { - fmt.Println("Warning: ignore_checksums set to true, downloading mismatched checksum file.") - d.Set("checksum_md5", localMD5) - d.Set("checksum_sha1", localSHA1) - d.Set("checksum_sha256", localSHA256) - d.Set("checksum_sha512", localSHA512) - } else { - if localMD5 != pkg.GetChecksumMd5() || localSHA1 != pkg.GetChecksumSha1() || localSHA256 != pkg.GetChecksumSha256() || localSHA512 != pkg.GetChecksumSha512() { - errMsg := "" - if localMD5 != pkg.GetChecksumMd5() { - errMsg += checksumMismatchError(localMD5, pkg.GetChecksumMd5(), "MD5") + "\n" - } - if localSHA1 != pkg.GetChecksumSha1() { - errMsg += checksumMismatchError(localSHA1, pkg.GetChecksumSha1(), "SHA1") + "\n" - } - if localSHA256 != pkg.GetChecksumSha256() { - errMsg += checksumMismatchError(localSHA256, pkg.GetChecksumSha256(), "SHA256") + "\n" - } - if localSHA512 != pkg.GetChecksumSha512() { - errMsg += checksumMismatchError(localSHA512, pkg.GetChecksumSha512(), "SHA512") + "\n" - } - return errors.New(errMsg) - } - } - } + if ignoreChecksum { + fmt.Println("Warning: ignore_checksums set to true, downloading mismatched checksum file.") + break } - d.Set("checksum_md5", localMD5) - d.Set("checksum_sha1", localSHA1) - d.Set("checksum_sha256", localSHA256) - d.Set("checksum_sha512", localSHA512) - } else { - d.Set("output_path", pkg.GetCdnUrl()) - d.Set("output_directory", "") + if checksumError = localChecksums.CompareWithPkg(pkg); checksumError != nil { + bustCache = true + retryTimes++ + } else { + break + } } + if checksumError != nil { + return checksumError + } + + d.Set("checksum_md5", localChecksums.MD5) + d.Set("checksum_sha1", localChecksums.SHA1) + d.Set("checksum_sha256", localChecksums.SHA256) + d.Set("checksum_sha512", localChecksums.SHA512) + return nil }