diff --git a/cloudsmith/data_source_package.go b/cloudsmith/data_source_package.go index a99f6b7..237be1a 100644 --- a/cloudsmith/data_source_package.go +++ b/cloudsmith/data_source_package.go @@ -1,16 +1,61 @@ package cloudsmith import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" "fmt" "io" "net/http" + "net/url" "os" "path" + "strconv" + "time" + cloudsmith_api "github.com/cloudsmith-io/cloudsmith-api-go" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" ) +type Checksums struct { + MD5 string + SHA1 string + SHA256 string + SHA512 string +} + +func (c Checksums) CompareWithPkg(pkg *cloudsmith_api.Package) error { + var errs []error + + if c.MD5 != pkg.GetChecksumMd5() { + errs = append(errs, fmt.Errorf(checksumMismatchError(c.MD5, pkg.GetChecksumMd5(), "MD5"))) + } + if c.SHA1 != pkg.GetChecksumSha1() { + errs = append(errs, fmt.Errorf(checksumMismatchError(c.SHA1, pkg.GetChecksumSha1(), "SHA1"))) + } + if c.SHA256 != pkg.GetChecksumSha256() { + errs = append(errs, fmt.Errorf(checksumMismatchError(c.SHA256, pkg.GetChecksumSha256(), "SHA256"))) + } + if c.SHA512 != pkg.GetChecksumSha512() { + errs = append(errs, fmt.Errorf(checksumMismatchError(c.SHA512, pkg.GetChecksumSha512(), "SHA512"))) + } + + var finalError error = nil + for _, err := range errs { + finalError = fmt.Errorf("%w\n", err) + } + + return finalError +} + +func checksumMismatchError(localChecksum string, remoteChecksum string, checksumType string) string { + formatString := fmt.Sprintf("Checksum mismatch (%s): expected=%s, got=%s", localChecksum, remoteChecksum, checksumType) + return formatString +} + func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { pc := m.(*providerConfig) namespace := requiredString(d, "namespace") @@ -18,6 +63,7 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { identifier := requiredString(d, "identifier") download := requiredBool(d, "download") downloadDir := requiredString(d, "download_dir") + ignoreChecksum := requiredBool(d, "ignore_checksums") req := pc.APIClient.PackagesApi.PackagesRead(pc.Auth, namespace, repository, identifier) pkg, _, err := pc.APIClient.PackagesApi.PackagesReadExecute(req) @@ -36,6 +82,7 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { d.Set("slug", pkg.GetSlug()) d.Set("slug_perm", pkg.GetSlugPerm()) d.Set("version", pkg.GetVersion()) + // Grab the checksum from API in case they don't want to download the file directly via terraform (when returning just the cdn_url) d.Set("checksum_md5", pkg.GetChecksumMd5()) d.Set("checksum_sha1", pkg.GetChecksumSha1()) d.Set("checksum_sha256", pkg.GetChecksumSha256()) @@ -43,23 +90,59 @@ func dataSourcePackageRead(d *schema.ResourceData, m interface{}) error { d.SetId(fmt.Sprintf("%s_%s_%s", namespace, repository, pkg.GetSlugPerm())) - if download { - outputPath, err := downloadPackage(pkg.GetCdnUrl(), downloadDir, pc) + if !download { + d.Set("output_path", pkg.GetCdnUrl()) + d.Set("output_directory", "") + return nil + } + + bustCache := false + retryTimes := 0 + var checksumError error = nil + var localChecksums Checksums + + for retryTimes < 2 { + outputPath, err := downloadPackage(pkg.GetCdnUrl(), downloadDir, pc, bustCache) if err != nil { return err } + d.Set("output_path", outputPath) d.Set("output_directory", downloadDir) - } else { - d.Set("output_path", pkg.GetCdnUrl()) - d.Set("output_directory", "") + + // Calculate checksums for the downloaded file + localChecksums, err = calculateChecksums(outputPath) + if err != nil { + return err + } + + if ignoreChecksum { + fmt.Println("Warning: ignore_checksums set to true, downloading mismatched checksum file.") + break + } + + if checksumError = localChecksums.CompareWithPkg(pkg); checksumError != nil { + bustCache = true + retryTimes++ + } else { + break + } } + if checksumError != nil { + return checksumError + } + + d.Set("checksum_md5", localChecksums.MD5) + d.Set("checksum_sha1", localChecksums.SHA1) + d.Set("checksum_sha256", localChecksums.SHA256) + d.Set("checksum_sha512", localChecksums.SHA512) + return nil } -func downloadPackage(url string, downloadDir string, pc *providerConfig) (string, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func downloadPackage(downloadUrl string, downloadDir string, pc *providerConfig, bustCache bool) (string, error) { + req, err := http.NewRequest(http.MethodGet, downloadUrl, nil) if err != nil { return "", err } @@ -67,6 +150,20 @@ func downloadPackage(url string, downloadDir string, pc *providerConfig) (string req.Header.Add("Authorization", fmt.Sprintf("Token %s", pc.GetAPIKey())) client := pc.APIClient.GetConfig().HTTPClient + if bustCache { + timestamp := time.Now().Unix() + parsedURL, err := url.Parse(downloadUrl) + if err != nil { + return "", err + } + + queryValues := parsedURL.Query() + queryValues.Set("time", strconv.FormatInt(timestamp, 10)) + parsedURL.RawQuery = queryValues.Encode() + + req.URL = parsedURL + } + resp, err := client.Do(req) if err != nil { return "", err @@ -74,11 +171,11 @@ func downloadPackage(url string, downloadDir string, pc *providerConfig) (string defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("failed to download file: %s, status code: %d", url, resp.StatusCode) + return "", fmt.Errorf("failed to download file: %s, status code: %d", downloadUrl, resp.StatusCode) } // Extract filename from CDN URL - filename := path.Base(url) + filename := path.Base(downloadUrl) outputPath := path.Join(downloadDir, filename) outputFile, err := os.Create(outputPath) @@ -95,6 +192,32 @@ func downloadPackage(url string, downloadDir string, pc *providerConfig) (string return outputPath, nil } +func calculateChecksums(filePath string) (Checksums, error) { + var checksums Checksums + + file, err := os.Open(filePath) + if err != nil { + return checksums, err + } + defer file.Close() + + md5hash := md5.New() + sha1hash := sha1.New() + sha256hash := sha256.New() + sha512hash := sha512.New() + + if _, err := io.Copy(io.MultiWriter(md5hash, sha1hash, sha256hash, sha512hash), file); err != nil { + return checksums, err + } + + checksums.MD5 = hex.EncodeToString(md5hash.Sum(nil)) + checksums.SHA1 = hex.EncodeToString(sha1hash.Sum(nil)) + checksums.SHA256 = hex.EncodeToString(sha256hash.Sum(nil)) + checksums.SHA512 = hex.EncodeToString(sha512hash.Sum(nil)) + + return checksums, nil +} + func dataSourcePackage() *schema.Resource { return &schema.Resource{ Read: dataSourcePackageRead, @@ -131,11 +254,23 @@ func dataSourcePackage() *schema.Resource { Optional: true, Default: false, }, + "download_dir": { + Type: schema.TypeString, + Description: "The directory where the file will be downloaded if download is set to true", + Optional: true, + Default: os.TempDir(), + }, "format": { Type: schema.TypeString, Description: "The format of the package", Computed: true, }, + "ignore_checksums": { + Type: schema.TypeBool, + Description: "Ignore checksums for the package", + Optional: true, + Default: false, + }, "identifier": { Type: schema.TypeString, Description: "The identifier for this repository.", @@ -144,27 +279,27 @@ func dataSourcePackage() *schema.Resource { }, "is_sync_awaiting": { Type: schema.TypeBool, - Description: "Is the package awaiting synchronisation", + Description: "Is the package awaiting synchronization", Computed: true, }, "is_sync_completed": { Type: schema.TypeBool, - Description: "Has the package synchronisation completed", + Description: "Has the package synchronization completed", Computed: true, }, "is_sync_failed": { Type: schema.TypeBool, - Description: "Has the package synchronisation failed", + Description: "Has the package synchronization failed", Computed: true, }, "is_sync_in_flight": { Type: schema.TypeBool, - Description: "Is the package synchronisation currently in-flight", + Description: "Is the package synchronization currently in-flight", Computed: true, }, "is_sync_in_progress": { Type: schema.TypeBool, - Description: "Is the package synchronisation currently in-progress", + Description: "Is the package synchronization currently in-progress", Computed: true, }, "name": { @@ -178,22 +313,22 @@ func dataSourcePackage() *schema.Resource { Required: true, ValidateFunc: validation.StringIsNotEmpty, }, - "output_path": { + "output_directory": { Type: schema.TypeString, - Description: "The location of the package", + Description: "The directory where the file is downloaded", Computed: true, }, - "download_dir": { - Type: schema.TypeString, - Description: "The directory where the file will be downloaded if download is set to true", - Optional: true, - Default: os.TempDir(), - }, - "output_directory": { + "output_path": { Type: schema.TypeString, - Description: "The directory where the file is downloaded", + Description: "The location of the package", Computed: true, }, + "repository": { + Type: schema.TypeString, + Description: "The repository of the package", + Required: true, + ValidateFunc: validation.StringIsNotEmpty, + }, "slug": { Type: schema.TypeString, Description: "The slug identifies the package in URIs.", @@ -205,12 +340,6 @@ func dataSourcePackage() *schema.Resource { "It will never change once a package has been created.", Computed: true, }, - "repository": { - Type: schema.TypeString, - Description: "The repository of the package", - Required: true, - ValidateFunc: validation.StringIsNotEmpty, - }, "version": { Type: schema.TypeString, Description: "The version of the package", diff --git a/cloudsmith/data_source_package_test.go b/cloudsmith/data_source_package_test.go index 5125bfd..8645b3d 100644 --- a/cloudsmith/data_source_package_test.go +++ b/cloudsmith/data_source_package_test.go @@ -36,7 +36,7 @@ func TestAccPackage_data(t *testing.T) { resource.TestCheckResourceAttr("cloudsmith_repository.test", "name", dsPackageTestRepository), // Custom TestCheckFunc to upload the package and wait for sync after repository creation func(s *terraform.State) error { - return uploadPackage(testAccProvider.Meta().(*providerConfig)) + return uploadPackage(testAccProvider.Meta().(*providerConfig), false) }, ), }, @@ -58,6 +58,39 @@ func TestAccPackage_data(t *testing.T) { if _, err := os.Stat(filePath); os.IsNotExist(err) { return fmt.Errorf("file does not exist at path: %s", filePath) } + defer func() { + // Remove the file after the check is done + if err := os.Remove(filePath); err != nil { + fmt.Printf("Error removing file: %s\n", err) + } + }() + expectedContent := "Hello world" + if err := checkFileContent(filePath, expectedContent); err != nil { + return fmt.Errorf("file content check failed: %w", err) + } + return nil + }, + func(s *terraform.State) error { + return uploadPackage(testAccProvider.Meta().(*providerConfig), true) + }, + ), + }, + { + Config: testAccPackageDataReadPackageDownloadRepublish(dsPackageTestNamespace, dsPackageTestRepository), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("data.cloudsmith_package.test", "namespace", dsPackageTestNamespace), + resource.TestCheckResourceAttr("data.cloudsmith_package.test", "repository", dsPackageTestRepository), + func(s *terraform.State) error { + filePath := filepath.Join(os.TempDir(), "hello.txt") + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return fmt.Errorf("file does not exist at path: %s", filePath) + } + + expectedContent := "Hello world updated content" + if err := checkFileContent(filePath, expectedContent); err != nil { + return fmt.Errorf("file content check failed: %w", err) + } + return nil }, ), @@ -65,9 +98,30 @@ func TestAccPackage_data(t *testing.T) { }, }) } +func checkFileContent(filePath string, expectedContent string) error { + content, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read file: %w", err) + } + + if string(content) != expectedContent { + return fmt.Errorf("file content does not match expected. Got: %s, Expected: %s", content, expectedContent) + } + + return nil +} + +func uploadPackage(pc *providerConfig, republish bool) error { + + var ( + fileContent []byte + ) -func uploadPackage(pc *providerConfig) error { - fileContent := []byte("Hello world") + if republish { + fileContent = []byte("Hello world updated content") + } else { + fileContent = []byte("Hello world") + } initPayload := cloudsmith.PackageFileUploadRequest{ Filename: "hello.txt", @@ -154,6 +208,7 @@ func testAccPackageDataSetup(namespace, repository string) string { resource "cloudsmith_repository" "test" { name = "%s" namespace = "%s" + replace_packages_by_default = true } `, repository, namespace) } @@ -163,6 +218,7 @@ func testAccPackageDataReadPackage(namespace, repository string) string { resource "cloudsmith_repository" "test" { name = "%s" namespace = "%s" + replace_packages_by_default = true } data "cloudsmith_package_list" "test" { @@ -183,6 +239,29 @@ func testAccPackageDataReadPackageDownload(namespace, repository string) string resource "cloudsmith_repository" "test" { name = "%s" namespace = "%s" + replace_packages_by_default = true + } + + data "cloudsmith_package_list" "test" { + repository = "%s" + namespace = "%s" + } + + data "cloudsmith_package" "test" { + repository = "%s" + namespace = "%s" + identifier = data.cloudsmith_package_list.test.packages[0].slug_perm + download = true + } + `, repository, namespace, repository, namespace, repository, namespace) +} + +func testAccPackageDataReadPackageDownloadRepublish(namespace, repository string) string { + return fmt.Sprintf(` + resource "cloudsmith_repository" "test" { + name = "%s" + namespace = "%s" + replace_packages_by_default = true } data "cloudsmith_package_list" "test" { diff --git a/docs/data-sources/package.md b/docs/data-sources/package.md index 71b743a..8d0b6ba 100644 --- a/docs/data-sources/package.md +++ b/docs/data-sources/package.md @@ -39,14 +39,15 @@ data "cloudsmith_package" "test" { - `identifier` (Required): The identifier for the package. - `download` (Optional): If set to true, the package will be downloaded. Defaults to false. If set to false, the CDN URL will be available in the `output_path`. - `download_dir` (Optional): The directory where the file will be downloaded to. If not set and `download` is set to `true`, it will default to the operating system's default temporary directory and save the file there. +- `ignore_checksums` (Optional): If set to `true`, any mismatched checksum from our API and local check will be ignored and download the package if `download` is set to `true`. ## Attribute Reference - `cdn_url`: The URL of the package to download. This attribute is computed and available only when the `download` argument is set to `false`. -- `checksum_md5`: MD5 hash of the package. -- `checksum_sha1`: SHA1 hash of the package. -- `checksum_sha256`: SHA256 hash of the package. -- `checksum_sha512`: SHA512 hash of the package. +- `checksum_md5`: MD5 hash of the downloaded package. If `download` is set to `false`, the checksum is returned from the package API instead. +- `checksum_sha1`: SHA1 hash of the downloaded package.If `download` is set to `false`, the checksum is returned from the package API instead. +- `checksum_sha256`: SHA256 hash of the downloaded package.If `download` is set to `false`, the checksum is returned from the package API instead. +- `checksum_sha512`: SHA512 hash of the downloaded package.If `download` is set to `false`, the checksum is returned from the package API instead. - `format`: The format of the package. - `is_sync_awaiting`: Indicates whether the package is awaiting synchronization. - `is_sync_completed`: Indicates whether the package synchronization has completed.