feat(obj): add lifecycle to object-storage

docs/guides/migration_guide_v2.md

@@ -296,7 +296,7 @@ The [above example](#scaleway_server-gt-scaleway_instance_server) shows how this
 
 The `scaleway_bucket` was moved to the `object` product in the `storage` product category.
 
-It's behaviour remained the same, but we also added an [`acl` attribute](../resources/object_bucket.md#acl).
+It's behaviour remained the same, but we also added an [`acl` attribute](../resources/object_bucket.md#the-acl).
 This attribute takes canned ACLs.
 
 ### LoadBalancer

docs/resources/object_bucket.md

@@ -21,17 +21,106 @@ resource "scaleway_object_bucket" "some_bucket" {
 }
 ```
 
+### Using object lifecycle
+
+```hcl
+resource "scaleway_object_bucket" "main"{
+  name = "mybuckectid"
+  region = "fr-par"
+  acl = "private"
+
+  # This lifecycle configuration rule will make that all objects that got a filter key that start with (path1/) be transferred
+  # from their default storage class (STANDARD, ONEZONE_IA) to GLACIER after 120 days counting 
+  # from their creation and then 365 days after that they will be expired and deleted.
+  lifecycle_rule {
+      id      = "id1"
+      prefix  = "path1/"
+      enabled = true
+
+      expiration {
+        days = 365
+      }
+
+      transition {
+        days          = 120
+        storage_class = "GLACIER"
+      }
+  }
+
+  # This lifecycle configuration rule specifies that all objects (identified by the key name prefix (path2/) in the rule)
+  # from their creation and then 50 days after that they will be expired and deleted.
+  lifecycle_rule {
+      id      = "id2"
+      prefix  = "path2/"
+      enabled = true
+
+      expiration {
+        days = "50"
+      }
+  }
+
+  # This lifecycle configuration rule remove any object with (path3/) prefix that match
+  # with the tags one day after creation.
+  lifecycle_rule {
+      id      = "id3"
+      prefix  = "path3/"
+      enabled = false
+
+      tags = {
+        "tagKey"    = "tagValue"
+        "terraform" = "hashicorp"
+      }
+
+      expiration {
+        days = "1"
+      }
+  }
+
+  # This lifecycle configuration rule specifies a tag-based filter (tag1/value1).
+  # This rule directs Scaleway S3 to transition objects S3 Glacier class soon after creation.
+  # It is also disable temporaly.
+  lifecycle_rule {
+      id      = "id4"
+      enabled = true
+
+      tags = {
+        "tag1"    = "value1"
+      }
+
+      transition {
+        days          = 0
+        storage_class = "GLACIER"
+      }
+  }
+
+  # This lifecycle configuration rule specifies with the AbortIncompleteMultipartUpload action to 
+  # stop incomplete multipart uploads (identified by the key name prefix (path5/) in the rule)
+  # if they aren't completed within a specified number of days after initiation.
+  # Note: It's not recommended using prefix/ for AbortIncompleteMultipartUpload as any incomplete multipart upload will be billed
+  lifecycle_rule {
+      #  prefix  = "path5/"
+      enabled = true
+      abort_incomplete_multipart_upload_days = 30
+  }
+}
+```
+
 ## Arguments Reference
 
+
 The following arguments are supported:
 
 * `name` - (Required) The name of the bucket.
 * `tags` - (Optional) A list of tags (key / value) for the bucket.
-* `acl` - (Optional) The [canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl_overview.html#canned-acl) you want to apply to the bucket.
+* `acl` - (Optional) The canned ACL you want to apply to the bucket.
 * `region` - (Optional) The [region](https://developers.scaleway.com/en/quickstart/#region-definition) in which the bucket should be created.
 * `versioning` - (Optional) A state of [versioning](https://docs.aws.amazon.com/AmazonS3/latest/dev/Versioning.html) (documented below)
 * `cors_rule` - (Optional) A rule of [Cross-Origin Resource Sharing](https://docs.aws.amazon.com/AmazonS3/latest/dev/cors.html) (documented below).
 
+## The ACL
+
+Please check the [canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl_overview.html#canned-acl)
+
 The `CORS` object supports the following:
 
 * `allowed_headers` (Optional) Specifies which headers are allowed.
@@ -40,6 +129,35 @@ The `CORS` object supports the following:
 * `expose_headers` (Optional) Specifies expose header in the response.
 * `max_age_seconds` (Optional) Specifies time in seconds that browser can cache the response for a preflight request.
 
+The `lifecycle_rule` (Optional) object supports the following:
+
+* `id` - (Optional) Unique identifier for the rule. Must be less than or equal to 255 characters in length.
+* `prefix` - (Optional) Object key prefix identifying one or more objects to which the rule applies.
+* `tags` - (Optional) Specifies object tags key and value.
+* `enabled` - (Required) The element value can be either Enabled or Disabled. If a rule is disabled, Scaleway S3 doesn't perform any of the actions defined in the rule.
+
+* `abort_incomplete_multipart_upload_days` (Optional) Specifies the number of days after initiating a multipart upload when the multipart upload must be completed.
+
+    * ~> **Important:** It's not recommended using `prefix` for `AbortIncompleteMultipartUpload` as any incomplete multipart upload will be billed
+
+* `expiration` - (Optional) Specifies a period in the object's expire (documented below).
+* `transition` - (Optional) Specifies a period in the object's transitions (documented below).
+
+At least one of `abort_incomplete_multipart_upload_days`, `expiration`, `transition` must be specified.
+
+The `expiration` object supports the following
+
+* `days` (Optional) Specifies the number of days after object creation when the specific rule action takes effect.
+
+~> **Important:**  If versioning is enabled, this rule only deletes the current version of an object.
+
+The `transition` object supports the following
+
+* `days` (Optional) Specifies the number of days after object creation when the specific rule action takes effect.
+* `storage_class` (Required) Specifies the Scaleway [storage class](https://www.scaleway.com/en/docs/storage/object/concepts/#storage-class) `STANDARD`, `GLACIER`, `ONEZONE_IA`  to which you want the object to transition.
+
+~> **Important:**  `ONEZONE_IA` is only available in `fr-par` region. The storage class `GLACIER` is not available in `pl-waw` region.
+
 The `versioning` object supports the following:
 
 * `enabled` - (Optional) Enable versioning. Once you version-enable a bucket, it can never return to an unversioned state. You can, however, suspend versioning on that bucket.
@@ -50,6 +168,7 @@ In addition to all above arguments, the following attribute is exported:
 
 * `id` - The unique name of the bucket.
 * `endpoint` - The endpoint URL of the bucket
+* `region` - The Scaleway region this bucket resides in.
 
 ## Import
 

go.mod

@@ -5,6 +5,7 @@ require (
 	github.com/dnaeon/go-vcr v1.2.0
 	github.com/dustin/go-humanize v1.0.0
 	github.com/google/go-cmp v0.5.7
+	github.com/hashicorp/aws-sdk-go-base v1.0.0
 	github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320
 	github.com/hashicorp/go-retryablehttp v0.7.0
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.10.1

go.sum

@@ -67,6 +67,7 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM=
 github.com/aws/aws-sdk-go v1.25.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aws/aws-sdk-go v1.42.49 h1:g7H76HJWcer7W2bKL+UHEEhj2jg51rQPcR1OO8ntWBY=
 github.com/aws/aws-sdk-go v1.42.49/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
@@ -108,6 +109,7 @@ github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
 github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -165,6 +167,8 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/aws-sdk-go-base v1.0.0 h1:J7MMLOfSoDWkusy+cSzKYG1/aFyCzYJmdE0mod3/WLw=
+github.com/hashicorp/aws-sdk-go-base v1.0.0/go.mod h1:2fRjWDv3jJBeN6mVWFHV6hFTNeFBx2gpDLQaZNxUVAY=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU=
@@ -237,6 +241,7 @@ github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZ
 github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -316,6 +321,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

scaleway/errors_object.go

@@ -0,0 +1,10 @@
+package scaleway
+
+// Error code constants missing from AWS Go SDK:
+// https://docs.aws.amazon.com/sdk-for-go/api/service/s3/#pkg-constants
+
+const (
+	ErrCodeNoSuchTagSet                 = "NoSuchTagSet"
+	ErrCodeNoSuchCORSConfiguration      = "NoSuchCORSConfiguration"
+	ErrCodeNoSuchLifecycleConfiguration = "NoSuchLifecycleConfiguration"
+)

scaleway/helpers.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/hashicorp/go-cty/cty"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
 	"github.com/scaleway/scaleway-sdk-go/namegenerator"
@@ -527,3 +528,13 @@ func diffSuppressFuncIgnoreCaseAndHyphen(k, old, new string, d *schema.ResourceD
 func diffSuppressFuncLocality(k, old, new string, d *schema.ResourceData) bool {
 	return expandID(old) == expandID(new)
 }
+
+// TimedOut returns true if the error represents a "wait timed out" condition.
+// Specifically, TimedOut returns true if the error matches all these conditions:
+//  * err is of type resource.TimeoutError
+//  * TimeoutError.LastError is nil
+func TimedOut(err error) bool {
+	// This explicitly does *not* match wrapped TimeoutErrors
+	timeoutErr, ok := err.(*resource.TimeoutError) //nolint:errorlint // Explicitly does *not* match wrapped TimeoutErrors
+	return ok && timeoutErr.LastError == nil
+}

scaleway/helpers_object.go

@@ -1,8 +1,11 @@
 package scaleway
 
 import (
+	"bytes"
+	"context"
 	"errors"
 	"fmt"
+	"hash/crc32"
 	"net/http"
 	"os"
 	"strings"
@@ -13,12 +16,15 @@ import (
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/hashicorp/aws-sdk-go-base/tfawserr"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/scaleway/scaleway-sdk-go/scw"
 )
 
 const (
 	defaultObjectBucketTimeout = 10 * time.Minute
+	retryOnAWSAPI              = 2 * time.Minute
 )
 
 func newS3Client(httpClient *http.Client, region, accessKey, secretKey string) (*s3.S3, error) {
@@ -195,3 +201,78 @@ func expandBucketCORS(rawCors []interface{}, bucket string) []*s3.CORSRule {
 	}
 	return rules
 }
+
+func transitionHash(v interface{}) int {
+	var buf bytes.Buffer
+	m, ok := v.(map[string]interface{})
+
+	if !ok {
+		return 0
+	}
+
+	if v, ok := m["days"]; ok {
+		buf.WriteString(fmt.Sprintf("%d-", v.(int)))
+	}
+	if v, ok := m["storage_class"]; ok {
+		buf.WriteString(fmt.Sprintf("%s-", v.(string)))
+	}
+	return StringHashcode(buf.String())
+}
+
+// StringHashcode hashes a string to a unique hashcode.
+//
+// crc32 returns a uint32, but for our use we need
+// and non-negative integer. Here we cast to an integer
+// and invert it if the result is negative.
+func StringHashcode(s string) int {
+	v := int(crc32.ChecksumIEEE([]byte(s)))
+	if v >= 0 {
+		return v
+	}
+	if -v >= 0 {
+		return -v
+	}
+	// v == MinInt
+	return 0
+}
+
+func retryOnAWSCode(ctx context.Context, code string, f func() (interface{}, error)) (interface{}, error) {
+	var resp interface{}
+	err := resource.RetryContext(ctx, retryOnAWSAPI, func() *resource.RetryError {
+		var err error
+		resp, err = f()
+		if err != nil {
+			if tfawserr.ErrCodeEquals(err, code) {
+				return resource.RetryableError(err)
+			}
+			return resource.NonRetryableError(err)
+		}
+		return nil
+	})
+
+	if TimedOut(err) {
+		resp, err = f()
+	}
+
+	return resp, err
+}
+
+const (
+	// TransitionStorageClassStandard is a TransitionStorageClass enum value
+	TransitionStorageClassStandard = "STANDARD"
+
+	// TransitionStorageClassGlacier is a TransitionStorageClass enum value
+	TransitionStorageClassGlacier = "GLACIER"
+
+	// TransitionStorageClassOnezoneIa is a TransitionStorageClass enum value
+	TransitionStorageClassOnezoneIa = "ONEZONE_IA"
+)
+
+// TransitionSCWStorageClassValues returns all elements of the TransitionStorageClass enum supported by scaleway
+func TransitionSCWStorageClassValues() []string {
+	return []string{
+		TransitionStorageClassStandard,
+		TransitionStorageClassGlacier,
+		TransitionStorageClassOnezoneIa,
+	}
+}

scaleway/helpers_test.go

@@ -226,3 +226,24 @@ func testCheckResourceAttrIP(name string, key string) resource.TestCheckFunc {
 		return nil
 	})
 }
+
+func TestStringHashcode(t *testing.T) {
+	v := "hello, world"
+	expected := StringHashcode(v)
+	for i := 0; i < 100; i++ {
+		actual := StringHashcode(v)
+		if actual != expected {
+			t.Fatalf("bad: %#v\n\t%#v", actual, expected)
+		}
+	}
+}
+
+func TestStringHashcode_positiveIndex(t *testing.T) {
+	// "2338615298" hashes to uint32(2147483648) which is math.MinInt32
+	ips := []string{"192.168.1.3", "192.168.1.5", "2338615298"}
+	for _, ip := range ips {
+		if index := StringHashcode(ip); index < 0 {
+			t.Fatalf("Bad Index %#v for ip %s", index, ip)
+		}
+	}
+}