ci: Add integration tests for Cloudflare R2 (#2423)

* ci: Add integration tests for Cloudflare R2 Signed-off-by: Xuanwo <github@xuanwo.io> * Set env Signed-off-by: Xuanwo <github@xuanwo.io> --------- Signed-off-by: Xuanwo <github@xuanwo.io>
apache · Jun 6, 2023 · bff9cd5 · bff9cd5
1 parent bb993f8
commit bff9cd5
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 1 deletion.
diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml
@@ -178,3 +178,36 @@ jobs:
           OPENDAL_S3_BUCKET: test
           OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000"
           OPENDAL_S3_ALLOW_ANONYMOUS: on
+
+  r2:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+        with:
+          need-nextest: true
+
+      - name: Load secret
+        id: op-load-secret
+        uses: 1password/load-secrets-action@v1
+        with:
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          OPENDAL_S3_TEST: op://services/r2/test
+          OPENDAL_S3_BUCKET: op://services/r2/bucket
+          OPENDAL_S3_ENDPOINT: op://services/r2/endpoint
+          OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id
+          OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key
+
+      - name: Test
+        shell: bash
+        working-directory: core
+        run: cargo nextest run s3
+        env:
+          OPENDAL_S3_REGION: auto
+          # This is the R2's limitation
+          # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
+          OPENDAL_S3_BATCH_MAX_OPERATIONS: 700
diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs
@@ -560,7 +560,11 @@ impl Builder for S3Builder {
             .filter(|v| *v == "on" || *v == "true")
             .map(|_| builder.allow_anonymous());
         map.get("default_storage_class")
-            .map(|v| builder.default_storage_class(v));
+            .map(|v: &String| builder.default_storage_class(v));
+        map.get("write_min_size")
+            .map(|v| builder.write_min_size(v.parse().expect("input must be a number")));
+        map.get("batch_max_operations")
+            .map(|v| builder.batch_max_operations(v.parse().expect("input must be a number")));
 
         builder
     }

diff --git a/core/src/services/s3/compatible_services.md b/core/src/services/s3/compatible_services.md
@@ -97,3 +97,17 @@ To connect to wasabi, we need to set:
 - `bucket`: The bucket name of wasabi.
 
 > Refer to [What are the service URLs for Wasabi's different storage regions?](https://wasabi-support.zendesk.com/hc/en-us/articles/360015106031) for more details.
+
+### Cloudflare R2
+
+[Cloudflare R2](https://developers.cloudflare.com/r2/) provides s3 compatible API.
+
+> Cloudflare R2 Storage allows developers to store large amounts of unstructured data without the costly egress bandwidth fees associated with typical cloud storage services.
+
+
+To connect to r2, we need to set:
+
+- `endpoint`: The endpoint of r2, for example: `https://<account_id>.r2.cloudflarestorage.com`
+- `bucket`: The bucket name of r2.
+- `region`: When you create a new bucket, the data location is set to Automatic by default. So please use `auto` for region.
+- `batch_max_operations`: R2's delete objects will return `Internal Error` if the batch is larger than `700`. Please set this value `<= 700` to make sure batch delete work as expected.