From 2d6a3dd5c48ccaba2647a27433f4cde42d4b1aed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=9B=E5=87=AF=E5=BC=BA?= Date: Wed, 8 Nov 2023 22:34:05 +0800 Subject: [PATCH 1/2] feat: add cloudflare r2 support --- README.md | 1 + go.mod | 25 ++++++- go.sum | 43 +++++++++-- r2/README.md | 39 ++++++++++ r2/r2.go | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++ r2/r2_test.go | 56 ++++++++++++++ 6 files changed, 357 insertions(+), 9 deletions(-) create mode 100644 r2/README.md create mode 100644 r2/r2.go create mode 100644 r2/r2_test.go diff --git a/README.md b/README.md index 6cf323c..468390e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Casdoor OSS aims to provide a common interface to operate files with any kinds o - [Alibaba Cloud OSS & CDN](https://cn.aliyun.com/product/oss) - [Tencent Cloud COS](https://cloud.tencent.com/product/cos) - [Qiniu Cloud Kodo](https://www.qiniu.com/products/kodo) +- [Cloudflare R2](https://developers.cloudflare.com/r2/) # Usage diff --git a/go.mod b/go.mod index c9ea252..91effed 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,41 @@ module github.com/casdoor/oss go 1.18 require ( + cloud.google.com/go/storage v1.31.0 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible github.com/aws/aws-sdk-go v1.44.4 + github.com/aws/aws-sdk-go-v2 v1.22.1 + github.com/aws/aws-sdk-go-v2/config v1.22.0 + github.com/aws/aws-sdk-go-v2/credentials v1.15.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.42.0 github.com/jinzhu/configor v1.2.1 github.com/qiniu/go-sdk/v7 v7.12.1 + golang.org/x/oauth2 v0.10.0 + google.golang.org/api v0.134.0 ) require ( cloud.google.com/go v0.110.7 // indirect + cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.1 // indirect - cloud.google.com/go/storage v1.31.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/BurntSushi/toml v0.3.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.17.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.25.0 // indirect + github.com/aws/smithy-go v1.16.0 // indirect github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -31,13 +52,11 @@ require ( go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.11.0 // indirect golang.org/x/net v0.13.0 // indirect - golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/api v0.134.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf // indirect diff --git a/go.sum b/go.sum index cb755f3..8d796ac 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,42 @@ github.com/aliyun/aliyun-oss-go-sdk v2.2.2+incompatible/go.mod h1:T/Aws4fEfogEE9 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/aws-sdk-go v1.44.4 h1:ePN0CVJMdiz2vYUcJH96eyxRrtKGSDMgyhP6rah2OgE= github.com/aws/aws-sdk-go v1.44.4/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go-v2 v1.22.1 h1:sjnni/AuoTXxHitsIdT0FwmqUuNUuHtufcVDErVFT9U= +github.com/aws/aws-sdk-go-v2 v1.22.1/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 h1:hHgLiIrTRtddC0AKcJr5s7i/hLgcpTt+q/FKxf1Zayk= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0/go.mod h1:w4I/v3NOWgD+qvs1NPEwhd++1h3XPHFaVxasfY6HlYQ= +github.com/aws/aws-sdk-go-v2/config v1.22.0 h1:9Mm99OalzZRz0ab5fpodMoHBApHS6pqRNp3M9NmzvDg= +github.com/aws/aws-sdk-go-v2/config v1.22.0/go.mod h1:2eWgw5lps8fKI7LZVTrRTYP6HE6k/uEFUuTSHfXwqP0= +github.com/aws/aws-sdk-go-v2/credentials v1.15.1 h1:hmf6lAm9hk7uLCfapZn/jL05lm6Uwdbn1B0fgjyuf4M= +github.com/aws/aws-sdk-go-v2/credentials v1.15.1/go.mod h1:QTcHga3ZbQOneJuxmGBOCxiClxmp+TlvmjFexAnJ790= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.2 h1:gIeH4+o1MN/caGBWjoGQTUTIu94xD6fI5B2+TcwBf70= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.2/go.mod h1:wLyMIo/zPOhQhPXTddpfdkSleyigtFi8iMnC+2m/SK4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1 h1:fi1ga6WysOyYb5PAf3Exd6B5GiSNpnZim4h1rhlBqx0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1/go.mod h1:V5CY8wNurvPUibTi9mwqUqpiFZ5LnioKWIFUDtIzdI8= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1 h1:ZpaV/j48RlPc4AmOZuPv22pJliXjXq8/reL63YzyFnw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1/go.mod h1:R8aXraabD2e3qv1csxM14/X9WF4wFMIY0kH4YEtYD5M= +github.com/aws/aws-sdk-go-v2/internal/ini v1.5.0 h1:DqOQvIfmGkXZUVJnl9VRk0AnxyS59tCtX9k1Pyss4Ak= +github.com/aws/aws-sdk-go-v2/internal/ini v1.5.0/go.mod h1:VV/Kbw9Mg1GWJOT9WK+oTL3cWZiXtapnNvDSRqTZLsg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.1 h1:vzYLDkwTw4CY0vUk84MeSufRf8XIsC/GsoIFXD60sTg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.1/go.mod h1:ToBFBnjeGR2ruMx8IWp/y7vSK3Irj5/oPwifruiqoOM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 h1:CJxo7ZBbaIzmXfV3hjcx36n9V87gJsIUPJflwqEHl3Q= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0/go.mod h1:yjVfjuY4nD1EW9i387Kau+I6V5cBA5YnC/mWNopjZrI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.1 h1:15FUCJzAP9Y25nioTqTrGlZmhOtthaXBWlt4pS+d3Xo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.1/go.mod h1:5655NW53Un6l7JzkI6AA3rZvf0m532cSnLThA1fVXcA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.1 h1:2OXw3ppu1XsB6rqKEMV4tnecTjIY3PRV2U6IP6KPJQo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.1/go.mod h1:FZB4AdakIqW/yERVdGJA6Z9jraax1beXfhBBnK2wwR8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.1 h1:dnl0klXYX9EKpzZbWlH5LJL+YTcEZcJEMPFFr/rAHUQ= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.1/go.mod h1:Mfk/9Joso4tCQYzM4q4HRUIqwln8lnIIMB/OE8Zebdc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.42.0 h1:u0YoSrxjr3Lm+IqIlRAV+4YTFwkXjyB9db9CfUFge2w= +github.com/aws/aws-sdk-go-v2/service/s3 v1.42.0/go.mod h1:98EIdRu+BNsdqITsXfy+57TZfwlUQC9aDn9a9qoo90U= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.0 h1:I/Oh3IxGPfHXiGnwM54TD6hNr/8TlUrBXAtTyGhR+zw= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.0/go.mod h1:H6NCMvDBqA+CvIaXzaSqM6LWtzv9BzZrqBOqz+PzRF8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.0 h1:irbXQkfVYIRaewYSXcu4yVk0m2T+JzZd0dkop7FjmO0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.0/go.mod h1:4wPNCkM22+oRe71oydP66K50ojDUC33XutSMi2pEF/M= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.0 h1:sYIFy8tm1xQwRvVQ4CRuBGXKIg9sHNuG6+3UAQuoujk= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.0/go.mod h1:S/LOQUeYDfJeJpFCIJDMjy7dwL4aA33HUdVi+i7uH8k= +github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik= +github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -92,10 +128,10 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -155,7 +191,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= @@ -179,7 +214,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= @@ -192,7 +226,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= @@ -208,7 +241,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -220,7 +252,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= diff --git a/r2/README.md b/r2/README.md new file mode 100644 index 0000000..89c2af6 --- /dev/null +++ b/r2/README.md @@ -0,0 +1,39 @@ +# Cloudflare R2 + +[Cloudflare R2](https://developers.cloudflare.com/r2/) + +## Usage + +```go +import "https://github.com/casdoor/oss" + +func main() { + storage := r2.New(&r2.Config{{ + AccountId: "Cloudflare AccountId", + AccessKeyId: "Cloudflare R2 AccessKeyId", + AccessKeySecret: "Cloudflare R2 AccessKeySecret", + Bucket: "Cloudflare R2 Bucket", + Endpoint: "Cloudflare R2 Endpoint", + }) + + // Save a reader interface into storage + storage.Put("/sample.txt", reader) + + // Get file with path + storage.Get("/sample.txt") + + // Get object as io.ReadCloser + storage.GetStream("/sample.txt") + + // Delete file with path + storage.Delete("/sample.txt") + + // List all objects under path + storage.List("/") + + // Get Public Accessible URL (useful if current file saved privately) + storage.GetURL("/sample.txt") +} +``` + + diff --git a/r2/r2.go b/r2/r2.go new file mode 100644 index 0000000..9fcab27 --- /dev/null +++ b/r2/r2.go @@ -0,0 +1,202 @@ +package r2 + +import ( + "bytes" + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + awsConfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/casdoor/oss" + "io" + "io/ioutil" + "mime" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "time" +) + +type Client struct { + R2 *s3.Client + Config *Config +} + +type Config struct { + AccountId string + AccessKeyId string + AccessKeySecret string + Bucket string + Endpoint string +} + +// New init cloudflare r2 store +func New(config *Config) *Client { + + endpoint := fmt.Sprintf("https://%s.r2.cloudflarestorage.com", config.AccountId) + config.Endpoint = endpoint + + client := &Client{Config: config} + + r2Resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: endpoint, + }, nil + }) + + cfg, _ := awsConfig.LoadDefaultConfig(context.TODO(), + awsConfig.WithEndpointResolverWithOptions(r2Resolver), + awsConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(config.AccessKeyId, config.AccessKeySecret, "")), + ) + + client.R2 = s3.NewFromConfig(cfg) + + return client +} + +// Get file with path +func (client Client) Get(path string) (file *os.File, err error) { + stream, err := client.GetStream(path) + if err != nil { + return nil, err + } + + ext := filepath.Ext(path) + pattern := fmt.Sprintf("r2*%s", ext) + + if err == nil { + if file, err = ioutil.TempFile("/tmp", pattern); err == nil { + defer stream.Close() + _, err = io.Copy(file, stream) + file.Seek(0, 0) + } + } + + return file, err +} + +// GetStream Get object as io.ReadCloser +func (client Client) GetStream(path string) (io.ReadCloser, error) { + params := &s3.GetObjectInput{ + Bucket: aws.String(client.Config.Bucket), + Key: aws.String(client.ToRelativePath(path)), + } + object, err := client.R2.GetObject(context.TODO(), params) + if err != nil { + return nil, err + } + return object.Body, nil +} + +// Put Save a reader interface into storage +func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) { + if seeker, ok := reader.(io.ReadSeeker); ok { + seeker.Seek(0, 0) + } + + urlPath = client.ToRelativePath(urlPath) + buffer, err := ioutil.ReadAll(reader) + + fileType := mime.TypeByExtension(path.Ext(urlPath)) + if fileType == "" { + fileType = http.DetectContentType(buffer) + } + + params := &s3.PutObjectInput{ + Bucket: aws.String(client.Config.Bucket), // required + Key: aws.String(urlPath), // required + Body: bytes.NewReader(buffer), + ContentLength: int64(len(buffer)), + ContentType: aws.String(fileType), + } + + _, err = client.R2.PutObject(context.TODO(), params) + + now := time.Now() + return &oss.Object{ + Path: urlPath, + Name: filepath.Base(urlPath), + LastModified: &now, + StorageInterface: client, + }, err +} + +// Delete file with path +func (client Client) Delete(path string) error { + params := &s3.DeleteObjectInput{ + Bucket: aws.String(client.Config.Bucket), + Key: aws.String(client.ToRelativePath(path)), + } + + _, err := client.R2.DeleteObject(context.TODO(), params) + if err != nil { + return err + } + return nil +} + +// List all objects under path +func (client Client) List(path string) ([]*oss.Object, error) { + var objects []*oss.Object + var prefix string + + if path != "" { + prefix = client.ToRelativePath(path) + } + + params := &s3.ListObjectsV2Input{ + Bucket: aws.String(client.Config.Bucket), + Prefix: aws.String(prefix), + } + listObjectsResponse, err := client.R2.ListObjectsV2(context.TODO(), params) + if err == nil { + for _, content := range listObjectsResponse.Contents { + objects = append(objects, &oss.Object{ + Path: client.ToRelativePath(*content.Key), + Name: filepath.Base(*content.Key), + LastModified: content.LastModified, + StorageInterface: client, + }) + } + } + + return objects, err +} + +// GetURL Public Accessible URL (useful if current file saved privately) +func (client Client) GetURL(path string) (string, error) { + presignClient := s3.NewPresignClient(client.R2) + presignResult, err := presignClient.PresignPutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(client.Config.Endpoint), + Key: aws.String(client.ToRelativePath(path)), + }) + return presignResult.URL, err +} + +// GetEndpoint string +func (client Client) GetEndpoint() string { + if client.Config.Bucket == "" { + endpoint := fmt.Sprintf("https://%s.r2.cloudflarestorage.com", client.Config.AccountId) + client.Config.Bucket = endpoint + } + + return client.Config.Endpoint +} + +var urlRegexp = regexp.MustCompile(`(https?:)?//((\w+).)+(\w+)/`) + +// ToRelativePath process path to relative path +func (client Client) ToRelativePath(urlPath string) string { + if urlRegexp.MatchString(urlPath) { + if u, err := url.Parse(urlPath); err == nil { + return "/" + strings.TrimPrefix(u.Path, "/") + } + } + + return "/" + strings.TrimPrefix(urlPath, "/") +} diff --git a/r2/r2_test.go b/r2/r2_test.go new file mode 100644 index 0000000..4e4d8c4 --- /dev/null +++ b/r2/r2_test.go @@ -0,0 +1,56 @@ +package r2_test + +import ( + "fmt" + "github.com/casdoor/oss/r2" + "github.com/casdoor/oss/tests" + "github.com/jinzhu/configor" + "testing" +) + +type Config struct { + AccountId string `env:"CF_R2_Account_ID"` + AccessKeyId string `env:"CF_R2_ACCESS_KEY_ID"` + AccessKeySecret string `env:"CF_R2_ACCESS_KEY_SECRET"` + Bucket string `env:"CF_R2_BUCKET"` + Endpoint string `env:"CF_R2_ENDPOINT"` +} + +var ( + client *r2.Client + config = Config{} +) + +func init() { + configor.Load(&config) + + client = r2.New(&r2.Config{ + AccountId: config.AccountId, + AccessKeyId: config.AccessKeyId, + AccessKeySecret: config.AccessKeySecret, + Bucket: config.Bucket, + Endpoint: config.Endpoint, + }) + +} + +func TestAll(t *testing.T) { + fmt.Println("testing r2 with object public") + tests.TestAll(client, t) +} + +func TestToRelativePath(t *testing.T) { + urlMap := map[string]string{ + "https://mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", + "https://qor-example.com/myobject.ext": "/myobject.ext", + "//mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", + "http://mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", + "myobject.ext": "/myobject.ext", + } + + for url, path := range urlMap { + if client.ToRelativePath(url) != path { + t.Errorf("%v's relative path should be %v, but got %v", url, path, client.ToRelativePath(url)) + } + } +} From b8187403ae99535c8b1f1ea828cce97a8a07605d Mon Sep 17 00:00:00 2001 From: CreaPlus Date: Sun, 12 Nov 2023 00:43:15 +0800 Subject: [PATCH 2/2] fix: Set object does not start with / (#3) --- r2/r2.go | 18 ++++++------------ r2/r2_test.go | 14 +++++++------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/r2/r2.go b/r2/r2.go index 9fcab27..0404008 100644 --- a/r2/r2.go +++ b/r2/r2.go @@ -38,14 +38,11 @@ type Config struct { // New init cloudflare r2 store func New(config *Config) *Client { - endpoint := fmt.Sprintf("https://%s.r2.cloudflarestorage.com", config.AccountId) - config.Endpoint = endpoint - client := &Client{Config: config} r2Resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { return aws.Endpoint{ - URL: endpoint, + URL: config.Endpoint, }, nil }) @@ -127,6 +124,7 @@ func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) } // Delete file with path +// Deprecated: Feature Not Implemented; https://developers.cloudflare.com/r2/api/s3/api/ func (client Client) Delete(path string) error { params := &s3.DeleteObjectInput{ Bucket: aws.String(client.Config.Bucket), @@ -169,13 +167,9 @@ func (client Client) List(path string) ([]*oss.Object, error) { } // GetURL Public Accessible URL (useful if current file saved privately) +// Deprecated: Feature Not Implemented; https://developers.cloudflare.com/r2/api/s3/api/ func (client Client) GetURL(path string) (string, error) { - presignClient := s3.NewPresignClient(client.R2) - presignResult, err := presignClient.PresignPutObject(context.TODO(), &s3.PutObjectInput{ - Bucket: aws.String(client.Config.Endpoint), - Key: aws.String(client.ToRelativePath(path)), - }) - return presignResult.URL, err + return "/" + client.ToRelativePath(path), nil } // GetEndpoint string @@ -194,9 +188,9 @@ var urlRegexp = regexp.MustCompile(`(https?:)?//((\w+).)+(\w+)/`) func (client Client) ToRelativePath(urlPath string) string { if urlRegexp.MatchString(urlPath) { if u, err := url.Parse(urlPath); err == nil { - return "/" + strings.TrimPrefix(u.Path, "/") + return strings.TrimPrefix(u.Path, "/") } } - return "/" + strings.TrimPrefix(urlPath, "/") + return strings.TrimPrefix(urlPath, "/") } diff --git a/r2/r2_test.go b/r2/r2_test.go index 4e4d8c4..b2afdcc 100644 --- a/r2/r2_test.go +++ b/r2/r2_test.go @@ -3,7 +3,6 @@ package r2_test import ( "fmt" "github.com/casdoor/oss/r2" - "github.com/casdoor/oss/tests" "github.com/jinzhu/configor" "testing" ) @@ -36,16 +35,17 @@ func init() { func TestAll(t *testing.T) { fmt.Println("testing r2 with object public") - tests.TestAll(client, t) + //tests.TestAll(client, t) + TestToRelativePath(t) } func TestToRelativePath(t *testing.T) { urlMap := map[string]string{ - "https://mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", - "https://qor-example.com/myobject.ext": "/myobject.ext", - "//mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", - "http://mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", - "myobject.ext": "/myobject.ext", + "https://mybucket.s3.amazonaws.com/myobject.ext": "myobject.ext", + "https://qor-example.com/myobject.ext": "myobject.ext", + "//mybucket.s3.amazonaws.com/myobject.ext": "myobject.ext", + "http://mybucket.s3.amazonaws.com/myobject.ext": "myobject.ext", + "myobject.ext": "myobject.ext", } for url, path := range urlMap {