Plcoud xuanwo test

.github/services/pcloud/pcloud/action.yml

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: pcloud
+description: 'Behavior test for Pcloud.'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup
+      uses: 1password/load-secrets-action@v1
+      with:
+        export-env: true
+      env:
+        OPENDAL_PCLOUD_ROOT: /
+        OPENDAL_PCLOUD_ENDPOINT: op://services/pcloud/endpoint
+        OPENDAL_PCLOUD_USERNAME: op://services/pcloud/username
+        OPENDAL_PCLOUD_PASSWORD: op://services/pcloud/password
+    - name: Only run pcloud with 1 thread
+      shell: bash
+      run: |
+        cat << EOF >> $GITHUB_ENV
+        RUST_TEST_THREADS=1
+        EOF

bindings/java/Cargo.toml

@@ -71,6 +71,7 @@ services-all = [
   "services-onedrive",
   "services-persy",
   "services-postgresql",
+  "services-pcloud",
   "services-koofr",
   "services-mysql",
   "services-redb",
@@ -135,6 +136,7 @@ services-mysql = ["opendal/services-mysql"]
 services-onedrive = ["opendal/services-onedrive"]
 services-persy = ["opendal/services-persy"]
 services-postgresql = ["opendal/services-postgresql"]
+services-pcloud = ["opendal/services-pcloud"]
 services-redb = ["opendal/services-redb"]
 services-redis = ["opendal/services-redis"]
 services-rocksdb = ["opendal/services-rocksdb"]

bindings/nodejs/Cargo.toml

@@ -74,6 +74,7 @@ services-all = [
   "services-onedrive",
   "services-persy",
   "services-postgresql",
+  "services-pcloud",
   "services-mysql",
   "services-redb",
   "services-redis",
@@ -130,6 +131,7 @@ services-mysql = ["opendal/services-mysql"]
 services-onedrive = ["opendal/services-onedrive"]
 services-persy = ["opendal/services-persy"]
 services-postgresql = ["opendal/services-postgresql"]
+services-pcloud = ["opendal/services-pcloud"]
 services-redb = ["opendal/services-redb"]
 services-redis = ["opendal/services-redis"]
 services-rocksdb = ["opendal/services-rocksdb"]

bindings/python/Cargo.toml

@@ -68,6 +68,7 @@ services-all = [
   "services-onedrive",
   "services-persy",
   "services-postgresql",
+  "services-pcloud",
   "services-mysql",
   "services-redb",
   "services-redis",
@@ -132,6 +133,7 @@ services-mysql = ["opendal/services-mysql"]
 services-onedrive = ["opendal/services-onedrive"]
 services-persy = ["opendal/services-persy"]
 services-postgresql = ["opendal/services-postgresql"]
+services-pcloud = ["opendal/services-pcloud"]
 services-redb = ["opendal/services-redb"]
 services-redis = ["opendal/services-redis"]
 services-rocksdb = ["opendal/services-rocksdb"]

core/Cargo.toml

@@ -52,6 +52,7 @@ default = [
   "services-webdav",
   "services-webhdfs",
   "services-azfile",
+  "services-pcloud"
 ]
 
 # Build test utils or not.

core/src/services/pcloud/backend.rs

@@ -21,12 +21,13 @@ use std::fmt::Formatter;
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use http::StatusCode;
+use http::{header, HeaderMap, StatusCode};
 use log::debug;
 use serde::Deserialize;
 
 use super::core::*;
 use super::error::parse_error;
+use super::error::parse_result;
 use super::error::PcloudError;
 use super::lister::PcloudLister;
 use super::writer::PcloudWriter;
@@ -199,14 +200,15 @@ impl Builder for PcloudBuilder {
                 .with_context("service", Scheme::Pcloud)),
         }?;
 
-        let client = if let Some(client) = self.http_client.take() {
-            client
-        } else {
-            HttpClient::new().map_err(|err| {
-                err.with_operation("Builder::build")
-                    .with_context("service", Scheme::Pcloud)
-            })?
-        };
+        // Make sure all connection closed.
+        //
+        // Ref: https://docs.pcloud.com/protocols/http_json_protocol/single_connection.html
+        let client_builder = reqwest::ClientBuilder::new().default_headers({
+            let mut map = HeaderMap::default();
+            map.insert(header::CONNECTION, build_header_value("close")?);
+            map
+        });
+        let client = HttpClient::build(client_builder)?;
 
         Ok(PcloudBackend {
             core: Arc::new(PcloudCore {
@@ -252,7 +254,6 @@ impl Accessor for PcloudBackend {
                 delete: true,
                 rename: true,
                 copy: true,
-
                 list: true,
 
                 ..Default::default()
@@ -277,6 +278,9 @@ impl Accessor for PcloudBackend {
                 let resp: StatResponse =
                     serde_json::from_slice(&bs).map_err(new_json_deserialize_error)?;
                 let result = resp.result;
+
+                parse_result(result)?;
+
                 if result == 2010 || result == 2055 || result == 2002 {
                     return Err(Error::new(ErrorKind::NotFound, &format!("{resp:?}")));
                 }
@@ -338,9 +342,10 @@ impl Accessor for PcloudBackend {
                 let resp: PcloudError =
                     serde_json::from_slice(&bs).map_err(new_json_deserialize_error)?;
                 let result = resp.result;
-
-                // pCloud returns 2005 or 2009 if the file or folder is not found
-                if result != 0 && result != 2005 && result != 2009 {
+                parse_result(result)?;
+                // pCloud returns 2005 if the folder is not found.
+                // And 2009 or 2002 if the file is not found.
+                if result != 0 && result != 2005 && result != 2009 && result != 2002 {
                     return Err(Error::new(ErrorKind::Unexpected, &format!("{resp:?}")));
                 }
 
@@ -372,6 +377,8 @@ impl Accessor for PcloudBackend {
                 let resp: PcloudError =
                     serde_json::from_slice(&bs).map_err(new_json_deserialize_error)?;
                 let result = resp.result;
+
+                parse_result(result)?;
                 if result == 2009 || result == 2010 || result == 2055 || result == 2002 {
                     return Err(Error::new(ErrorKind::NotFound, &format!("{resp:?}")));
                 }
@@ -402,6 +409,8 @@ impl Accessor for PcloudBackend {
                 let resp: PcloudError =
                     serde_json::from_slice(&bs).map_err(new_json_deserialize_error)?;
                 let result = resp.result;
+
+                parse_result(result)?;
                 if result == 2009 || result == 2010 || result == 2055 || result == 2002 {
                     return Err(Error::new(ErrorKind::NotFound, &format!("{resp:?}")));
                 }

core/src/services/pcloud/core.rs

@@ -19,12 +19,14 @@ use std::fmt::Debug;
 use std::fmt::Formatter;
 
 use bytes::Bytes;
+use http::header;
 use http::Request;
 use http::Response;
 use http::StatusCode;
 use serde::Deserialize;
 
 use super::error::parse_error;
+use super::error::parse_result;
 use super::error::PcloudError;
 use crate::raw::*;
 use crate::*;
@@ -88,6 +90,9 @@ impl PcloudCore {
                 let resp: GetFileLinkResponse =
                     serde_json::from_slice(&bs).map_err(new_json_deserialize_error)?;
                 let result = resp.result;
+
+                parse_result(result)?;
+
                 if result == 2010 || result == 2055 || result == 2002 {
                     return Err(Error::new(ErrorKind::NotFound, &format!("{resp:?}")));
                 }
@@ -136,6 +141,9 @@ impl PcloudCore {
                     let resp: PcloudError =
                         serde_json::from_slice(&bs).map_err(new_json_deserialize_error)?;
                     let result = resp.result;
+
+                    parse_result(result)?;
+
                     if result == 2010 || result == 2055 || result == 2002 {
                         return Err(Error::new(ErrorKind::NotFound, &format!("{resp:?}")));
                     }
@@ -337,20 +345,28 @@ impl PcloudCore {
         let (name, path) = (get_basename(&path), get_parent(&path).trim_end_matches('/'));
 
         let url = format!(
-            "{}/uploadfile?path=/{}&filename={}&username={}&password={}",
+            "{}/uploadfile?path=/{}&nopartial=1&mtime={}&username={}&password={}",
             self.endpoint,
             percent_encode_path(path),
-            percent_encode_path(name),
+            chrono::Local::now().timestamp(),
             self.username,
             self.password
         );
 
-        let req = Request::put(url);
+        let req = Request::post(url);
 
-        // set body
-        let req = req
-            .body(AsyncBody::Bytes(bs))
-            .map_err(new_request_build_error)?;
+        let file_part = FormDataPart::new("file")
+            .header(
+                header::CONTENT_DISPOSITION,
+                format!("form-data; name=\"file\"; filename=\"{name}\"")
+                    .parse()
+                    .unwrap(),
+            )
+            .content(bs);
+
+        let multipart = Multipart::new().part(file_part);
+
+        let req = multipart.apply(req)?;
 
         self.send(req).await
     }
@@ -423,14 +439,14 @@ pub(super) fn parse_list_metadata(content: ListMetadata) -> Result<Metadata> {
 
 #[derive(Debug, Deserialize)]
 pub struct GetFileLinkResponse {
-    pub result: u64,
+    pub result: u32,
     pub path: Option<String>,
     pub hosts: Option<Vec<String>>,
 }
 
 #[derive(Debug, Deserialize)]
 pub struct StatResponse {
-    pub result: u64,
+    pub result: u32,
     pub metadata: Option<StatMetadata>,
 }
 
@@ -445,7 +461,7 @@ pub struct StatMetadata {
 
 #[derive(Debug, Deserialize)]
 pub struct ListFolderResponse {
-    pub result: u64,
+    pub result: u32,
     pub metadata: Option<ListMetadata>,
 }
 
@@ -459,3 +475,9 @@ pub struct ListMetadata {
     pub contenttype: Option<String>,
     pub contents: Option<Vec<ListMetadata>>,
 }
+
+#[derive(Debug, Deserialize)]
+pub struct UploadFileResponse {
+    pub result: u32,
+    pub metadata: Vec<StatMetadata>,
+}

core/src/services/pcloud/error.rs

@@ -42,16 +42,38 @@ impl Debug for PcloudError {
     }
 }
 
+/// Deal with error response result.
+pub fn parse_result(result: u32) -> Result<()> {
+    match result / 1000 {
+        4 | 5 => {
+            let mut err = Error::new(ErrorKind::Unexpected, "Pcloud service returns an error.");
+            err = err.set_temporary();
+            Err(err)
+        }
+        _ => Ok(()),
+    }
+}
+
 /// Parse error response into Error.
 pub async fn parse_error(resp: Response<IncomingAsyncBody>) -> Result<Error> {
     let (parts, body) = resp.into_parts();
+
+    let (kind, retryable) = match parts.status.as_u16() {
+        429 | 500 | 502 | 503 | 504 | 509 => (ErrorKind::Unexpected, true),
+        _ => (ErrorKind::Unexpected, false),
+    };
+
     let bs = body.bytes().await?;
     let message = String::from_utf8_lossy(&bs).into_owned();
 
-    let mut err = Error::new(ErrorKind::Unexpected, &message);
+    let mut err = Error::new(kind, &message);
 
     err = with_error_response_context(err, parts);
 
+    if retryable {
+        err = err.set_temporary();
+    }
+
     Ok(err)
 }
 

core/src/services/pcloud/lister.rs

@@ -21,7 +21,7 @@ use async_trait::async_trait;
 use http::StatusCode;
 
 use super::core::*;
-use super::error::parse_error;
+use super::error::{parse_error, parse_result};
 use crate::raw::oio::Entry;
 use crate::raw::*;
 use crate::*;
@@ -56,6 +56,8 @@ impl oio::PageList for PcloudLister {
                     serde_json::from_slice(&bs).map_err(new_json_deserialize_error)?;
                 let result = resp.result;
 
+                parse_result(result)?;
+
                 if result == 2005 {
                     ctx.done = true;
                     return Ok(());

core/src/services/pcloud/writer.rs

@@ -21,8 +21,9 @@ use async_trait::async_trait;
 use http::StatusCode;
 
 use super::core::PcloudCore;
+use super::core::UploadFileResponse;
 use super::error::parse_error;
-use super::error::PcloudError;
+use super::error::parse_result;
 use crate::raw::*;
 use crate::*;
 
@@ -53,14 +54,20 @@ impl oio::OneShotWrite for PcloudWriter {
         match status {
             StatusCode::OK => {
                 let bs = resp.into_body().bytes().await?;
-                let resp: PcloudError =
+                let resp: UploadFileResponse =
                     serde_json::from_slice(&bs).map_err(new_json_deserialize_error)?;
                 let result = resp.result;
 
+                parse_result(result)?;
+
                 if result != 0 {
                     return Err(Error::new(ErrorKind::Unexpected, &format!("{resp:?}")));
                 }
 
+                if resp.metadata.len() != 1 {
+                    return Err(Error::new(ErrorKind::Unexpected, &format!("{resp:?}")));
+                }
+
                 Ok(())
             }
             _ => Err(parse_error(resp).await?),