storage: support external config merge

Signed-off-by: Yan Song <yansong.ys@antgroup.com>

Cargo.toml

@@ -95,6 +95,7 @@ default = [
     "backend-s3",
     "backend-http-proxy",
     "backend-localdisk",
+    "backend-external",
 ]
 virtiofs = [
     "nydus-service/virtiofs",
@@ -115,6 +116,7 @@ backend-localdisk = [
 backend-oss = ["nydus-storage/backend-oss"]
 backend-registry = ["nydus-storage/backend-registry"]
 backend-s3 = ["nydus-storage/backend-s3"]
+backend-external = ["nydus-storage/backend-external"]
 
 [workspace]
 members = [

api/src/config.rs

@@ -25,6 +25,9 @@ pub struct ConfigV2 {
     pub id: String,
     /// Configuration information for storage backend.
     pub backend: Option<BackendConfigV2>,
+    /// Configuration for external storage backends, order insensitivity.
+    #[serde(default)]
+    pub external_backends: Vec<ExternalBackendConfig>,
     /// Configuration information for local cache system.
     pub cache: Option<CacheConfigV2>,
     /// Configuration information for RAFS filesystem.
@@ -40,6 +43,7 @@ impl Default for ConfigV2 {
             version: 2,
             id: String::new(),
             backend: None,
+            external_backends: Vec::new(),
             cache: None,
             rafs: None,
             internal: ConfigV2Internal::default(),
@@ -54,6 +58,7 @@ impl ConfigV2 {
             version: 2,
             id: id.to_string(),
             backend: None,
+            external_backends: Vec::new(),
             cache: None,
             rafs: None,
             internal: ConfigV2Internal::default(),
@@ -959,6 +964,9 @@ pub struct BlobCacheEntryConfigV2 {
     /// Configuration information for storage backend.
     #[serde(default)]
     pub backend: BackendConfigV2,
+    /// Configuration for external storage backends, order insensitivity.
+    #[serde(default)]
+    pub external_backends: Vec<ExternalBackendConfig>,
     /// Configuration information for local cache system.
     #[serde(default)]
     pub cache: CacheConfigV2,
@@ -1022,6 +1030,7 @@ impl From<&BlobCacheEntryConfigV2> for ConfigV2 {
             version: c.version,
             id: c.id.clone(),
             backend: Some(c.backend.clone()),
+            external_backends: c.external_backends.clone(),
             cache: Some(c.cache.clone()),
             rafs: None,
             internal: ConfigV2Internal::default(),
@@ -1292,6 +1301,19 @@ struct CacheConfig {
     pub prefetch_config: BlobPrefetchConfig,
 }
 
+/// Additional configuration information for external backend, its items
+/// will be merged to the configuration from image.
+#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
+pub struct ExternalBackendConfig {
+    /// External backend identifier to merge.
+    pub patch: HashMap<String, String>,
+    /// External backend type.
+    #[serde(rename = "type")]
+    pub kind: String,
+    /// External backend config items to merge.
+    pub config: HashMap<String, String>,
+}
+
 impl TryFrom<&CacheConfig> for CacheConfigV2 {
     type Error = std::io::Error;
 
@@ -1333,6 +1355,9 @@ struct FactoryConfig {
     pub id: String,
     /// Configuration for storage backend.
     pub backend: BackendConfig,
+    /// Configuration for external storage backends, order insensitivity.
+    #[serde(default)]
+    pub external_backends: Vec<ExternalBackendConfig>,
     /// Configuration for blob cache manager.
     #[serde(default)]
     pub cache: CacheConfig,
@@ -1393,6 +1418,7 @@ impl TryFrom<RafsConfig> for ConfigV2 {
             version: 2,
             id: v.device.id,
             backend: Some(backend),
+            external_backends: v.device.external_backends,
             cache: Some(cache),
             rafs: Some(rafs),
             internal: ConfigV2Internal::default(),
@@ -1482,6 +1508,9 @@ pub(crate) struct BlobCacheEntryConfig {
     ///
     /// Possible value: `LocalFsConfig`, `RegistryConfig`, `OssConfig`, `LocalDiskConfig`.
     backend_config: Value,
+    /// Configuration for external storage backends, order insensitivity.
+    #[serde(default)]
+    external_backends: Vec<ExternalBackendConfig>,
     /// Type of blob cache, corresponding to `FactoryConfig::CacheConfig::cache_type`.
     ///
     /// Possible value: "fscache", "filecache".
@@ -1517,6 +1546,7 @@ impl TryFrom<&BlobCacheEntryConfig> for BlobCacheEntryConfigV2 {
             version: 2,
             id: v.id.clone(),
             backend: (&backend_config).try_into()?,
+            external_backends: v.external_backends.clone(),
             cache: (&cache_config).try_into()?,
             metadata_path: v.metadata_path.clone(),
         })

builder/src/core/context.rs

@@ -1579,6 +1579,7 @@ mod tests {
                 registry: None,
                 http_proxy: None,
             }),
+            external_backends: Vec::new(),
             id: "id".to_owned(),
             cache: None,
             rafs: None,

clib/Cargo.toml

@@ -26,3 +26,4 @@ backend-oss = ["nydus-storage/backend-oss"]
 backend-registry = ["nydus-storage/backend-registry"]
 backend-http-proxy = ["nydus-storage/backend-http-proxy"]
 backend-localdisk = ["nydus-storage/backend-localdisk"]
+backend-external = ["nydus-storage/backend-external"]

storage/Cargo.toml

@@ -44,6 +44,8 @@ url = { version = "2.1.1", optional = true }
 vm-memory = "0.10"
 fuse-backend-rs = "^0.11.0"
 gpt = { version = "3.1.0", optional = true }
+chrono = { version = "0.4", features = ["serde"] }
+serde_bytes = "0.11"
 
 nydus-api = { version = "0.3", path = "../api" }
 nydus-utils = { version = "0.4", path = "../utils", features = [
@@ -67,6 +69,7 @@ backend-oss = ["base64", "httpdate", "hmac", "sha1", "reqwest", "url"]
 backend-registry = ["base64", "reqwest", "url"]
 backend-s3 = ["base64", "hmac", "http", "reqwest", "sha2", "time", "url"]
 backend-http-proxy = ["hyper", "hyperlocal", "http", "reqwest", "url"]
+backend-external = ["base64", "reqwest", "url"]
 dedup = ["rusqlite", "r2d2", "r2d2_sqlite"]
 prefetch-rate-limit = ["leaky-bucket"]
 

storage/src/backend/connection.rs

@@ -74,6 +74,7 @@ pub(crate) struct ConnectionConfig {
     pub timeout: u32,
     pub connect_timeout: u32,
     pub retry_limit: u8,
+    pub redirect: bool,
 }
 
 impl Default for ConnectionConfig {
@@ -85,6 +86,7 @@ impl Default for ConnectionConfig {
             timeout: 5,
             connect_timeout: 5,
             retry_limit: 0,
+            redirect: false,
         }
     }
 }
@@ -98,6 +100,7 @@ impl From<OssConfig> for ConnectionConfig {
             timeout: c.timeout,
             connect_timeout: c.connect_timeout,
             retry_limit: c.retry_limit,
+            redirect: false,
         }
     }
 }
@@ -111,6 +114,7 @@ impl From<S3Config> for ConnectionConfig {
             timeout: c.timeout,
             connect_timeout: c.connect_timeout,
             retry_limit: c.retry_limit,
+            redirect: false,
         }
     }
 }
@@ -124,6 +128,7 @@ impl From<RegistryConfig> for ConnectionConfig {
             timeout: c.timeout,
             connect_timeout: c.connect_timeout,
             retry_limit: c.retry_limit,
+            redirect: false,
         }
     }
 }
@@ -137,6 +142,7 @@ impl From<HttpProxyConfig> for ConnectionConfig {
             timeout: c.timeout,
             connect_timeout: c.connect_timeout,
             retry_limit: c.retry_limit,
+            redirect: false,
         }
     }
 }
@@ -610,8 +616,12 @@ impl Connection {
 
         let mut cb = Client::builder()
             .timeout(timeout)
-            .connect_timeout(connect_timeout)
-            .redirect(Policy::none());
+            .connect_timeout(connect_timeout);
+        if config.redirect {
+            cb = cb.redirect(Policy::default());
+        } else {
+            cb = cb.redirect(Policy::none());
+        }
 
         if config.skip_verify {
             cb = cb.danger_accept_invalid_certs(true);