feat: Add Webdav (a.k.a HTTP Cache) support

.github/workflows/integration-tests.yml

@@ -293,3 +293,50 @@ jobs:
           ${SCCACHE_PATH} --show-stats
 
           ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]"
+
+  webdav:
+    runs-on: ubuntu-latest
+    needs: build
+
+    env:
+      SCCACHE_WEBDAV_ENDPOINT: "http://127.0.0.1:8080"
+      RUSTC_WRAPPER: /home/runner/.cargo/bin/sccache
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Start nginx
+        shell: bash
+        run: |
+          mkdir /tmp/static
+          nginx -c `pwd`/tests/nginx_http_cache.conf
+
+      - name: Install rust
+        uses: ./.github/actions/rust-toolchain
+        with:
+          toolchain: "stable"
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: integration-tests
+          path: /home/runner/.cargo/bin/
+      - name: Chmod for binary
+        run: chmod +x ${SCCACHE_PATH}
+
+      - name: Test
+        run: cargo clean && cargo build
+
+      - name: Output
+        run: |
+          ${SCCACHE_PATH} --show-stats
+
+          ${SCCACHE_PATH} --show-stats | grep webdav
+
+      - name: Test Twice for Cache Read
+        run: cargo clean && cargo build
+
+      - name: Output
+        run: |
+          ${SCCACHE_PATH} --show-stats
+
+          ${SCCACHE_PATH} --show-stats | grep -e "Cache hits\s*[1-9]"

Cargo.toml

@@ -32,7 +32,7 @@ bincode = "1"
 blake3 = "1"
 byteorder = "1.0"
 bytes = "1"
-opendal = { version= "0.26.2", optional=true }
+opendal = { version= "0.27.0", optional=true }
 reqsign = {version="0.8.1", optional=true}
 clap = { version = "4.0.32", features = ["derive", "env", "wrap_help"] }
 directories = "4.0.1"
@@ -117,11 +117,12 @@ features = [
 
 [features]
 default = ["all"]
-all = ["dist-client", "redis", "s3", "memcached", "gcs", "azure", "gha"]
+all = ["dist-client", "redis", "s3", "memcached", "gcs", "azure", "gha", "webdav"]
 azure = ["opendal","reqsign"]
 s3 = ["opendal","reqsign"]
 gcs = ["opendal","reqsign", "url", "reqwest/blocking"]
 gha = ["opendal"]
+webdav = ["opendal"]
 memcached = ["opendal/services-memcached"]
 native-zlib = []
 redis = ["url", "opendal/services-redis"]

README.md

@@ -34,6 +34,7 @@ Table of Contents (ToC)
   * [Google Cloud Storage](docs/Gcs.md)
   * [Azure](docs/Azure.md)
   * [GitHub Actions](docs/GHA.md)
+  * [WebDAV (Ccache/Bazel/Gradle compatible)](docs/Webdav.md)
 * [Debugging](#debugging)
 * [Interaction with GNU `make` jobserver](#interaction-with-gnu-make-jobserver)
 * [Known Caveats](#known-caveats)

docs/Webdav.md

@@ -0,0 +1,11 @@
+# WebDAV
+
+Set `SCCACHE_WEBDAV_ENDPOINT` to a wevdav service endpoint to store cache in a webdav service. Set `SCCACHE_WEBDAV_KEY_PREFIX` to specify the key prefix of cache.
+
+The webdav cache is compatible with:
+
+- [Ccache HTTP storage backend](https://ccache.dev/manual/4.7.4.html#_http_storage_backend)
+- [Bazel Remote Caching](https://bazel.build/remote/caching).
+- [Gradle Build Cache](https://docs.gradle.org/current/userguide/build_cache.html)
+
+Users can set `SCCACHE_WEBDAV_ENDPOINT` to those services directly.

src/cache/cache.rs

@@ -25,14 +25,17 @@ use crate::cache::memcached::MemcachedCache;
 use crate::cache::redis::RedisCache;
 #[cfg(feature = "s3")]
 use crate::cache::s3::S3Cache;
+#[cfg(feature = "webdav")]
+use crate::cache::webdav::WebdavCache;
 use crate::config::Config;
 #[cfg(any(
     feature = "azure",
     feature = "gcs",
     feature = "gha",
     feature = "memcached",
     feature = "redis",
-    feature = "s3"
+    feature = "s3",
+    feature = "webdav"
 ))]
 use crate::config::{self, CacheType};
 use std::fmt;
@@ -546,6 +549,13 @@ pub fn storage_from_config(
 
                 return Ok(Arc::new(storage));
             }
+            #[cfg(feature = "webdav")]
+            CacheType::Webdav(ref c) => {
+                debug!("Init webdav cache with endpoint {}", c.endpoint);
+                let storage = WebdavCache::build(&c.endpoint, &c.key_prefix)
+                    .map_err(|err| anyhow!("create webdav cache failed: {err:?}"))?;
+                return Ok(Arc::new(storage));
+            }
             #[allow(unreachable_patterns)]
             _ => bail!("cache type is not enabled"),
         }

src/cache/mod.rs

@@ -27,5 +27,7 @@ pub mod memcached;
 pub mod redis;
 #[cfg(feature = "s3")]
 pub mod s3;
+#[cfg(feature = "webdav")]
+pub mod webdav;
 
 pub use crate::cache::cache::*;

src/cache/webdav.rs

@@ -0,0 +1,33 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::errors::*;
+use opendal::layers::LoggingLayer;
+use opendal::services::Webdav;
+use opendal::Operator;
+
+/// A cache that stores entries in a Webdav.
+pub struct WebdavCache;
+
+impl WebdavCache {
+    /// Create a new `WebdavCache`.
+    pub fn build(endpoint: &str, key_prefix: &str) -> Result<Operator> {
+        let mut builder = Webdav::default();
+        builder.endpoint(endpoint);
+        builder.root(key_prefix);
+
+        let op = Operator::create(builder)?
+            .layer(LoggingLayer::default())
+            .finish();
+        Ok(op)
+    }
+}

src/config.rs

@@ -224,6 +224,13 @@ pub struct RedisCacheConfig {
     pub url: String,
 }
 
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct WebdavCacheConfig {
+    pub endpoint: String,
+    pub key_prefix: String,
+}
+
 #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct S3CacheConfig {
@@ -243,6 +250,7 @@ pub enum CacheType {
     Memcached(MemcachedCacheConfig),
     Redis(RedisCacheConfig),
     S3(S3CacheConfig),
+    Webdav(WebdavCacheConfig),
 }
 
 #[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
@@ -255,6 +263,7 @@ pub struct CacheConfigs {
     pub memcached: Option<MemcachedCacheConfig>,
     pub redis: Option<RedisCacheConfig>,
     pub s3: Option<S3CacheConfig>,
+    pub webdav: Option<WebdavCacheConfig>,
 }
 
 impl CacheConfigs {
@@ -269,6 +278,7 @@ impl CacheConfigs {
             memcached,
             redis,
             s3,
+            webdav,
         } = self;
 
         let cache_type = s3
@@ -277,7 +287,8 @@ impl CacheConfigs {
             .or_else(|| memcached.map(CacheType::Memcached))
             .or_else(|| gcs.map(CacheType::GCS))
             .or_else(|| gha.map(CacheType::GHA))
-            .or_else(|| azure.map(CacheType::Azure));
+            .or_else(|| azure.map(CacheType::Azure))
+            .or_else(|| webdav.map(CacheType::Webdav));
 
         let fallback = disk.unwrap_or_default();
 
@@ -294,6 +305,7 @@ impl CacheConfigs {
             memcached,
             redis,
             s3,
+            webdav,
         } = other;
 
         if azure.is_some() {
@@ -317,6 +329,9 @@ impl CacheConfigs {
         if s3.is_some() {
             self.s3 = s3
         }
+        if webdav.is_some() {
+            self.webdav = webdav
+        }
     }
 }
 
@@ -637,6 +652,24 @@ fn config_from_env() -> Result<EnvConfig> {
         None
     };
 
+    // ======= WebDAV =======
+    let webdav = if let Ok(endpoint) = env::var("SCCACHE_WEBDAV_ENDPOINT") {
+        let key_prefix = env::var("SCCACHE_WEBDAV_KEY_PREFIX")
+            .ok()
+            .as_ref()
+            .map(|s| s.trim_end_matches('/'))
+            .filter(|s| !s.is_empty())
+            .unwrap_or_default()
+            .to_owned();
+
+        Some(WebdavCacheConfig {
+            endpoint,
+            key_prefix,
+        })
+    } else {
+        None
+    };
+
     // ======= Local =======
     let disk_dir = env::var_os("SCCACHE_DIR").map(PathBuf::from);
     let disk_sz = env::var("SCCACHE_CACHE_SIZE")
@@ -660,6 +693,7 @@ fn config_from_env() -> Result<EnvConfig> {
         memcached,
         redis,
         s3,
+        webdav,
     };
 
     Ok(EnvConfig { cache })
@@ -1126,6 +1160,10 @@ endpoint = "s3-us-east-1.amazonaws.com"
 use_ssl = true
 key_prefix = "s3prefix"
 no_credentials = true
+
+[cache.webdav]
+endpoint = "http://127.0.0.1:8080"
+key_prefix = "webdavprefix"
 "#;
 
     let file_config: FileConfig = toml::from_str(CONFIG_STR).expect("Is valid toml.");
@@ -1165,6 +1203,10 @@ no_credentials = true
                     key_prefix: "s3prefix".into(),
                     no_credentials: true,
                 }),
+                webdav: Some(WebdavCacheConfig {
+                    endpoint: "http://127.0.0.1:8080".to_string(),
+                    key_prefix: "webdavprefix".into(),
+                })
             },
             dist: DistConfig {
                 auth: DistAuth::Token {

tests/harness/mod.rs

@@ -140,6 +140,7 @@ pub fn sccache_client_cfg(tmpdir: &Path) -> sccache::config::FileConfig {
             memcached: None,
             redis: None,
             s3: None,
+            webdav: None,
         },
         dist: sccache::config::DistConfig {
             auth: Default::default(), // dangerously_insecure