feat(cache/oss): add support for oss (#2046)

* feat(cache/oss): add support for oss

Signed-off-by: Chojan Shang <psiace@apache.org>
Co-authored-by: Sylvestre Ledru <sylvestre@debian.org>

Cargo.toml

@@ -148,13 +148,15 @@ all = [
   "azure",
   "gha",
   "webdav",
+  "oss",
 ]
 azure = ["opendal", "reqsign"]
 default = ["all"]
 gcs = ["opendal", "reqsign", "url", "reqwest/blocking"]
 gha = ["opendal"]
 memcached = ["opendal/services-memcached"]
 native-zlib = []
+oss = ["opendal", "reqsign"]
 redis = ["url", "opendal/services-redis"]
 s3 = ["opendal", "reqsign"]
 webdav = ["opendal"]

README.md

@@ -42,6 +42,7 @@ Table of Contents (ToC)
   * [Azure](docs/Azure.md)
   * [GitHub Actions](docs/GHA.md)
   * [WebDAV (Ccache/Bazel/Gradle compatible)](docs/Webdav.md)
+  * [Alibaba OSS](docs/OSS.md)
 
 ---
 
@@ -295,3 +296,4 @@ Storage Options
 * [Azure](docs/Azure.md)
 * [GitHub Actions](docs/GHA.md)
 * [WebDAV (Ccache/Bazel/Gradle compatible)](docs/Webdav.md)
+* [Alibaba OSS](docs/OSS.md)

docs/OSS.md

@@ -0,0 +1,13 @@
+# OSS
+
+If you want to use _Object Storage Service_ (aka OSS) by Alibaba for the sccache cache, you need to set the `SCCACHE_OSS_BUCKET` environment variable to the name of the OSS bucket to use.
+
+You **must** specify the endpoint URL using the `SCCACHE_OSS_ENDPOINT` environment variable. More details about [OSS endpoints](https://www.alibabacloud.com/help/en/oss/user-guide/regions-and-endpoints).
+
+You can also define a prefix that will be prepended to the keys of all cache objects created and read within the OSS bucket, effectively creating a scope. To do that use the `SCCACHE_OSS_KEY_PREFIX` environment variable. This can be useful when sharing a bucket with another application.
+
+## Credentials
+
+Sccache is able to load credentials from environment variables: `ALIBABA_CLOUD_ACCESS_KEY_ID` and `ALIBABA_CLOUD_ACCESS_KEY_SECRET`.
+
+Alternatively, the `SCCACHE_OSS_NO_CREDENTIALS` environment variable can be set to use public readonly access to the OSS bucket, without the need for credentials. Valid values for this environment variable are `true`, `1`, `false`, and `0`. This can be useful for implementing a readonly cache for pull requests, which typically cannot be given access to credentials for security reasons.

src/cache/cache.rs

@@ -21,6 +21,8 @@ use crate::cache::gcs::{GCSCache, RWMode};
 use crate::cache::gha::GHACache;
 #[cfg(feature = "memcached")]
 use crate::cache::memcached::MemcachedCache;
+#[cfg(feature = "oss")]
+use crate::cache::oss::OSSCache;
 #[cfg(feature = "redis")]
 use crate::cache::redis::RedisCache;
 #[cfg(feature = "s3")]
@@ -36,7 +38,8 @@ use crate::config::Config;
     feature = "memcached",
     feature = "redis",
     feature = "s3",
-    feature = "webdav"
+    feature = "webdav",
+    feature = "oss"
 ))]
 use crate::config::{self, CacheType};
 use async_trait::async_trait;
@@ -653,6 +656,23 @@ pub fn storage_from_config(
 
                 return Ok(Arc::new(storage));
             }
+            #[cfg(feature = "oss")]
+            CacheType::OSS(ref c) => {
+                debug!(
+                    "Init oss cache with bucket {}, endpoint {:?}",
+                    c.bucket, c.endpoint
+                );
+
+                let storage = OSSCache::build(
+                    &c.bucket,
+                    &c.key_prefix,
+                    c.endpoint.as_deref(),
+                    c.no_credentials,
+                )
+                .map_err(|err| anyhow!("create oss cache failed: {err:?}"))?;
+
+                return Ok(Arc::new(storage));
+            }
             #[allow(unreachable_patterns)]
             _ => bail!("cache type is not enabled"),
         }

src/cache/mod.rs

@@ -23,6 +23,8 @@ pub mod gcs;
 pub mod gha;
 #[cfg(feature = "memcached")]
 pub mod memcached;
+#[cfg(feature = "oss")]
+pub mod oss;
 #[cfg(feature = "redis")]
 pub mod redis;
 #[cfg(feature = "s3")]

src/cache/oss.rs

@@ -0,0 +1,48 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use opendal::layers::LoggingLayer;
+use opendal::services::Oss;
+use opendal::Operator;
+
+use crate::errors::*;
+
+pub struct OSSCache;
+
+// Implement the Object Storage Service for Alibaba cloud
+impl OSSCache {
+    pub fn build(
+        bucket: &str,
+        key_prefix: &str,
+        endpoint: Option<&str>,
+        no_credentials: bool,
+    ) -> Result<Operator> {
+        let mut builder = Oss::default();
+        builder.bucket(bucket);
+        builder.root(key_prefix);
+
+        if let Some(endpoint) = endpoint {
+            builder.endpoint(endpoint);
+        }
+
+        if no_credentials {
+            // Allow anonymous access to OSS so that OpenDAL will not
+            // throw error when no credentials are provided.
+            builder.allow_anonymous();
+        }
+
+        let op = Operator::new(builder)?
+            .layer(LoggingLayer::default())
+            .finish();
+        Ok(op)
+    }
+}

src/config.rs

@@ -261,6 +261,15 @@ pub struct S3CacheConfig {
     pub server_side_encryption: Option<bool>,
 }
 
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct OSSCacheConfig {
+    pub bucket: String,
+    pub key_prefix: String,
+    pub endpoint: Option<String>,
+    pub no_credentials: bool,
+}
+
 #[derive(Debug, PartialEq, Eq)]
 pub enum CacheType {
     Azure(AzureCacheConfig),
@@ -270,6 +279,7 @@ pub enum CacheType {
     Redis(RedisCacheConfig),
     S3(S3CacheConfig),
     Webdav(WebdavCacheConfig),
+    OSS(OSSCacheConfig),
 }
 
 #[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
@@ -283,6 +293,7 @@ pub struct CacheConfigs {
     pub redis: Option<RedisCacheConfig>,
     pub s3: Option<S3CacheConfig>,
     pub webdav: Option<WebdavCacheConfig>,
+    pub oss: Option<OSSCacheConfig>,
 }
 
 impl CacheConfigs {
@@ -298,6 +309,7 @@ impl CacheConfigs {
             redis,
             s3,
             webdav,
+            oss,
         } = self;
 
         let cache_type = s3
@@ -307,7 +319,8 @@ impl CacheConfigs {
             .or_else(|| gcs.map(CacheType::GCS))
             .or_else(|| gha.map(CacheType::GHA))
             .or_else(|| azure.map(CacheType::Azure))
-            .or_else(|| webdav.map(CacheType::Webdav));
+            .or_else(|| webdav.map(CacheType::Webdav))
+            .or_else(|| oss.map(CacheType::OSS));
 
         let fallback = disk.unwrap_or_default();
 
@@ -325,6 +338,7 @@ impl CacheConfigs {
             redis,
             s3,
             webdav,
+            oss,
         } = other;
 
         if azure.is_some() {
@@ -351,6 +365,10 @@ impl CacheConfigs {
         if webdav.is_some() {
             self.webdav = webdav
         }
+
+        if oss.is_some() {
+            self.oss = oss
+        }
     }
 }
 
@@ -720,6 +738,44 @@ fn config_from_env() -> Result<EnvConfig> {
         None
     };
 
+    // ======= OSS =======
+    let oss = if let Ok(bucket) = env::var("SCCACHE_OSS_BUCKET") {
+        let endpoint = env::var("SCCACHE_OSS_ENDPOINT").ok();
+        let key_prefix = env::var("SCCACHE_OSS_KEY_PREFIX")
+            .ok()
+            .as_ref()
+            .map(|s| s.trim_end_matches('/'))
+            .filter(|s| !s.is_empty())
+            .unwrap_or_default()
+            .to_owned();
+
+        let no_credentials =
+            env::var("SCCACHE_OSS_NO_CREDENTIALS").map_or(Ok(false), |val| match val.as_str() {
+                "true" | "1" => Ok(true),
+                "false" | "0" => Ok(false),
+                _ => bail!("SCCACHE_OSS_NO_CREDENTIALS must be 'true', '1', 'false', or '0'."),
+            })?;
+
+        Some(OSSCacheConfig {
+            bucket,
+            endpoint,
+            key_prefix,
+            no_credentials,
+        })
+    } else {
+        None
+    };
+
+    if oss
+        .as_ref()
+        .map(|oss| oss.no_credentials)
+        .unwrap_or_default()
+        && (env::var_os("ALIBABA_CLOUD_ACCESS_KEY_ID").is_some()
+            || env::var_os("ALIBABA_CLOUD_ACCESS_KEY_SECRET").is_some())
+    {
+        bail!("If setting OSS credentials, SCCACHE_OSS_NO_CREDENTIALS must not be set.");
+    }
+
     // ======= Local =======
     let disk_dir = env::var_os("SCCACHE_DIR").map(PathBuf::from);
     let disk_sz = env::var("SCCACHE_CACHE_SIZE")
@@ -759,6 +815,7 @@ fn config_from_env() -> Result<EnvConfig> {
         redis,
         s3,
         webdav,
+        oss,
     };
 
     Ok(EnvConfig { cache })
@@ -1305,6 +1362,12 @@ key_prefix = "webdavprefix"
 username = "webdavusername"
 password = "webdavpassword"
 token = "webdavtoken"
+
+[cache.oss]
+bucket = "name"
+endpoint = "oss-us-east-1.aliyuncs.com"
+key_prefix = "ossprefix"
+no_credentials = true
 "#;
 
     let file_config: FileConfig = toml::from_str(CONFIG_STR).expect("Is valid toml.");
@@ -1353,7 +1416,13 @@ token = "webdavtoken"
                     username: Some("webdavusername".to_string()),
                     password: Some("webdavpassword".to_string()),
                     token: Some("webdavtoken".to_string()),
-                })
+                }),
+                oss: Some(OSSCacheConfig {
+                    bucket: "name".to_owned(),
+                    endpoint: Some("oss-us-east-1.aliyuncs.com".to_owned()),
+                    key_prefix: "ossprefix".into(),
+                    no_credentials: true,
+                }),
             },
             dist: DistConfig {
                 auth: DistAuth::Token {

tests/harness/mod.rs

@@ -155,6 +155,7 @@ pub fn sccache_client_cfg(
             redis: None,
             s3: None,
             webdav: None,
+            oss: None,
         },
         dist: sccache::config::DistConfig {
             auth: Default::default(), // dangerously_insecure