diff --git a/Cargo.toml b/Cargo.toml index 5a558c5d1..b16858353 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -148,6 +148,7 @@ all = [ "azure", "gha", "webdav", + "oss", ] azure = ["opendal", "reqsign"] default = ["all"] @@ -155,6 +156,7 @@ gcs = ["opendal", "reqsign", "url", "reqwest/blocking"] gha = ["opendal"] memcached = ["opendal/services-memcached"] native-zlib = [] +oss = ["opendal", "reqsign"] redis = ["url", "opendal/services-redis"] s3 = ["opendal", "reqsign"] webdav = ["opendal"] diff --git a/README.md b/README.md index f70854c5e..63d8c2b96 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Table of Contents (ToC) * [Azure](docs/Azure.md) * [GitHub Actions](docs/GHA.md) * [WebDAV (Ccache/Bazel/Gradle compatible)](docs/Webdav.md) + * [Alibaba OSS](docs/OSS.md) --- @@ -295,3 +296,4 @@ Storage Options * [Azure](docs/Azure.md) * [GitHub Actions](docs/GHA.md) * [WebDAV (Ccache/Bazel/Gradle compatible)](docs/Webdav.md) +* [Alibaba OSS](docs/OSS.md) diff --git a/docs/OSS.md b/docs/OSS.md new file mode 100644 index 000000000..89e8ab070 --- /dev/null +++ b/docs/OSS.md @@ -0,0 +1,13 @@ +# OSS + +If you want to use _Object Storage Service_ (aka OSS) by Alibaba for the sccache cache, you need to set the `SCCACHE_OSS_BUCKET` environment variable to the name of the OSS bucket to use. + +You **must** specify the endpoint URL using the `SCCACHE_OSS_ENDPOINT` environment variable. More details about [OSS endpoints](https://www.alibabacloud.com/help/en/oss/user-guide/regions-and-endpoints). + +You can also define a prefix that will be prepended to the keys of all cache objects created and read within the OSS bucket, effectively creating a scope. To do that use the `SCCACHE_OSS_KEY_PREFIX` environment variable. This can be useful when sharing a bucket with another application. + +## Credentials + +Sccache is able to load credentials from environment variables: `ALIBABA_CLOUD_ACCESS_KEY_ID` and `ALIBABA_CLOUD_ACCESS_KEY_SECRET`. + +Alternatively, the `SCCACHE_OSS_NO_CREDENTIALS` environment variable can be set to use public readonly access to the OSS bucket, without the need for credentials. Valid values for this environment variable are `true`, `1`, `false`, and `0`. This can be useful for implementing a readonly cache for pull requests, which typically cannot be given access to credentials for security reasons. diff --git a/src/cache/cache.rs b/src/cache/cache.rs index db1bee7bb..0c09a78ea 100644 --- a/src/cache/cache.rs +++ b/src/cache/cache.rs @@ -21,6 +21,8 @@ use crate::cache::gcs::{GCSCache, RWMode}; use crate::cache::gha::GHACache; #[cfg(feature = "memcached")] use crate::cache::memcached::MemcachedCache; +#[cfg(feature = "oss")] +use crate::cache::oss::OSSCache; #[cfg(feature = "redis")] use crate::cache::redis::RedisCache; #[cfg(feature = "s3")] @@ -36,7 +38,8 @@ use crate::config::Config; feature = "memcached", feature = "redis", feature = "s3", - feature = "webdav" + feature = "webdav", + feature = "oss" ))] use crate::config::{self, CacheType}; use async_trait::async_trait; @@ -653,6 +656,23 @@ pub fn storage_from_config( return Ok(Arc::new(storage)); } + #[cfg(feature = "oss")] + CacheType::OSS(ref c) => { + debug!( + "Init oss cache with bucket {}, endpoint {:?}", + c.bucket, c.endpoint + ); + + let storage = OSSCache::build( + &c.bucket, + &c.key_prefix, + c.endpoint.as_deref(), + c.no_credentials, + ) + .map_err(|err| anyhow!("create oss cache failed: {err:?}"))?; + + return Ok(Arc::new(storage)); + } #[allow(unreachable_patterns)] _ => bail!("cache type is not enabled"), } diff --git a/src/cache/mod.rs b/src/cache/mod.rs index 772aa343f..3dc86baf7 100644 --- a/src/cache/mod.rs +++ b/src/cache/mod.rs @@ -23,6 +23,8 @@ pub mod gcs; pub mod gha; #[cfg(feature = "memcached")] pub mod memcached; +#[cfg(feature = "oss")] +pub mod oss; #[cfg(feature = "redis")] pub mod redis; #[cfg(feature = "s3")] diff --git a/src/cache/oss.rs b/src/cache/oss.rs new file mode 100644 index 000000000..e98e8ccc2 --- /dev/null +++ b/src/cache/oss.rs @@ -0,0 +1,48 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use opendal::layers::LoggingLayer; +use opendal::services::Oss; +use opendal::Operator; + +use crate::errors::*; + +pub struct OSSCache; + +// Implement the Object Storage Service for Alibaba cloud +impl OSSCache { + pub fn build( + bucket: &str, + key_prefix: &str, + endpoint: Option<&str>, + no_credentials: bool, + ) -> Result { + let mut builder = Oss::default(); + builder.bucket(bucket); + builder.root(key_prefix); + + if let Some(endpoint) = endpoint { + builder.endpoint(endpoint); + } + + if no_credentials { + // Allow anonymous access to OSS so that OpenDAL will not + // throw error when no credentials are provided. + builder.allow_anonymous(); + } + + let op = Operator::new(builder)? + .layer(LoggingLayer::default()) + .finish(); + Ok(op) + } +} diff --git a/src/config.rs b/src/config.rs index b39b5547b..036e2e603 100644 --- a/src/config.rs +++ b/src/config.rs @@ -261,6 +261,15 @@ pub struct S3CacheConfig { pub server_side_encryption: Option, } +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OSSCacheConfig { + pub bucket: String, + pub key_prefix: String, + pub endpoint: Option, + pub no_credentials: bool, +} + #[derive(Debug, PartialEq, Eq)] pub enum CacheType { Azure(AzureCacheConfig), @@ -270,6 +279,7 @@ pub enum CacheType { Redis(RedisCacheConfig), S3(S3CacheConfig), Webdav(WebdavCacheConfig), + OSS(OSSCacheConfig), } #[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)] @@ -283,6 +293,7 @@ pub struct CacheConfigs { pub redis: Option, pub s3: Option, pub webdav: Option, + pub oss: Option, } impl CacheConfigs { @@ -298,6 +309,7 @@ impl CacheConfigs { redis, s3, webdav, + oss, } = self; let cache_type = s3 @@ -307,7 +319,8 @@ impl CacheConfigs { .or_else(|| gcs.map(CacheType::GCS)) .or_else(|| gha.map(CacheType::GHA)) .or_else(|| azure.map(CacheType::Azure)) - .or_else(|| webdav.map(CacheType::Webdav)); + .or_else(|| webdav.map(CacheType::Webdav)) + .or_else(|| oss.map(CacheType::OSS)); let fallback = disk.unwrap_or_default(); @@ -325,6 +338,7 @@ impl CacheConfigs { redis, s3, webdav, + oss, } = other; if azure.is_some() { @@ -351,6 +365,10 @@ impl CacheConfigs { if webdav.is_some() { self.webdav = webdav } + + if oss.is_some() { + self.oss = oss + } } } @@ -720,6 +738,44 @@ fn config_from_env() -> Result { None }; + // ======= OSS ======= + let oss = if let Ok(bucket) = env::var("SCCACHE_OSS_BUCKET") { + let endpoint = env::var("SCCACHE_OSS_ENDPOINT").ok(); + let key_prefix = env::var("SCCACHE_OSS_KEY_PREFIX") + .ok() + .as_ref() + .map(|s| s.trim_end_matches('/')) + .filter(|s| !s.is_empty()) + .unwrap_or_default() + .to_owned(); + + let no_credentials = + env::var("SCCACHE_OSS_NO_CREDENTIALS").map_or(Ok(false), |val| match val.as_str() { + "true" | "1" => Ok(true), + "false" | "0" => Ok(false), + _ => bail!("SCCACHE_OSS_NO_CREDENTIALS must be 'true', '1', 'false', or '0'."), + })?; + + Some(OSSCacheConfig { + bucket, + endpoint, + key_prefix, + no_credentials, + }) + } else { + None + }; + + if oss + .as_ref() + .map(|oss| oss.no_credentials) + .unwrap_or_default() + && (env::var_os("ALIBABA_CLOUD_ACCESS_KEY_ID").is_some() + || env::var_os("ALIBABA_CLOUD_ACCESS_KEY_SECRET").is_some()) + { + bail!("If setting OSS credentials, SCCACHE_OSS_NO_CREDENTIALS must not be set."); + } + // ======= Local ======= let disk_dir = env::var_os("SCCACHE_DIR").map(PathBuf::from); let disk_sz = env::var("SCCACHE_CACHE_SIZE") @@ -759,6 +815,7 @@ fn config_from_env() -> Result { redis, s3, webdav, + oss, }; Ok(EnvConfig { cache }) @@ -1305,6 +1362,12 @@ key_prefix = "webdavprefix" username = "webdavusername" password = "webdavpassword" token = "webdavtoken" + +[cache.oss] +bucket = "name" +endpoint = "oss-us-east-1.aliyuncs.com" +key_prefix = "ossprefix" +no_credentials = true "#; let file_config: FileConfig = toml::from_str(CONFIG_STR).expect("Is valid toml."); @@ -1353,7 +1416,13 @@ token = "webdavtoken" username: Some("webdavusername".to_string()), password: Some("webdavpassword".to_string()), token: Some("webdavtoken".to_string()), - }) + }), + oss: Some(OSSCacheConfig { + bucket: "name".to_owned(), + endpoint: Some("oss-us-east-1.aliyuncs.com".to_owned()), + key_prefix: "ossprefix".into(), + no_credentials: true, + }), }, dist: DistConfig { auth: DistAuth::Token { diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index b46fc033d..b663f5166 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -155,6 +155,7 @@ pub fn sccache_client_cfg( redis: None, s3: None, webdav: None, + oss: None, }, dist: sccache::config::DistConfig { auth: Default::default(), // dangerously_insecure