diff --git a/Cargo.lock b/Cargo.lock index 6e1e670..c86c8d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,9 +125,9 @@ checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8" [[package]] name = "arc-swap" @@ -440,9 +440,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.1.15" +version = "1.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +checksum = "a93fe60e2fc87b6ba2c117f67ae14f66e3fc7d6a1e612a25adb238cc980eadb3" dependencies = [ "jobserver", "libc", @@ -633,9 +633,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -1485,9 +1485,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", @@ -1666,7 +1666,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.4.1", - "hyper-rustls 0.27.2", + "hyper-rustls 0.27.3", "hyper-util", "ring", "rustls-pki-types", @@ -2229,9 +2229,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.3.1+3.3.1" +version = "300.3.2+3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7259953d42a81bf137fbbd73bd30a8e1914d6dce43c2b90ed575783a22608b91" +checksum = "a211a18d945ef7e648cc6e0058f4c548ee46aab922ea203e0d30e966ea23647b" dependencies = [ "cc", ] @@ -3396,9 +3396,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.35" +version = "0.38.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f" +checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" dependencies = [ "bitflags 2.6.0", "errno", @@ -3435,9 +3435,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" dependencies = [ "openssl-probe", "rustls-pemfile 2.1.3", @@ -3926,9 +3926,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "symbolic-common" -version = "12.10.1" +version = "12.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1944ea8afd197111bca0c0edea1e1f56abb3edd030e240c1035cc0e3ff51fec" +checksum = "9c1db5ac243c7d7f8439eb3b8f0357888b37cf3732957e91383b0ad61756374e" dependencies = [ "debugid", "memmap2", @@ -3938,9 +3938,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.10.1" +version = "12.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddaccaf1bf8e73c4f64f78dbb30aadd6965c71faa4ff3fba33f8d7296cf94a87" +checksum = "ea26e430c27d4a8a5dea4c4b81440606c7c1a415bd611451ef6af8c81416afc3" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -4221,9 +4221,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -4245,9 +4245,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", diff --git a/TODO.md b/TODO.md index eb82e1d..68ecfc0 100644 --- a/TODO.md +++ b/TODO.md @@ -3,6 +3,8 @@ - [ ] log rotate - [ ] secret storage - [ ] support include comnand for configuraion +- [ ] accept encoding adjustment plugin +- [x] support purge http cache - [x] support docker service discovery - [x] upstream and location update notification - [x] support validate config for plugin diff --git a/src/logger/mod.rs b/src/logger/mod.rs index 69a3bd0..f2d55fb 100644 --- a/src/logger/mod.rs +++ b/src/logger/mod.rs @@ -20,7 +20,6 @@ use std::path::Path; use std::sync::Mutex; use tracing::{info, Level}; use tracing_subscriber::fmt::writer::BoxMakeWriter; - use crate::util; #[derive(Default, Debug)] diff --git a/src/plugin/cache.rs b/src/plugin/cache.rs index 6cbbaeb..59228bd 100644 --- a/src/plugin/cache.rs +++ b/src/plugin/cache.rs @@ -22,14 +22,16 @@ use crate::config::{ }; use crate::http_extra::HttpResponse; use crate::state::State; +use crate::util; use async_trait::async_trait; -use bytes::{BufMut, BytesMut}; +use bytes::{BufMut, Bytes, BytesMut}; use bytesize::ByteSize; -use http::Method; +use http::{Method, StatusCode}; use humantime::parse_duration; -use once_cell::sync::OnceCell; +use once_cell::sync::{Lazy, OnceCell}; use pingora::cache::eviction::simple_lru::Manager; use pingora::cache::eviction::EvictionManager; +use pingora::cache::key::CacheHashKey; use pingora::cache::lock::CacheLock; use pingora::cache::predictor::{CacheablePredictor, Predictor}; use pingora::proxy::Session; @@ -54,6 +56,7 @@ pub struct Cache { max_ttl: Option, namespace: Option, headers: Option>, + purge_ip_rules: util::IpRules, hash_value: String, } @@ -177,6 +180,9 @@ impl TryFrom<&PluginConf> for Cache { None }; + let purge_ip_rules = + util::IpRules::new(&get_str_slice_conf(value, "purge_ip_list")); + let params = Self { hash_value, http_cache: cache, @@ -188,6 +194,7 @@ impl TryFrom<&PluginConf> for Cache { max_file_size: max_file_size.as_u64() as usize, namespace, headers, + purge_ip_rules, }; if params.plugin_step != PluginStep::Request { return Err(Error::Invalid { @@ -207,6 +214,9 @@ impl Cache { } } +static METHOD_PURGE: Lazy = + Lazy::new(|| Method::from_bytes(b"PURGE").unwrap()); + #[async_trait] impl Plugin for Cache { #[inline] @@ -224,26 +234,13 @@ impl Plugin for Cache { return Ok(None); } // cache only support get or head - if ![Method::GET, Method::HEAD].contains(&session.req_header().method) { + let method = &session.req_header().method; + if ![Method::GET, Method::HEAD, METHOD_PURGE.to_owned()] + .contains(method) + { return Ok(None); } - // max age of cache control - ctx.cache_max_ttl = self.max_ttl; - session.cache.enable( - self.http_cache, - self.eviction, - self.predictor, - self.lock, - ); - // set max size of cache response body - if self.max_file_size > 0 { - session.cache.set_max_file_size_bytes(self.max_file_size); - } - if let Some(stats) = self.http_cache.stats() { - ctx.cache_reading = Some(stats.reading); - ctx.cache_writing = Some(stats.writing); - } let mut keys = BytesMut::with_capacity(64); if let Some(namespace) = &self.namespace { keys.put(namespace.as_bytes()); @@ -264,6 +261,52 @@ impl Plugin for Cache { debug!("Cache prefix: {prefix}"); ctx.cache_prefix = Some(prefix); } + if method == METHOD_PURGE.to_owned() { + let found = match self + .purge_ip_rules + .matched(&util::get_client_ip(session)) + { + Ok(matched) => matched, + Err(e) => { + return Ok(Some(HttpResponse::bad_request( + e.to_string().into(), + ))); + }, + }; + if !found { + return Ok(Some(HttpResponse { + status: StatusCode::FORBIDDEN, + body: Bytes::from_static(b"Forbidden, ip is not allowed"), + ..Default::default() + })); + } + + let key = util::get_cache_key( + &ctx.cache_prefix.clone().unwrap_or_default(), + Method::GET.as_ref(), + &session.req_header().uri, + ); + self.http_cache.cached.remove(&key.combined()).await?; + return Ok(Some(HttpResponse::no_content())); + } + + // max age of cache control + ctx.cache_max_ttl = self.max_ttl; + + session.cache.enable( + self.http_cache, + self.eviction, + self.predictor, + self.lock, + ); + // set max size of cache response body + if self.max_file_size > 0 { + session.cache.set_max_file_size_bytes(self.max_file_size); + } + if let Some(stats) = self.http_cache.stats() { + ctx.cache_reading = Some(stats.reading); + ctx.cache_writing = Some(stats.writing); + } Ok(None) } diff --git a/src/plugin/ip_restriction.rs b/src/plugin/ip_restriction.rs index 5d1e7c7..efad9ff 100644 --- a/src/plugin/ip_restriction.rs +++ b/src/plugin/ip_restriction.rs @@ -23,16 +23,12 @@ use crate::util; use async_trait::async_trait; use bytes::Bytes; use http::StatusCode; -use ipnet::IpNet; use pingora::proxy::Session; -use std::net::IpAddr; -use std::str::FromStr; use tracing::debug; pub struct IpRestriction { plugin_step: PluginStep, - ip_net_list: Vec, - ip_list: Vec, + ip_rules: util::IpRules, restriction_category: String, forbidden_resp: HttpResponse, hash_value: String, @@ -44,15 +40,8 @@ impl TryFrom<&PluginConf> for IpRestriction { let hash_value = get_hash_key(value); let step = get_step_conf(value); - let mut ip_net_list = vec![]; - let mut ip_list = vec![]; - for item in get_str_slice_conf(value, "ip_list") { - if let Ok(value) = IpNet::from_str(&item) { - ip_net_list.push(value); - } else { - ip_list.push(item); - } - } + let ip_rules = + util::IpRules::new(&get_str_slice_conf(value, "ip_list")); let mut message = get_str_conf(value, "message"); if message.is_empty() { message = "Request is forbidden".to_string(); @@ -60,8 +49,7 @@ impl TryFrom<&PluginConf> for IpRestriction { let params = Self { hash_value, plugin_step: step, - ip_list, - ip_net_list, + ip_rules, restriction_category: get_str_conf(value, "type"), forbidden_resp: HttpResponse { status: StatusCode::FORBIDDEN, @@ -112,20 +100,15 @@ impl Plugin for IpRestriction { ip }; - let found = if self.ip_list.contains(&ip) { - true - } else { - match ip.parse::() { - Ok(addr) => { - self.ip_net_list.iter().any(|item| item.contains(&addr)) - }, - Err(e) => { - return Ok(Some(HttpResponse::bad_request( - e.to_string().into(), - ))); - }, - } + let found = match self.ip_rules.matched(&ip) { + Ok(matched) => matched, + Err(e) => { + return Ok(Some(HttpResponse::bad_request( + e.to_string().into(), + ))); + }, }; + // deny ip let allow = if self.restriction_category == "deny" { !found @@ -167,15 +150,9 @@ type = "deny" ) .unwrap(); assert_eq!("request", params.plugin_step.to_string()); - assert_eq!("192.168.1.1,10.1.1.1", params.ip_list.join(",")); assert_eq!( - "1.1.1.0/24,2.1.1.0/24", - params - .ip_net_list - .iter() - .map(|item| item.to_string()) - .collect::>() - .join(",") + r#"IpRules { ip_net_list: [1.1.1.0/24, 2.1.1.0/24], ip_list: ["192.168.1.1", "10.1.1.1"] }"#, + format!("{:?}", params.ip_rules) ); let result = IpRestriction::try_from( diff --git a/src/proxy/server.rs b/src/proxy/server.rs index 2cb30e7..16aa25d 100644 --- a/src/proxy/server.rs +++ b/src/proxy/server.rs @@ -682,10 +682,10 @@ impl ProxyHttp for Server { session: &Session, ctx: &mut Self::CTX, ) -> pingora::Result { - let key = CacheKey::new( - ctx.cache_prefix.clone().unwrap_or_default(), - format!("{}", session.req_header().uri), - "".to_string(), + let key = util::get_cache_key( + &ctx.cache_prefix.clone().unwrap_or_default(), + session.req_header().method.as_ref(), + &session.req_header().uri, ); debug!(key = format!("{key:?}"), "cache key callback"); Ok(key) @@ -1144,7 +1144,7 @@ mod tests { }, ); assert_eq!( - r#"Ok(CacheKey { namespace: "ss:", primary: "/vicanso/pingap?size=1", primary_bin_override: None, variance: None, user_tag: "" })"#, + r#"Ok(CacheKey { namespace: "GET:ss:", primary: "/vicanso/pingap?size=1", primary_bin_override: None, variance: None, user_tag: "" })"#, format!("{key:?}") ); } diff --git a/src/util/ip.rs b/src/util/ip.rs new file mode 100644 index 0000000..d65c739 --- /dev/null +++ b/src/util/ip.rs @@ -0,0 +1,50 @@ +// Copyright 2024 Tree xie. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use ipnet::IpNet; +use std::net::{AddrParseError, IpAddr}; +use std::str::FromStr; + +#[derive(Clone, Debug)] +pub struct IpRules { + ip_net_list: Vec, + ip_list: Vec, +} + +impl IpRules { + pub fn new(values: &Vec) -> Self { + let mut ip_net_list = vec![]; + let mut ip_list = vec![]; + for item in values { + if let Ok(value) = IpNet::from_str(item) { + ip_net_list.push(value); + } else { + ip_list.push(item.to_string()); + } + } + Self { + ip_net_list, + ip_list, + } + } + pub fn matched(&self, ip: &String) -> Result { + let found = if self.ip_list.contains(ip) { + true + } else { + let addr = ip.parse::()?; + self.ip_net_list.iter().any(|item| item.contains(&addr)) + }; + Ok(found) + } +} diff --git a/src/util/mod.rs b/src/util/mod.rs index 1e47d77..b43d1e8 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -14,15 +14,20 @@ use base64::{engine::general_purpose::STANDARD, Engine}; use bytes::BytesMut; -use http::HeaderName; +use http::{HeaderName, Uri}; use once_cell::sync::Lazy; use path_absolutize::*; +use pingora::cache::CacheKey; use pingora::tls::ssl::SslVersion; use pingora::{http::RequestHeader, proxy::Session}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use std::{path::Path, str::FromStr}; use substring::Substring; +mod ip; + +pub use ip::IpRules; + const NAME: &str = env!("CARGO_PKG_NAME"); const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -239,6 +244,15 @@ pub fn get_host(header: &RequestHeader) -> Option<&str> { None } +pub fn get_cache_key(prefix: &str, method: &str, uri: &Uri) -> CacheKey { + let namespace = if prefix.is_empty() { + method + } else { + &format!("{method}:{prefix}") + }; + CacheKey::new(namespace, uri.to_string(), "") +} + /// Get the content length from http request header. pub fn get_content_length(header: &RequestHeader) -> Option { if let Some(content_length) = diff --git a/web/src/components/form-plugin.tsx b/web/src/components/form-plugin.tsx index d7f2f31..cb42d12 100644 --- a/web/src/components/form-plugin.tsx +++ b/web/src/components/form-plugin.tsx @@ -758,6 +758,14 @@ export function FormPluginField({ addLabel: t("form.cacheHeadersAdd"), span: 12, }, + { + category: "textlist", + key: "purge_ip_list", + label: t("form.cachePurgeIpList"), + addLabel: t("form.cachePurgeIpListAdd"), + id: "purge-ip-restriction-list", + span: 12, + }, ); break; } diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 763cba1..33e6dd0 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -243,6 +243,8 @@ export default { "form.cacheEviction": "Enable Evicted From Storage", "form.cachePredictor": "Enable Predictor", "form.cacheMaxTtl": "Max Ttl Of Cache(e.g. 10m)", + "form.cachePurgeIpList": "The ip list are allowed to purge cache", + "form.cachePurgeIpListAdd": "Add more ip or ip net", // cors "form.corsPath": "The Gegexp Path For Cors(optional)", "form.corsAllowOrigin": "The Allow Origin", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index a098222..b150cfd 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -238,6 +238,8 @@ export default { "form.cacheEviction": "是否启用缓存清除方式", "form.cachePredictor": "是否启用缓存预测方式", "form.cacheMaxTtl": "缓存的最长有效期(如10m)", + "form.cachePurgeIpList": "允许清除缓存的IP列表", + "form.cachePurgeIpListAdd": "添加更多的IP或IP段", // cors "form.corsPath": "设置支持cors的正则路径(可选)", "form.corsAllowOrigin": "允许的来源",