From 9b3f8bdc89c162caec76bbf003333d8ac6b4fb31 Mon Sep 17 00:00:00 2001 From: Hiram Chirino Date: Mon, 3 Jun 2024 11:17:59 -0400 Subject: [PATCH] Add some load testing tools to the sandbox. * support passing cargo build args to the Docker image builder, allowing us to enable all features. * add a distributed deployment option in the sandbox * update the deployments to have envoy depend on limitador instead of limitador depending on envoy. Signed-off-by: Hiram Chirino --- Cargo.lock | 460 +++++++++++++++++- Cargo.toml | 2 +- Dockerfile | 20 +- limitador-server/sandbox/.gitignore | 1 + limitador-server/sandbox/Makefile | 25 +- limitador-server/sandbox/README.md | 67 ++- .../sandbox/docker-compose-envoy.yaml | 1 + .../docker-compose-limitador-disk.yaml | 5 +- .../docker-compose-limitador-distributed.yaml | 24 + .../docker-compose-limitador-memory.yaml | 5 +- ...docker-compose-limitador-redis-cached.yaml | 4 +- .../docker-compose-limitador-redis-otel.yaml | 4 +- .../docker-compose-limitador-redis-tls.yaml | 4 +- .../docker-compose-limitador-redis.yaml | 4 +- limitador-server/sandbox/limits.yaml | 9 + limitador-server/sandbox/load-test.json | 18 + limitador-server/sandbox/loadtest/Cargo.toml | 8 + limitador-server/sandbox/loadtest/src/main.rs | 26 + 18 files changed, 660 insertions(+), 27 deletions(-) create mode 100644 limitador-server/sandbox/docker-compose-limitador-distributed.yaml create mode 100644 limitador-server/sandbox/load-test.json create mode 100644 limitador-server/sandbox/loadtest/Cargo.toml create mode 100644 limitador-server/sandbox/loadtest/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 49b09ca5..91356f83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -149,7 +149,7 @@ dependencies = [ "bytes", "bytestring", "cfg-if", - "cookie", + "cookie 0.16.2", "derive_more", "encoding_rs", "futures-core", @@ -234,6 +234,21 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anes" version = "0.1.6" @@ -300,6 +315,25 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "async-compression" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -551,13 +585,22 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chrono" version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ + "android-tzdata", + "iana-time-zone", "num-traits", + "windows-targets 0.52.5", ] [[package]] @@ -688,6 +731,34 @@ dependencies = [ "version_check", ] +[[package]] +name = "cookie" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7efb37c3e1ccb1ff97164ad95ac1606e8ccd35b3fa0a7d99a304c7f4a428cc24" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "387461abbc748185c3a6e1673d826918b450b87ff22639429c694619a83b6cf6" +dependencies = [ + "cookie 0.17.0", + "idna 0.3.0", + "log", + "publicsuffix", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -816,6 +887,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctrlc" +version = "3.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" +dependencies = [ + "nix", + "windows-sys 0.52.0", +] + [[package]] name = "dashmap" version = "5.5.3" @@ -829,6 +910,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" + [[package]] name = "deranged" version = "0.3.11" @@ -861,6 +948,12 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "either" version = "1.11.0" @@ -932,6 +1025,18 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "spin", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1095,6 +1200,58 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "goose" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfeedc01d217935e1371901ea642ea6693d68893c67336d146eba23be04bf11" +dependencies = [ + "async-trait", + "chrono", + "ctrlc", + "downcast-rs", + "flume", + "futures", + "gumdrop", + "http 0.2.12", + "itertools 0.11.0", + "lazy_static", + "log", + "num-format", + "rand", + "regex", + "reqwest", + "serde", + "serde_json", + "simplelog", + "strum 0.25.0", + "strum_macros 0.25.3", + "tokio", + "tokio-tungstenite", + "tungstenite", + "url", +] + +[[package]] +name = "gumdrop" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bc700f989d2f6f0248546222d9b4258f5b02a171a431f8285a81c08142629e3" +dependencies = [ + "gumdrop_derive", +] + +[[package]] +name = "gumdrop_derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "729f9bd3449d77e7831a18abfb7ba2f99ee813dfd15b8c2167c9a54ba20aa99d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "h2" version = "0.3.26" @@ -1324,6 +1481,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.28", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "hyper-tls" version = "0.6.0" @@ -1360,6 +1530,39 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.5.0" @@ -1436,6 +1639,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -1632,6 +1844,14 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +[[package]] +name = "loadtest" +version = "0.1.0" +dependencies = [ + "goose", + "tokio", +] + [[package]] name = "local-channel" version = "0.1.5" @@ -1715,7 +1935,7 @@ dependencies = [ "base64 0.22.1", "http-body-util", "hyper 1.3.1", - "hyper-tls", + "hyper-tls 0.6.0", "hyper-util", "indexmap 2.2.6", "ipnet", @@ -1811,6 +2031,15 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -1829,6 +2058,18 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -1883,6 +2124,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + [[package]] name = "num-traits" version = "0.2.18" @@ -1902,6 +2153,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "object" version = "0.32.2" @@ -2149,8 +2409,8 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "strum", - "strum_macros", + "strum 0.24.1", + "strum_macros 0.24.3", "syn 1.0.109", ] @@ -2391,6 +2651,22 @@ dependencies = [ "prost", ] +[[package]] +name = "psl-types" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + +[[package]] +name = "publicsuffix" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a8c1bda5ae1af7f99a2962e49df150414a43d62404644d98dd5c3a93d07457" +dependencies = [ + "idna 0.3.0", + "psl-types", +] + [[package]] name = "quanta" version = "0.12.3" @@ -2574,6 +2850,50 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "async-compression", + "base64 0.21.7", + "bytes", + "cookie 0.17.0", + "cookie_store", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-tls 0.5.0", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rmp" version = "0.8.12" @@ -2640,6 +2960,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustversion" version = "1.0.15" @@ -2836,6 +3165,17 @@ dependencies = [ "libc", ] +[[package]] +name = "simplelog" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0" +dependencies = [ + "log", + "termcolor", + "time", +] + [[package]] name = "sketches-ddsketch" version = "0.2.2" @@ -2894,6 +3234,12 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +[[package]] +name = "strum" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" + [[package]] name = "strum_macros" version = "0.24.3" @@ -2907,6 +3253,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum_macros" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.59", +] + [[package]] name = "syn" version = "1.0.109" @@ -2950,6 +3309,27 @@ dependencies = [ "windows", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -2968,6 +3348,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "1.0.58" @@ -3006,7 +3395,9 @@ checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", + "libc", "num-conv", + "num_threads", "powerfmt", "serde", "time-core", @@ -3126,6 +3517,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.10" @@ -3317,6 +3720,25 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 0.2.12", + "httparse", + "log", + "rand", + "sha1", + "thiserror", + "url", + "utf-8", +] + [[package]] name = "typenum" version = "1.17.0" @@ -3372,7 +3794,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna", + "idna 0.5.0", "percent-encoding", ] @@ -3382,6 +3804,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8parse" version = "0.2.1" @@ -3466,6 +3894,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.92" @@ -3704,6 +4144,16 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "zerocopy" version = "0.7.32" diff --git a/Cargo.toml b/Cargo.toml index 817ad31c..a04b17d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["limitador", "limitador-server"] +members = ["limitador", "limitador-server", "limitador-server/sandbox/loadtest"] resolver = "2" [profile.release] diff --git a/Dockerfile b/Dockerfile index ee6f94de..9814bdc5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # Use bullseye as build image instead of Bookworm as ubi9 does not not have GLIBCXX_3.4.30 # https://access.redhat.com/solutions/6969351 -FROM --platform=${BUILDPLATFORM} rust:1.78.0-bullseye as limitador-build +FROM rust:1.78.0-bullseye as limitador-build RUN apt update && apt upgrade -y \ && apt install -y protobuf-compiler clang @@ -12,12 +12,26 @@ RUN apt update && apt upgrade -y \ WORKDIR /usr/src/limitador ARG GITHUB_SHA +ARG CARGO_ARGS ENV GITHUB_SHA=${GITHUB_SHA:-unknown} ENV RUSTFLAGS="-C target-feature=-crt-static" -COPY . . +# We set the env here just to make sure that the build is invalidated if the args change +ENV CARGO_ARGS=${CARGO_ARGS} -RUN cargo build --release +# The following allows us to cache the Cargo dependency downloads with image layers +COPY Cargo.toml Cargo.lock ./ +COPY limitador/Cargo.toml ./limitador/ +COPY limitador-server/Cargo.toml ./limitador-server/ +COPY limitador-server/sandbox/loadtest/Cargo.toml ./limitador-server/sandbox/loadtest/ +RUN mkdir -p limitador-server/src && echo 'fn main() {}' > limitador-server/src/main.rs +RUN mkdir -p limitador-server/sandbox/loadtest/src && echo 'fn main() {}' > limitador-server/sandbox/loadtest/src/main.rs +RUN cargo build --release ${CARGO_ARGS} + +COPY ./limitador ./limitador +COPY ./limitador-server ./limitador-server + +RUN cargo build --release ${CARGO_ARGS} # ------------------------------------------------------------------------------ # Run Stage diff --git a/limitador-server/sandbox/.gitignore b/limitador-server/sandbox/.gitignore index 90006a67..35d09850 100644 --- a/limitador-server/sandbox/.gitignore +++ b/limitador-server/sandbox/.gitignore @@ -3,3 +3,4 @@ *.key *.pem *.csr +report.html \ No newline at end of file diff --git a/limitador-server/sandbox/Makefile b/limitador-server/sandbox/Makefile index c9f476d6..2463b9a3 100644 --- a/limitador-server/sandbox/Makefile +++ b/limitador-server/sandbox/Makefile @@ -45,10 +45,16 @@ deploy-redis-otel: clean ## Uses Redis to store counters, instrumented with open deploy-disk: clean ## Uses disk to store counters $(DOCKER) compose -f docker-compose-envoy.yaml -f docker-compose-limitador-disk.yaml up +deploy-distributed: clean ## Counters are held in Limitador (ephemeral) but replicated to other Limitador servers. + $(DOCKER) compose -f docker-compose-envoy.yaml -f docker-compose-limitador-distributed.yaml up + ##@ Helper targets -build: ## Build "limitador-testing" image - $(DOCKER) build -t limitador-testing -f ../../Dockerfile ../../ +build: clean ## Build the "limitador-testing" image + $(DOCKER) compose -f docker-compose-limitador-memory.yaml build + +build-all-features: clean ## Build the image "limitador-testing-all-features" image + $(DOCKER) compose -f docker-compose-limitador-distributed.yaml build ca: ## Create CA cert openssl genrsa -out ca.key 2048 @@ -88,6 +94,21 @@ $(GRPCURL): .PHONY: grpcurl grpcurl: $(GRPCURL) ## Download grpcurl locally if necessary. +.PHONY: ghz +ghz: + $(call go-install-tool,$(PROJECT_PATH)/bin/ghz,github.com/bojand/ghz/cmd/ghz@latest) + +RPS?=1000 +.PHONY: load-test +load-test: ghz + # see https://ghz.sh/docs/load for usage details + $(PROJECT_PATH)/bin/ghz 127.0.0.1:18081 --insecure \ + --call envoy.service.ratelimit.v3.RateLimitService.ShouldRateLimit \ + --async --concurrency=50 \ + --rps=$(RPS) \ + --total=$(RPS)0 \ + --data-file load-test.json + # go-install-tool will 'go install' any package $2 and install it to $1. define go-install-tool @[ -f $(1) ] || { \ diff --git a/limitador-server/sandbox/README.md b/limitador-server/sandbox/README.md index ba62374f..b8fb2ed8 100644 --- a/limitador-server/sandbox/README.md +++ b/limitador-server/sandbox/README.md @@ -17,14 +17,39 @@ Check out `make help` for all the targets. ### Deployment options -| Limitador's configuration | Command | Info | -| ------------- | ----- | ----- | -| In-memory configuration | `make deploy-in-memory` | Counters are held in Limitador (ephemeral) | -| Redis | `make deploy-redis` | Uses Redis to store counters | -| Redis Secured | `make deploy-redis-tls` | Uses Redis with TLS and password protected to store counters | -| Redis Cached | `make deploy-redis-cached` | Uses Redis to store counters, with an in-memory cache | -| Redis Otel Instrumented | `make deploy-redis-otel` | Uses redis to store counters, [instrumented with opentelemetry](redis-otel/README.md) | -| Disk | `make deploy-disk` | Uses disk to store counters | +| Limitador's configuration | Command | Info | +|--------------------------| ----- |----------------------------------------------------------------------------------------------------------------| +| In-memory configuration | `make deploy-in-memory` | Counters are held in Limitador (ephemeral) | +| Redis | `make deploy-redis` | Uses Redis to store counters | +| Redis Secured | `make deploy-redis-tls` | Uses Redis with TLS and password protected to store counters | +| Redis Cached | `make deploy-redis-cached` | Uses Redis to store counters, with an in-memory cache | +| Redis Otel Instrumented | `make deploy-redis-otel` | Uses redis to store counters, [instrumented with opentelemetry](redis-otel/README.md) | +| Disk | `make deploy-disk` | Uses disk to store counters | +| Distributed | `make deploy-distributed` | Counters are held in Limitador (ephemeral) but replicated to other Limitador servers. | + + +### Running Multi Node Distributed Deployments + +The `make deploy-distributed` target can be connected to other Limitador servers but requires you to set the `PEER_ID` and `PEER_URLS` environment variables when you run the target. + +If you have 3 servers you want to replicate between, you would run the following commands: + +```bash +# on server where: hostname=server1 +PEER_ID=`hostname` PEER_URLS="http://server2:15001 http://server3:15001" make deploy-distributed +``` + +```bash +# on server where: hostname=server2 +PEER_ID=`hostname` PEER_URLS="http://server1:15001 http://server3:15001" make deploy-distributed +``` + +```bash +# on server where: hostname=server3 +PEER_ID=`hostname` PEER_URLS="http://server1:15001 http://server2:15001" make deploy-distributed +``` + +The `PEER_ID` just need to be unique between the servers, and the `PEER_URLS` should be a space-separated list of the other servers' URLs. ### Limitador's admin HTTP endpoint @@ -75,6 +100,10 @@ bin/grpcurl -plaintext -d @ 127.0.0.1:18081 envoy.service.ratelimit.v3.RateLimit { "key": "req.method", "value": "POST" + }, + { + "key": "req.path", + "value": "/" } ] } @@ -97,6 +126,10 @@ while :; do bin/grpcurl -plaintext -d @ 127.0.0.1:18081 envoy.service.ratelimit. { "key": "req.method", "value": "POST" + }, + { + "key": "req.path", + "value": "/" } ] } @@ -113,6 +146,24 @@ EOM curl -i -H "Host: example.com" http://127.0.0.1:18000/get ``` +### Load Testing the GRPC RateLimitService directly + +This load test will use `grpcurl`. You need [Go SDK](https://golang.org/doc/install) installed. + +Run a load test a 5000 requests per second (RPS) for 10 seconds: + +```bash +RPS=5000 make load-test +``` + +### Load Testing via Envoy Proxy + +```bash +cargo run --package loadtest --release -- --report-file=report.htm +``` + +The report will be saved in `report.htm` file. + ### Limitador Image By default, the sandbox will run Limitador's `limitador-testing:latest` image. diff --git a/limitador-server/sandbox/docker-compose-envoy.yaml b/limitador-server/sandbox/docker-compose-envoy.yaml index c2305db8..bfbfca17 100644 --- a/limitador-server/sandbox/docker-compose-envoy.yaml +++ b/limitador-server/sandbox/docker-compose-envoy.yaml @@ -5,6 +5,7 @@ services: image: envoyproxy/envoy:v1.20-latest depends_on: - upstream + - limitador command: - /usr/local/bin/envoy - --config-path diff --git a/limitador-server/sandbox/docker-compose-limitador-disk.yaml b/limitador-server/sandbox/docker-compose-limitador-disk.yaml index b62397a1..f84a7c73 100644 --- a/limitador-server/sandbox/docker-compose-limitador-disk.yaml +++ b/limitador-server/sandbox/docker-compose-limitador-disk.yaml @@ -3,8 +3,9 @@ version: '3.8' services: limitador: image: ${LIMITADOR_IMAGE:-limitador-testing} - depends_on: - - envoy + build: + context: ../.. + dockerfile: Dockerfile command: - limitador-server - --rls-ip diff --git a/limitador-server/sandbox/docker-compose-limitador-distributed.yaml b/limitador-server/sandbox/docker-compose-limitador-distributed.yaml new file mode 100644 index 00000000..c0d83038 --- /dev/null +++ b/limitador-server/sandbox/docker-compose-limitador-distributed.yaml @@ -0,0 +1,24 @@ +--- +version: '3.8' +services: + limitador: + image: limitador-testing-all-features + build: + context: ../.. + dockerfile: Dockerfile + args: + - CARGO_ARGS=--all-features + command: | + limitador-server --rls-ip 0.0.0.0 --rls-port 8081 --http-ip 0.0.0.0 --http-port "8080" + -vv --grpc-reflection-service /opt/kuadrant/limits/limits.yaml + distributed ${PEER_ID:-node1} 0.0.0.0:5001 ${PEER_URLS:-} + expose: + - "8080" + - "8081" + - "5001" + ports: + - "18080:8080" + - "18081:8081" + - "15001:5001" + volumes: + - ./limits.yaml:/opt/kuadrant/limits/limits.yaml diff --git a/limitador-server/sandbox/docker-compose-limitador-memory.yaml b/limitador-server/sandbox/docker-compose-limitador-memory.yaml index eec05ecf..133d7ffa 100644 --- a/limitador-server/sandbox/docker-compose-limitador-memory.yaml +++ b/limitador-server/sandbox/docker-compose-limitador-memory.yaml @@ -3,8 +3,9 @@ version: '3.8' services: limitador: image: ${LIMITADOR_IMAGE:-limitador-testing} - depends_on: - - envoy + build: + context: ../.. + dockerfile: Dockerfile command: - limitador-server - --rls-ip diff --git a/limitador-server/sandbox/docker-compose-limitador-redis-cached.yaml b/limitador-server/sandbox/docker-compose-limitador-redis-cached.yaml index 0fc384cf..7c257bbe 100644 --- a/limitador-server/sandbox/docker-compose-limitador-redis-cached.yaml +++ b/limitador-server/sandbox/docker-compose-limitador-redis-cached.yaml @@ -3,8 +3,10 @@ version: '3.8' services: limitador: image: ${LIMITADOR_IMAGE:-limitador-testing} + build: + context: ../.. + dockerfile: Dockerfile depends_on: - - envoy - redis command: - limitador-server diff --git a/limitador-server/sandbox/docker-compose-limitador-redis-otel.yaml b/limitador-server/sandbox/docker-compose-limitador-redis-otel.yaml index 169820b5..8c61f7aa 100644 --- a/limitador-server/sandbox/docker-compose-limitador-redis-otel.yaml +++ b/limitador-server/sandbox/docker-compose-limitador-redis-otel.yaml @@ -3,9 +3,11 @@ version: '3.8' services: limitador: image: ${LIMITADOR_IMAGE:-limitador-testing} + build: + context: ../.. + dockerfile: Dockerfile depends_on: - jaeger - - envoy - redis command: - limitador-server diff --git a/limitador-server/sandbox/docker-compose-limitador-redis-tls.yaml b/limitador-server/sandbox/docker-compose-limitador-redis-tls.yaml index 1046b25b..5c603a5e 100644 --- a/limitador-server/sandbox/docker-compose-limitador-redis-tls.yaml +++ b/limitador-server/sandbox/docker-compose-limitador-redis-tls.yaml @@ -3,8 +3,10 @@ version: '3.8' services: limitador: image: ${LIMITADOR_IMAGE:-limitador-testing} + build: + context: ../.. + dockerfile: Dockerfile depends_on: - - envoy - redis command: - limitador-server diff --git a/limitador-server/sandbox/docker-compose-limitador-redis.yaml b/limitador-server/sandbox/docker-compose-limitador-redis.yaml index 611e0f1b..f5fc3b02 100644 --- a/limitador-server/sandbox/docker-compose-limitador-redis.yaml +++ b/limitador-server/sandbox/docker-compose-limitador-redis.yaml @@ -3,8 +3,10 @@ version: '3.8' services: limitador: image: ${LIMITADOR_IMAGE:-limitador-testing} + build: + context: ../.. + dockerfile: Dockerfile depends_on: - - envoy - redis command: - limitador-server diff --git a/limitador-server/sandbox/limits.yaml b/limitador-server/sandbox/limits.yaml index 0f7b90b2..68b6df8d 100644 --- a/limitador-server/sandbox/limits.yaml +++ b/limitador-server/sandbox/limits.yaml @@ -4,10 +4,19 @@ seconds: 60 conditions: - "req.method == 'GET'" + - "req.path != '/json'" variables: [] - namespace: test_namespace max_value: 5 seconds: 60 conditions: - "req.method == 'POST'" + - "req.path != '/json'" + variables: [] +- namespace: test_namespace + max_value: 50000 + seconds: 10 + conditions: + - "req.method == 'GET'" + - "req.path == '/json'" variables: [] diff --git a/limitador-server/sandbox/load-test.json b/limitador-server/sandbox/load-test.json new file mode 100644 index 00000000..b23422cc --- /dev/null +++ b/limitador-server/sandbox/load-test.json @@ -0,0 +1,18 @@ +{ + "domain": "test_namespace", + "hits_addend": 1, + "descriptors": [ + { + "entries": [ + { + "key": "req.method", + "value": "GET" + }, + { + "key": "req.path", + "value": "/json" + } + ] + } + ] +} \ No newline at end of file diff --git a/limitador-server/sandbox/loadtest/Cargo.toml b/limitador-server/sandbox/loadtest/Cargo.toml new file mode 100644 index 00000000..9f056990 --- /dev/null +++ b/limitador-server/sandbox/loadtest/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "loadtest" +version = "0.1.0" +edition = "2021" + +[dependencies] +goose = "^0.17" +tokio = "^1.12" diff --git a/limitador-server/sandbox/loadtest/src/main.rs b/limitador-server/sandbox/loadtest/src/main.rs new file mode 100644 index 00000000..41c2bd55 --- /dev/null +++ b/limitador-server/sandbox/loadtest/src/main.rs @@ -0,0 +1,26 @@ +use goose::prelude::*; + +async fn loadtest_get_json(user: &mut GooseUser) -> TransactionResult { + let _goose_metrics = user.get("/json").await?; + + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<(), GooseError> { + GooseAttack::initialize()? + .register_scenario( + scenario!("LoadtestTransactions").register_transaction(transaction!(loadtest_get_json)), + ) + .set_default(GooseDefault::Host, "http://localhost:18000")? + .set_default(GooseDefault::HatchRate, "2")? + .set_default( + GooseDefault::CoordinatedOmissionMitigation, + GooseCoordinatedOmissionMitigation::Average, + )? + .set_default(GooseDefault::RunTime, 20)? + .execute() + .await?; + + Ok(()) +}