Add some load testing tools to the sandbox.

* support passing cargo build args to the Docker image builder, allowing us to enable all features.
* add a distributed deployment option in the sandbox
* update the deployments to have envoy depend on limitador instead of limitador depending on envoy.

Signed-off-by: Hiram Chirino <hiram@hiramchirino.com>

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["limitador", "limitador-server"]
+members = ["limitador", "limitador-server", "limitador-server/sandbox/loadtest"]
 resolver = "2"
 
 [profile.release]

Dockerfile

@@ -4,20 +4,34 @@
 
 # Use bullseye as build image instead of Bookworm as ubi9 does not not have GLIBCXX_3.4.30
 # https://access.redhat.com/solutions/6969351
-FROM --platform=${BUILDPLATFORM} rust:1.78.0-bullseye as limitador-build
+FROM rust:1.78.0-bullseye as limitador-build
 
 RUN apt update && apt upgrade -y \
     && apt install -y protobuf-compiler clang
 
 WORKDIR /usr/src/limitador
 
 ARG GITHUB_SHA
+ARG CARGO_ARGS
 ENV GITHUB_SHA=${GITHUB_SHA:-unknown}
 ENV RUSTFLAGS="-C target-feature=-crt-static"
 
-COPY . .
+# We set the env here just to make sure that the build is invalidated if the args change
+ENV CARGO_ARGS=${CARGO_ARGS}
 
-RUN cargo build --release
+# The following allows us to cache the Cargo dependency downloads with image layers
+COPY Cargo.toml Cargo.lock ./
+COPY limitador/Cargo.toml ./limitador/
+COPY limitador-server/Cargo.toml ./limitador-server/
+COPY limitador-server/sandbox/loadtest/Cargo.toml ./limitador-server/sandbox/loadtest/
+RUN mkdir -p limitador-server/src && echo 'fn main() {}' > limitador-server/src/main.rs
+RUN mkdir -p limitador-server/sandbox/loadtest/src && echo 'fn main() {}' > limitador-server/sandbox/loadtest/src/main.rs
+RUN cargo build --release ${CARGO_ARGS}
+
+COPY ./limitador ./limitador
+COPY ./limitador-server ./limitador-server
+
+RUN cargo build --release ${CARGO_ARGS}
 
 # ------------------------------------------------------------------------------
 # Run Stage

limitador-server/sandbox/.gitignore

@@ -3,3 +3,4 @@
 *.key
 *.pem
 *.csr
+report.html

limitador-server/sandbox/Makefile

@@ -45,10 +45,16 @@ deploy-redis-otel: clean ## Uses Redis to store counters, instrumented with open
 deploy-disk: clean ## Uses disk to store counters
 	$(DOCKER) compose -f docker-compose-envoy.yaml -f docker-compose-limitador-disk.yaml up
 
+deploy-distributed: clean ## Counters are held in Limitador (ephemeral) but replicated to other Limitador servers.
+	$(DOCKER) compose -f docker-compose-envoy.yaml -f docker-compose-limitador-distributed.yaml up
+
 ##@ Helper targets
 
-build: ## Build "limitador-testing" image
-	$(DOCKER) build -t limitador-testing -f ../../Dockerfile ../../
+build: clean ## Build the "limitador-testing" image
+	$(DOCKER) compose -f docker-compose-limitador-memory.yaml build
+
+build-all-features: clean ## Build the image "limitador-testing-all-features" image
+	$(DOCKER) compose -f docker-compose-limitador-distributed.yaml build
 
 ca: ## Create CA cert
 	openssl genrsa -out ca.key 2048
@@ -88,6 +94,21 @@ $(GRPCURL):
 .PHONY: grpcurl
 grpcurl: $(GRPCURL) ## Download grpcurl locally if necessary.
 
+.PHONY: ghz
+ghz:
+	$(call go-install-tool,$(PROJECT_PATH)/bin/ghz,github.com/bojand/ghz/cmd/ghz@latest) 
+
+RPS?=1000
+.PHONY: load-test
+load-test: ghz
+	# see https://ghz.sh/docs/load for usage details
+	$(PROJECT_PATH)/bin/ghz 127.0.0.1:18081 --insecure \
+	--call envoy.service.ratelimit.v3.RateLimitService.ShouldRateLimit \
+	--async --concurrency=50 \
+	--rps=$(RPS) \
+	--total=$(RPS)0 \
+	--data-file load-test.json
+
 # go-install-tool will 'go install' any package $2 and install it to $1.
 define go-install-tool
 @[ -f $(1) ] || { \

limitador-server/sandbox/README.md

@@ -17,14 +17,39 @@ Check out `make help` for all the targets.
 
 ### Deployment options
 
-| Limitador's configuration | Command | Info |
-| ------------- | ----- | ----- |
-| In-memory configuration | `make deploy-in-memory` | Counters are held in Limitador (ephemeral) |
-| Redis | `make deploy-redis` | Uses Redis to store counters |
-| Redis Secured | `make deploy-redis-tls` | Uses Redis with TLS and password protected to store counters |
-| Redis Cached | `make deploy-redis-cached` | Uses Redis to store counters, with an in-memory cache |
-| Redis Otel Instrumented | `make deploy-redis-otel` | Uses redis to store counters, [instrumented with opentelemetry](redis-otel/README.md) |
-| Disk | `make deploy-disk` | Uses disk to store counters |
+| Limitador's configuration | Command | Info                                                                                                           |
+|--------------------------| ----- |----------------------------------------------------------------------------------------------------------------|
+| In-memory configuration  | `make deploy-in-memory` | Counters are held in Limitador (ephemeral)                                                                     |
+| Redis                    | `make deploy-redis` | Uses Redis to store counters                                                                                   |
+| Redis Secured            | `make deploy-redis-tls` | Uses Redis with TLS and password protected to store counters                                                   |
+| Redis Cached             | `make deploy-redis-cached` | Uses Redis to store counters, with an in-memory cache                                                          |
+| Redis Otel Instrumented  | `make deploy-redis-otel` | Uses redis to store counters, [instrumented with opentelemetry](redis-otel/README.md)                          |
+| Disk                     | `make deploy-disk` | Uses disk to store counters                                                                                    |
+| Distributed | `make deploy-distributed` | Counters are held in Limitador (ephemeral) but replicated to other Limitador servers. |
+
+
+### Running Multi Node Distributed Deployments
+
+The `make deploy-distributed` target can be connected to other Limitador servers but requires you to set the `PEER_ID` and `PEER_URLS` environment variables when you run the target.
+
+If you have 3 servers you want to replicate between, you would run the following commands:
+
+```bash
+# on server where: hostname=server1
+PEER_ID=`hostname` PEER_URLS="http://server2:15001 http://server3:15001" make deploy-distributed
+```
+
+```bash
+# on server where: hostname=server2
+PEER_ID=`hostname` PEER_URLS="http://server1:15001 http://server3:15001" make deploy-distributed
+```
+
+```bash
+# on server where: hostname=server3
+PEER_ID=`hostname` PEER_URLS="http://server1:15001 http://server2:15001" make deploy-distributed
+```
+
+The `PEER_ID` just need to be unique between the servers, and the `PEER_URLS` should be a space-separated list of the other servers' URLs.
 
 ### Limitador's admin HTTP endpoint
 
@@ -75,6 +100,10 @@ bin/grpcurl -plaintext -d @ 127.0.0.1:18081 envoy.service.ratelimit.v3.RateLimit
                 {
                     "key": "req.method",
                     "value": "POST"
+                },
+                {
+                    "key": "req.path",
+                    "value": "/"
                 }
             ]
         }
@@ -97,6 +126,10 @@ while :; do bin/grpcurl -plaintext -d @ 127.0.0.1:18081 envoy.service.ratelimit.
                 {
                     "key": "req.method",
                     "value": "POST"
+                },
+                {
+                    "key": "req.path",
+                    "value": "/"
                 }
             ]
         }
@@ -113,6 +146,24 @@ EOM
 curl -i -H "Host: example.com" http://127.0.0.1:18000/get
 ```
 
+### Load Testing the GRPC RateLimitService directly
+
+This load test will use `grpcurl`. You need [Go SDK](https://golang.org/doc/install) installed.
+
+Run a load test a 5000 requests per second (RPS) for 10 seconds:
+
+```bash
+RPS=5000 make load-test
+```
+
+### Load Testing via Envoy Proxy
+
+```bash
+cargo run --package loadtest --release -- --report-file=report.htm
+```
+
+The report will be saved in `report.htm` file.
+
 ### Limitador Image
 
 By default, the sandbox will run Limitador's `limitador-testing:latest` image.

limitador-server/sandbox/docker-compose-envoy.yaml

@@ -5,6 +5,7 @@ services:
     image: envoyproxy/envoy:v1.20-latest
     depends_on:
       - upstream
+      - limitador
     command:
       - /usr/local/bin/envoy
       - --config-path

limitador-server/sandbox/docker-compose-limitador-disk.yaml

@@ -3,8 +3,9 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
-    depends_on:
-      - envoy
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     command:
       - limitador-server
       - --rls-ip

limitador-server/sandbox/docker-compose-limitador-distributed.yaml

@@ -0,0 +1,24 @@
+---
+version: '3.8'
+services:
+  limitador:
+    image: limitador-testing-all-features
+    build:
+      context: ../..
+      dockerfile: Dockerfile
+      args:
+        - CARGO_ARGS=--all-features
+    command: |
+      limitador-server --rls-ip 0.0.0.0 --rls-port 8081 --http-ip 0.0.0.0 --http-port "8080" 
+      -vv --grpc-reflection-service /opt/kuadrant/limits/limits.yaml 
+      distributed ${PEER_ID:-node1} 0.0.0.0:5001 ${PEER_URLS:-}
+    expose:
+      - "8080"
+      - "8081"
+      - "5001"
+    ports:
+      - "18080:8080"
+      - "18081:8081"
+      - "15001:5001"
+    volumes:
+      - ./limits.yaml:/opt/kuadrant/limits/limits.yaml

limitador-server/sandbox/docker-compose-limitador-memory.yaml

@@ -3,8 +3,9 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
-    depends_on:
-      - envoy
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     command:
       - limitador-server
       - --rls-ip

limitador-server/sandbox/docker-compose-limitador-redis-cached.yaml

@@ -3,8 +3,10 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     depends_on:
-      - envoy
       - redis
     command:
       - limitador-server

limitador-server/sandbox/docker-compose-limitador-redis-otel.yaml

@@ -3,9 +3,11 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     depends_on:
       - jaeger
-      - envoy
       - redis
     command:
       - limitador-server

limitador-server/sandbox/docker-compose-limitador-redis-tls.yaml

@@ -3,8 +3,10 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     depends_on:
-      - envoy
       - redis
     command:
       - limitador-server

limitador-server/sandbox/docker-compose-limitador-redis.yaml

@@ -3,8 +3,10 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     depends_on:
-      - envoy
       - redis
     command:
       - limitador-server

limitador-server/sandbox/limits.yaml

@@ -4,10 +4,19 @@
   seconds: 60
   conditions:
     - "req.method == 'GET'"
+    - "req.path != '/json'"
   variables: []
 - namespace: test_namespace
   max_value: 5
   seconds: 60
   conditions:
     - "req.method == 'POST'"
+    - "req.path != '/json'"
+  variables: []
+- namespace: test_namespace
+  max_value: 50000
+  seconds: 10
+  conditions:
+    - "req.method == 'GET'"
+    - "req.path == '/json'"
   variables: []

limitador-server/sandbox/load-test.json

@@ -0,0 +1,18 @@
+{
+    "domain": "test_namespace",
+    "hits_addend": 1,
+    "descriptors": [
+        {   
+            "entries": [
+                {
+                    "key": "req.method",
+                    "value": "GET"
+                },
+                {
+                    "key": "req.path",
+                    "value": "/json"
+                }
+            ]
+        }
+    ]
+}

limitador-server/sandbox/loadtest/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "loadtest"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+goose = "^0.17"
+tokio = "^1.12"

limitador-server/sandbox/loadtest/src/main.rs

@@ -0,0 +1,26 @@
+use goose::prelude::*;
+
+async fn loadtest_get_json(user: &mut GooseUser) -> TransactionResult {
+    let _goose_metrics = user.get("/json").await?;
+
+    Ok(())
+}
+
+#[tokio::main]
+async fn main() -> Result<(), GooseError> {
+    GooseAttack::initialize()?
+        .register_scenario(
+            scenario!("LoadtestTransactions").register_transaction(transaction!(loadtest_get_json)),
+        )
+        .set_default(GooseDefault::Host, "http://localhost:18000")?
+        .set_default(GooseDefault::HatchRate, "2")?
+        .set_default(
+            GooseDefault::CoordinatedOmissionMitigation,
+            GooseCoordinatedOmissionMitigation::Average,
+        )?
+        .set_default(GooseDefault::RunTime, 20)?
+        .execute()
+        .await?;
+
+    Ok(())
+}