Merge pull request #351 from chirino/sandbox-distributed

Add some load testing tools to the sandbox
Kuadrant · Jun 13, 2024 · d44ebdb · d44ebdb
2 parents 72c2b12 + d4ee1f6
commit d44ebdb
Show file tree

Hide file tree

Showing 21 changed files with 2,382 additions and 21 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -4,20 +4,32 @@
 
 # Use bullseye as build image instead of Bookworm as ubi9 does not not have GLIBCXX_3.4.30
 # https://access.redhat.com/solutions/6969351
-FROM --platform=${BUILDPLATFORM} rust:1.78.0-bullseye as limitador-build
+FROM rust:1.78.0-bullseye as limitador-build
 
 RUN apt update && apt upgrade -y \
     && apt install -y protobuf-compiler clang
 
 WORKDIR /usr/src/limitador
 
 ARG GITHUB_SHA
+ARG CARGO_ARGS
 ENV GITHUB_SHA=${GITHUB_SHA:-unknown}
 ENV RUSTFLAGS="-C target-feature=-crt-static"
 
-COPY . .
+# We set the env here just to make sure that the build is invalidated if the args change
+ENV CARGO_ARGS=${CARGO_ARGS}
 
-RUN cargo build --release
+# The following allows us to cache the Cargo dependency downloads with image layers
+COPY Cargo.toml Cargo.lock ./
+COPY limitador/Cargo.toml ./limitador/
+COPY limitador-server/Cargo.toml ./limitador-server/
+RUN mkdir -p limitador-server/src && echo 'fn main() {}' > limitador-server/src/main.rs
+RUN cargo build --release ${CARGO_ARGS}
+
+COPY ./limitador ./limitador
+COPY ./limitador-server ./limitador-server
+
+RUN cargo build --release ${CARGO_ARGS}
 
 # ------------------------------------------------------------------------------
 # Run Stage

diff --git a/limitador-server/sandbox/.gitignore b/limitador-server/sandbox/.gitignore
@@ -3,3 +3,4 @@
 *.key
 *.pem
 *.csr
+report.html
diff --git a/limitador-server/sandbox/Makefile b/limitador-server/sandbox/Makefile
@@ -45,10 +45,19 @@ deploy-redis-otel: clean ## Uses Redis to store counters, instrumented with open
 deploy-disk: clean ## Uses disk to store counters
 	$(DOCKER) compose -f docker-compose-envoy.yaml -f docker-compose-limitador-disk.yaml up
 
+deploy-distributed: clean ## Counters are held in Limitador (ephemeral) but replicated to other Limitador servers.
+	$(DOCKER) compose -f docker-compose-envoy.yaml -f docker-compose-limitador-distributed.yaml up
+
+deploy-distributed-3-node: clean ## Counters are held in Limitador (ephemeral) but replicated to 3 other local Limitador servers.
+	$(DOCKER) compose -f docker-compose-envoy-3-node.yaml -f docker-compose-limitador-distributed-3-node.yaml up
+
 ##@ Helper targets
 
-build: ## Build "limitador-testing" image
-	$(DOCKER) build -t limitador-testing -f ../../Dockerfile ../../
+build: clean ## Build the "limitador-testing" image
+	$(DOCKER) compose -f docker-compose-limitador-memory.yaml build
+
+build-all-features: clean ## Build the image "limitador-testing-all-features" image
+	$(DOCKER) compose -f docker-compose-limitador-distributed.yaml build
 
 ca: ## Create CA cert
 	openssl genrsa -out ca.key 2048
@@ -88,6 +97,21 @@ $(GRPCURL):
 .PHONY: grpcurl
 grpcurl: $(GRPCURL) ## Download grpcurl locally if necessary.
 
+.PHONY: ghz
+ghz:
+	$(call go-install-tool,$(PROJECT_PATH)/bin/ghz,github.com/bojand/ghz/cmd/ghz@latest) 
+
+RPS?=1000
+.PHONY: load-test
+load-test: ghz
+	# see https://ghz.sh/docs/load for usage details
+	$(PROJECT_PATH)/bin/ghz 127.0.0.1:18081 --insecure \
+	--call envoy.service.ratelimit.v3.RateLimitService.ShouldRateLimit \
+	--async --concurrency=50 \
+	--rps=$(RPS) \
+	--total=$(RPS)0 \
+	--data-file load-test.json
+
 # go-install-tool will 'go install' any package $2 and install it to $1.
 define go-install-tool
 @[ -f $(1) ] || { \

diff --git a/limitador-server/sandbox/README.md b/limitador-server/sandbox/README.md
@@ -17,14 +17,40 @@ Check out `make help` for all the targets.
 
 ### Deployment options
 
-| Limitador's configuration | Command | Info |
-| ------------- | ----- | ----- |
-| In-memory configuration | `make deploy-in-memory` | Counters are held in Limitador (ephemeral) |
-| Redis | `make deploy-redis` | Uses Redis to store counters |
-| Redis Secured | `make deploy-redis-tls` | Uses Redis with TLS and password protected to store counters |
-| Redis Cached | `make deploy-redis-cached` | Uses Redis to store counters, with an in-memory cache |
-| Redis Otel Instrumented | `make deploy-redis-otel` | Uses redis to store counters, [instrumented with opentelemetry](redis-otel/README.md) |
-| Disk | `make deploy-disk` | Uses disk to store counters |
+| Limitador's configuration | Command | Info                                                                                                           |
+|--------------------------| ----- |----------------------------------------------------------------------------------------------------------------|
+| In-memory configuration  | `make deploy-in-memory` | Counters are held in Limitador (ephemeral)                                                                     |
+| Redis                    | `make deploy-redis` | Uses Redis to store counters                                                                                   |
+| Redis Secured            | `make deploy-redis-tls` | Uses Redis with TLS and password protected to store counters                                                   |
+| Redis Cached             | `make deploy-redis-cached` | Uses Redis to store counters, with an in-memory cache                                                          |
+| Redis Otel Instrumented  | `make deploy-redis-otel` | Uses redis to store counters, [instrumented with opentelemetry](redis-otel/README.md)                          |
+| Disk                     | `make deploy-disk` | Uses disk to store counters                                                                                    |
+| Distributed | `make deploy-distributed` | Counters are held in Limitador (ephemeral) but replicated to other Limitador servers. |
+
+| Distributed 3 Node | `make deploy-distributed-3-node` | Counters are held in Limitador (ephemeral) but replicated to 3 other Limitador servers. |
+
+### Running Multi Node Distributed Deployments
+
+The `make deploy-distributed` target can be connected to other Limitador servers but requires you to set the `PEER_ID` and `PEER_URLS` environment variables when you run the target.
+
+If you have 3 servers you want to replicate between, you would run the following commands:
+
+```bash
+# on server where: hostname=server1
+PEER_ID=`hostname` PEER_URLS="http://server2:15001 http://server3:15001" make deploy-distributed
+```
+
+```bash
+# on server where: hostname=server2
+PEER_ID=`hostname` PEER_URLS="http://server1:15001 http://server3:15001" make deploy-distributed
+```
+
+```bash
+# on server where: hostname=server3
+PEER_ID=`hostname` PEER_URLS="http://server1:15001 http://server2:15001" make deploy-distributed
+```
+
+The `PEER_ID` just need to be unique between the servers, and the `PEER_URLS` should be a space-separated list of the other servers' URLs.
 
 ### Limitador's admin HTTP endpoint
 
@@ -75,6 +101,10 @@ bin/grpcurl -plaintext -d @ 127.0.0.1:18081 envoy.service.ratelimit.v3.RateLimit
                 {
                     "key": "req.method",
                     "value": "POST"
+                },
+                {
+                    "key": "req.path",
+                    "value": "/"
                 }
             ]
         }
@@ -97,6 +127,10 @@ while :; do bin/grpcurl -plaintext -d @ 127.0.0.1:18081 envoy.service.ratelimit.
                 {
                     "key": "req.method",
                     "value": "POST"
+                },
+                {
+                    "key": "req.path",
+                    "value": "/"
                 }
             ]
         }
@@ -113,6 +147,24 @@ EOM
 curl -i -H "Host: example.com" http://127.0.0.1:18000/get
 ```
 
+### Load Testing the GRPC RateLimitService directly
+
+This load test will use `grpcurl`. You need [Go SDK](https://golang.org/doc/install) installed.
+
+Run a load test a 5000 requests per second (RPS) for 10 seconds:
+
+```bash
+RPS=5000 make load-test
+```
+
+### Load Testing via Envoy Proxy
+
+```bash
+cargo run --manifest-path loadtest/Cargo.toml  --package loadtest --release -- --report-file=report.htm
+```
+
+The report will be saved in `report.htm` file.
+
 ### Limitador Image
 
 By default, the sandbox will run Limitador's `limitador-testing:latest` image.

diff --git a/limitador-server/sandbox/docker-compose-envoy-3-node.yaml b/limitador-server/sandbox/docker-compose-envoy-3-node.yaml
@@ -0,0 +1,74 @@
+---
+version: '3.8'
+services:
+  envoy:
+    image: envoyproxy/envoy:v1.20-latest
+    depends_on:
+      - upstream
+      - limitador
+    command:
+      - /usr/local/bin/envoy
+      - --config-path
+      - /etc/envoy.yaml
+      - --log-level
+      - info
+      - --component-log-level
+      - http:debug,router:debug
+      - --service-cluster
+      - proxy
+    expose:
+      - "80"
+      - "8001"
+    ports:
+      - "18000:80"
+      - "18001:8001"
+    volumes:
+      - ./envoy.yaml:/etc/envoy.yaml
+  envoy2:
+    image: envoyproxy/envoy:v1.20-latest
+    depends_on:
+      - upstream
+      - limitador
+    command:
+      - /usr/local/bin/envoy
+      - --config-path
+      - /etc/envoy.yaml
+      - --log-level
+      - info
+      - --component-log-level
+      - http:debug,router:debug
+      - --service-cluster
+      - proxy
+    expose:
+      - "80"
+      - "8001"
+    ports:
+      - "18100:80"
+      - "18101:8001"
+    volumes:
+      - ./envoy.yaml:/etc/envoy.yaml
+  envoy3:
+    image: envoyproxy/envoy:v1.20-latest
+    depends_on:
+      - upstream
+      - limitador
+    command:
+      - /usr/local/bin/envoy
+      - --config-path
+      - /etc/envoy.yaml
+      - --log-level
+      - info
+      - --component-log-level
+      - http:debug,router:debug
+      - --service-cluster
+      - proxy
+    expose:
+      - "80"
+      - "8001"
+    ports:
+      - "18200:80"
+      - "18201:8001"
+    volumes:
+      - ./envoy.yaml:/etc/envoy.yaml
+  upstream:
+    image: kennethreitz/httpbin
diff --git a/limitador-server/sandbox/docker-compose-envoy.yaml b/limitador-server/sandbox/docker-compose-envoy.yaml
@@ -5,6 +5,7 @@ services:
     image: envoyproxy/envoy:v1.20-latest
     depends_on:
       - upstream
+      - limitador
     command:
       - /usr/local/bin/envoy
       - --config-path

diff --git a/limitador-server/sandbox/docker-compose-limitador-disk.yaml b/limitador-server/sandbox/docker-compose-limitador-disk.yaml
@@ -3,8 +3,9 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
-    depends_on:
-      - envoy
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     command:
       - limitador-server
       - --rls-ip

diff --git a/limitador-server/sandbox/docker-compose-limitador-distributed-3-node.yaml b/limitador-server/sandbox/docker-compose-limitador-distributed-3-node.yaml
@@ -0,0 +1,58 @@
+---
+version: '3.8'
+services:
+  limitador:
+    image: limitador-testing-all-features
+    build:
+      context: ../..
+      dockerfile: Dockerfile
+      args:
+        - CARGO_ARGS=--all-features
+    command: |
+      limitador-server --rls-ip 0.0.0.0 --rls-port 8081 --http-ip 0.0.0.0 --http-port "8080" 
+      -vv --grpc-reflection-service /opt/kuadrant/limits/limits.yaml 
+      distributed limitador 0.0.0.0:5001 http://limitador2:5001 http://limitador3:5001
+    expose:
+      - "8080"
+      - "8081"
+      - "5001"
+    ports:
+      - "18080:8080"
+      - "18081:8081"
+      - "15001:5001"
+    volumes:
+      - ./limits.yaml:/opt/kuadrant/limits/limits.yaml
+  limitador2:
+    image: limitador-testing-all-features
+    build:
+      context: ../..
+      dockerfile: Dockerfile
+      args:
+        - CARGO_ARGS=--all-features
+    command: |
+      limitador-server --rls-ip 0.0.0.0 --rls-port 8081 --http-ip 0.0.0.0 --http-port "8080" 
+      -vv --grpc-reflection-service /opt/kuadrant/limits/limits.yaml 
+      distributed limitador2 0.0.0.0:5001 http://limitador:5001 http://limitador3:5001
+    expose:
+      - "8080"
+      - "8081"
+      - "5001"
+    volumes:
+      - ./limits.yaml:/opt/kuadrant/limits/limits.yaml
+  limitador3:
+    image: limitador-testing-all-features
+    build:
+      context: ../..
+      dockerfile: Dockerfile
+      args:
+        - CARGO_ARGS=--all-features
+    command: |
+      limitador-server --rls-ip 0.0.0.0 --rls-port 8081 --http-ip 0.0.0.0 --http-port "8080" 
+      -vv --grpc-reflection-service /opt/kuadrant/limits/limits.yaml 
+      distributed limitador3 0.0.0.0:5001 http://limitador:5001 http://limitador2:5001
+    expose:
+      - "8080"
+      - "8081"
+      - "5001"
+    volumes:
+      - ./limits.yaml:/opt/kuadrant/limits/limits.yaml
diff --git a/limitador-server/sandbox/docker-compose-limitador-distributed.yaml b/limitador-server/sandbox/docker-compose-limitador-distributed.yaml
@@ -0,0 +1,24 @@
+---
+version: '3.8'
+services:
+  limitador:
+    image: limitador-testing-all-features
+    build:
+      context: ../..
+      dockerfile: Dockerfile
+      args:
+        - CARGO_ARGS=--all-features
+    command: |
+      limitador-server --rls-ip 0.0.0.0 --rls-port 8081 --http-ip 0.0.0.0 --http-port "8080" 
+      -vv --grpc-reflection-service /opt/kuadrant/limits/limits.yaml 
+      distributed ${PEER_ID:-node1} 0.0.0.0:5001 ${PEER_URLS:-}
+    expose:
+      - "8080"
+      - "8081"
+      - "5001"
+    ports:
+      - "18080:8080"
+      - "18081:8081"
+      - "15001:5001"
+    volumes:
+      - ./limits.yaml:/opt/kuadrant/limits/limits.yaml
diff --git a/limitador-server/sandbox/docker-compose-limitador-memory.yaml b/limitador-server/sandbox/docker-compose-limitador-memory.yaml
@@ -3,8 +3,9 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
-    depends_on:
-      - envoy
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     command:
       - limitador-server
       - --rls-ip

diff --git a/limitador-server/sandbox/docker-compose-limitador-redis-cached.yaml b/limitador-server/sandbox/docker-compose-limitador-redis-cached.yaml
@@ -3,8 +3,10 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     depends_on:
-      - envoy
       - redis
     command:
       - limitador-server

diff --git a/limitador-server/sandbox/docker-compose-limitador-redis-otel.yaml b/limitador-server/sandbox/docker-compose-limitador-redis-otel.yaml
@@ -3,9 +3,11 @@ version: '3.8'
 services:
   limitador:
     image: ${LIMITADOR_IMAGE:-limitador-testing}
+    build:
+      context: ../..
+      dockerfile: Dockerfile
     depends_on:
       - jaeger
-      - envoy
       - redis
     command:
       - limitador-server