diff --git a/tools/grpcurl-query-ingesters/README.md b/tools/grpcurl-query-ingesters/README.md index 6936f29c76b..ed27fee25a8 100644 --- a/tools/grpcurl-query-ingesters/README.md +++ b/tools/grpcurl-query-ingesters/README.md @@ -4,6 +4,7 @@ A simple hacky script + tool to download chunks from ingesters and dump their co # How to use it +1. Edit `pkg/ingester/client/ingester.proto` and change the `import "github.com/grafana/mimir/pkg/mimirpb/mimir.proto"` statement to `import "pkg/mimirpb/mimir.proto"` 1. Edit `download-chunks-from-ingesters-query.json` with the label matchers and time range to query. -2. Edit `download-chunks-from-ingesters.sh` with the configuration about the Kubernetes namespace and Mimir tenant to query. -3. Once you've got the dump (1 file per ingester), run the go tool in this directory to print the dump content of 1+ files. +1. Edit `download-chunks-from-ingesters.sh` with the configuration about the Kubernetes namespace and Mimir tenant to query. +1. Once you've got the dump (1 file per ingester), run the go tool in this directory to print the dump content of 1+ files. diff --git a/tools/grpcurl-query-ingesters/download-chunks-from-ingesters.sh b/tools/grpcurl-query-ingesters/download-chunks-from-ingesters.sh old mode 100644 new mode 100755 index 14bb1b32b82..33c57b22ab5 --- a/tools/grpcurl-query-ingesters/download-chunks-from-ingesters.sh +++ b/tools/grpcurl-query-ingesters/download-chunks-from-ingesters.sh @@ -7,7 +7,10 @@ K8S_NAMESPACE="" MIMIR_TENANT_ID="" # End of configuration. -mkdir -p chunks-dump/ +SCRIPT_DIR=$(realpath "$(dirname "${0}")") +OUTPUT_DIR="chunks-dump" + +mkdir -p "$OUTPUT_DIR" # Get list of pods. PODS=$(kubectl --context "$K8S_CONTEXT" -n "$K8S_NAMESPACE" get pods --no-headers | grep ingester | awk '{print $1}') @@ -22,14 +25,18 @@ for POD in $PODS; do # Wait some time sleep 5 - cat query.json | grpcurl \ + # HACK + # If you get an error resolving the reference to "github.com/grafana/mimir/pkg/mimirpb/mimir.proto" in + # pkg/ingester/client/ingester.proto, you need to manually modify the import statement to be just + # "pkg/mimirpb/mimir.proto". + cat "$SCRIPT_DIR/download-chunks-from-ingesters-query.json" | grpcurl \ -d @ \ -H "X-Scope-OrgID: $MIMIR_TENANT_ID" \ - -proto pkg/ingester/client/ingester.proto \ - -import-path . \ - -import-path ./vendor \ - -plaintext \ - localhost:9095 "cortex.Ingester/QueryStream" > "chunks-dump/$POD" + -proto pkg/ingester/client/ingester.proto \ + -import-path "$SCRIPT_DIR" \ + -import-path "$SCRIPT_DIR/vendor" \ + -plaintext \ + localhost:9095 "cortex.Ingester/QueryStream" > "$OUTPUT_DIR/$POD" kill $KUBECTL_PID wait $KUBECTL_PID diff --git a/tools/grpcurl-query-ingesters/main.go b/tools/grpcurl-query-ingesters/main.go index a8f2623e262..1ca068b01ee 100644 --- a/tools/grpcurl-query-ingesters/main.go +++ b/tools/grpcurl-query-ingesters/main.go @@ -5,8 +5,10 @@ package main import ( "bytes" "encoding/json" + "errors" "flag" "fmt" + "io" "os" "time" @@ -23,31 +25,41 @@ func main() { } for _, arg := range args { - res, err := parseFile(arg) + resps, err := parseFile(arg) if err != nil { fmt.Println("Failed to parse file:", err.Error()) os.Exit(1) } - dumpResponse(res) + for _, res := range resps { + dumpResponse(res) + } } } -func parseFile(file string) (QueryStreamResponse, error) { - res := QueryStreamResponse{} +func parseFile(file string) ([]QueryStreamResponse, error) { + resps := []QueryStreamResponse{} fileData, err := os.ReadFile(file) if err != nil { - return res, err + return nil, err } // Decode file. decoder := json.NewDecoder(bytes.NewReader(fileData)) - if err := decoder.Decode(&res); err != nil { - return res, err + for { + res := QueryStreamResponse{} + if err := decoder.Decode(&res); err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err + } + + resps = append(resps, res) } - return res, nil + return resps, nil } func dumpResponse(res QueryStreamResponse) {