From c46de61e988aade3af52646fca6fe05e86fe4cbc Mon Sep 17 00:00:00 2001 From: Tate Date: Thu, 13 Jun 2024 17:05:20 -0600 Subject: [PATCH 01/20] Add logging of requests to file and/or to stdout --- Dockerfile | 4 +- Makefile | 10 +- README.md | 4 + internal/app/cmd/cmd.go | 46 ++++--- internal/config.go | 32 ++--- internal/config_test.go | 4 +- internal/server/http/dump.go | 116 ++++++++++++++++++ internal/server/http/server.go | 55 ++++++--- internal/server/http/server_test.go | 80 ++++++++++-- .../testdata/imposters/test_request.imp.yaml | 9 +- 10 files changed, 302 insertions(+), 58 deletions(-) create mode 100644 internal/server/http/dump.go diff --git a/Dockerfile b/Dockerfile index 85e5ad7..dd3171e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,12 +4,14 @@ LABEL MAINTAINER = 'Friends of Go (it@friendsofgo.tech)' ARG TARGETOS=linux ARG TARGETARCH=amd64 +ARG TAG="" RUN apk add --update git RUN apk add ca-certificates WORKDIR /go/src/github.com/friendsofgo/killgrave COPY . . -RUN go mod tidy && TAG=$(git describe --tags --abbrev=0) \ +RUN go mod tidy \ + && if [ -z "$TAG" ]; then TAG=$(git describe --tags --abbrev=0); fi \ && LDFLAGS=$(echo "-s -w -X github.com/friendsofgo/killgrave/internal/app/cmd._version="docker-$TAG) \ && CGO_ENABLED=0 GOOS="${TARGETOS}" GOARCH="${TARGETARCH}" go build -a -installsuffix cgo -o /go/bin/killgrave -ldflags "$LDFLAGS" cmd/killgrave/main.go diff --git a/Makefile b/Makefile index b773558..6f58600 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,11 @@ .PHONY: build build: - go build -ldflags "-s -w -X 'github.com/friendsofgo/killgrave/internal/app/cmd._version=`git rev-parse --abbrev-ref HEAD`-`git rev-parse --short HEAD`'" -o bin/killgrave cmd/killgrave/main.go \ No newline at end of file + go build -ldflags "-s -w -X 'github.com/friendsofgo/killgrave/internal/app/cmd._version=`git rev-parse --abbrev-ref HEAD`-`git rev-parse --short HEAD`'" -o bin/killgrave cmd/killgrave/main.go + +.PHONY: build-docker +build-docker: + docker build --build-arg TAG=$(TAG) -t killgrave:$(TAG) . + +.PHONY: test +test: + go test -v -vet=off -race ./... diff --git a/README.md b/README.md index 8b12570..7c93d66 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,8 @@ Flags: -s, --secure Run mock server using TLS (https) -v, --version Version of Killgrave -w, --watcher File watcher will reload the server on each file change + -V, --verbose Print out more detailed logging + -d, --dump-requests-path Print requests out to specified file ``` ### Using Killgrave by config file @@ -182,6 +184,8 @@ cors: allow_credentials: true watcher: true secure: true +verbose: false +dump-requests-path: "/abc/def.log ``` As you can see, you can configure all the options in a very easy way. For the above example, the file tree looks as follows, with the current working directory being `mymock`. diff --git a/internal/app/cmd/cmd.go b/internal/app/cmd/cmd.go index ac41447..80c4547 100644 --- a/internal/app/cmd/cmd.go +++ b/internal/app/cmd/cmd.go @@ -28,22 +28,26 @@ const ( _defaultProxyMode = killgrave.ProxyNone _defaultStrictSlash = true - _impostersFlag = "imposters" - _configFlag = "config" - _hostFlag = "host" - _portFlag = "port" - _watcherFlag = "watcher" - _secureFlag = "secure" - _proxyModeFlag = "proxy-mode" - _proxyURLFlag = "proxy-url" + _impostersFlag = "imposters" + _configFlag = "config" + _hostFlag = "host" + _portFlag = "port" + _watcherFlag = "watcher" + _secureFlag = "secure" + _proxyModeFlag = "proxy-mode" + _proxyURLFlag = "proxy-url" + _verboseFlag = "verbose" + _dumpRequestsPathFlag = "dump-requests-path" ) var ( - errGetDataFromImpostersFlag = errors.New("error trying to get data from imposters flag") - errGetDataFromHostFlag = errors.New("error trying to get data from host flag") - errGetDataFromPortFlag = errors.New("error trying to get data from port flag") - errGetDataFromSecureFlag = errors.New("error trying to get data from secure flag") - errMandatoryURL = errors.New("the field proxy-url is mandatory if you selected a proxy mode") + errGetDataFromImpostersFlag = errors.New("error trying to get data from imposters flag") + errGetDataFromHostFlag = errors.New("error trying to get data from host flag") + errGetDataFromPortFlag = errors.New("error trying to get data from port flag") + errGetDataFromSecureFlag = errors.New("error trying to get data from secure flag") + errGetDataFromVerboseFlag = errors.New("error trying to get data from verbose flag") + errGetDataFromDumpRequestsPathFlag = errors.New("error trying to get data from dump-requests-path flag") + errMandatoryURL = errors.New("the field proxy-url is mandatory if you selected a proxy mode") ) // NewKillgraveCmd returns cobra.Command to run killgrave command @@ -78,6 +82,8 @@ func NewKillgraveCmd() *cobra.Command { rootCmd.Flags().BoolP(_secureFlag, "s", false, "Run mock server using TLS (https)") rootCmd.Flags().StringP(_proxyModeFlag, "m", _defaultProxyMode.String(), "Proxy mode, the options are all, missing or none") rootCmd.Flags().StringP(_proxyURLFlag, "u", "", "The url where the proxy will redirect to") + rootCmd.Flags().BoolP(_verboseFlag, "V", false, "More verbose logging, adds request dumps to logs") + rootCmd.Flags().StringP(_dumpRequestsPathFlag, "d", "", "Path the requests will be dumped to") rootCmd.SetVersionTemplate("Killgrave version: {{.Version}}\n") @@ -133,6 +139,8 @@ func runServer(cfg killgrave.Config) server.Server { proxyServer, cfg.Secure, imposterFs, + cfg.Verbose, + cfg.DumpRequestsPath, ) if err := s.Build(); err != nil { log.Fatal(err) @@ -183,7 +191,17 @@ func prepareConfig(cmd *cobra.Command) (killgrave.Config, error) { return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromSecureFlag) } - cfg, err := killgrave.NewConfig(impostersPath, host, port, secure) + verbose, err := cmd.Flags().GetBool(_verboseFlag) + if err != nil { + return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromVerboseFlag) + } + + dumpRequestsPath, err := cmd.Flags().GetString(_dumpRequestsPathFlag) + if err != nil { + return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromDumpRequestsPathFlag) + } + + cfg, err := killgrave.NewConfig(impostersPath, host, port, secure, verbose, dumpRequestsPath) if err != nil { return killgrave.Config{}, err } diff --git a/internal/config.go b/internal/config.go index e2b1e70..c6b6df1 100644 --- a/internal/config.go +++ b/internal/config.go @@ -3,7 +3,7 @@ package killgrave import ( "errors" "fmt" - "io/ioutil" + "io" "os" "path" @@ -12,13 +12,15 @@ import ( // Config representation of config file yaml type Config struct { - ImpostersPath string `yaml:"imposters_path"` - Port int `yaml:"port"` - Host string `yaml:"host"` - CORS ConfigCORS `yaml:"cors"` - Proxy ConfigProxy `yaml:"proxy"` - Secure bool `yaml:"secure"` - Watcher bool `yaml:"watcher"` + ImpostersPath string `yaml:"imposters_path"` + Port int `yaml:"port"` + Host string `yaml:"host"` + CORS ConfigCORS `yaml:"cors"` + Proxy ConfigProxy `yaml:"proxy"` + Secure bool `yaml:"secure"` + Watcher bool `yaml:"watcher"` + Verbose bool `yaml:"verbose"` + DumpRequestsPath string `yaml:"dump_requests_path"` } // ConfigCORS representation of section CORS of the yaml @@ -111,7 +113,7 @@ func (cfg *Config) ConfigureProxy(proxyMode ProxyMode, proxyURL string) { type ConfigOpt func(cfg *Config) error // NewConfig initialize the config -func NewConfig(impostersPath, host string, port int, secure bool) (Config, error) { +func NewConfig(impostersPath, host string, port int, secure, verbose bool, dumpRequestsPath string) (Config, error) { if impostersPath == "" { return Config{}, errEmptyImpostersPath } @@ -125,10 +127,12 @@ func NewConfig(impostersPath, host string, port int, secure bool) (Config, error } cfg := Config{ - ImpostersPath: impostersPath, - Host: host, - Port: port, - Secure: secure, + ImpostersPath: impostersPath, + Host: host, + Port: port, + Secure: secure, + Verbose: verbose, + DumpRequestsPath: dumpRequestsPath, } return cfg, nil @@ -146,7 +150,7 @@ func NewConfigFromFile(cfgPath string) (Config, error) { defer configFile.Close() var cfg Config - bytes, _ := ioutil.ReadAll(configFile) + bytes, _ := io.ReadAll(configFile) if err := yaml.Unmarshal(bytes, &cfg); err != nil { return Config{}, fmt.Errorf("%w: error while unmarshalling configFile file %s, using default configuration instead", err, cfgPath) } diff --git a/internal/config_test.go b/internal/config_test.go index 38c0076..6fb96df 100644 --- a/internal/config_test.go +++ b/internal/config_test.go @@ -203,7 +203,7 @@ func TestNewConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewConfig(tt.args.impostersPath, tt.args.host, tt.args.port, false) + got, err := NewConfig(tt.args.impostersPath, tt.args.host, tt.args.port, false, false, "") assert.Equal(t, tt.err, err) assert.Equal(t, tt.want, got) }) @@ -221,7 +221,7 @@ func TestConfig_ConfigureProxy(t *testing.T) { }, } - got, err := NewConfig("imposters", "localhost", 80, false) + got, err := NewConfig("imposters", "localhost", 80, false, false, "") assert.NoError(t, err) got.ConfigureProxy(ProxyAll, "https://friendsofgo.tech") diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go new file mode 100644 index 0000000..b73693e --- /dev/null +++ b/internal/server/http/dump.go @@ -0,0 +1,116 @@ +package http + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "net/http/httputil" + "os" + "strings" +) + +// RequestData struct to hold request data +type RequestData struct { + Method string `json:"method"` + Host string `json:"host"` + URL string `json:"url"` + Header map[string][]string `json:"header"` + Body string `json:"body"` +} + +func GetRequestData(r *http.Request) *RequestData { + bodyBytes, err := io.ReadAll(r.Body) + if err != nil { + log.Printf("Error reading request body: %v", err) + return nil + } + r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) // Reset the body + + return &RequestData{ + Method: r.Method, + Host: r.Host, + URL: r.URL.String(), + Header: r.Header, + Body: string(bodyBytes), + } +} + +func LogRequest(r *RequestData, s *Server) { + if !s.verbose { + return + } + + // rebuild the request + req, err := http.NewRequest(r.Method, r.URL, bytes.NewBufferString(r.Body)) + req.Host = r.Host + if err != nil { + log.Printf("failed to create request for logging: %+v", err) + return + } + for key, values := range r.Header { + for _, value := range values { + req.Header.Add(key, value) + } + } + + dumped, err := httputil.DumpRequest(req, true) + if err != nil { + log.Printf("failed to dump request: %v", err) + } + if dumped != nil { + log.Println(string(dumped)) + } + +} + +func RecordRequest(r *RequestData, s *Server) { + if len(s.dumpRequestsPath) < 1 || s.dumpCh == nil { + return + } + s.dumpCh <- r +} + +// Goroutine function to write requests to a JSON file +func RequestWriter(filePath string, requestChan <-chan *RequestData) { + file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatalf("Failed to open file: %+v", err) + } + defer file.Close() + + encoder := json.NewEncoder(file) + for requestData := range requestChan { + if err := encoder.Encode(requestData); err != nil { + log.Printf("Failed to write to file: %+v", err) + } + } +} + +// GetRecordedRequests reads the requests from the file and returns them as a slice of RequestData +func getRecordedRequests(filePath string) ([]RequestData, error) { + // Read the file contents + fileContent, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("error reading file: %w", err) + } + + // Split the contents by the newline separator + requestDumps := strings.Split(string(fileContent), "\n") + requestsData := []RequestData{} + for _, requestDump := range requestDumps { + if requestDump == "" { + continue + } + // Unmarshal the JSON string into the RequestData struct + rd := RequestData{} + err := json.Unmarshal([]byte(requestDump), &rd) + if err != nil { + return nil, fmt.Errorf("error unmarshaling JSON: %w", err) + } + requestsData = append(requestsData, rd) + } + return requestsData, nil +} diff --git a/internal/server/http/server.go b/internal/server/http/server.go index f34d438..ae6f941 100644 --- a/internal/server/http/server.go +++ b/internal/server/http/server.go @@ -31,23 +31,28 @@ type ServerOpt func(s *Server) // Server definition of mock server type Server struct { - impostersPath string - router *mux.Router - httpServer *http.Server - proxy *Proxy - secure bool - imposterFs ImposterFs + impostersPath string + router *mux.Router + httpServer *http.Server + proxy *Proxy + secure bool + imposterFs ImposterFs + verbose bool + dumpRequestsPath string + dumpCh chan *RequestData } // NewServer initialize the mock server -func NewServer(p string, r *mux.Router, httpServer *http.Server, proxyServer *Proxy, secure bool, fs ImposterFs) Server { +func NewServer(p string, r *mux.Router, httpServer *http.Server, proxyServer *Proxy, secure bool, fs ImposterFs, verbose bool, dumpRequestsPath string) Server { return Server{ - impostersPath: p, - router: r, - httpServer: httpServer, - proxy: proxyServer, - secure: secure, - imposterFs: fs, + impostersPath: p, + router: r, + httpServer: httpServer, + proxy: proxyServer, + secure: secure, + imposterFs: fs, + verbose: verbose, + dumpRequestsPath: dumpRequestsPath, } } @@ -95,6 +100,12 @@ func (s *Server) Build() error { var impostersCh = make(chan []Imposter) var done = make(chan struct{}) + // only intantiate the request dump if we need it + if s.dumpCh == nil && len(s.dumpRequestsPath) > 0 { + s.dumpCh = make(chan *RequestData, 100) + go RequestWriter(s.dumpRequestsPath, s.dumpCh) + } + go func() { s.imposterFs.FindImposters(s.impostersPath, impostersCh) done <- struct{}{} @@ -162,7 +173,7 @@ func (s *Server) Shutdown() error { func (s *Server) addImposterHandler(imposters []Imposter) { for _, imposter := range imposters { - r := s.router.HandleFunc(imposter.Request.Endpoint, ImposterHandler(imposter)). + r := s.router.HandleFunc(imposter.Request.Endpoint, s.handlerMiddleWare(imposter)). Methods(imposter.Request.Method). MatcherFunc(MatcherBySchema(imposter)) @@ -180,6 +191,18 @@ func (s *Server) addImposterHandler(imposters []Imposter) { } } -func (s *Server) handleAll(h http.HandlerFunc) { - s.router.PathPrefix("/").HandlerFunc(h) +func (s *Server) handlerMiddleWare(i Imposter) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // handle anything we need before the request is used + // We need to grab the request data before the request is used since the body may be empty + requestData := GetRequestData(r) + + // handle the imposter reqest + handler := ImposterHandler(i) + handler(w, r) + + // handle anything we need after the request + LogRequest(requestData, s) + RecordRequest(requestData, s) + } } diff --git a/internal/server/http/server_test.go b/internal/server/http/server_test.go index ba16c0c..93a9484 100644 --- a/internal/server/http/server_test.go +++ b/internal/server/http/server_test.go @@ -1,14 +1,16 @@ package http import ( + "bytes" "crypto/tls" "errors" "io" - "io/ioutil" "log" "net/http" "net/http/httptest" "os" + "path/filepath" + "strings" "testing" "time" @@ -19,7 +21,7 @@ import ( ) func TestMain(m *testing.M) { - log.SetOutput(ioutil.Discard) + log.SetOutput(io.Discard) os.Exit(m.Run()) } @@ -31,9 +33,9 @@ func TestServer_Build(t *testing.T) { server Server err error }{ - {"imposter directory not found", NewServer("failImposterPath", nil, &http.Server{}, &Proxy{}, false, imposterFs), errors.New("hello")}, - {"malformed json", NewServer("test/testdata/malformatted_imposters", nil, &http.Server{}, &Proxy{}, false, imposterFs), nil}, - {"valid imposter", NewServer("test/testdata/imposters", mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs), nil}, + {"imposter directory not found", NewServer("failImposterPath", nil, &http.Server{}, &Proxy{}, false, imposterFs, false, ""), errors.New("hello")}, + {"malformed json", NewServer("test/testdata/malformatted_imposters", nil, &http.Server{}, &Proxy{}, false, imposterFs, false, ""), nil}, + {"valid imposter", NewServer("test/testdata/imposters", mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, false, ""), nil}, } for _, tt := range serverData { @@ -60,7 +62,7 @@ func TestBuildProxyMode(t *testing.T) { proxyServer, err := NewProxy(proxyServer.URL, mode) assert.Nil(t, err) imposterFs := NewImposterFS(afero.NewOsFs()) - server := NewServer("test/testdata/imposters", router, httpServer, proxyServer, false, imposterFs) + server := NewServer("test/testdata/imposters", router, httpServer, proxyServer, false, imposterFs, false, "") return &server, func() { httpServer.Close() } @@ -113,7 +115,7 @@ func TestBuildProxyMode(t *testing.T) { s.router.ServeHTTP(w, req) response := w.Result() - body, _ := ioutil.ReadAll(response.Body) + body, _ := io.ReadAll(response.Body) assert.Equal(t, tc.body, string(body)) assert.Equal(t, tc.status, response.StatusCode) @@ -136,7 +138,7 @@ func TestBuildSecureMode(t *testing.T) { proxyServer, err := NewProxy(proxyServer.URL, mode) assert.Nil(t, err) imposterFs := NewImposterFS(afero.NewOsFs()) - server := NewServer("test/testdata/imposters_secure", router, httpServer, proxyServer, true, imposterFs) + server := NewServer("test/testdata/imposters_secure", router, httpServer, proxyServer, true, imposterFs, false, "") return &server, func() { httpServer.Close() } @@ -187,7 +189,7 @@ func TestBuildSecureMode(t *testing.T) { defer response.Body.Close() - body, err := ioutil.ReadAll(response.Body) + body, err := io.ReadAll(response.Body) if err != nil { return false } @@ -197,3 +199,63 @@ func TestBuildSecureMode(t *testing.T) { }) } } + +func TestBuildLogRequests(t *testing.T) { + var buf bytes.Buffer + log.SetOutput(&buf) + defer func() { + log.SetOutput(os.Stderr) + }() + + imposterFs := NewImposterFS(afero.NewOsFs()) + server := NewServer("test/testdata/imposters", mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, true, "") + err := server.Build() + assert.NoError(t, err) + + expectedBody := "Dumped" + expectedLog := "GET /yamlTestDumpRequest HTTP/1.1\r\nHost: example.com\r\n\r\nDumped" + w := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(expectedBody)) + + server.router.ServeHTTP(w, req) + response := w.Result() + assert.Equal(t, http.StatusOK, response.StatusCode, "Expected status code: %v, got: %v", http.StatusOK, response.StatusCode) + + // verify the request is dumped in the logs + assert.Contains(t, buf.String(), expectedLog, "Expect request dumped on logs failed") +} + +func TestBuildRecordRequests(t *testing.T) { + var buf bytes.Buffer + log.SetOutput(&buf) + defer func() { + log.SetOutput(os.Stderr) + }() + tempDir, err := os.MkdirTemp("", "testdir") + assert.NoError(t, err, "Failed to create temporary directory") + defer os.RemoveAll(tempDir) + dumpFile := filepath.Join(tempDir, "dump_requests.log") + + imposterFs := NewImposterFS(afero.NewOsFs()) + w := httptest.NewRecorder() + server := NewServer("test/testdata/imposters", mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, true, dumpFile) + err = server.Build() + assert.NoError(t, err) + + expectedBodies := []string{"Dumped1", "Dumped2"} + req1 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(expectedBodies[0])) + req2 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(expectedBodies[1])) + server.router.ServeHTTP(w, req1) + server.router.ServeHTTP(w, req2) + + // wait for channel to print out the requests + time.Sleep(1 * time.Second) + + // check recoreded request dumps + reqs, err := getRecordedRequests(dumpFile) + assert.NoError(t, err, "Failed to read requests from file") + assert.Equal(t, 2, len(reqs), "Expect 2 requests to be dumped in file failed") + for i, expectedBody := range expectedBodies { + assert.Equal(t, expectedBody, reqs[i].Body, "Expect request body to be dumped in file failed") + } +} diff --git a/internal/server/http/test/testdata/imposters/test_request.imp.yaml b/internal/server/http/test/testdata/imposters/test_request.imp.yaml index 0d49b8e..fcb2f70 100644 --- a/internal/server/http/test/testdata/imposters/test_request.imp.yaml +++ b/internal/server/http/test/testdata/imposters/test_request.imp.yaml @@ -4,4 +4,11 @@ endpoint: /yamlTestRequest response: status: 200 - body: "Yaml Handled" \ No newline at end of file + body: "Yaml Handled" +- request: + method: GET + endpoint: /yamlTestDumpRequest + dump: true + response: + status: 200 + body: "Yaml Dump Handled" From e8dff5fccc940519d3296115d3abb51623585970 Mon Sep 17 00:00:00 2001 From: Tate Date: Fri, 14 Jun 2024 15:49:44 -0600 Subject: [PATCH 02/20] merge conflict fixes --- internal/server/http/server_test.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/internal/server/http/server_test.go b/internal/server/http/server_test.go index 036262c..5f6ead7 100644 --- a/internal/server/http/server_test.go +++ b/internal/server/http/server_test.go @@ -26,7 +26,7 @@ func TestMain(m *testing.M) { func TestServer_Build(t *testing.T) { newServer := func(fs ImposterFs) Server { - return NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, fs) + return NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, fs, false, "") } testCases := map[string]struct { @@ -70,7 +70,7 @@ func TestBuildProxyMode(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters") require.NoError(t, err) - server := NewServer(router, httpServer, proxyServer, false, imposterFs) + server := NewServer(router, httpServer, proxyServer, false, imposterFs, false, "") return &server, func() error { return httpServer.Close() } @@ -151,7 +151,7 @@ func TestBuildSecureMode(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters_secure") require.NoError(t, err) - server := NewServer(router, httpServer, proxyServer, true, imposterFs) + server := NewServer(router, httpServer, proxyServer, true, imposterFs, false, "") return &server, func() { httpServer.Close() } @@ -221,9 +221,10 @@ func TestBuildLogRequests(t *testing.T) { log.SetOutput(os.Stderr) }() - imposterFs := NewImposterFS(afero.NewOsFs()) - server := NewServer("test/testdata/imposters", mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, true, "") - err := server.Build() + imposterFs, err := NewImposterFS("test/testdata/imposters") + assert.NoError(t, err) + server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, true, "") + err = server.Build() assert.NoError(t, err) expectedBody := "Dumped" @@ -250,9 +251,10 @@ func TestBuildRecordRequests(t *testing.T) { defer os.RemoveAll(tempDir) dumpFile := filepath.Join(tempDir, "dump_requests.log") - imposterFs := NewImposterFS(afero.NewOsFs()) + imposterFs, err := NewImposterFS("test/testdata/imposters") + assert.NoError(t, err) w := httptest.NewRecorder() - server := NewServer("test/testdata/imposters", mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, true, dumpFile) + server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, true, dumpFile) err = server.Build() assert.NoError(t, err) From 3c18bdc8d89f7f4a7ff2f7d14517f6ebd20b5312 Mon Sep 17 00:00:00 2001 From: Tate Date: Sat, 22 Jun 2024 09:47:08 -0600 Subject: [PATCH 03/20] Handle error in config for io.ReadAll --- internal/config.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/config.go b/internal/config.go index c6b6df1..81dfb01 100644 --- a/internal/config.go +++ b/internal/config.go @@ -150,7 +150,10 @@ func NewConfigFromFile(cfgPath string) (Config, error) { defer configFile.Close() var cfg Config - bytes, _ := io.ReadAll(configFile) + bytes, err := io.ReadAll(configFile) + if err != nil { + return Config{}, fmt.Errorf("%w: error while reading configFile file %s, using default configuration instead", err, cfgPath) + } if err := yaml.Unmarshal(bytes, &cfg); err != nil { return Config{}, fmt.Errorf("%w: error while unmarshalling configFile file %s, using default configuration instead", err, cfgPath) } From 72b3f4e58185d1dc0427eb12caa58ca411b47cc0 Mon Sep 17 00:00:00 2001 From: Tate Date: Sat, 22 Jun 2024 09:49:02 -0600 Subject: [PATCH 04/20] - Sync file wrties in the RequestWriter - Handle cases where the dumpCh is full - Use bytes.NewReader --- internal/server/http/dump.go | 22 +++++++++++++++------- internal/server/http/server.go | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go index b73693e..b0629fa 100644 --- a/internal/server/http/dump.go +++ b/internal/server/http/dump.go @@ -14,11 +14,11 @@ import ( // RequestData struct to hold request data type RequestData struct { - Method string `json:"method"` - Host string `json:"host"` - URL string `json:"url"` - Header map[string][]string `json:"header"` - Body string `json:"body"` + Method string `json:"method"` + Host string `json:"host"` + URL string `json:"url"` + Header http.Header `json:"header"` + Body string `json:"body"` } func GetRequestData(r *http.Request) *RequestData { @@ -27,7 +27,7 @@ func GetRequestData(r *http.Request) *RequestData { log.Printf("Error reading request body: %v", err) return nil } - r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) // Reset the body + r.Body = io.NopCloser(bytes.NewReader(bodyBytes)) // Reset the body return &RequestData{ Method: r.Method, @@ -70,7 +70,14 @@ func RecordRequest(r *RequestData, s *Server) { if len(s.dumpRequestsPath) < 1 || s.dumpCh == nil { return } - s.dumpCh <- r + select { + case s.dumpCh <- r: + // Successfully sent the request data to the channel + default: + // Handle the case where the channel is full + log.Println("Channel is full, dropping request and logging it instead:") + LogRequest(r, &Server{verbose: true}) + } } // Goroutine function to write requests to a JSON file @@ -86,6 +93,7 @@ func RequestWriter(filePath string, requestChan <-chan *RequestData) { if err := encoder.Encode(requestData); err != nil { log.Printf("Failed to write to file: %+v", err) } + file.Sync() } } diff --git a/internal/server/http/server.go b/internal/server/http/server.go index 89da077..9e8538e 100644 --- a/internal/server/http/server.go +++ b/internal/server/http/server.go @@ -95,7 +95,7 @@ func (s *Server) Build() error { // only intantiate the request dump if we need it if s.dumpCh == nil && len(s.dumpRequestsPath) > 0 { - s.dumpCh = make(chan *RequestData, 100) + s.dumpCh = make(chan *RequestData, 1000) go RequestWriter(s.dumpRequestsPath, s.dumpCh) } From acaa33c762ae837fa4b5a453fd400e85795b1429 Mon Sep 17 00:00:00 2001 From: Tate Date: Mon, 24 Jun 2024 16:24:48 -0600 Subject: [PATCH 05/20] Wrap request gorilla/handlers#CustomLoggingHandler to log requests Base64 encode the body and attach it to the log that is output if there is a body --- internal/app/cmd/cmd.go | 5 +- internal/server/http/dump.go | 190 +++++++++++++++++++++++----- internal/server/http/server.go | 29 ++--- internal/server/http/server_test.go | 102 +++++++++++---- 4 files changed, 246 insertions(+), 80 deletions(-) diff --git a/internal/app/cmd/cmd.go b/internal/app/cmd/cmd.go index fcfa67e..c2c0d6f 100644 --- a/internal/app/cmd/cmd.go +++ b/internal/app/cmd/cmd.go @@ -11,7 +11,6 @@ import ( killgrave "github.com/friendsofgo/killgrave/internal" server "github.com/friendsofgo/killgrave/internal/server/http" - "github.com/gorilla/handlers" "github.com/gorilla/mux" "github.com/radovskyb/watcher" "github.com/spf13/cobra" @@ -121,8 +120,7 @@ func runServer(cfg killgrave.Config) server.Server { httpAddr := fmt.Sprintf("%s:%d", cfg.Host, cfg.Port) httpServer := http.Server{ - Addr: httpAddr, - Handler: handlers.CORS(server.PrepareAccessControl(cfg.CORS)...)(router), + Addr: httpAddr, } proxyServer, err := server.NewProxy(cfg.Proxy.Url, cfg.Proxy.Mode) @@ -141,6 +139,7 @@ func runServer(cfg killgrave.Config) server.Server { proxyServer, cfg.Secure, imposterFs, + server.PrepareAccessControl(cfg.CORS), cfg.Verbose, cfg.DumpRequestsPath, ) diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go index b0629fa..861bfb4 100644 --- a/internal/server/http/dump.go +++ b/internal/server/http/dump.go @@ -2,14 +2,21 @@ package http import ( "bytes" + "encoding/base64" "encoding/json" "fmt" "io" "log" + "net" "net/http" - "net/http/httputil" + "net/url" "os" + "strconv" "strings" + "time" + "unicode/utf8" + + "github.com/gorilla/handlers" ) // RequestData struct to hold request data @@ -21,63 +28,184 @@ type RequestData struct { Body string `json:"body"` } -func GetRequestData(r *http.Request) *RequestData { - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - log.Printf("Error reading request body: %v", err) - return nil - } - r.Body = io.NopCloser(bytes.NewReader(bodyBytes)) // Reset the body - +func getRequestData(r *http.Request, body string) *RequestData { return &RequestData{ Method: r.Method, Host: r.Host, URL: r.URL.String(), Header: r.Header, - Body: string(bodyBytes), + Body: body, } } -func LogRequest(r *RequestData, s *Server) { - if !s.verbose { - return - } +// Copied from the gorilla/handlers package +const lowerhex = "0123456789abcdef" - // rebuild the request - req, err := http.NewRequest(r.Method, r.URL, bytes.NewBufferString(r.Body)) - req.Host = r.Host - if err != nil { - log.Printf("failed to create request for logging: %+v", err) - return +// Copied from the gorilla/handlers package +func appendQuoted(buf []byte, s string) []byte { + var runeTmp [utf8.UTFMax]byte + for width := 0; len(s) > 0; s = s[width:] { + r := rune(s[0]) + width = 1 + if r >= utf8.RuneSelf { + r, width = utf8.DecodeRuneInString(s) + } + if width == 1 && r == utf8.RuneError { + buf = append(buf, `\x`...) + buf = append(buf, lowerhex[s[0]>>4]) + buf = append(buf, lowerhex[s[0]&0xF]) + continue + } + if r == rune('"') || r == '\\' { // always backslashed + buf = append(buf, '\\') + buf = append(buf, byte(r)) + continue + } + if strconv.IsPrint(r) { + n := utf8.EncodeRune(runeTmp[:], r) + buf = append(buf, runeTmp[:n]...) + continue + } + switch r { + case '\a': + buf = append(buf, `\a`...) + case '\b': + buf = append(buf, `\b`...) + case '\f': + buf = append(buf, `\f`...) + case '\n': + buf = append(buf, `\n`...) + case '\r': + buf = append(buf, `\r`...) + case '\t': + buf = append(buf, `\t`...) + case '\v': + buf = append(buf, `\v`...) + default: + switch { + case r < ' ': + buf = append(buf, `\x`...) + buf = append(buf, lowerhex[s[0]>>4]) + buf = append(buf, lowerhex[s[0]&0xF]) + case r > utf8.MaxRune: + r = 0xFFFD + fallthrough + case r < 0x10000: + buf = append(buf, `\u`...) + for s := 12; s >= 0; s -= 4 { + buf = append(buf, lowerhex[r>>uint(s)&0xF]) + } + default: + buf = append(buf, `\U`...) + for s := 28; s >= 0; s -= 4 { + buf = append(buf, lowerhex[r>>uint(s)&0xF]) + } + } + } } - for key, values := range r.Header { - for _, value := range values { - req.Header.Add(key, value) + return buf +} + +// Copied from the gorilla/handlers package +func buildCommonLogLine(req *http.Request, url url.URL, ts time.Time, status int, size int) []byte { + username := "-" + if url.User != nil { + if name := url.User.Username(); name != "" { + username = name } } - dumped, err := httputil.DumpRequest(req, true) + host, _, err := net.SplitHostPort(req.RemoteAddr) if err != nil { - log.Printf("failed to dump request: %v", err) + host = req.RemoteAddr } - if dumped != nil { - log.Println(string(dumped)) + + uri := req.RequestURI + + // Requests using the CONNECT method over HTTP/2.0 must use + // the authority field (aka r.Host) to identify the target. + // Refer: https://httpwg.github.io/specs/rfc7540.html#CONNECT + if req.ProtoMajor == 2 && req.Method == "CONNECT" { + uri = req.Host + } + if uri == "" { + uri = url.RequestURI() + } + + buf := make([]byte, 0, 3*(len(host)+len(username)+len(req.Method)+len(uri)+len(req.Proto)+50)/2) + buf = append(buf, host...) + buf = append(buf, " - "...) + buf = append(buf, username...) + buf = append(buf, " ["...) + buf = append(buf, ts.Format("02/Jan/2006:15:04:05 -0700")...) + buf = append(buf, `] "`...) + buf = append(buf, req.Method...) + buf = append(buf, " "...) + buf = appendQuoted(buf, uri) + buf = append(buf, " "...) + buf = append(buf, req.Proto...) + buf = append(buf, `" `...) + buf = append(buf, strconv.Itoa(status)...) + buf = append(buf, " "...) + buf = append(buf, strconv.Itoa(size)...) + return buf +} + +// copied from gorilla/handlers and modified to add the body +// writeLog writes a log entry for req to w in Apache Common Log Format. +// ts is the timestamp with which the entry should be logged. +// status and size are used to provide the response HTTP status and size. +func writeLog(writer io.Writer, params handlers.LogFormatterParams, body string) { + buf := buildCommonLogLine(params.Request, params.URL, params.TimeStamp, params.StatusCode, params.Size) + // Append body if present + if len(body) > 0 { + buf = append(buf, " "...) + buf = append(buf, body...) } + buf = append(buf, '\n') + writer.Write(buf) +} +// CustomLoggingHandler provides a way to supply a custom log formatter +// while taking advantage of the mechanisms in this package +func CustomLoggingHandler(out io.Writer, h http.Handler, s *Server) http.Handler { + return handlers.CustomLoggingHandler(out, h, func(writer io.Writer, params handlers.LogFormatterParams) { + bodyBytes, err := io.ReadAll(params.Request.Body) + if err != nil { + log.Printf("Error reading request body: %v\n", err) + return + } + params.Request.Body = io.NopCloser(bytes.NewReader(bodyBytes)) // Reset the body + base64Body := base64.StdEncoding.EncodeToString(bodyBytes) + verbose := s.verbose + + // record request, if error set verbose to true to log current request since + // it didn't make it into a full channel + err = recordRequest(params.Request, s, base64Body) + if err != nil { + verbose = true + } + + if verbose { + writeLog(writer, params, base64Body) + } + }) } -func RecordRequest(r *RequestData, s *Server) { +func recordRequest(r *http.Request, s *Server, body string) error { if len(s.dumpRequestsPath) < 1 || s.dumpCh == nil { - return + return nil } + rd := getRequestData(r, body) select { - case s.dumpCh <- r: + case s.dumpCh <- rd: // Successfully sent the request data to the channel default: // Handle the case where the channel is full log.Println("Channel is full, dropping request and logging it instead:") - LogRequest(r, &Server{verbose: true}) + return fmt.Errorf("request dump channel is full") } + return nil } // Goroutine function to write requests to a JSON file diff --git a/internal/server/http/server.go b/internal/server/http/server.go index 9e8538e..8bff734 100644 --- a/internal/server/http/server.go +++ b/internal/server/http/server.go @@ -34,19 +34,21 @@ type Server struct { proxy *Proxy secure bool imposterFs ImposterFs + corsOptions []handlers.CORSOption verbose bool dumpRequestsPath string dumpCh chan *RequestData } // NewServer initialize the mock server -func NewServer(r *mux.Router, httpServer *http.Server, proxyServer *Proxy, secure bool, fs ImposterFs, verbose bool, dumpRequestsPath string) Server { +func NewServer(r *mux.Router, httpServer *http.Server, proxyServer *Proxy, secure bool, fs ImposterFs, CORSOptions []handlers.CORSOption, verbose bool, dumpRequestsPath string) Server { return Server{ router: r, httpServer: httpServer, proxy: proxyServer, secure: secure, imposterFs: fs, + corsOptions: CORSOptions, verbose: verbose, dumpRequestsPath: dumpRequestsPath, } @@ -90,6 +92,13 @@ func (s *Server) Build() error { return nil } + // setup the logging handler + var handler http.Handler = s.router + if s.verbose || (s.dumpCh != nil && len(s.dumpRequestsPath) > 0) { + handler = CustomLoggingHandler(log.Writer(), handler, s) + } + s.httpServer.Handler = handlers.CORS(s.corsOptions...)(handler) + var impostersCh = make(chan []Imposter) var done = make(chan struct{}) @@ -166,7 +175,7 @@ func (s *Server) Shutdown() error { func (s *Server) addImposterHandler(imposters []Imposter) { for _, imposter := range imposters { - r := s.router.HandleFunc(imposter.Request.Endpoint, s.handlerMiddleWare(imposter)). + r := s.router.HandleFunc(imposter.Request.Endpoint, ImposterHandler(imposter)). Methods(imposter.Request.Method). MatcherFunc(MatcherBySchema(imposter)) @@ -183,19 +192,3 @@ func (s *Server) addImposterHandler(imposters []Imposter) { } } } - -func (s *Server) handlerMiddleWare(i Imposter) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - // handle anything we need before the request is used - // We need to grab the request data before the request is used since the body may be empty - requestData := GetRequestData(r) - - // handle the imposter reqest - handler := ImposterHandler(i) - handler(w, r) - - // handle anything we need after the request - LogRequest(requestData, s) - RecordRequest(requestData, s) - } -} diff --git a/internal/server/http/server_test.go b/internal/server/http/server_test.go index 5f6ead7..e62239f 100644 --- a/internal/server/http/server_test.go +++ b/internal/server/http/server_test.go @@ -3,6 +3,7 @@ package http import ( "bytes" "crypto/tls" + "encoding/base64" "io" "log" "net/http" @@ -26,7 +27,7 @@ func TestMain(m *testing.M) { func TestServer_Build(t *testing.T) { newServer := func(fs ImposterFs) Server { - return NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, fs, false, "") + return NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, fs, nil, false, "") } testCases := map[string]struct { @@ -70,7 +71,7 @@ func TestBuildProxyMode(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters") require.NoError(t, err) - server := NewServer(router, httpServer, proxyServer, false, imposterFs, false, "") + server := NewServer(router, httpServer, proxyServer, false, imposterFs, nil, false, "") return &server, func() error { return httpServer.Close() } @@ -151,7 +152,7 @@ func TestBuildSecureMode(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters_secure") require.NoError(t, err) - server := NewServer(router, httpServer, proxyServer, true, imposterFs, false, "") + server := NewServer(router, httpServer, proxyServer, true, imposterFs, nil, false, "") return &server, func() { httpServer.Close() } @@ -215,29 +216,70 @@ func TestBuildSecureMode(t *testing.T) { } func TestBuildLogRequests(t *testing.T) { - var buf bytes.Buffer - log.SetOutput(&buf) - defer func() { - log.SetOutput(os.Stderr) - }() + testCases := map[string]struct { + method string + path string + body string + expectedLog string + expectedStatus int + }{ + "GET valid imposter request": { + method: "GET", + path: "/yamlTestDumpRequest", + body: "Dumped", + expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17 RHVtcGVk\n", + expectedStatus: http.StatusOK, + }, + "GET valid imposter request no body": { + method: "GET", + path: "/yamlTestDumpRequest", + body: "", + expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17\n", + expectedStatus: http.StatusOK, + }, + "GET invalid imposter request": { + method: "GET", + path: "/doesnotexist", + body: "Dumped", + expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19 RHVtcGVk\n", + expectedStatus: http.StatusNotFound, + }, + "GET invalid imposter request no body": { + method: "GET", + path: "/doesnotexist", + body: "", + expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19\n", + expectedStatus: http.StatusNotFound, + }, + } + for name, tc := range testCases { + name := name + tc := tc + t.Run(name, func(t *testing.T) { + var buf bytes.Buffer + log.SetOutput(&buf) + defer func() { + log.SetOutput(os.Stderr) + }() + + imposterFs, err := NewImposterFS("test/testdata/imposters") + assert.NoError(t, err) + server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, nil, true, "") + err = server.Build() + assert.NoError(t, err) - imposterFs, err := NewImposterFS("test/testdata/imposters") - assert.NoError(t, err) - server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, true, "") - err = server.Build() - assert.NoError(t, err) + w := httptest.NewRecorder() + req := httptest.NewRequest(tc.method, tc.path, strings.NewReader(tc.body)) - expectedBody := "Dumped" - expectedLog := "GET /yamlTestDumpRequest HTTP/1.1\r\nHost: example.com\r\n\r\nDumped" - w := httptest.NewRecorder() - req := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(expectedBody)) + server.httpServer.Handler.ServeHTTP(w, req) - server.router.ServeHTTP(w, req) - response := w.Result() - assert.Equal(t, http.StatusOK, response.StatusCode, "Expected status code: %v, got: %v", http.StatusOK, response.StatusCode) + response := w.Result() + assert.Equal(t, tc.expectedStatus, response.StatusCode, "Expected status code: %v, got: %v", tc.expectedStatus, response.StatusCode) - // verify the request is dumped in the logs - assert.Contains(t, buf.String(), expectedLog, "Expect request dumped on logs failed") + // verify the request is dumped in the logs + assert.Contains(t, buf.String(), tc.expectedLog, "Expect request dumped on logs failed") + }) + } } func TestBuildRecordRequests(t *testing.T) { @@ -254,15 +296,16 @@ func TestBuildRecordRequests(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters") assert.NoError(t, err) w := httptest.NewRecorder() - server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, true, dumpFile) + server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, nil, true, dumpFile) err = server.Build() assert.NoError(t, err) - expectedBodies := []string{"Dumped1", "Dumped2"} - req1 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(expectedBodies[0])) - req2 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(expectedBodies[1])) - server.router.ServeHTTP(w, req1) - server.router.ServeHTTP(w, req2) + inputBodies := []string{"Dumped1", ""} + expectedBodies := []string{"RHVtcGVkMQ==", ""} + req1 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(inputBodies[0])) + req2 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(inputBodies[1])) + server.httpServer.Handler.ServeHTTP(w, req1) + server.httpServer.Handler.ServeHTTP(w, req2) // wait for channel to print out the requests time.Sleep(1 * time.Second) @@ -273,5 +316,8 @@ func TestBuildRecordRequests(t *testing.T) { assert.Equal(t, 2, len(reqs), "Expect 2 requests to be dumped in file failed") for i, expectedBody := range expectedBodies { assert.Equal(t, expectedBody, reqs[i].Body, "Expect request body to be dumped in file failed") + decodedString, err := base64.StdEncoding.DecodeString(reqs[i].Body) + assert.NoError(t, err, "Failed to decode base64 string") + assert.Equal(t, inputBodies[i], string(decodedString), "Expect request body to be dumped in file failed") } } From cd93b1f34dd79b62eefcbf550470e06cca5aa2dc Mon Sep 17 00:00:00 2001 From: Tate Date: Mon, 24 Jun 2024 16:25:23 -0600 Subject: [PATCH 06/20] clean up docker build warning --- .gitignore | 2 ++ Dockerfile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d608424..e720419 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ bin *.out coverage.txt +.vscode + imposters !internal/server/http/test/testdata/imposters/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index dd3171e..d22dbbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM golang:1.21-alpine AS build -LABEL MAINTAINER = 'Friends of Go (it@friendsofgo.tech)' +LABEL MAINTAINER='Friends of Go (it@friendsofgo.tech)' ARG TARGETOS=linux ARG TARGETARCH=amd64 From b7bafe8e89d8c62380b078992801798a8f2eca5b Mon Sep 17 00:00:00 2001 From: Tate Date: Mon, 24 Jun 2024 16:33:57 -0600 Subject: [PATCH 07/20] use -v and update readme for version --- README.md | 4 ++-- internal/app/cmd/cmd.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7c93d66..f5e1013 100644 --- a/README.md +++ b/README.md @@ -154,9 +154,9 @@ Flags: -m, --proxy-mode string Proxy mode, the options are all, missing or none (default "none") -u, --proxy-url string The url where the proxy will redirect to -s, --secure Run mock server using TLS (https) - -v, --version Version of Killgrave + --version Version of Killgrave -w, --watcher File watcher will reload the server on each file change - -V, --verbose Print out more detailed logging + -v, --verbose Print out more detailed logging -d, --dump-requests-path Print requests out to specified file ``` diff --git a/internal/app/cmd/cmd.go b/internal/app/cmd/cmd.go index c2c0d6f..1d78cd0 100644 --- a/internal/app/cmd/cmd.go +++ b/internal/app/cmd/cmd.go @@ -80,7 +80,7 @@ func NewKillgraveCmd() *cobra.Command { rootCmd.Flags().BoolP(_secureFlag, "s", false, "Run mock server using TLS (https)") rootCmd.Flags().StringP(_proxyModeFlag, "m", _defaultProxyMode.String(), "Proxy mode, the options are all, missing or none") rootCmd.Flags().StringP(_proxyURLFlag, "u", "", "The url where the proxy will redirect to") - rootCmd.Flags().BoolP(_verboseFlag, "V", false, "More verbose logging, adds request dumps to logs") + rootCmd.Flags().BoolP(_verboseFlag, "v", false, "More verbose logging, adds request dumps to logs") rootCmd.Flags().StringP(_dumpRequestsPathFlag, "d", "", "Path the requests will be dumped to") rootCmd.SetVersionTemplate("Killgrave version: {{.Version}}\n") From ac2569aa80c9be529051471288302c639d0bc888 Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 25 Jun 2024 07:44:16 -0600 Subject: [PATCH 08/20] Only set the body as binary when the Content-Type Header is a binary type. --- internal/server/http/dump.go | 29 ++++++++++++++++++++++++++--- internal/server/http/server_test.go | 22 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go index 861bfb4..b9b4255 100644 --- a/internal/server/http/dump.go +++ b/internal/server/http/dump.go @@ -166,6 +166,25 @@ func writeLog(writer io.Writer, params handlers.LogFormatterParams, body string) writer.Write(buf) } +// isBinaryContent checks to see if the body is a common binary content type +func isBinaryContent(r *http.Request) bool { + contentType := r.Header.Get("Content-Type") + binaryContentTypes := []string{ + "application/octet-stream", + "image/", + "audio/", + "video/", + "application/pdf", + } + + for _, binaryType := range binaryContentTypes { + if strings.HasPrefix(contentType, binaryType) { + return true + } + } + return false +} + // CustomLoggingHandler provides a way to supply a custom log formatter // while taking advantage of the mechanisms in this package func CustomLoggingHandler(out io.Writer, h http.Handler, s *Server) http.Handler { @@ -176,18 +195,22 @@ func CustomLoggingHandler(out io.Writer, h http.Handler, s *Server) http.Handler return } params.Request.Body = io.NopCloser(bytes.NewReader(bodyBytes)) // Reset the body - base64Body := base64.StdEncoding.EncodeToString(bodyBytes) + body := string(bodyBytes) + // if content is binary, encode it to base64 + if isBinaryContent(params.Request) { + body = base64.StdEncoding.EncodeToString(bodyBytes) + } verbose := s.verbose // record request, if error set verbose to true to log current request since // it didn't make it into a full channel - err = recordRequest(params.Request, s, base64Body) + err = recordRequest(params.Request, s, body) if err != nil { verbose = true } if verbose { - writeLog(writer, params, base64Body) + writeLog(writer, params, body) } }) } diff --git a/internal/server/http/server_test.go b/internal/server/http/server_test.go index e62239f..b60daa5 100644 --- a/internal/server/http/server_test.go +++ b/internal/server/http/server_test.go @@ -219,6 +219,7 @@ func TestBuildLogRequests(t *testing.T) { testCases := map[string]struct { method string path string + contentType string body string expectedLog string expectedStatus int @@ -226,6 +227,15 @@ func TestBuildLogRequests(t *testing.T) { "GET valid imposter request": { method: "GET", path: "/yamlTestDumpRequest", + contentType: "text/plain", + body: "Dumped", + expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17 Dumped\n", + expectedStatus: http.StatusOK, + }, + "GET valid imposter binary request": { + method: "GET", + path: "/yamlTestDumpRequest", + contentType: "application/octet-stream", body: "Dumped", expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17 RHVtcGVk\n", expectedStatus: http.StatusOK, @@ -233,6 +243,7 @@ func TestBuildLogRequests(t *testing.T) { "GET valid imposter request no body": { method: "GET", path: "/yamlTestDumpRequest", + contentType: "text/plain", body: "", expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17\n", expectedStatus: http.StatusOK, @@ -240,6 +251,15 @@ func TestBuildLogRequests(t *testing.T) { "GET invalid imposter request": { method: "GET", path: "/doesnotexist", + contentType: "text/plain", + body: "Dumped", + expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19 Dumped\n", + expectedStatus: http.StatusNotFound, + }, + "GET invalid imposter binary request": { + method: "GET", + path: "/doesnotexist", + contentType: "video/mp4", body: "Dumped", expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19 RHVtcGVk\n", expectedStatus: http.StatusNotFound, @@ -247,6 +267,7 @@ func TestBuildLogRequests(t *testing.T) { "GET invalid imposter request no body": { method: "GET", path: "/doesnotexist", + contentType: "text/plain", body: "", expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19\n", expectedStatus: http.StatusNotFound, @@ -270,6 +291,7 @@ func TestBuildLogRequests(t *testing.T) { w := httptest.NewRecorder() req := httptest.NewRequest(tc.method, tc.path, strings.NewReader(tc.body)) + req.Header.Set("Content-Type", tc.contentType) server.httpServer.Handler.ServeHTTP(w, req) From 8c2e34cf03349b7bd927a26b065c54b5db5e1c1f Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 25 Jun 2024 09:22:48 -0600 Subject: [PATCH 09/20] Make binaryContentTypes global so it isn't building it every time. --- internal/server/http/dump.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go index b9b4255..1e9ee81 100644 --- a/internal/server/http/dump.go +++ b/internal/server/http/dump.go @@ -19,6 +19,14 @@ import ( "github.com/gorilla/handlers" ) +var binaryContentTypes = []string{ + "application/octet-stream", + "image/", + "audio/", + "video/", + "application/pdf", +} + // RequestData struct to hold request data type RequestData struct { Method string `json:"method"` @@ -169,14 +177,6 @@ func writeLog(writer io.Writer, params handlers.LogFormatterParams, body string) // isBinaryContent checks to see if the body is a common binary content type func isBinaryContent(r *http.Request) bool { contentType := r.Header.Get("Content-Type") - binaryContentTypes := []string{ - "application/octet-stream", - "image/", - "audio/", - "video/", - "application/pdf", - } - for _, binaryType := range binaryContentTypes { if strings.HasPrefix(contentType, binaryType) { return true From fcc3cb7b5ebcac6feb9193107dd63ca8e9c88e01 Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 25 Jun 2024 10:24:35 -0600 Subject: [PATCH 10/20] replace -v --verbose with -l --log-level with 3 settings, 0 default, 1 adds requests, 2 adds request body --- README.md | 8 ++-- internal/app/cmd/cmd.go | 19 ++++++---- internal/config.go | 6 +-- internal/config_test.go | 4 +- internal/server/http/dump.go | 52 +++++++++++++++----------- internal/server/http/server.go | 20 +++++----- internal/server/http/server_test.go | 58 ++++++++++++++++++++++------- 7 files changed, 106 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index f5e1013..2c30e9a 100644 --- a/README.md +++ b/README.md @@ -154,9 +154,9 @@ Flags: -m, --proxy-mode string Proxy mode, the options are all, missing or none (default "none") -u, --proxy-url string The url where the proxy will redirect to -s, --secure Run mock server using TLS (https) - --version Version of Killgrave + -v, --version Version of Killgrave -w, --watcher File watcher will reload the server on each file change - -v, --verbose Print out more detailed logging + -l, --log-level The higher the log level the more detail you get (default 0, 1 adds requests, 2 adds request body) -d, --dump-requests-path Print requests out to specified file ``` @@ -184,8 +184,8 @@ cors: allow_credentials: true watcher: true secure: true -verbose: false -dump-requests-path: "/abc/def.log +log_level: 1 +dump_requests_path: "/abc/def.log ``` As you can see, you can configure all the options in a very easy way. For the above example, the file tree looks as follows, with the current working directory being `mymock`. diff --git a/internal/app/cmd/cmd.go b/internal/app/cmd/cmd.go index 1d78cd0..f634e79 100644 --- a/internal/app/cmd/cmd.go +++ b/internal/app/cmd/cmd.go @@ -25,6 +25,7 @@ const ( _defaultPort = 3000 _defaultProxyMode = killgrave.ProxyNone _defaultStrictSlash = true + _defaultLogLevel = 0 _impostersFlag = "imposters" _configFlag = "config" @@ -34,7 +35,7 @@ const ( _secureFlag = "secure" _proxyModeFlag = "proxy-mode" _proxyURLFlag = "proxy-url" - _verboseFlag = "verbose" + _logLevel = "log-level" _dumpRequestsPathFlag = "dump-requests-path" ) @@ -43,7 +44,8 @@ var ( errGetDataFromHostFlag = errors.New("error trying to get data from host flag") errGetDataFromPortFlag = errors.New("error trying to get data from port flag") errGetDataFromSecureFlag = errors.New("error trying to get data from secure flag") - errGetDataFromVerboseFlag = errors.New("error trying to get data from verbose flag") + errGetDataFromLogLevelFlag = errors.New("error trying to get data from log-level flag") + errGetDataLogLevelInvalid = errors.New("error setting log-level, must be between 0 and 2 inclusive") errGetDataFromDumpRequestsPathFlag = errors.New("error trying to get data from dump-requests-path flag") errMandatoryURL = errors.New("the field proxy-url is mandatory if you selected a proxy mode") ) @@ -80,7 +82,7 @@ func NewKillgraveCmd() *cobra.Command { rootCmd.Flags().BoolP(_secureFlag, "s", false, "Run mock server using TLS (https)") rootCmd.Flags().StringP(_proxyModeFlag, "m", _defaultProxyMode.String(), "Proxy mode, the options are all, missing or none") rootCmd.Flags().StringP(_proxyURLFlag, "u", "", "The url where the proxy will redirect to") - rootCmd.Flags().BoolP(_verboseFlag, "v", false, "More verbose logging, adds request dumps to logs") + rootCmd.Flags().IntP(_logLevel, "l", _defaultLogLevel, "Log level, the options are 0, 1, 2. Default is 0, 1 adds requests, 2 adds request body") rootCmd.Flags().StringP(_dumpRequestsPathFlag, "d", "", "Path the requests will be dumped to") rootCmd.SetVersionTemplate("Killgrave version: {{.Version}}\n") @@ -140,7 +142,7 @@ func runServer(cfg killgrave.Config) server.Server { cfg.Secure, imposterFs, server.PrepareAccessControl(cfg.CORS), - cfg.Verbose, + cfg.LogLevel, cfg.DumpRequestsPath, ) if err := s.Build(); err != nil { @@ -192,9 +194,12 @@ func prepareConfig(cmd *cobra.Command) (killgrave.Config, error) { return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromSecureFlag) } - verbose, err := cmd.Flags().GetBool(_verboseFlag) + logLevel, err := cmd.Flags().GetInt(_logLevel) if err != nil { - return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromVerboseFlag) + return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromLogLevelFlag) + } + if logLevel < 0 || logLevel > 2 { + return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataLogLevelInvalid) } dumpRequestsPath, err := cmd.Flags().GetString(_dumpRequestsPathFlag) @@ -202,7 +207,7 @@ func prepareConfig(cmd *cobra.Command) (killgrave.Config, error) { return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromDumpRequestsPathFlag) } - cfg, err := killgrave.NewConfig(impostersPath, host, port, secure, verbose, dumpRequestsPath) + cfg, err := killgrave.NewConfig(impostersPath, host, port, secure, logLevel, dumpRequestsPath) if err != nil { return killgrave.Config{}, err } diff --git a/internal/config.go b/internal/config.go index 81dfb01..cdb620e 100644 --- a/internal/config.go +++ b/internal/config.go @@ -19,7 +19,7 @@ type Config struct { Proxy ConfigProxy `yaml:"proxy"` Secure bool `yaml:"secure"` Watcher bool `yaml:"watcher"` - Verbose bool `yaml:"verbose"` + LogLevel int `yaml:"log_level"` DumpRequestsPath string `yaml:"dump_requests_path"` } @@ -113,7 +113,7 @@ func (cfg *Config) ConfigureProxy(proxyMode ProxyMode, proxyURL string) { type ConfigOpt func(cfg *Config) error // NewConfig initialize the config -func NewConfig(impostersPath, host string, port int, secure, verbose bool, dumpRequestsPath string) (Config, error) { +func NewConfig(impostersPath, host string, port int, secure bool, logLevel int, dumpRequestsPath string) (Config, error) { if impostersPath == "" { return Config{}, errEmptyImpostersPath } @@ -131,7 +131,7 @@ func NewConfig(impostersPath, host string, port int, secure, verbose bool, dumpR Host: host, Port: port, Secure: secure, - Verbose: verbose, + LogLevel: logLevel, DumpRequestsPath: dumpRequestsPath, } diff --git a/internal/config_test.go b/internal/config_test.go index 6fb96df..3230e8a 100644 --- a/internal/config_test.go +++ b/internal/config_test.go @@ -203,7 +203,7 @@ func TestNewConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewConfig(tt.args.impostersPath, tt.args.host, tt.args.port, false, false, "") + got, err := NewConfig(tt.args.impostersPath, tt.args.host, tt.args.port, false, 0, "") assert.Equal(t, tt.err, err) assert.Equal(t, tt.want, got) }) @@ -221,7 +221,7 @@ func TestConfig_ConfigureProxy(t *testing.T) { }, } - got, err := NewConfig("imposters", "localhost", 80, false, false, "") + got, err := NewConfig("imposters", "localhost", 80, false, 0, "") assert.NoError(t, err) got.ConfigureProxy(ProxyAll, "https://friendsofgo.tech") diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go index 1e9ee81..6f21602 100644 --- a/internal/server/http/dump.go +++ b/internal/server/http/dump.go @@ -185,40 +185,50 @@ func isBinaryContent(r *http.Request) bool { return false } +func shouldRecordRequest(s *Server) bool { + return len(s.dumpRequestsPath) > 0 && s.dumpCh != nil +} + +func getBody(r *http.Request, s *Server) string { + if s.logLevel == 0 && !shouldRecordRequest(s) { + return "" + } + bodyBytes, err := io.ReadAll(r.Body) + if err != nil { + log.Printf("Error reading request body: %v\n", err) + return "" + } + r.Body = io.NopCloser(bytes.NewReader(bodyBytes)) // Reset the body + body := string(bodyBytes) + // if content is binary, encode it to base64 + if isBinaryContent(r) { + body = base64.StdEncoding.EncodeToString(bodyBytes) + } + return body +} + // CustomLoggingHandler provides a way to supply a custom log formatter // while taking advantage of the mechanisms in this package func CustomLoggingHandler(out io.Writer, h http.Handler, s *Server) http.Handler { return handlers.CustomLoggingHandler(out, h, func(writer io.Writer, params handlers.LogFormatterParams) { - bodyBytes, err := io.ReadAll(params.Request.Body) - if err != nil { - log.Printf("Error reading request body: %v\n", err) - return - } - params.Request.Body = io.NopCloser(bytes.NewReader(bodyBytes)) // Reset the body - body := string(bodyBytes) - // if content is binary, encode it to base64 - if isBinaryContent(params.Request) { - body = base64.StdEncoding.EncodeToString(bodyBytes) - } - verbose := s.verbose + body := getBody(params.Request, s) - // record request, if error set verbose to true to log current request since - // it didn't make it into a full channel - err = recordRequest(params.Request, s, body) - if err != nil { - verbose = true + var err error + if shouldRecordRequest(s) { + err = recordRequest(params.Request, s, body) } - if verbose { + // log the request based on the log level + // if err is set, log the request, but only add the body if the log level is 2 or higher + if s.logLevel >= 2 { writeLog(writer, params, body) + } else if err != nil || s.logLevel > 0 { + writeLog(writer, params, "") } }) } func recordRequest(r *http.Request, s *Server, body string) error { - if len(s.dumpRequestsPath) < 1 || s.dumpCh == nil { - return nil - } rd := getRequestData(r, body) select { case s.dumpCh <- rd: diff --git a/internal/server/http/server.go b/internal/server/http/server.go index 8bff734..d9008bd 100644 --- a/internal/server/http/server.go +++ b/internal/server/http/server.go @@ -35,13 +35,13 @@ type Server struct { secure bool imposterFs ImposterFs corsOptions []handlers.CORSOption - verbose bool + logLevel int dumpRequestsPath string dumpCh chan *RequestData } // NewServer initialize the mock server -func NewServer(r *mux.Router, httpServer *http.Server, proxyServer *Proxy, secure bool, fs ImposterFs, CORSOptions []handlers.CORSOption, verbose bool, dumpRequestsPath string) Server { +func NewServer(r *mux.Router, httpServer *http.Server, proxyServer *Proxy, secure bool, fs ImposterFs, CORSOptions []handlers.CORSOption, logLevel int, dumpRequestsPath string) Server { return Server{ router: r, httpServer: httpServer, @@ -49,7 +49,7 @@ func NewServer(r *mux.Router, httpServer *http.Server, proxyServer *Proxy, secur secure: secure, imposterFs: fs, corsOptions: CORSOptions, - verbose: verbose, + logLevel: logLevel, dumpRequestsPath: dumpRequestsPath, } } @@ -92,9 +92,15 @@ func (s *Server) Build() error { return nil } + // only intantiate the request dump if we need it + if s.dumpCh == nil && len(s.dumpRequestsPath) > 0 { + s.dumpCh = make(chan *RequestData, 1000) + go RequestWriter(s.dumpRequestsPath, s.dumpCh) + } + // setup the logging handler var handler http.Handler = s.router - if s.verbose || (s.dumpCh != nil && len(s.dumpRequestsPath) > 0) { + if s.logLevel > 0 || shouldRecordRequest(s) { handler = CustomLoggingHandler(log.Writer(), handler, s) } s.httpServer.Handler = handlers.CORS(s.corsOptions...)(handler) @@ -102,12 +108,6 @@ func (s *Server) Build() error { var impostersCh = make(chan []Imposter) var done = make(chan struct{}) - // only intantiate the request dump if we need it - if s.dumpCh == nil && len(s.dumpRequestsPath) > 0 { - s.dumpCh = make(chan *RequestData, 1000) - go RequestWriter(s.dumpRequestsPath, s.dumpCh) - } - go func() { s.imposterFs.FindImposters(impostersCh) done <- struct{}{} diff --git a/internal/server/http/server_test.go b/internal/server/http/server_test.go index b60daa5..f788fd0 100644 --- a/internal/server/http/server_test.go +++ b/internal/server/http/server_test.go @@ -3,7 +3,6 @@ package http import ( "bytes" "crypto/tls" - "encoding/base64" "io" "log" "net/http" @@ -27,7 +26,7 @@ func TestMain(m *testing.M) { func TestServer_Build(t *testing.T) { newServer := func(fs ImposterFs) Server { - return NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, fs, nil, false, "") + return NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, fs, nil, 0, "") } testCases := map[string]struct { @@ -71,7 +70,7 @@ func TestBuildProxyMode(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters") require.NoError(t, err) - server := NewServer(router, httpServer, proxyServer, false, imposterFs, nil, false, "") + server := NewServer(router, httpServer, proxyServer, false, imposterFs, nil, 0, "") return &server, func() error { return httpServer.Close() } @@ -152,7 +151,7 @@ func TestBuildSecureMode(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters_secure") require.NoError(t, err) - server := NewServer(router, httpServer, proxyServer, true, imposterFs, nil, false, "") + server := NewServer(router, httpServer, proxyServer, true, imposterFs, nil, 0, "") return &server, func() { httpServer.Close() } @@ -221,6 +220,7 @@ func TestBuildLogRequests(t *testing.T) { path string contentType string body string + logLevel int expectedLog string expectedStatus int }{ @@ -229,6 +229,16 @@ func TestBuildLogRequests(t *testing.T) { path: "/yamlTestDumpRequest", contentType: "text/plain", body: "Dumped", + logLevel: 1, + expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17\n", + expectedStatus: http.StatusOK, + }, + "GET valid imposter request with body": { + method: "GET", + path: "/yamlTestDumpRequest", + contentType: "text/plain", + body: "Dumped", + logLevel: 2, expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17 Dumped\n", expectedStatus: http.StatusOK, }, @@ -237,6 +247,16 @@ func TestBuildLogRequests(t *testing.T) { path: "/yamlTestDumpRequest", contentType: "application/octet-stream", body: "Dumped", + logLevel: 1, + expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17\n", + expectedStatus: http.StatusOK, + }, + "GET valid imposter binary request with body": { + method: "GET", + path: "/yamlTestDumpRequest", + contentType: "application/octet-stream", + body: "Dumped", + logLevel: 2, expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17 RHVtcGVk\n", expectedStatus: http.StatusOK, }, @@ -245,6 +265,7 @@ func TestBuildLogRequests(t *testing.T) { path: "/yamlTestDumpRequest", contentType: "text/plain", body: "", + logLevel: 2, expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17\n", expectedStatus: http.StatusOK, }, @@ -253,14 +274,25 @@ func TestBuildLogRequests(t *testing.T) { path: "/doesnotexist", contentType: "text/plain", body: "Dumped", + logLevel: 1, + expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19\n", + expectedStatus: http.StatusNotFound, + }, + "GET invalid imposter request with body": { + method: "GET", + path: "/doesnotexist", + contentType: "text/plain", + body: "Dumped", + logLevel: 2, expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19 Dumped\n", expectedStatus: http.StatusNotFound, }, - "GET invalid imposter binary request": { + "GET invalid imposter binary request with body": { method: "GET", path: "/doesnotexist", contentType: "video/mp4", body: "Dumped", + logLevel: 2, expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19 RHVtcGVk\n", expectedStatus: http.StatusNotFound, }, @@ -269,6 +301,7 @@ func TestBuildLogRequests(t *testing.T) { path: "/doesnotexist", contentType: "text/plain", body: "", + logLevel: 2, expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19\n", expectedStatus: http.StatusNotFound, }, @@ -285,7 +318,7 @@ func TestBuildLogRequests(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters") assert.NoError(t, err) - server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, nil, true, "") + server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, nil, tc.logLevel, "") err = server.Build() assert.NoError(t, err) @@ -305,6 +338,7 @@ func TestBuildLogRequests(t *testing.T) { } func TestBuildRecordRequests(t *testing.T) { + var buf bytes.Buffer log.SetOutput(&buf) defer func() { @@ -318,14 +352,13 @@ func TestBuildRecordRequests(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters") assert.NoError(t, err) w := httptest.NewRecorder() - server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, nil, true, dumpFile) + server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, nil, 0, dumpFile) err = server.Build() assert.NoError(t, err) - inputBodies := []string{"Dumped1", ""} - expectedBodies := []string{"RHVtcGVkMQ==", ""} - req1 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(inputBodies[0])) - req2 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(inputBodies[1])) + expectedBodies := []string{"Dumped1", ""} + req1 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(expectedBodies[0])) + req2 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(expectedBodies[1])) server.httpServer.Handler.ServeHTTP(w, req1) server.httpServer.Handler.ServeHTTP(w, req2) @@ -338,8 +371,5 @@ func TestBuildRecordRequests(t *testing.T) { assert.Equal(t, 2, len(reqs), "Expect 2 requests to be dumped in file failed") for i, expectedBody := range expectedBodies { assert.Equal(t, expectedBody, reqs[i].Body, "Expect request body to be dumped in file failed") - decodedString, err := base64.StdEncoding.DecodeString(reqs[i].Body) - assert.NoError(t, err, "Failed to decode base64 string") - assert.Equal(t, inputBodies[i], string(decodedString), "Expect request body to be dumped in file failed") } } From 7c5b2d5eca76ec0d9be62b7f077fbf8920fe7bbe Mon Sep 17 00:00:00 2001 From: Tate Exon Date: Mon, 22 Jul 2024 13:01:45 -0600 Subject: [PATCH 11/20] Remove some changes around build and git ignore for this PR --- .gitignore | 2 -- Dockerfile | 6 ++---- Makefile | 10 +--------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index e720419..d608424 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,6 @@ bin *.out coverage.txt -.vscode - imposters !internal/server/http/test/testdata/imposters/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d22dbbe..85e5ad7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,15 @@ FROM golang:1.21-alpine AS build -LABEL MAINTAINER='Friends of Go (it@friendsofgo.tech)' +LABEL MAINTAINER = 'Friends of Go (it@friendsofgo.tech)' ARG TARGETOS=linux ARG TARGETARCH=amd64 -ARG TAG="" RUN apk add --update git RUN apk add ca-certificates WORKDIR /go/src/github.com/friendsofgo/killgrave COPY . . -RUN go mod tidy \ - && if [ -z "$TAG" ]; then TAG=$(git describe --tags --abbrev=0); fi \ +RUN go mod tidy && TAG=$(git describe --tags --abbrev=0) \ && LDFLAGS=$(echo "-s -w -X github.com/friendsofgo/killgrave/internal/app/cmd._version="docker-$TAG) \ && CGO_ENABLED=0 GOOS="${TARGETOS}" GOARCH="${TARGETARCH}" go build -a -installsuffix cgo -o /go/bin/killgrave -ldflags "$LDFLAGS" cmd/killgrave/main.go diff --git a/Makefile b/Makefile index 6f58600..b773558 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,3 @@ .PHONY: build build: - go build -ldflags "-s -w -X 'github.com/friendsofgo/killgrave/internal/app/cmd._version=`git rev-parse --abbrev-ref HEAD`-`git rev-parse --short HEAD`'" -o bin/killgrave cmd/killgrave/main.go - -.PHONY: build-docker -build-docker: - docker build --build-arg TAG=$(TAG) -t killgrave:$(TAG) . - -.PHONY: test -test: - go test -v -vet=off -race ./... + go build -ldflags "-s -w -X 'github.com/friendsofgo/killgrave/internal/app/cmd._version=`git rev-parse --abbrev-ref HEAD`-`git rev-parse --short HEAD`'" -o bin/killgrave cmd/killgrave/main.go \ No newline at end of file From dbd4701a4b12519ba3126b38e158258ce2db03b3 Mon Sep 17 00:00:00 2001 From: Tate Exon Date: Mon, 22 Jul 2024 13:06:08 -0600 Subject: [PATCH 12/20] Remove deprecation fix from this PR --- internal/config.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/config.go b/internal/config.go index cdb620e..9299cd1 100644 --- a/internal/config.go +++ b/internal/config.go @@ -3,7 +3,7 @@ package killgrave import ( "errors" "fmt" - "io" + "io/ioutil" "os" "path" @@ -150,10 +150,7 @@ func NewConfigFromFile(cfgPath string) (Config, error) { defer configFile.Close() var cfg Config - bytes, err := io.ReadAll(configFile) - if err != nil { - return Config{}, fmt.Errorf("%w: error while reading configFile file %s, using default configuration instead", err, cfgPath) - } + bytes, _ := ioutil.ReadAll(configFile) if err := yaml.Unmarshal(bytes, &cfg); err != nil { return Config{}, fmt.Errorf("%w: error while unmarshalling configFile file %s, using default configuration instead", err, cfgPath) } From b0dbb5f1818d762d77a41cf6b469b0166c749647 Mon Sep 17 00:00:00 2001 From: Tate Exon Date: Tue, 23 Jul 2024 11:42:58 -0600 Subject: [PATCH 13/20] Add a serverconfig package Use server config package in cmd, NewServer, and tests Move gorilla handlers logging copy paste to its own package Add gorilla license to the license file --- LICENSE | 26 ++++ internal/app/cmd/cmd.go | 23 ++- internal/gorilla/handlers/logging.go | 141 ++++++++++++++++++ internal/server/http/dump.go | 206 ++++---------------------- internal/server/http/server.go | 76 ++++++---- internal/server/http/server_test.go | 46 +++++- internal/serverconfig/serverconfig.go | 33 +++++ 7 files changed, 337 insertions(+), 214 deletions(-) create mode 100644 internal/gorilla/handlers/logging.go create mode 100644 internal/serverconfig/serverconfig.go diff --git a/LICENSE b/LICENSE index b95d5b3..2189f36 100644 --- a/LICENSE +++ b/LICENSE @@ -19,3 +19,29 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +For code copied from github.com/gorilla/handlers in ./internal/gorilla directory: + +Copyright (c) 2013 The Gorilla Handlers Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/internal/app/cmd/cmd.go b/internal/app/cmd/cmd.go index f634e79..55cbf4a 100644 --- a/internal/app/cmd/cmd.go +++ b/internal/app/cmd/cmd.go @@ -3,6 +3,7 @@ package cmd import ( "errors" "fmt" + "io" "log" "net/http" "os" @@ -11,6 +12,7 @@ import ( killgrave "github.com/friendsofgo/killgrave/internal" server "github.com/friendsofgo/killgrave/internal/server/http" + sc "github.com/friendsofgo/killgrave/internal/serverconfig" "github.com/gorilla/mux" "github.com/radovskyb/watcher" "github.com/spf13/cobra" @@ -35,7 +37,7 @@ const ( _secureFlag = "secure" _proxyModeFlag = "proxy-mode" _proxyURLFlag = "proxy-url" - _logLevel = "log-level" + _logLevelFlag = "log-level" _dumpRequestsPathFlag = "dump-requests-path" ) @@ -82,7 +84,7 @@ func NewKillgraveCmd() *cobra.Command { rootCmd.Flags().BoolP(_secureFlag, "s", false, "Run mock server using TLS (https)") rootCmd.Flags().StringP(_proxyModeFlag, "m", _defaultProxyMode.String(), "Proxy mode, the options are all, missing or none") rootCmd.Flags().StringP(_proxyURLFlag, "u", "", "The url where the proxy will redirect to") - rootCmd.Flags().IntP(_logLevel, "l", _defaultLogLevel, "Log level, the options are 0, 1, 2. Default is 0, 1 adds requests, 2 adds request body") + rootCmd.Flags().IntP(_logLevelFlag, "l", _defaultLogLevel, "Log level, the options are 0, 1, 2. Default is 0, 1 adds requests, 2 adds request body") rootCmd.Flags().StringP(_dumpRequestsPathFlag, "d", "", "Path the requests will be dumped to") rootCmd.SetVersionTemplate("Killgrave version: {{.Version}}\n") @@ -135,15 +137,24 @@ func runServer(cfg killgrave.Config) server.Server { log.Fatal(err) } + var logWriter io.Writer + if len(cfg.DumpRequestsPath) > 0 && cfg.LogLevel > 0 { + file, err := os.OpenFile(cfg.DumpRequestsPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatalf("Failed to open file: %+v", err) + } + logWriter = file + } + s := server.NewServer( router, &httpServer, proxyServer, cfg.Secure, imposterFs, - server.PrepareAccessControl(cfg.CORS), - cfg.LogLevel, - cfg.DumpRequestsPath, + sc.WithCORSOptions(server.PrepareAccessControl(cfg.CORS)), + sc.WithLogLevel(cfg.LogLevel), + sc.WithLogWriter(logWriter), ) if err := s.Build(); err != nil { log.Fatal(err) @@ -194,7 +205,7 @@ func prepareConfig(cmd *cobra.Command) (killgrave.Config, error) { return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromSecureFlag) } - logLevel, err := cmd.Flags().GetInt(_logLevel) + logLevel, err := cmd.Flags().GetInt(_logLevelFlag) if err != nil { return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromLogLevelFlag) } diff --git a/internal/gorilla/handlers/logging.go b/internal/gorilla/handlers/logging.go new file mode 100644 index 0000000..3f9931c --- /dev/null +++ b/internal/gorilla/handlers/logging.go @@ -0,0 +1,141 @@ +package handlers + +import ( + "io" + "net" + "net/http" + "net/url" + "strconv" + "time" + "unicode/utf8" + + "github.com/gorilla/handlers" +) + +// Copied from https://github.com/gorilla/handlers/blob/v1.5.1/logging.go at commit sha d453effd20e6817a8acfd9d278dadd21371da0ba +// so we can create a custom handler for logging use cases + +const lowerhex = "0123456789abcdef" + +func AppendQuoted(buf []byte, s string) []byte { + var runeTmp [utf8.UTFMax]byte + for width := 0; len(s) > 0; s = s[width:] { + r := rune(s[0]) + width = 1 + if r >= utf8.RuneSelf { + r, width = utf8.DecodeRuneInString(s) + } + if width == 1 && r == utf8.RuneError { + buf = append(buf, `\x`...) + buf = append(buf, lowerhex[s[0]>>4]) + buf = append(buf, lowerhex[s[0]&0xF]) + continue + } + if r == rune('"') || r == '\\' { // always backslashed + buf = append(buf, '\\') + buf = append(buf, byte(r)) + continue + } + if strconv.IsPrint(r) { + n := utf8.EncodeRune(runeTmp[:], r) + buf = append(buf, runeTmp[:n]...) + continue + } + switch r { + case '\a': + buf = append(buf, `\a`...) + case '\b': + buf = append(buf, `\b`...) + case '\f': + buf = append(buf, `\f`...) + case '\n': + buf = append(buf, `\n`...) + case '\r': + buf = append(buf, `\r`...) + case '\t': + buf = append(buf, `\t`...) + case '\v': + buf = append(buf, `\v`...) + default: + switch { + case r < ' ': + buf = append(buf, `\x`...) + buf = append(buf, lowerhex[s[0]>>4]) + buf = append(buf, lowerhex[s[0]&0xF]) + case r > utf8.MaxRune: + r = 0xFFFD + fallthrough + case r < 0x10000: + buf = append(buf, `\u`...) + for s := 12; s >= 0; s -= 4 { + buf = append(buf, lowerhex[r>>uint(s)&0xF]) + } + default: + buf = append(buf, `\U`...) + for s := 28; s >= 0; s -= 4 { + buf = append(buf, lowerhex[r>>uint(s)&0xF]) + } + } + } + } + return buf +} + +func BuildCommonLogLine(req *http.Request, url url.URL, ts time.Time, status int, size int) []byte { + username := "-" + if url.User != nil { + if name := url.User.Username(); name != "" { + username = name + } + } + + host, _, err := net.SplitHostPort(req.RemoteAddr) + if err != nil { + host = req.RemoteAddr + } + + uri := req.RequestURI + + // Requests using the CONNECT method over HTTP/2.0 must use + // the authority field (aka r.Host) to identify the target. + // Refer: https://httpwg.github.io/specs/rfc7540.html#CONNECT + if req.ProtoMajor == 2 && req.Method == "CONNECT" { + uri = req.Host + } + if uri == "" { + uri = url.RequestURI() + } + + buf := make([]byte, 0, 3*(len(host)+len(username)+len(req.Method)+len(uri)+len(req.Proto)+50)/2) + buf = append(buf, host...) + buf = append(buf, " - "...) + buf = append(buf, username...) + buf = append(buf, " ["...) + buf = append(buf, ts.Format("02/Jan/2006:15:04:05 -0700")...) + buf = append(buf, `] "`...) + buf = append(buf, req.Method...) + buf = append(buf, " "...) + buf = AppendQuoted(buf, uri) + buf = append(buf, " "...) + buf = append(buf, req.Proto...) + buf = append(buf, `" `...) + buf = append(buf, strconv.Itoa(status)...) + buf = append(buf, " "...) + buf = append(buf, strconv.Itoa(size)...) + return buf +} + +// copied from gorilla/handlers and modified to add the body +// writeLog writes a log entry for req to w in Apache Common Log Format. +// ts is the timestamp with which the entry should be logged. +// status and size are used to provide the response HTTP status and size. +func WriteLog(writer io.Writer, params handlers.LogFormatterParams, body string) { + buf := BuildCommonLogLine(params.Request, params.URL, params.TimeStamp, params.StatusCode, params.Size) + // Append body if present + if len(body) > 0 { + buf = append(buf, " "...) + buf = append(buf, body...) + } + buf = append(buf, '\n') + writer.Write(buf) +} diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go index 6f21602..30c9583 100644 --- a/internal/server/http/dump.go +++ b/internal/server/http/dump.go @@ -2,20 +2,18 @@ package http import ( "bytes" + "context" "encoding/base64" "encoding/json" "fmt" "io" "log" - "net" "net/http" - "net/url" "os" - "strconv" "strings" - "time" - "unicode/utf8" + "sync" + internal_handlers "github.com/friendsofgo/killgrave/internal/gorilla/handlers" "github.com/gorilla/handlers" ) @@ -46,134 +44,6 @@ func getRequestData(r *http.Request, body string) *RequestData { } } -// Copied from the gorilla/handlers package -const lowerhex = "0123456789abcdef" - -// Copied from the gorilla/handlers package -func appendQuoted(buf []byte, s string) []byte { - var runeTmp [utf8.UTFMax]byte - for width := 0; len(s) > 0; s = s[width:] { - r := rune(s[0]) - width = 1 - if r >= utf8.RuneSelf { - r, width = utf8.DecodeRuneInString(s) - } - if width == 1 && r == utf8.RuneError { - buf = append(buf, `\x`...) - buf = append(buf, lowerhex[s[0]>>4]) - buf = append(buf, lowerhex[s[0]&0xF]) - continue - } - if r == rune('"') || r == '\\' { // always backslashed - buf = append(buf, '\\') - buf = append(buf, byte(r)) - continue - } - if strconv.IsPrint(r) { - n := utf8.EncodeRune(runeTmp[:], r) - buf = append(buf, runeTmp[:n]...) - continue - } - switch r { - case '\a': - buf = append(buf, `\a`...) - case '\b': - buf = append(buf, `\b`...) - case '\f': - buf = append(buf, `\f`...) - case '\n': - buf = append(buf, `\n`...) - case '\r': - buf = append(buf, `\r`...) - case '\t': - buf = append(buf, `\t`...) - case '\v': - buf = append(buf, `\v`...) - default: - switch { - case r < ' ': - buf = append(buf, `\x`...) - buf = append(buf, lowerhex[s[0]>>4]) - buf = append(buf, lowerhex[s[0]&0xF]) - case r > utf8.MaxRune: - r = 0xFFFD - fallthrough - case r < 0x10000: - buf = append(buf, `\u`...) - for s := 12; s >= 0; s -= 4 { - buf = append(buf, lowerhex[r>>uint(s)&0xF]) - } - default: - buf = append(buf, `\U`...) - for s := 28; s >= 0; s -= 4 { - buf = append(buf, lowerhex[r>>uint(s)&0xF]) - } - } - } - } - return buf -} - -// Copied from the gorilla/handlers package -func buildCommonLogLine(req *http.Request, url url.URL, ts time.Time, status int, size int) []byte { - username := "-" - if url.User != nil { - if name := url.User.Username(); name != "" { - username = name - } - } - - host, _, err := net.SplitHostPort(req.RemoteAddr) - if err != nil { - host = req.RemoteAddr - } - - uri := req.RequestURI - - // Requests using the CONNECT method over HTTP/2.0 must use - // the authority field (aka r.Host) to identify the target. - // Refer: https://httpwg.github.io/specs/rfc7540.html#CONNECT - if req.ProtoMajor == 2 && req.Method == "CONNECT" { - uri = req.Host - } - if uri == "" { - uri = url.RequestURI() - } - - buf := make([]byte, 0, 3*(len(host)+len(username)+len(req.Method)+len(uri)+len(req.Proto)+50)/2) - buf = append(buf, host...) - buf = append(buf, " - "...) - buf = append(buf, username...) - buf = append(buf, " ["...) - buf = append(buf, ts.Format("02/Jan/2006:15:04:05 -0700")...) - buf = append(buf, `] "`...) - buf = append(buf, req.Method...) - buf = append(buf, " "...) - buf = appendQuoted(buf, uri) - buf = append(buf, " "...) - buf = append(buf, req.Proto...) - buf = append(buf, `" `...) - buf = append(buf, strconv.Itoa(status)...) - buf = append(buf, " "...) - buf = append(buf, strconv.Itoa(size)...) - return buf -} - -// copied from gorilla/handlers and modified to add the body -// writeLog writes a log entry for req to w in Apache Common Log Format. -// ts is the timestamp with which the entry should be logged. -// status and size are used to provide the response HTTP status and size. -func writeLog(writer io.Writer, params handlers.LogFormatterParams, body string) { - buf := buildCommonLogLine(params.Request, params.URL, params.TimeStamp, params.StatusCode, params.Size) - // Append body if present - if len(body) > 0 { - buf = append(buf, " "...) - buf = append(buf, body...) - } - buf = append(buf, '\n') - writer.Write(buf) -} - // isBinaryContent checks to see if the body is a common binary content type func isBinaryContent(r *http.Request) bool { contentType := r.Header.Get("Content-Type") @@ -186,11 +56,11 @@ func isBinaryContent(r *http.Request) bool { } func shouldRecordRequest(s *Server) bool { - return len(s.dumpRequestsPath) > 0 && s.dumpCh != nil + return s.serverCfg.LogWriter != nil && s.dumpCh != nil } func getBody(r *http.Request, s *Server) string { - if s.logLevel == 0 && !shouldRecordRequest(s) { + if s.serverCfg.LogLevel == 0 && !shouldRecordRequest(s) { return "" } bodyBytes, err := io.ReadAll(r.Body) @@ -220,10 +90,10 @@ func CustomLoggingHandler(out io.Writer, h http.Handler, s *Server) http.Handler // log the request based on the log level // if err is set, log the request, but only add the body if the log level is 2 or higher - if s.logLevel >= 2 { - writeLog(writer, params, body) - } else if err != nil || s.logLevel > 0 { - writeLog(writer, params, "") + if s.serverCfg.LogLevel >= 2 { + internal_handlers.WriteLog(writer, params, body) + } else if err != nil || s.serverCfg.LogLevel > 0 { + internal_handlers.WriteLog(writer, params, "") } }) } @@ -242,44 +112,28 @@ func recordRequest(r *http.Request, s *Server, body string) error { } // Goroutine function to write requests to a JSON file -func RequestWriter(filePath string, requestChan <-chan *RequestData) { - file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - log.Fatalf("Failed to open file: %+v", err) - } - defer file.Close() - - encoder := json.NewEncoder(file) - for requestData := range requestChan { - if err := encoder.Encode(requestData); err != nil { - log.Printf("Failed to write to file: %+v", err) - } - file.Sync() - } -} - -// GetRecordedRequests reads the requests from the file and returns them as a slice of RequestData -func getRecordedRequests(filePath string) ([]RequestData, error) { - // Read the file contents - fileContent, err := os.ReadFile(filePath) - if err != nil { - return nil, fmt.Errorf("error reading file: %w", err) - } +func RequestWriter(ctx context.Context, wg *sync.WaitGroup, writer io.Writer, requestChan <-chan *RequestData) { + defer wg.Done() + // defer file.Close() + + encoder := json.NewEncoder(writer) + for { + select { + case requestData := <-requestChan: + if requestData == nil { + return // channel closed + } - // Split the contents by the newline separator - requestDumps := strings.Split(string(fileContent), "\n") - requestsData := []RequestData{} - for _, requestDump := range requestDumps { - if requestDump == "" { - continue - } - // Unmarshal the JSON string into the RequestData struct - rd := RequestData{} - err := json.Unmarshal([]byte(requestDump), &rd) - if err != nil { - return nil, fmt.Errorf("error unmarshaling JSON: %w", err) + if err := encoder.Encode(requestData); err != nil { + log.Printf("Failed to write to file: %+v", err) + fmt.Printf("Failed to write to file: %+v", err) + } + // Type assertion to call Sync if writer is *os.File + if f, ok := writer.(*os.File); ok { + f.Sync() + } + case <-ctx.Done(): + return // context cancelled } - requestsData = append(requestsData, rd) } - return requestsData, nil } diff --git a/internal/server/http/server.go b/internal/server/http/server.go index d9008bd..7176d7c 100644 --- a/internal/server/http/server.go +++ b/internal/server/http/server.go @@ -6,8 +6,11 @@ import ( _ "embed" "log" "net/http" + "os" + "sync" killgrave "github.com/friendsofgo/killgrave/internal" + sc "github.com/friendsofgo/killgrave/internal/serverconfig" "github.com/gorilla/handlers" "github.com/gorilla/mux" ) @@ -24,33 +27,37 @@ var ( defaultCORSExposedHeaders = []string{"Cache-Control", "Content-Language", "Content-Type", "Expires", "Last-Modified", "Pragma"} ) -// ServerOpt function that allow modify the current server -type ServerOpt func(s *Server) - // Server definition of mock server type Server struct { - router *mux.Router - httpServer *http.Server - proxy *Proxy - secure bool - imposterFs ImposterFs - corsOptions []handlers.CORSOption - logLevel int - dumpRequestsPath string - dumpCh chan *RequestData + router *mux.Router + httpServer *http.Server + proxy *Proxy + secure bool + imposterFs ImposterFs + serverCfg *sc.ServerConfig + dumpCh chan *RequestData + wg *sync.WaitGroup + ctx context.Context + cancel context.CancelFunc } // NewServer initialize the mock server -func NewServer(r *mux.Router, httpServer *http.Server, proxyServer *Proxy, secure bool, fs ImposterFs, CORSOptions []handlers.CORSOption, logLevel int, dumpRequestsPath string) Server { +func NewServer(r *mux.Router, httpServer *http.Server, proxyServer *Proxy, secure bool, fs ImposterFs, options ...sc.ServerOption) Server { + ctx, cancel := context.WithCancel(context.Background()) + cfg := &sc.ServerConfig{} + for _, opt := range options { + opt(cfg) + } return Server{ - router: r, - httpServer: httpServer, - proxy: proxyServer, - secure: secure, - imposterFs: fs, - corsOptions: CORSOptions, - logLevel: logLevel, - dumpRequestsPath: dumpRequestsPath, + router: r, + httpServer: httpServer, + proxy: proxyServer, + secure: secure, + imposterFs: fs, + serverCfg: cfg, + wg: &sync.WaitGroup{}, + ctx: ctx, + cancel: cancel, } } @@ -93,17 +100,19 @@ func (s *Server) Build() error { } // only intantiate the request dump if we need it - if s.dumpCh == nil && len(s.dumpRequestsPath) > 0 { + if s.dumpCh == nil && s.serverCfg.LogWriter != nil { s.dumpCh = make(chan *RequestData, 1000) - go RequestWriter(s.dumpRequestsPath, s.dumpCh) + // Start the RequestWriter goroutine with context + s.wg.Add(1) + go RequestWriter(s.ctx, s.wg, s.serverCfg.LogWriter, s.dumpCh) } // setup the logging handler var handler http.Handler = s.router - if s.logLevel > 0 || shouldRecordRequest(s) { + if s.serverCfg.LogLevel > 0 || shouldRecordRequest(s) { handler = CustomLoggingHandler(log.Writer(), handler, s) } - s.httpServer.Handler = handlers.CORS(s.corsOptions...)(handler) + s.httpServer.Handler = handlers.CORS(s.serverCfg.CORSOptions...)(handler) var impostersCh = make(chan []Imposter) var done = make(chan struct{}) @@ -166,10 +175,21 @@ func (s *Server) run(secure bool) error { // Shutdown shutdown the current http server func (s *Server) Shutdown() error { log.Println("stopping server...") - if err := s.httpServer.Shutdown(context.TODO()); err != nil { + if err := s.httpServer.Shutdown(s.ctx); err != nil { log.Fatalf("Server Shutdown Failed:%+v", err) } + // Cancel the context to stop the RequestWriter goroutine + s.cancel() + + // wait for all goroutines to finish + s.wg.Wait() + if s.serverCfg.LogWriter != nil { + if f, ok := s.serverCfg.LogWriter.(*os.File); ok { + f.Close() + } + } + return nil } @@ -192,3 +212,7 @@ func (s *Server) addImposterHandler(imposters []Imposter) { } } } + +func (s *Server) handleAll(h http.HandlerFunc) { + s.router.PathPrefix("/").HandlerFunc(h) +} diff --git a/internal/server/http/server_test.go b/internal/server/http/server_test.go index f788fd0..0c9e53d 100644 --- a/internal/server/http/server_test.go +++ b/internal/server/http/server_test.go @@ -3,6 +3,8 @@ package http import ( "bytes" "crypto/tls" + "encoding/json" + "fmt" "io" "log" "net/http" @@ -14,6 +16,7 @@ import ( "time" killgrave "github.com/friendsofgo/killgrave/internal" + sc "github.com/friendsofgo/killgrave/internal/serverconfig" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -26,7 +29,7 @@ func TestMain(m *testing.M) { func TestServer_Build(t *testing.T) { newServer := func(fs ImposterFs) Server { - return NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, fs, nil, 0, "") + return NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, fs) } testCases := map[string]struct { @@ -70,7 +73,7 @@ func TestBuildProxyMode(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters") require.NoError(t, err) - server := NewServer(router, httpServer, proxyServer, false, imposterFs, nil, 0, "") + server := NewServer(router, httpServer, proxyServer, false, imposterFs) return &server, func() error { return httpServer.Close() } @@ -151,7 +154,7 @@ func TestBuildSecureMode(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters_secure") require.NoError(t, err) - server := NewServer(router, httpServer, proxyServer, true, imposterFs, nil, 0, "") + server := NewServer(router, httpServer, proxyServer, true, imposterFs) return &server, func() { httpServer.Close() } @@ -318,7 +321,7 @@ func TestBuildLogRequests(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters") assert.NoError(t, err) - server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, nil, tc.logLevel, "") + server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, sc.WithLogLevel(tc.logLevel)) err = server.Build() assert.NoError(t, err) @@ -338,7 +341,6 @@ func TestBuildLogRequests(t *testing.T) { } func TestBuildRecordRequests(t *testing.T) { - var buf bytes.Buffer log.SetOutput(&buf) defer func() { @@ -352,7 +354,12 @@ func TestBuildRecordRequests(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters") assert.NoError(t, err) w := httptest.NewRecorder() - server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, nil, 0, dumpFile) + file, err := os.OpenFile(dumpFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatalf("Failed to open file: %+v", err) + } + defer file.Close() + server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, sc.WithLogWriter(file), sc.WithLogLevel(2)) err = server.Build() assert.NoError(t, err) @@ -373,3 +380,30 @@ func TestBuildRecordRequests(t *testing.T) { assert.Equal(t, expectedBody, reqs[i].Body, "Expect request body to be dumped in file failed") } } + +// TODO: make so we don't need to read from a file by abstracting test to write with io.Writer instead of a file somehow +// getRecordedRequests reads the requests from the file and returns them as a slice of RequestData +func getRecordedRequests(filePath string) ([]RequestData, error) { + // Read the file contents + fileContent, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("error reading file: %w", err) + } + + // Split the contents by the newline separator + requestDumps := strings.Split(string(fileContent), "\n") + requestsData := []RequestData{} + for _, requestDump := range requestDumps { + if requestDump == "" { + continue + } + // Unmarshal the JSON string into the RequestData struct + rd := RequestData{} + err := json.Unmarshal([]byte(requestDump), &rd) + if err != nil { + return nil, fmt.Errorf("error unmarshaling JSON: %w", err) + } + requestsData = append(requestsData, rd) + } + return requestsData, nil +} diff --git a/internal/serverconfig/serverconfig.go b/internal/serverconfig/serverconfig.go new file mode 100644 index 0000000..ec1d187 --- /dev/null +++ b/internal/serverconfig/serverconfig.go @@ -0,0 +1,33 @@ +package serverconfig + +import ( + "io" + + "github.com/gorilla/handlers" +) + +type ServerConfig struct { + CORSOptions []handlers.CORSOption + LogLevel int + LogWriter io.Writer +} + +type ServerOption func(*ServerConfig) + +func WithCORSOptions(corsOptions []handlers.CORSOption) ServerOption { + return func(cfg *ServerConfig) { + cfg.CORSOptions = corsOptions + } +} + +func WithLogLevel(logLevel int) ServerOption { + return func(cfg *ServerConfig) { + cfg.LogLevel = logLevel + } +} + +func WithLogWriter(logWriter io.Writer) ServerOption { + return func(cfg *ServerConfig) { + cfg.LogWriter = logWriter + } +} From 0a345b057ca2a9dcb5d867ae9a8df56a6cf22fb1 Mon Sep 17 00:00:00 2001 From: Tate Exon Date: Tue, 23 Jul 2024 12:13:40 -0600 Subject: [PATCH 14/20] Better handle the log file close defer now that everything is an io.Writer --- internal/app/cmd/cmd.go | 22 +++++++++++----------- internal/config.go | 2 ++ internal/server/http/dump.go | 4 +--- internal/server/http/server.go | 9 ++------- internal/server/http/server_test.go | 1 - 5 files changed, 16 insertions(+), 22 deletions(-) diff --git a/internal/app/cmd/cmd.go b/internal/app/cmd/cmd.go index 55cbf4a..1958dff 100644 --- a/internal/app/cmd/cmd.go +++ b/internal/app/cmd/cmd.go @@ -3,7 +3,6 @@ package cmd import ( "errors" "fmt" - "io" "log" "net/http" "os" @@ -98,6 +97,16 @@ func runHTTP(cmd *cobra.Command, cfg killgrave.Config) error { signal.Notify(done, syscall.SIGINT, syscall.SIGTERM) + // setup the log file and its deferred close if needed + if len(cfg.DumpRequestsPath) > 0 && cfg.LogLevel > 0 { + file, err := os.OpenFile(cfg.DumpRequestsPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatalf("Failed to open file: %+v", err) + } + cfg.LogWriter = file + defer file.Close() + } + srv := runServer(cfg) watcherFlag, _ := cmd.Flags().GetBool(_watcherFlag) @@ -137,15 +146,6 @@ func runServer(cfg killgrave.Config) server.Server { log.Fatal(err) } - var logWriter io.Writer - if len(cfg.DumpRequestsPath) > 0 && cfg.LogLevel > 0 { - file, err := os.OpenFile(cfg.DumpRequestsPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - log.Fatalf("Failed to open file: %+v", err) - } - logWriter = file - } - s := server.NewServer( router, &httpServer, @@ -154,7 +154,7 @@ func runServer(cfg killgrave.Config) server.Server { imposterFs, sc.WithCORSOptions(server.PrepareAccessControl(cfg.CORS)), sc.WithLogLevel(cfg.LogLevel), - sc.WithLogWriter(logWriter), + sc.WithLogWriter(cfg.LogWriter), ) if err := s.Build(); err != nil { log.Fatal(err) diff --git a/internal/config.go b/internal/config.go index 9299cd1..303dba7 100644 --- a/internal/config.go +++ b/internal/config.go @@ -3,6 +3,7 @@ package killgrave import ( "errors" "fmt" + "io" "io/ioutil" "os" "path" @@ -21,6 +22,7 @@ type Config struct { Watcher bool `yaml:"watcher"` LogLevel int `yaml:"log_level"` DumpRequestsPath string `yaml:"dump_requests_path"` + LogWriter io.Writer } // ConfigCORS representation of section CORS of the yaml diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go index 30c9583..6d39f97 100644 --- a/internal/server/http/dump.go +++ b/internal/server/http/dump.go @@ -114,7 +114,6 @@ func recordRequest(r *http.Request, s *Server, body string) error { // Goroutine function to write requests to a JSON file func RequestWriter(ctx context.Context, wg *sync.WaitGroup, writer io.Writer, requestChan <-chan *RequestData) { defer wg.Done() - // defer file.Close() encoder := json.NewEncoder(writer) for { @@ -126,9 +125,8 @@ func RequestWriter(ctx context.Context, wg *sync.WaitGroup, writer io.Writer, re if err := encoder.Encode(requestData); err != nil { log.Printf("Failed to write to file: %+v", err) - fmt.Printf("Failed to write to file: %+v", err) } - // Type assertion to call Sync if writer is *os.File + // call Sync if writer is *os.File if f, ok := writer.(*os.File); ok { f.Sync() } diff --git a/internal/server/http/server.go b/internal/server/http/server.go index 7176d7c..a15a51d 100644 --- a/internal/server/http/server.go +++ b/internal/server/http/server.go @@ -6,7 +6,6 @@ import ( _ "embed" "log" "net/http" - "os" "sync" killgrave "github.com/friendsofgo/killgrave/internal" @@ -99,9 +98,10 @@ func (s *Server) Build() error { return nil } - // only intantiate the request dump if we need it + // only instantiate the request dump if we need it if s.dumpCh == nil && s.serverCfg.LogWriter != nil { s.dumpCh = make(chan *RequestData, 1000) + // Start the RequestWriter goroutine with context s.wg.Add(1) go RequestWriter(s.ctx, s.wg, s.serverCfg.LogWriter, s.dumpCh) @@ -184,11 +184,6 @@ func (s *Server) Shutdown() error { // wait for all goroutines to finish s.wg.Wait() - if s.serverCfg.LogWriter != nil { - if f, ok := s.serverCfg.LogWriter.(*os.File); ok { - f.Close() - } - } return nil } diff --git a/internal/server/http/server_test.go b/internal/server/http/server_test.go index 0c9e53d..4887dad 100644 --- a/internal/server/http/server_test.go +++ b/internal/server/http/server_test.go @@ -381,7 +381,6 @@ func TestBuildRecordRequests(t *testing.T) { } } -// TODO: make so we don't need to read from a file by abstracting test to write with io.Writer instead of a file somehow // getRecordedRequests reads the requests from the file and returns them as a slice of RequestData func getRecordedRequests(filePath string) ([]RequestData, error) { // Read the file contents From 64596954272c3451a09d03a6e22b0fa1518b8a23 Mon Sep 17 00:00:00 2001 From: Tate Exon Date: Tue, 23 Jul 2024 12:37:25 -0600 Subject: [PATCH 15/20] reverse the order so we default to base64 encoded body --- internal/server/http/dump.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go index 6d39f97..d887b10 100644 --- a/internal/server/http/dump.go +++ b/internal/server/http/dump.go @@ -69,10 +69,11 @@ func getBody(r *http.Request, s *Server) string { return "" } r.Body = io.NopCloser(bytes.NewReader(bodyBytes)) // Reset the body - body := string(bodyBytes) - // if content is binary, encode it to base64 - if isBinaryContent(r) { - body = base64.StdEncoding.EncodeToString(bodyBytes) + + body := base64.StdEncoding.EncodeToString(bodyBytes) + // if content is not binary, get it as a string + if !isBinaryContent(r) { + body = string(bodyBytes) } return body } From fdf0d401bcb8bf66d274cb5da54d93aa3f006e08 Mon Sep 17 00:00:00 2001 From: Tate Exon Date: Tue, 23 Jul 2024 12:47:17 -0600 Subject: [PATCH 16/20] remove log that added no value --- internal/server/http/dump.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go index d887b10..c036159 100644 --- a/internal/server/http/dump.go +++ b/internal/server/http/dump.go @@ -106,8 +106,7 @@ func recordRequest(r *http.Request, s *Server, body string) error { // Successfully sent the request data to the channel default: // Handle the case where the channel is full - log.Println("Channel is full, dropping request and logging it instead:") - return fmt.Errorf("request dump channel is full") + return fmt.Errorf("request dump channel is full, could not write request") } return nil } From cf5ca239348c3f4ecce82537555bec5ab1ce585c Mon Sep 17 00:00:00 2001 From: Tate Exon Date: Wed, 24 Jul 2024 09:17:20 -0600 Subject: [PATCH 17/20] Remove gorilla code copied in --- LICENSE | 26 ----- internal/gorilla/handlers/logging.go | 141 --------------------------- 2 files changed, 167 deletions(-) delete mode 100644 internal/gorilla/handlers/logging.go diff --git a/LICENSE b/LICENSE index 2189f36..b95d5b3 100644 --- a/LICENSE +++ b/LICENSE @@ -19,29 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -For code copied from github.com/gorilla/handlers in ./internal/gorilla directory: - -Copyright (c) 2013 The Gorilla Handlers Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/internal/gorilla/handlers/logging.go b/internal/gorilla/handlers/logging.go deleted file mode 100644 index 3f9931c..0000000 --- a/internal/gorilla/handlers/logging.go +++ /dev/null @@ -1,141 +0,0 @@ -package handlers - -import ( - "io" - "net" - "net/http" - "net/url" - "strconv" - "time" - "unicode/utf8" - - "github.com/gorilla/handlers" -) - -// Copied from https://github.com/gorilla/handlers/blob/v1.5.1/logging.go at commit sha d453effd20e6817a8acfd9d278dadd21371da0ba -// so we can create a custom handler for logging use cases - -const lowerhex = "0123456789abcdef" - -func AppendQuoted(buf []byte, s string) []byte { - var runeTmp [utf8.UTFMax]byte - for width := 0; len(s) > 0; s = s[width:] { - r := rune(s[0]) - width = 1 - if r >= utf8.RuneSelf { - r, width = utf8.DecodeRuneInString(s) - } - if width == 1 && r == utf8.RuneError { - buf = append(buf, `\x`...) - buf = append(buf, lowerhex[s[0]>>4]) - buf = append(buf, lowerhex[s[0]&0xF]) - continue - } - if r == rune('"') || r == '\\' { // always backslashed - buf = append(buf, '\\') - buf = append(buf, byte(r)) - continue - } - if strconv.IsPrint(r) { - n := utf8.EncodeRune(runeTmp[:], r) - buf = append(buf, runeTmp[:n]...) - continue - } - switch r { - case '\a': - buf = append(buf, `\a`...) - case '\b': - buf = append(buf, `\b`...) - case '\f': - buf = append(buf, `\f`...) - case '\n': - buf = append(buf, `\n`...) - case '\r': - buf = append(buf, `\r`...) - case '\t': - buf = append(buf, `\t`...) - case '\v': - buf = append(buf, `\v`...) - default: - switch { - case r < ' ': - buf = append(buf, `\x`...) - buf = append(buf, lowerhex[s[0]>>4]) - buf = append(buf, lowerhex[s[0]&0xF]) - case r > utf8.MaxRune: - r = 0xFFFD - fallthrough - case r < 0x10000: - buf = append(buf, `\u`...) - for s := 12; s >= 0; s -= 4 { - buf = append(buf, lowerhex[r>>uint(s)&0xF]) - } - default: - buf = append(buf, `\U`...) - for s := 28; s >= 0; s -= 4 { - buf = append(buf, lowerhex[r>>uint(s)&0xF]) - } - } - } - } - return buf -} - -func BuildCommonLogLine(req *http.Request, url url.URL, ts time.Time, status int, size int) []byte { - username := "-" - if url.User != nil { - if name := url.User.Username(); name != "" { - username = name - } - } - - host, _, err := net.SplitHostPort(req.RemoteAddr) - if err != nil { - host = req.RemoteAddr - } - - uri := req.RequestURI - - // Requests using the CONNECT method over HTTP/2.0 must use - // the authority field (aka r.Host) to identify the target. - // Refer: https://httpwg.github.io/specs/rfc7540.html#CONNECT - if req.ProtoMajor == 2 && req.Method == "CONNECT" { - uri = req.Host - } - if uri == "" { - uri = url.RequestURI() - } - - buf := make([]byte, 0, 3*(len(host)+len(username)+len(req.Method)+len(uri)+len(req.Proto)+50)/2) - buf = append(buf, host...) - buf = append(buf, " - "...) - buf = append(buf, username...) - buf = append(buf, " ["...) - buf = append(buf, ts.Format("02/Jan/2006:15:04:05 -0700")...) - buf = append(buf, `] "`...) - buf = append(buf, req.Method...) - buf = append(buf, " "...) - buf = AppendQuoted(buf, uri) - buf = append(buf, " "...) - buf = append(buf, req.Proto...) - buf = append(buf, `" `...) - buf = append(buf, strconv.Itoa(status)...) - buf = append(buf, " "...) - buf = append(buf, strconv.Itoa(size)...) - return buf -} - -// copied from gorilla/handlers and modified to add the body -// writeLog writes a log entry for req to w in Apache Common Log Format. -// ts is the timestamp with which the entry should be logged. -// status and size are used to provide the response HTTP status and size. -func WriteLog(writer io.Writer, params handlers.LogFormatterParams, body string) { - buf := BuildCommonLogLine(params.Request, params.URL, params.TimeStamp, params.StatusCode, params.Size) - // Append body if present - if len(body) > 0 { - buf = append(buf, " "...) - buf = append(buf, body...) - } - buf = append(buf, '\n') - writer.Write(buf) -} From 5cdd79758d767f828b3f89774971671be452579a Mon Sep 17 00:00:00 2001 From: Tate Exon Date: Wed, 24 Jul 2024 09:18:41 -0600 Subject: [PATCH 18/20] Add custom logger as json output Convert channel to *[]byte to make handling future log formats easier --- internal/server/http/dump.go | 58 ++++++++++++++++------------- internal/server/http/server.go | 4 +- internal/server/http/server_test.go | 18 ++++----- 3 files changed, 44 insertions(+), 36 deletions(-) diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go index c036159..8871def 100644 --- a/internal/server/http/dump.go +++ b/internal/server/http/dump.go @@ -5,7 +5,6 @@ import ( "context" "encoding/base64" "encoding/json" - "fmt" "io" "log" "net/http" @@ -13,7 +12,6 @@ import ( "strings" "sync" - internal_handlers "github.com/friendsofgo/killgrave/internal/gorilla/handlers" "github.com/gorilla/handlers" ) @@ -31,19 +29,30 @@ type RequestData struct { Host string `json:"host"` URL string `json:"url"` Header http.Header `json:"header"` - Body string `json:"body"` + Status int `json:"statusCode,omitempty"` + Body string `json:"body,omitempty"` } -func getRequestData(r *http.Request, body string) *RequestData { +func getRequestData(r *http.Request, status int, body string) *RequestData { return &RequestData{ Method: r.Method, Host: r.Host, URL: r.URL.String(), Header: r.Header, + Status: status, Body: body, } } +// ToJSON converts the RequestData struct to JSON. +func (rd *RequestData) toJSON() ([]byte, error) { + jsonData, err := json.Marshal(rd) + if err != nil { + return nil, err + } + return jsonData, nil +} + // isBinaryContent checks to see if the body is a common binary content type func isBinaryContent(r *http.Request) bool { contentType := r.Header.Get("Content-Type") @@ -83,39 +92,40 @@ func getBody(r *http.Request, s *Server) string { func CustomLoggingHandler(out io.Writer, h http.Handler, s *Server) http.Handler { return handlers.CustomLoggingHandler(out, h, func(writer io.Writer, params handlers.LogFormatterParams) { body := getBody(params.Request, s) + requestData := getRequestData(params.Request, params.StatusCode, body) + + // log the request + if s.serverCfg.LogLevel > 0 { + // if we add other formats to log in we can switch here and return the bytes + data, err := requestData.toJSON() + if err != nil { + log.Printf("Error encoding request data: %+v\n", err) + return + } - var err error - if shouldRecordRequest(s) { - err = recordRequest(params.Request, s, body) - } - - // log the request based on the log level - // if err is set, log the request, but only add the body if the log level is 2 or higher - if s.serverCfg.LogLevel >= 2 { - internal_handlers.WriteLog(writer, params, body) - } else if err != nil || s.serverCfg.LogLevel > 0 { - internal_handlers.WriteLog(writer, params, "") + data = append(data, '\n') + writer.Write(data) + if shouldRecordRequest(s) { + recordRequest(&data, s) + } } }) } -func recordRequest(r *http.Request, s *Server, body string) error { - rd := getRequestData(r, body) +func recordRequest(request *[]byte, s *Server) { select { - case s.dumpCh <- rd: + case s.dumpCh <- request: // Successfully sent the request data to the channel default: // Handle the case where the channel is full - return fmt.Errorf("request dump channel is full, could not write request") + log.Println("request dump channel is full, could not write request") } - return nil } // Goroutine function to write requests to a JSON file -func RequestWriter(ctx context.Context, wg *sync.WaitGroup, writer io.Writer, requestChan <-chan *RequestData) { +func RequestWriter(ctx context.Context, wg *sync.WaitGroup, writer io.Writer, requestChan <-chan *[]byte) { defer wg.Done() - encoder := json.NewEncoder(writer) for { select { case requestData := <-requestChan: @@ -123,9 +133,7 @@ func RequestWriter(ctx context.Context, wg *sync.WaitGroup, writer io.Writer, re return // channel closed } - if err := encoder.Encode(requestData); err != nil { - log.Printf("Failed to write to file: %+v", err) - } + writer.Write(*requestData) // call Sync if writer is *os.File if f, ok := writer.(*os.File); ok { f.Sync() diff --git a/internal/server/http/server.go b/internal/server/http/server.go index a15a51d..e1476bf 100644 --- a/internal/server/http/server.go +++ b/internal/server/http/server.go @@ -34,7 +34,7 @@ type Server struct { secure bool imposterFs ImposterFs serverCfg *sc.ServerConfig - dumpCh chan *RequestData + dumpCh chan *[]byte wg *sync.WaitGroup ctx context.Context cancel context.CancelFunc @@ -100,7 +100,7 @@ func (s *Server) Build() error { // only instantiate the request dump if we need it if s.dumpCh == nil && s.serverCfg.LogWriter != nil { - s.dumpCh = make(chan *RequestData, 1000) + s.dumpCh = make(chan *[]byte, 1000) // Start the RequestWriter goroutine with context s.wg.Add(1) diff --git a/internal/server/http/server_test.go b/internal/server/http/server_test.go index 4887dad..4ad2f50 100644 --- a/internal/server/http/server_test.go +++ b/internal/server/http/server_test.go @@ -233,7 +233,7 @@ func TestBuildLogRequests(t *testing.T) { contentType: "text/plain", body: "Dumped", logLevel: 1, - expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17\n", + expectedLog: "{\"method\":\"GET\",\"host\":\"example.com\",\"url\":\"/yamlTestDumpRequest\",\"header\":{\"Content-Type\":[\"text/plain\"]},\"statusCode\":200,\"body\":\"Dumped\"}\n", expectedStatus: http.StatusOK, }, "GET valid imposter request with body": { @@ -242,7 +242,7 @@ func TestBuildLogRequests(t *testing.T) { contentType: "text/plain", body: "Dumped", logLevel: 2, - expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17 Dumped\n", + expectedLog: "{\"method\":\"GET\",\"host\":\"example.com\",\"url\":\"/yamlTestDumpRequest\",\"header\":{\"Content-Type\":[\"text/plain\"]},\"statusCode\":200,\"body\":\"Dumped\"}\n", expectedStatus: http.StatusOK, }, "GET valid imposter binary request": { @@ -251,7 +251,7 @@ func TestBuildLogRequests(t *testing.T) { contentType: "application/octet-stream", body: "Dumped", logLevel: 1, - expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17\n", + expectedLog: "{\"method\":\"GET\",\"host\":\"example.com\",\"url\":\"/yamlTestDumpRequest\",\"header\":{\"Content-Type\":[\"application/octet-stream\"]},\"statusCode\":200,\"body\":\"RHVtcGVk\"}\n", expectedStatus: http.StatusOK, }, "GET valid imposter binary request with body": { @@ -260,7 +260,7 @@ func TestBuildLogRequests(t *testing.T) { contentType: "application/octet-stream", body: "Dumped", logLevel: 2, - expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17 RHVtcGVk\n", + expectedLog: "{\"method\":\"GET\",\"host\":\"example.com\",\"url\":\"/yamlTestDumpRequest\",\"header\":{\"Content-Type\":[\"application/octet-stream\"]},\"statusCode\":200,\"body\":\"RHVtcGVk\"}\n", expectedStatus: http.StatusOK, }, "GET valid imposter request no body": { @@ -269,7 +269,7 @@ func TestBuildLogRequests(t *testing.T) { contentType: "text/plain", body: "", logLevel: 2, - expectedLog: "GET /yamlTestDumpRequest HTTP/1.1\" 200 17\n", + expectedLog: "{\"method\":\"GET\",\"host\":\"example.com\",\"url\":\"/yamlTestDumpRequest\",\"header\":{\"Content-Type\":[\"text/plain\"]},\"statusCode\":200}\n", expectedStatus: http.StatusOK, }, "GET invalid imposter request": { @@ -278,7 +278,7 @@ func TestBuildLogRequests(t *testing.T) { contentType: "text/plain", body: "Dumped", logLevel: 1, - expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19\n", + expectedLog: "{\"method\":\"GET\",\"host\":\"example.com\",\"url\":\"/doesnotexist\",\"header\":{\"Content-Type\":[\"text/plain\"]},\"statusCode\":404,\"body\":\"Dumped\"}\n", expectedStatus: http.StatusNotFound, }, "GET invalid imposter request with body": { @@ -287,7 +287,7 @@ func TestBuildLogRequests(t *testing.T) { contentType: "text/plain", body: "Dumped", logLevel: 2, - expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19 Dumped\n", + expectedLog: "{\"method\":\"GET\",\"host\":\"example.com\",\"url\":\"/doesnotexist\",\"header\":{\"Content-Type\":[\"text/plain\"]},\"statusCode\":404,\"body\":\"Dumped\"}\n", expectedStatus: http.StatusNotFound, }, "GET invalid imposter binary request with body": { @@ -296,7 +296,7 @@ func TestBuildLogRequests(t *testing.T) { contentType: "video/mp4", body: "Dumped", logLevel: 2, - expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19 RHVtcGVk\n", + expectedLog: "{\"method\":\"GET\",\"host\":\"example.com\",\"url\":\"/doesnotexist\",\"header\":{\"Content-Type\":[\"video/mp4\"]},\"statusCode\":404,\"body\":\"RHVtcGVk\"}\n", expectedStatus: http.StatusNotFound, }, "GET invalid imposter request no body": { @@ -305,7 +305,7 @@ func TestBuildLogRequests(t *testing.T) { contentType: "text/plain", body: "", logLevel: 2, - expectedLog: "GET /doesnotexist HTTP/1.1\" 404 19\n", + expectedLog: "{\"method\":\"GET\",\"host\":\"example.com\",\"url\":\"/doesnotexist\",\"header\":{\"Content-Type\":[\"text/plain\"]},\"statusCode\":404}\n", expectedStatus: http.StatusNotFound, }, } From 3dacb9d7d1e1b496fccb3290d201c3220dc119ec Mon Sep 17 00:00:00 2001 From: Tate Exon Date: Wed, 24 Jul 2024 09:52:16 -0600 Subject: [PATCH 19/20] Add max to the body that can be logged out add configuration for this --- README.md | 2 ++ internal/app/cmd/cmd.go | 12 +++++++++++- internal/config.go | 4 +++- internal/config_test.go | 4 ++-- internal/server/http/dump.go | 5 +++++ internal/server/http/server_test.go | 10 ++++++---- internal/serverconfig/serverconfig.go | 7 +++++++ 7 files changed, 36 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2c30e9a..ca1aa1f 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,7 @@ Flags: -v, --version Version of Killgrave -w, --watcher File watcher will reload the server on each file change -l, --log-level The higher the log level the more detail you get (default 0, 1 adds requests, 2 adds request body) + --log-body-max The maximum size of body that will be returned, will cut off if body is longer, default size is 512 -d, --dump-requests-path Print requests out to specified file ``` @@ -185,6 +186,7 @@ cors: watcher: true secure: true log_level: 1 +log_body_max: 256 dump_requests_path: "/abc/def.log ``` diff --git a/internal/app/cmd/cmd.go b/internal/app/cmd/cmd.go index 1958dff..1fa9a11 100644 --- a/internal/app/cmd/cmd.go +++ b/internal/app/cmd/cmd.go @@ -27,6 +27,7 @@ const ( _defaultProxyMode = killgrave.ProxyNone _defaultStrictSlash = true _defaultLogLevel = 0 + _defaultLogBodyMax = 512 _impostersFlag = "imposters" _configFlag = "config" @@ -37,6 +38,7 @@ const ( _proxyModeFlag = "proxy-mode" _proxyURLFlag = "proxy-url" _logLevelFlag = "log-level" + _logBodyMaxFlag = "log-body-max" _dumpRequestsPathFlag = "dump-requests-path" ) @@ -47,6 +49,7 @@ var ( errGetDataFromSecureFlag = errors.New("error trying to get data from secure flag") errGetDataFromLogLevelFlag = errors.New("error trying to get data from log-level flag") errGetDataLogLevelInvalid = errors.New("error setting log-level, must be between 0 and 2 inclusive") + errGetDataFromLogBodyMaxFlag = errors.New("error trying to get data from log-body-max flag") errGetDataFromDumpRequestsPathFlag = errors.New("error trying to get data from dump-requests-path flag") errMandatoryURL = errors.New("the field proxy-url is mandatory if you selected a proxy mode") ) @@ -84,6 +87,7 @@ func NewKillgraveCmd() *cobra.Command { rootCmd.Flags().StringP(_proxyModeFlag, "m", _defaultProxyMode.String(), "Proxy mode, the options are all, missing or none") rootCmd.Flags().StringP(_proxyURLFlag, "u", "", "The url where the proxy will redirect to") rootCmd.Flags().IntP(_logLevelFlag, "l", _defaultLogLevel, "Log level, the options are 0, 1, 2. Default is 0, 1 adds requests, 2 adds request body") + rootCmd.Flags().Int(_logBodyMaxFlag, _defaultLogBodyMax, fmt.Sprintf("The maximum size of body that will be returned, will cut off if body is longer, default size is %s", _defaultLogBodyMax)) rootCmd.Flags().StringP(_dumpRequestsPathFlag, "d", "", "Path the requests will be dumped to") rootCmd.SetVersionTemplate("Killgrave version: {{.Version}}\n") @@ -154,6 +158,7 @@ func runServer(cfg killgrave.Config) server.Server { imposterFs, sc.WithCORSOptions(server.PrepareAccessControl(cfg.CORS)), sc.WithLogLevel(cfg.LogLevel), + sc.WithLogBodyMax(cfg.LogBodyMax), sc.WithLogWriter(cfg.LogWriter), ) if err := s.Build(); err != nil { @@ -213,12 +218,17 @@ func prepareConfig(cmd *cobra.Command) (killgrave.Config, error) { return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataLogLevelInvalid) } + logBodyMax, err := cmd.Flags().GetInt(_logBodyMaxFlag) + if err != nil { + return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromLogBodyMaxFlag) + } + dumpRequestsPath, err := cmd.Flags().GetString(_dumpRequestsPathFlag) if err != nil { return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromDumpRequestsPathFlag) } - cfg, err := killgrave.NewConfig(impostersPath, host, port, secure, logLevel, dumpRequestsPath) + cfg, err := killgrave.NewConfig(impostersPath, host, port, secure, logLevel, logBodyMax, dumpRequestsPath) if err != nil { return killgrave.Config{}, err } diff --git a/internal/config.go b/internal/config.go index 303dba7..766f2a7 100644 --- a/internal/config.go +++ b/internal/config.go @@ -21,6 +21,7 @@ type Config struct { Secure bool `yaml:"secure"` Watcher bool `yaml:"watcher"` LogLevel int `yaml:"log_level"` + LogBodyMax int `yaml:"log_body_max"` DumpRequestsPath string `yaml:"dump_requests_path"` LogWriter io.Writer } @@ -115,7 +116,7 @@ func (cfg *Config) ConfigureProxy(proxyMode ProxyMode, proxyURL string) { type ConfigOpt func(cfg *Config) error // NewConfig initialize the config -func NewConfig(impostersPath, host string, port int, secure bool, logLevel int, dumpRequestsPath string) (Config, error) { +func NewConfig(impostersPath, host string, port int, secure bool, logLevel, logBodyMax int, dumpRequestsPath string) (Config, error) { if impostersPath == "" { return Config{}, errEmptyImpostersPath } @@ -134,6 +135,7 @@ func NewConfig(impostersPath, host string, port int, secure bool, logLevel int, Port: port, Secure: secure, LogLevel: logLevel, + LogBodyMax: logBodyMax, DumpRequestsPath: dumpRequestsPath, } diff --git a/internal/config_test.go b/internal/config_test.go index 3230e8a..daf7883 100644 --- a/internal/config_test.go +++ b/internal/config_test.go @@ -203,7 +203,7 @@ func TestNewConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewConfig(tt.args.impostersPath, tt.args.host, tt.args.port, false, 0, "") + got, err := NewConfig(tt.args.impostersPath, tt.args.host, tt.args.port, false, 0, 0, "") assert.Equal(t, tt.err, err) assert.Equal(t, tt.want, got) }) @@ -221,7 +221,7 @@ func TestConfig_ConfigureProxy(t *testing.T) { }, } - got, err := NewConfig("imposters", "localhost", 80, false, 0, "") + got, err := NewConfig("imposters", "localhost", 80, false, 0, 0, "") assert.NoError(t, err) got.ConfigureProxy(ProxyAll, "https://friendsofgo.tech") diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go index 8871def..1aa9fed 100644 --- a/internal/server/http/dump.go +++ b/internal/server/http/dump.go @@ -79,6 +79,11 @@ func getBody(r *http.Request, s *Server) string { } r.Body = io.NopCloser(bytes.NewReader(bodyBytes)) // Reset the body + // trim if larger than the limit allowed + if len(bodyBytes) > s.serverCfg.LogBodyMax { + bodyBytes = bodyBytes[:s.serverCfg.LogBodyMax] + } + body := base64.StdEncoding.EncodeToString(bodyBytes) // if content is not binary, get it as a string if !isBinaryContent(r) { diff --git a/internal/server/http/server_test.go b/internal/server/http/server_test.go index 4ad2f50..c00a449 100644 --- a/internal/server/http/server_test.go +++ b/internal/server/http/server_test.go @@ -321,7 +321,7 @@ func TestBuildLogRequests(t *testing.T) { imposterFs, err := NewImposterFS("test/testdata/imposters") assert.NoError(t, err) - server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, sc.WithLogLevel(tc.logLevel)) + server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, sc.WithLogLevel(tc.logLevel), sc.WithLogBodyMax(512)) err = server.Build() assert.NoError(t, err) @@ -359,15 +359,17 @@ func TestBuildRecordRequests(t *testing.T) { log.Fatalf("Failed to open file: %+v", err) } defer file.Close() - server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, sc.WithLogWriter(file), sc.WithLogLevel(2)) + server := NewServer(mux.NewRouter(), &http.Server{}, &Proxy{}, false, imposterFs, sc.WithLogWriter(file), sc.WithLogLevel(2), sc.WithLogBodyMax(8)) err = server.Build() assert.NoError(t, err) - expectedBodies := []string{"Dumped1", ""} + expectedBodies := []string{"Dumped1", "", "longer t"} req1 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(expectedBodies[0])) req2 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader(expectedBodies[1])) + req3 := httptest.NewRequest("GET", "/yamlTestDumpRequest", strings.NewReader("longer than allowed")) server.httpServer.Handler.ServeHTTP(w, req1) server.httpServer.Handler.ServeHTTP(w, req2) + server.httpServer.Handler.ServeHTTP(w, req3) // wait for channel to print out the requests time.Sleep(1 * time.Second) @@ -375,7 +377,7 @@ func TestBuildRecordRequests(t *testing.T) { // check recoreded request dumps reqs, err := getRecordedRequests(dumpFile) assert.NoError(t, err, "Failed to read requests from file") - assert.Equal(t, 2, len(reqs), "Expect 2 requests to be dumped in file failed") + assert.Equal(t, 3, len(reqs), "Expect 2 requests to be dumped in file failed") for i, expectedBody := range expectedBodies { assert.Equal(t, expectedBody, reqs[i].Body, "Expect request body to be dumped in file failed") } diff --git a/internal/serverconfig/serverconfig.go b/internal/serverconfig/serverconfig.go index ec1d187..6da11f9 100644 --- a/internal/serverconfig/serverconfig.go +++ b/internal/serverconfig/serverconfig.go @@ -9,6 +9,7 @@ import ( type ServerConfig struct { CORSOptions []handlers.CORSOption LogLevel int + LogBodyMax int LogWriter io.Writer } @@ -26,6 +27,12 @@ func WithLogLevel(logLevel int) ServerOption { } } +func WithLogBodyMax(logBodyMax int) ServerOption { + return func(cfg *ServerConfig) { + cfg.LogBodyMax = logBodyMax + } +} + func WithLogWriter(logWriter io.Writer) ServerOption { return func(cfg *ServerConfig) { cfg.LogWriter = logWriter From 78a8ae0658afdcc35cff382e006c1e2f315142ad Mon Sep 17 00:00:00 2001 From: Tate Exon Date: Wed, 24 Jul 2024 10:06:30 -0600 Subject: [PATCH 20/20] Use a limited reader to only read up to the allowed maximum bytes from the body --- internal/server/http/dump.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/internal/server/http/dump.go b/internal/server/http/dump.go index 1aa9fed..5e8b1e9 100644 --- a/internal/server/http/dump.go +++ b/internal/server/http/dump.go @@ -72,17 +72,15 @@ func getBody(r *http.Request, s *Server) string { if s.serverCfg.LogLevel == 0 && !shouldRecordRequest(s) { return "" } - bodyBytes, err := io.ReadAll(r.Body) + // Use io.LimitedReader to read up to the max allowed bytes + limitedReader := &io.LimitedReader{R: r.Body, N: int64(s.serverCfg.LogBodyMax)} + bodyBytes, err := io.ReadAll(limitedReader) if err != nil { log.Printf("Error reading request body: %v\n", err) return "" } - r.Body = io.NopCloser(bytes.NewReader(bodyBytes)) // Reset the body - - // trim if larger than the limit allowed - if len(bodyBytes) > s.serverCfg.LogBodyMax { - bodyBytes = bodyBytes[:s.serverCfg.LogBodyMax] - } + // Reset the body + r.Body = io.NopCloser(io.MultiReader(bytes.NewReader(bodyBytes), r.Body)) body := base64.StdEncoding.EncodeToString(bodyBytes) // if content is not binary, get it as a string