Add logging and recording of requests

README.md

@@ -156,6 +156,9 @@ Flags:
   -s, --secure              Run mock server using TLS (https)
   -v, --version             Version of Killgrave
   -w, --watcher             File watcher will reload the server on each file change
+  -l, --log-level           The higher the log level the more detail you get (default 0, 1 adds requests, 2 adds request body)
+      --log-body-max        The maximum size of body that will be returned, will cut off if body is longer, default size is 512
+  -d, --dump-requests-path  Print requests out to specified file
 ```
 
 ### Using Killgrave by config file
@@ -182,6 +185,9 @@ cors:
   allow_credentials: true
 watcher: true
 secure: true
+log_level: 1
+log_body_max: 256
+dump_requests_path: "/abc/def.log
 ```
 
 As you can see, you can configure all the options in a very easy way. For the above example, the file tree looks as follows, with the current working directory being `mymock`.

internal/app/cmd/cmd.go

@@ -11,7 +11,7 @@ import (
 
 	killgrave "github.com/friendsofgo/killgrave/internal"
 	server "github.com/friendsofgo/killgrave/internal/server/http"
-	"github.com/gorilla/handlers"
+	sc "github.com/friendsofgo/killgrave/internal/serverconfig"
 	"github.com/gorilla/mux"
 	"github.com/radovskyb/watcher"
 	"github.com/spf13/cobra"
@@ -26,23 +26,32 @@ const (
 	_defaultPort          = 3000
 	_defaultProxyMode     = killgrave.ProxyNone
 	_defaultStrictSlash   = true
-
-	_impostersFlag = "imposters"
-	_configFlag    = "config"
-	_hostFlag      = "host"
-	_portFlag      = "port"
-	_watcherFlag   = "watcher"
-	_secureFlag    = "secure"
-	_proxyModeFlag = "proxy-mode"
-	_proxyURLFlag  = "proxy-url"
+	_defaultLogLevel      = 0
+	_defaultLogBodyMax    = 512
+
+	_impostersFlag        = "imposters"
+	_configFlag           = "config"
+	_hostFlag             = "host"
+	_portFlag             = "port"
+	_watcherFlag          = "watcher"
+	_secureFlag           = "secure"
+	_proxyModeFlag        = "proxy-mode"
+	_proxyURLFlag         = "proxy-url"
+	_logLevelFlag         = "log-level"
+	_logBodyMaxFlag       = "log-body-max"
+	_dumpRequestsPathFlag = "dump-requests-path"
 )
 
 var (
-	errGetDataFromImpostersFlag = errors.New("error trying to get data from imposters flag")
-	errGetDataFromHostFlag      = errors.New("error trying to get data from host flag")
-	errGetDataFromPortFlag      = errors.New("error trying to get data from port flag")
-	errGetDataFromSecureFlag    = errors.New("error trying to get data from secure flag")
-	errMandatoryURL             = errors.New("the field proxy-url is mandatory if you selected a proxy mode")
+	errGetDataFromImpostersFlag        = errors.New("error trying to get data from imposters flag")
+	errGetDataFromHostFlag             = errors.New("error trying to get data from host flag")
+	errGetDataFromPortFlag             = errors.New("error trying to get data from port flag")
+	errGetDataFromSecureFlag           = errors.New("error trying to get data from secure flag")
+	errGetDataFromLogLevelFlag         = errors.New("error trying to get data from log-level flag")
+	errGetDataLogLevelInvalid          = errors.New("error setting log-level, must be between 0 and 2 inclusive")
+	errGetDataFromLogBodyMaxFlag       = errors.New("error trying to get data from log-body-max flag")
+	errGetDataFromDumpRequestsPathFlag = errors.New("error trying to get data from dump-requests-path flag")
+	errMandatoryURL                    = errors.New("the field proxy-url is mandatory if you selected a proxy mode")
 )
 
 // NewKillgraveCmd returns cobra.Command to run killgrave command
@@ -77,6 +86,9 @@ func NewKillgraveCmd() *cobra.Command {
 	rootCmd.Flags().BoolP(_secureFlag, "s", false, "Run mock server using TLS (https)")
 	rootCmd.Flags().StringP(_proxyModeFlag, "m", _defaultProxyMode.String(), "Proxy mode, the options are all, missing or none")
 	rootCmd.Flags().StringP(_proxyURLFlag, "u", "", "The url where the proxy will redirect to")
+	rootCmd.Flags().IntP(_logLevelFlag, "l", _defaultLogLevel, "Log level, the options are 0, 1, 2. Default is 0, 1 adds requests, 2 adds request body")
+	rootCmd.Flags().Int(_logBodyMaxFlag, _defaultLogBodyMax, fmt.Sprintf("The maximum size of body that will be returned, will cut off if body is longer, default size is %s", _defaultLogBodyMax))
+	rootCmd.Flags().StringP(_dumpRequestsPathFlag, "d", "", "Path the requests will be dumped to")
 
 	rootCmd.SetVersionTemplate("Killgrave version: {{.Version}}\n")
 
@@ -89,6 +101,16 @@ func runHTTP(cmd *cobra.Command, cfg killgrave.Config) error {
 
 	signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)
 
+	// setup the log file and its deferred close if needed
+	if len(cfg.DumpRequestsPath) > 0 && cfg.LogLevel > 0 {
+		file, err := os.OpenFile(cfg.DumpRequestsPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			log.Fatalf("Failed to open file: %+v", err)
+		}
+		cfg.LogWriter = file
+		defer file.Close()
+	}
+
 	srv := runServer(cfg)
 
 	watcherFlag, _ := cmd.Flags().GetBool(_watcherFlag)
@@ -115,8 +137,7 @@ func runServer(cfg killgrave.Config) server.Server {
 	httpAddr := fmt.Sprintf("%s:%d", cfg.Host, cfg.Port)
 
 	httpServer := http.Server{
-		Addr:    httpAddr,
-		Handler: handlers.CORS(server.PrepareAccessControl(cfg.CORS)...)(router),
+		Addr: httpAddr,
 	}
 
 	proxyServer, err := server.NewProxy(cfg.Proxy.Url, cfg.Proxy.Mode)
@@ -135,6 +156,10 @@ func runServer(cfg killgrave.Config) server.Server {
 		proxyServer,
 		cfg.Secure,
 		imposterFs,
+		sc.WithCORSOptions(server.PrepareAccessControl(cfg.CORS)),
+		sc.WithLogLevel(cfg.LogLevel),
+		sc.WithLogBodyMax(cfg.LogBodyMax),
+		sc.WithLogWriter(cfg.LogWriter),
 	)
 	if err := s.Build(); err != nil {
 		log.Fatal(err)
@@ -185,7 +210,25 @@ func prepareConfig(cmd *cobra.Command) (killgrave.Config, error) {
 		return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromSecureFlag)
 	}
 
-	cfg, err := killgrave.NewConfig(impostersPath, host, port, secure)
+	logLevel, err := cmd.Flags().GetInt(_logLevelFlag)
+	if err != nil {
+		return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromLogLevelFlag)
+	}
+	if logLevel < 0 || logLevel > 2 {
+		return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataLogLevelInvalid)
+	}
+
+	logBodyMax, err := cmd.Flags().GetInt(_logBodyMaxFlag)
+	if err != nil {
+		return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromLogBodyMaxFlag)
+	}
+
+	dumpRequestsPath, err := cmd.Flags().GetString(_dumpRequestsPathFlag)
+	if err != nil {
+		return killgrave.Config{}, fmt.Errorf("%v: %w", err, errGetDataFromDumpRequestsPathFlag)
+	}
+
+	cfg, err := killgrave.NewConfig(impostersPath, host, port, secure, logLevel, logBodyMax, dumpRequestsPath)
 	if err != nil {
 		return killgrave.Config{}, err
 	}

internal/config.go

@@ -3,6 +3,7 @@ package killgrave
 import (
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -12,13 +13,17 @@ import (
 
 // Config representation of config file yaml
 type Config struct {
-	ImpostersPath string      `yaml:"imposters_path"`
-	Port          int         `yaml:"port"`
-	Host          string      `yaml:"host"`
-	CORS          ConfigCORS  `yaml:"cors"`
-	Proxy         ConfigProxy `yaml:"proxy"`
-	Secure        bool        `yaml:"secure"`
-	Watcher       bool        `yaml:"watcher"`
+	ImpostersPath    string      `yaml:"imposters_path"`
+	Port             int         `yaml:"port"`
+	Host             string      `yaml:"host"`
+	CORS             ConfigCORS  `yaml:"cors"`
+	Proxy            ConfigProxy `yaml:"proxy"`
+	Secure           bool        `yaml:"secure"`
+	Watcher          bool        `yaml:"watcher"`
+	LogLevel         int         `yaml:"log_level"`
+	LogBodyMax       int         `yaml:"log_body_max"`
+	DumpRequestsPath string      `yaml:"dump_requests_path"`
+	LogWriter        io.Writer
 }
 
 // ConfigCORS representation of section CORS of the yaml
@@ -111,7 +116,7 @@ func (cfg *Config) ConfigureProxy(proxyMode ProxyMode, proxyURL string) {
 type ConfigOpt func(cfg *Config) error
 
 // NewConfig initialize the config
-func NewConfig(impostersPath, host string, port int, secure bool) (Config, error) {
+func NewConfig(impostersPath, host string, port int, secure bool, logLevel, logBodyMax int, dumpRequestsPath string) (Config, error) {
 	if impostersPath == "" {
 		return Config{}, errEmptyImpostersPath
 	}
@@ -125,10 +130,13 @@ func NewConfig(impostersPath, host string, port int, secure bool) (Config, error
 	}
 
 	cfg := Config{
-		ImpostersPath: impostersPath,
-		Host:          host,
-		Port:          port,
-		Secure:        secure,
+		ImpostersPath:    impostersPath,
+		Host:             host,
+		Port:             port,
+		Secure:           secure,
+		LogLevel:         logLevel,
+		LogBodyMax:       logBodyMax,
+		DumpRequestsPath: dumpRequestsPath,
 	}
 
 	return cfg, nil

internal/config_test.go

@@ -203,7 +203,7 @@ func TestNewConfig(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := NewConfig(tt.args.impostersPath, tt.args.host, tt.args.port, false)
+			got, err := NewConfig(tt.args.impostersPath, tt.args.host, tt.args.port, false, 0, 0, "")
 			assert.Equal(t, tt.err, err)
 			assert.Equal(t, tt.want, got)
 		})
@@ -221,7 +221,7 @@ func TestConfig_ConfigureProxy(t *testing.T) {
 		},
 	}
 
-	got, err := NewConfig("imposters", "localhost", 80, false)
+	got, err := NewConfig("imposters", "localhost", 80, false, 0, 0, "")
 	assert.NoError(t, err)
 
 	got.ConfigureProxy(ProxyAll, "https://friendsofgo.tech")

internal/server/http/dump.go

@@ -0,0 +1,148 @@
+package http
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/gorilla/handlers"
+)
+
+var binaryContentTypes = []string{
+	"application/octet-stream",
+	"image/",
+	"audio/",
+	"video/",
+	"application/pdf",
+}
+
+// RequestData struct to hold request data
+type RequestData struct {
+	Method string      `json:"method"`
+	Host   string      `json:"host"`
+	URL    string      `json:"url"`
+	Header http.Header `json:"header"`
+	Status int         `json:"statusCode,omitempty"`
+	Body   string      `json:"body,omitempty"`
+}
+
+func getRequestData(r *http.Request, status int, body string) *RequestData {
+	return &RequestData{
+		Method: r.Method,
+		Host:   r.Host,
+		URL:    r.URL.String(),
+		Header: r.Header,
+		Status: status,
+		Body:   body,
+	}
+}
+
+// ToJSON converts the RequestData struct to JSON.
+func (rd *RequestData) toJSON() ([]byte, error) {
+	jsonData, err := json.Marshal(rd)
+	if err != nil {
+		return nil, err
+	}
+	return jsonData, nil
+}
+
+// isBinaryContent checks to see if the body is a common binary content type
+func isBinaryContent(r *http.Request) bool {
+	contentType := r.Header.Get("Content-Type")
+	for _, binaryType := range binaryContentTypes {
+		if strings.HasPrefix(contentType, binaryType) {
+			return true
+		}
+	}
+	return false
+}
+
+func shouldRecordRequest(s *Server) bool {
+	return s.serverCfg.LogWriter != nil && s.dumpCh != nil
+}
+
+func getBody(r *http.Request, s *Server) string {
+	if s.serverCfg.LogLevel == 0 && !shouldRecordRequest(s) {
+		return ""
+	}
+	// Use io.LimitedReader to read up to the max allowed bytes
+	limitedReader := &io.LimitedReader{R: r.Body, N: int64(s.serverCfg.LogBodyMax)}
+	bodyBytes, err := io.ReadAll(limitedReader)
+	if err != nil {
+		log.Printf("Error reading request body: %v\n", err)
+		return ""
+	}
+	// Reset the body
+	r.Body = io.NopCloser(io.MultiReader(bytes.NewReader(bodyBytes), r.Body))
+
+	body := base64.StdEncoding.EncodeToString(bodyBytes)
+	// if content is not binary, get it as a string
+	if !isBinaryContent(r) {
+		body = string(bodyBytes)
+	}
+	return body
+}
+
+// CustomLoggingHandler provides a way to supply a custom log formatter
+// while taking advantage of the mechanisms in this package
+func CustomLoggingHandler(out io.Writer, h http.Handler, s *Server) http.Handler {
+	return handlers.CustomLoggingHandler(out, h, func(writer io.Writer, params handlers.LogFormatterParams) {
+		body := getBody(params.Request, s)
+		requestData := getRequestData(params.Request, params.StatusCode, body)
+
+		// log the request
+		if s.serverCfg.LogLevel > 0 {
+			// if we add other formats to log in we can switch here and return the bytes
+			data, err := requestData.toJSON()
+			if err != nil {
+				log.Printf("Error encoding request data: %+v\n", err)
+				return
+			}
+
+			data = append(data, '\n')
+			writer.Write(data)
+			if shouldRecordRequest(s) {
+				recordRequest(&data, s)
+			}
+		}
+	})
+}
+
+func recordRequest(request *[]byte, s *Server) {
+	select {
+	case s.dumpCh <- request:
+		// Successfully sent the request data to the channel
+	default:
+		// Handle the case where the channel is full
+		log.Println("request dump channel is full, could not write request")
+	}
+}
+
+// Goroutine function to write requests to a JSON file
+func RequestWriter(ctx context.Context, wg *sync.WaitGroup, writer io.Writer, requestChan <-chan *[]byte) {
+	defer wg.Done()
+
+	for {
+		select {
+		case requestData := <-requestChan:
+			if requestData == nil {
+				return // channel closed
+			}
+
+			writer.Write(*requestData)
+			// call Sync if writer is *os.File
+			if f, ok := writer.(*os.File); ok {
+				f.Sync()
+			}
+		case <-ctx.Done():
+			return // context cancelled
+		}
+	}
+}