From 4391ed163c1194948e897aaa91aee73ea7d00472 Mon Sep 17 00:00:00 2001 From: qinzhonghe Date: Sat, 29 Jan 2022 17:40:14 +0800 Subject: [PATCH] Release/v0.2.6 --- context.go | 2 +- downloader.go | 3 ++- downloader_test.go | 11 +++++++---- dupefilters.go | 6 +++--- dupefilters_test.go | 2 +- request.go | 5 +++-- settings.go | 10 ---------- 7 files changed, 17 insertions(+), 22 deletions(-) diff --git a/context.go b/context.go index c701a11..e48b8dd 100644 --- a/context.go +++ b/context.go @@ -38,7 +38,7 @@ type ContextOption func(c *Context) func NewContext(request *Request, opts ...ContextOption) *Context { ctx := &Context{ Request: request, - parent: context.TODO(), + parent: nil, CtxId: GetUUID(), DownloadResult: NewDownloadResult(), } diff --git a/downloader.go b/downloader.go index 5944ecb..a3162ff 100644 --- a/downloader.go +++ b/downloader.go @@ -12,6 +12,7 @@ package tegenaria import ( + "bytes" "context" "crypto/tls" "errors" @@ -278,7 +279,7 @@ func (d *SpiderDownloader) Download(ctx *Context, result chan<- *Context) { // Build the request here and pass in the context information var asCtxKey ctxKey = "key" valCtx := context.WithValue(ctx, asCtxKey, ctxValue) - req, err := http.NewRequestWithContext(valCtx, ctx.Request.Method, u.String(), ctx.Request.BodyReader) + req, err := http.NewRequestWithContext(valCtx, ctx.Request.Method, u.String(), bytes.NewReader(ctx.Request.Body)) if err != nil { downloadLog.Errorf(fmt.Sprintf("Create request error %s", err.Error())) ctx.DownloadResult.Error = NewError(ctx.CtxId, err, ErrorWithRequest(ctx.Request)) diff --git a/downloader_test.go b/downloader_test.go index b6d7708..57b826f 100644 --- a/downloader_test.go +++ b/downloader_test.go @@ -85,7 +85,8 @@ func newTestServer() *httptest.Server { c.String(200, "POST") } else { - c.String(200, string(data)) + s := string(data) + c.String(200, s) } }) router.GET("/testGetCookie", func(c *gin.Context) { @@ -188,11 +189,12 @@ func TestRequestGet(t *testing.T) { func TestRequestPost(t *testing.T) { body := map[string]interface{}{ - "key": "value", + "key1": "value1", } server := newTestServer() request := NewRequest(server.URL+"/testPOST", POST, testParser, RequestWithRequestBody(body)) + var MainCtx context.Context = context.Background() cancelCtx, cancel := context.WithCancel(MainCtx) @@ -217,8 +219,9 @@ func TestRequestPost(t *testing.T) { t.Errorf("response status = %d; expected %d", resp.Status, 200) } - if resp.String() != "POST" { - t.Errorf("response text = %s; expected %s", resp.String(), "POST") + data,_ := resp.Json() + if data["key1"].(string) != "value1" { + t.Errorf("response text = %v; expected %s", data, body) } diff --git a/dupefilters.go b/dupefilters.go index 534a30f..968397d 100644 --- a/dupefilters.go +++ b/dupefilters.go @@ -86,10 +86,10 @@ func (f *RFPDupeFilter) Fingerprint(request *Request) ([]byte, error) { if err != nil { return nil, err } - // get request body + // read request body if request.Body != nil { - body := request.Body - sha.Write(body) + + sha.Write(request.Body) } // to handle request header if len(request.Header) != 0 { diff --git a/dupefilters_test.go b/dupefilters_test.go index 08fcfe1..a0c774d 100644 --- a/dupefilters_test.go +++ b/dupefilters_test.go @@ -47,7 +47,6 @@ func TestDoBodyDupeFilter(t *testing.T) { request2 := NewRequest(server.URL+"/testHeader", GET, testParser, RequestWithRequestHeader(headers),RequestWithRequestBody(body)) request3 := NewRequest(server.URL+"/testHeader2", GET, testParser, RequestWithRequestHeader(headers)) request4 := NewRequest(server.URL+"/testHeader", GET, testParser, RequestWithRequestHeader(headers),RequestWithRequestBody(body)) - duplicates := NewRFPDupeFilter(1024*1024, 5) if r1, err := duplicates.DoDupeFilter(request1); r1||err!=nil { t.Errorf("Request1 error expected=%v, get=%v", false, true) @@ -64,4 +63,5 @@ func TestDoBodyDupeFilter(t *testing.T) { t.Errorf("Request4 error expected=%v, get=%v", true, false) } + } diff --git a/request.go b/request.go index 7fd59b0..706e65d 100644 --- a/request.go +++ b/request.go @@ -72,8 +72,9 @@ var reqLog *logrus.Entry = GetLogger("request") func RequestWithRequestBody(body map[string]interface{}) Option { return func(r *Request) { var err error - body, err := jsoniter.Marshal(body) - r.BodyReader = bytes.NewBuffer(body) + + r.Body, err = jsoniter.Marshal(body) + r.BodyReader = bytes.NewBuffer(r.Body) if err != nil { reqLog.Errorf("set request body err %s", err.Error()) panic(fmt.Sprintf("set request body err %s", err.Error())) diff --git a/settings.go b/settings.go index 3dda661..7a3ca01 100644 --- a/settings.go +++ b/settings.go @@ -12,7 +12,6 @@ package tegenaria import ( - "fmt" "os" "github.com/spf13/viper" @@ -31,10 +30,6 @@ type Configuration struct { Log *Logger `ymal:"log"` } -func (l *Logger) GetValue(key string) (string, error) { - return "", nil -} - var Config *Configuration = &Configuration{ Log: &Logger{ Path: "/var/log", @@ -43,11 +38,6 @@ var Config *Configuration = &Configuration{ } func load() bool { - defer func() { - if p := recover(); p != nil { - fmt.Printf("fatal error config file: %s", p) - } - }() str, _ := os.Getwd() runtimeViper := viper.New()