Merge branch 'release/v0.2.6' into 'master'

Release/v0.2.6

See merge request wecode/Tegenaria!15

context.go

@@ -38,7 +38,7 @@ type ContextOption func(c *Context)
 func NewContext(request *Request, opts ...ContextOption) *Context {
 	ctx := &Context{
 		Request: request,
-		parent:            context.TODO(),
+		parent:            nil,
 		CtxId:          GetUUID(),
 		DownloadResult: NewDownloadResult(),
 	}

downloader.go

@@ -12,6 +12,7 @@
 package tegenaria
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"errors"
@@ -278,7 +279,7 @@ func (d *SpiderDownloader) Download(ctx *Context, result chan<- *Context) {
 	// Build the request here and pass in the context information
 	var asCtxKey ctxKey = "key"
 	valCtx := context.WithValue(ctx, asCtxKey, ctxValue)
-	req, err := http.NewRequestWithContext(valCtx, ctx.Request.Method, u.String(), ctx.Request.BodyReader)
+	req, err := http.NewRequestWithContext(valCtx, ctx.Request.Method, u.String(), bytes.NewReader(ctx.Request.Body))
 	if err != nil {
 		downloadLog.Errorf(fmt.Sprintf("Create request error %s", err.Error()))
 		ctx.DownloadResult.Error = NewError(ctx.CtxId, err, ErrorWithRequest(ctx.Request))

downloader_test.go

@@ -85,7 +85,8 @@ func newTestServer() *httptest.Server {
 
 				c.String(200, "POST")
 			} else {
-				c.String(200, string(data))
+				s := string(data)
+				c.String(200, s)
 			}
 		})
 		router.GET("/testGetCookie", func(c *gin.Context) {
@@ -188,11 +189,12 @@ func TestRequestGet(t *testing.T) {
 
 func TestRequestPost(t *testing.T) {
 	body := map[string]interface{}{
-		"key": "value",
+		"key1": "value1",
 	}
 	server := newTestServer()
 
 	request := NewRequest(server.URL+"/testPOST", POST, testParser, RequestWithRequestBody(body))
+
 	var MainCtx context.Context = context.Background()
 
 	cancelCtx, cancel := context.WithCancel(MainCtx)
@@ -217,8 +219,9 @@ func TestRequestPost(t *testing.T) {
 		t.Errorf("response status = %d; expected %d", resp.Status, 200)
 
 	}
-	if resp.String() != "POST" {
-		t.Errorf("response text = %s; expected %s", resp.String(), "POST")
+	data,_ := resp.Json()
+	if data["key1"].(string) != "value1" {
+		t.Errorf("response text = %v; expected %s", data, body)
 
 	}
 

dupefilters.go

@@ -86,10 +86,10 @@ func (f *RFPDupeFilter) Fingerprint(request *Request) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	// get request body
+	// read request body
 	if request.Body != nil {
-		body := request.Body
-		sha.Write(body)
+
+		sha.Write(request.Body)
 	}
 	// to handle request header
 	if len(request.Header) != 0 {

dupefilters_test.go

@@ -47,7 +47,6 @@ func TestDoBodyDupeFilter(t *testing.T) {
 	request2 := NewRequest(server.URL+"/testHeader", GET, testParser, RequestWithRequestHeader(headers),RequestWithRequestBody(body))
 	request3 := NewRequest(server.URL+"/testHeader2", GET, testParser, RequestWithRequestHeader(headers))
 	request4 := NewRequest(server.URL+"/testHeader", GET, testParser, RequestWithRequestHeader(headers),RequestWithRequestBody(body))
-
 	duplicates := NewRFPDupeFilter(1024*1024, 5)
 	if r1, err := duplicates.DoDupeFilter(request1); r1||err!=nil {
 		t.Errorf("Request1 error expected=%v, get=%v", false, true)
@@ -64,4 +63,5 @@ func TestDoBodyDupeFilter(t *testing.T) {
 		t.Errorf("Request4 error expected=%v, get=%v", true, false)
 
 	}
+
 }

request.go

@@ -72,8 +72,9 @@ var reqLog *logrus.Entry = GetLogger("request")
 func RequestWithRequestBody(body map[string]interface{}) Option {
 	return func(r *Request) {
 		var err error
-		body, err := jsoniter.Marshal(body)
-		r.BodyReader = bytes.NewBuffer(body)
+
+		r.Body, err = jsoniter.Marshal(body)
+		r.BodyReader = bytes.NewBuffer(r.Body)
 		if err != nil {
 			reqLog.Errorf("set request body err %s", err.Error())
 			panic(fmt.Sprintf("set request body err %s", err.Error()))

settings.go

@@ -12,7 +12,6 @@
 package tegenaria
 
 import (
-	"fmt"
 	"os"
 
 	"github.com/spf13/viper"
@@ -31,10 +30,6 @@ type Configuration struct {
 	Log *Logger `ymal:"log"`
 }
 
-func (l *Logger) GetValue(key string) (string, error) {
-	return "", nil
-}
-
 var Config *Configuration = &Configuration{
 	Log: &Logger{
 		Path:  "/var/log",
@@ -43,11 +38,6 @@ var Config *Configuration = &Configuration{
 }
 
 func load() bool {
-	defer func() {
-		if p := recover(); p != nil {
-			fmt.Printf("fatal error config file: %s", p)
-		}
-	}()
 	str, _ := os.Getwd()
 	runtimeViper := viper.New()