[enh] speed up collector.scrape() and collector.fetch() | benchmark: …

…1.57s -> 1.44s

colly.go

@@ -365,11 +365,7 @@ func (c *Collector) scrape(u, method string, depth int, requestData io.Reader, c
 		}
 	}
 	if hdr == nil {
-		hdr = make(http.Header)
-		hdr.Set("User-Agent", c.UserAgent)
-		if method == "POST" {
-			hdr.Add("Content-Type", "application/x-www-form-urlencoded")
-		}
+		hdr = http.Header{"User-Agent": []string{c.UserAgent}}
 	}
 	rc, ok := requestData.(io.ReadCloser)
 	if !ok && requestData != nil {
@@ -754,7 +750,7 @@ func (c *Collector) handleOnResponse(r *Response) {
 }
 
 func (c *Collector) handleOnHTML(resp *Response) {
-	if !strings.Contains(strings.ToLower(resp.Headers.Get("Content-Type")), "html") || len(c.htmlCallbacks) == 0 {
+	if len(c.htmlCallbacks) == 0 || !strings.Contains(strings.ToLower(resp.Headers.Get("Content-Type")), "html") {
 		return
 	}
 	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(resp.Body))
@@ -778,8 +774,11 @@ func (c *Collector) handleOnHTML(resp *Response) {
 }
 
 func (c *Collector) handleOnXML(resp *Response) {
+	if len(c.xmlCallbacks) == 0 {
+		return
+	}
 	contentType := strings.ToLower(resp.Headers.Get("Content-Type"))
-	if (!strings.Contains(contentType, "html") && !strings.Contains(contentType, "xml")) || len(c.xmlCallbacks) == 0 {
+	if !strings.Contains(contentType, "html") && !strings.Contains(contentType, "xml") {
 		return
 	}
 

colly_test.go

@@ -27,6 +27,7 @@ func newTestServer() *httptest.Server {
 	mux := http.NewServeMux()
 
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
 		w.Write(serverIndexResponse)
 	})