Skip to content

Commit

Permalink
Proxy URL for request
Browse files Browse the repository at this point in the history
Add ProxyURL to Request
Add ProxyURL via context
Update example
  • Loading branch information
pyjac committed Sep 17, 2018
1 parent a3a4941 commit 78b4030
Show file tree
Hide file tree
Showing 4 changed files with 17 additions and 1 deletion.
1 change: 1 addition & 0 deletions _examples/proxy_switcher/proxy_switcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ func main() {

// Print the response
c.OnResponse(func(r *colly.Response) {
log.Printf("Proxy Address: %s\n", r.Request.ProxyURL)
log.Printf("%s\n", bytes.Replace(r.Body, []byte("\n"), nil, -1))
})

Expand Down
10 changes: 10 additions & 0 deletions colly.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,13 @@ type cookieJarSerializer struct {

var collectorCounter uint32

// The key type is unexported to prevent collisions with context keys defined in
// other packages.
type key int

// ProxyURLKey is the context key for the request proxy address.
const ProxyURLKey key = 0

var (
// ErrForbiddenDomain is the error thrown if visiting
// a domain which is not allowed in AllowedDomains
Expand Down Expand Up @@ -587,6 +594,9 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct
request.URL = req.URL
request.Headers = &req.Header
}
if proxyURL, ok := req.Context().Value(ProxyURLKey).(string); ok {
request.ProxyURL = proxyURL
}
atomic.AddUint32(&c.responseCount, 1)
response.Ctx = ctx
response.Request = request
Expand Down
5 changes: 4 additions & 1 deletion proxy/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package proxy

import (
"context"
"net/http"
"net/url"
"sync/atomic"
Expand All @@ -27,9 +28,11 @@ type roundRobinSwitcher struct {
index uint32
}

func (r *roundRobinSwitcher) GetProxy(_ *http.Request) (*url.URL, error) {
func (r *roundRobinSwitcher) GetProxy(pr *http.Request) (*url.URL, error) {
u := r.proxyURLs[r.index%uint32(len(r.proxyURLs))]
atomic.AddUint32(&r.index, 1)
ctx := context.WithValue(pr.Context(), colly.ProxyURLKey, u.String())
*pr = *pr.WithContext(ctx)
return u, nil
}

Expand Down
2 changes: 2 additions & 0 deletions request.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ type Request struct {
collector *Collector
abort bool
baseURL *url.URL
// ProxyURL is the proxy address that handles the request
ProxyURL string
}

type serializableRequest struct {
Expand Down

0 comments on commit 78b4030

Please sign in to comment.