Check error of robots.txt GET request

colly.go

@@ -567,15 +567,17 @@ func (c *Collector) isDomainAllowed(domain string) bool {
 func (c *Collector) checkRobots(u *url.URL) error {
 	// var robot *robotstxt.RobotsData
 	// var ok bool
-	var err error
 
 	c.lock.RLock()
 	robot, ok := c.robotsMap[u.Host]
 	c.lock.RUnlock()
 
 	if !ok {
 		// no robots file cached
-		resp, _ := c.backend.Client.Get(u.Scheme + "://" + u.Host + "/robots.txt")
+		resp, err := c.backend.Client.Get(u.Scheme + "://" + u.Host + "/robots.txt")
+		if err != nil {
+			return err
+		}
 		robot, err = robotstxt.FromResponse(resp)
 		if err != nil {
 			return err

colly_test.go

@@ -544,6 +544,19 @@ func TestIgnoreRobotsWhenDisallowed(t *testing.T) {
 
 }
 
+func TestConnectionErrorOnRobotsTxtResultsInError(t *testing.T) {
+	ts := newTestServer()
+	ts.Close() // immediately close the server to force a connection error
+
+	c := NewCollector()
+	c.IgnoreRobotsTxt = false
+	err := c.Visit(ts.URL)
+
+	if err == nil {
+		t.Fatal("Error expected")
+	}
+}
+
 func TestEnvSettings(t *testing.T) {
 	ts := newTestServer()
 	defer ts.Close()