catch error on http request timeout

lpradovera · Feb 2, 2011 · bd5bf64 · bd5bf64
1 parent 2c974c2
commit bd5bf64
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 4 deletions.
diff --git a/lib/anemone/http.rb b/lib/anemone/http.rb
@@ -43,7 +43,7 @@ def fetch_pages(url, referer = nil, depth = nil)
         end
 
         return pages
-      rescue => e
+      rescue Exception => e
         if verbose?
           puts e.inspect
           puts e.backtrace
@@ -116,7 +116,8 @@ def get_response(url, referer = nil)
         response_time = ((finish - start) * 1000).round
         @cookie_store.merge!(response['Set-Cookie']) if accept_cookies?
         return response, response_time
-      rescue EOFError
+      rescue Timeout::Error, Net::HTTPBadResponse, EOFError => e
+        puts e.inspect if verbose?
         refresh_connection(url)
         retries += 1
         retry unless retries > 3

diff --git a/spec/core_spec.rb b/spec/core_spec.rb
@@ -233,8 +233,11 @@ module Anemone
     describe Storage::PStore do
       it_should_behave_like "crawl"
 
-      before(:each) do
+      before(:all) do
         @test_file = 'test.pstore'
+      end
+
+      before(:each) do
         File.delete(@test_file) if File.exists?(@test_file)
         @opts = {:storage => Storage.PStore(@test_file)}
       end
@@ -247,8 +250,11 @@ module Anemone
     describe Storage::TokyoCabinet do
       it_should_behave_like "crawl"
 
-      before(:each) do
+      before(:all) do
         @test_file = 'test.tch'
+      end
+
+      before(:each) do
         File.delete(@test_file) if File.exists?(@test_file)
         @opts = {:storage => @store = Storage.TokyoCabinet(@test_file)}
       end