JariBakken: Cache Gecko SDKs in ~/.crazyfun.

Only SDKs passing an MD5 in the Rakefile will be cached. r13443
mpichlin · Aug 18, 2011 · 3a51f97 · 3a51f97
1 parent 596fbf9
commit 3a51f97
Show file tree

Hide file tree

Showing 3 changed files with 88 additions and 25 deletions.
diff --git a/Rakefile b/Rakefile
@@ -252,18 +252,47 @@ ie_generate_type_mapping(:name => "ie_result_type_java",
 
 
 GeckoSDKs.new do |sdks|
-  sdks.add 'third_party/gecko-1.9.2/linux', 'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/1.9.2.19/sdk/xulrunner-1.9.2.19.en-US.linux-i686.sdk.tar.bz2'
-  sdks.add 'third_party/gecko-2/linux'    , 'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/2.0/sdk/xulrunner-2.0.en-US.linux-i686.sdk.tar.bz2'
-  sdks.add 'third_party/gecko-2/linux64'  , 'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/2.0/sdk/xulrunner-2.0.en-US.linux-x86_64.sdk.tar.bz2'
-  sdks.add 'third_party/gecko-2/mac'      , 'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/2.0/sdk/xulrunner-2.0.en-US.mac-x86_64.sdk.tar.bz2'
-  sdks.add 'third_party/gecko-2/win32'    , 'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/2.0/sdk/xulrunner-2.0.en-US.win32.sdk.zip'
-  sdks.add 'third_party/gecko-5/linux'    , 'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/5.0/sdk/xulrunner-5.0.en-US.linux-i686.sdk.tar.bz2'
-  sdks.add 'third_party/gecko-5/linux64'  , 'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/5.0/sdk/xulrunner-5.0.en-US.linux-x86_64.sdk.tar.bz2'
-  sdks.add 'third_party/gecko-5/mac'      , 'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/5.0/sdk/xulrunner-5.0.en-US.mac-i386.sdk.tar.bz2'
-  sdks.add 'third_party/gecko-5/win32'    , 'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/5.0/sdk/xulrunner-5.0.en-US.win32.sdk.zip'
-  sdks.add 'third_party/gecko-6/linux'    , 'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/6.0/sdk/xulrunner-6.0.en-US.linux-i686.sdk.tar.bz2'
-  sdks.add 'third_party/gecko-6/linux64'  , 'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/6.0/sdk/xulrunner-6.0.en-US.linux-x86_64.sdk.tar.bz2'
-  sdks.add 'third_party/gecko-6/win32'    , 'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/6.0/sdk/xulrunner-6.0.en-US.win32.sdk.zip'
+  sdks.add 'third_party/gecko-1.9.2/linux', 
+           'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/3.6.20/sdk/xulrunner-3.6.20.en-US.linux-i686.sdk.tar.bz2'
+
+  sdks.add 'third_party/gecko-2/linux', 
+           'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/2.0/sdk/xulrunner-2.0.en-US.linux-i686.sdk.tar.bz2',
+           '1ec6039ee99596551845f27d4bc83436'
+
+  sdks.add 'third_party/gecko-2/linux64', 
+           'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/2.0/sdk/xulrunner-2.0.en-US.linux-x86_64.sdk.tar.bz2',
+           '101eb57d3f76f77e9c94d3cb25a8d56c'
+
+  sdks.add 'third_party/gecko-2/mac', 
+           'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/2.0/sdk/xulrunner-2.0.en-US.mac-x86_64.sdk.tar.bz2'
+
+  sdks.add 'third_party/gecko-2/win32', 
+           'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/2.0/sdk/xulrunner-2.0.en-US.win32.sdk.zip',
+           '5cfa95a2d46334ce6283a772eff19382'
+
+  sdks.add 'third_party/gecko-5/linux',
+           'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/5.0/sdk/xulrunner-5.0.en-US.linux-i686.sdk.tar.bz2',
+           '1c980270364eedea841b471578ebe4d8'
+
+  sdks.add 'third_party/gecko-5/linux64', 
+           'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/5.0/sdk/xulrunner-5.0.en-US.linux-x86_64.sdk.tar.bz2',
+           'fd193614e8dbe8f574e36c9f24eedf7a'
+
+  sdks.add 'third_party/gecko-5/mac',
+           'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/5.0/sdk/xulrunner-5.0.en-US.mac-i386.sdk.tar.bz2'
+
+  sdks.add 'third_party/gecko-5/win32', 
+           'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/5.0/sdk/xulrunner-5.0.en-US.win32.sdk.zip',
+           '8894612028e1e28e428d748d50e9bc70'
+
+  sdks.add 'third_party/gecko-6/linux', 
+           'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/6.0/sdk/xulrunner-6.0.en-US.linux-i686.sdk.tar.bz2'
+
+  sdks.add 'third_party/gecko-6/linux64', 
+           'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/6.0/sdk/xulrunner-6.0.en-US.linux-x86_64.sdk.tar.bz2'
+
+  sdks.add 'third_party/gecko-6/win32', 
+           'http://releases.mozilla.org/pub/mozilla.org/xulrunner/releases/6.0/sdk/xulrunner-6.0.en-US.win32.sdk.zip'
 end
 
 task :'selenium-server_zip' do

diff --git a/rake-tasks/downloader.rb b/rake-tasks/downloader.rb
@@ -1,5 +1,6 @@
 require "net/http"
 require "uri"
+require 'digest/md5'
 
 class Downloader
   class Error < StandardError; end
@@ -27,7 +28,7 @@ def initialize(source, destination)
   end
 
   def download!
-    response = Net::HTTP.get_response(@url) do |response|
+    resp = Net::HTTP.get_response(@url) do |response|
       total = response.content_length
       progress = 0
       segment_count = 0
@@ -45,8 +46,8 @@ def download!
       end
     end
 
-    unless response.kind_of? Net::HTTPSuccess
-      raise Error, "#{response.code} for #{@url}"
+    unless resp.kind_of? Net::HTTPSuccess
+      raise Error, "#{resp.code} for #{@url}"
     end
 
     complete_progress
@@ -56,11 +57,40 @@ def download!
 
   def report_progress(progress, total)
     percent = (progress.to_f / total.to_f) * 100
-    print "#{CL_RESET}Downloading #{@url}: #{percent.to_i}% (#{progress} / #{total})"
+    print "#{CL_RESET}Downloading #{basename}: #{percent.to_i}% (#{progress} / #{total})"
     $stdout.flush
   end
 
   def complete_progress
     puts CL_RESET
   end
+
+  def basename
+    @basename ||= File.basename @url.to_s
+  end
 end
+
+class CachingDownloader < Downloader
+  CACHE_DIR = File.expand_path("~/.crazyfun")
+
+  def self.fetch(url, hash)
+    basename = File.basename(url)
+    cached_path = File.join(CACHE_DIR, basename)
+
+    if File.exist?(cached_path)
+      cached_hash = Digest::MD5.file(cached_path).hexdigest
+      if cached_hash == hash
+        $stderr.puts "Using cached Gecko SDK: #{cached_path}"
+        return cached_path
+      end
+
+      $stderr.puts "MD5 mismatch: #{cached_path.inspect}, overwriting.."
+    end
+
+    super(url, cached_path)
+  end
+
+  def self.digest(path)
+
+  end
+end
diff --git a/rake-tasks/gecko_sdks.rb b/rake-tasks/gecko_sdks.rb
@@ -8,26 +8,30 @@ def initialize
 
     yield self if block_given?
 
-    @sdks.each do |path, url|
-      define_rule(path, url)
+    @sdks.each do |path, url, md5|
+      define_rule(path, url, md5)
     end
   end
 
-  def add(path, url)
-    @sdks << [path, url]
+  def add(path, url, md5 = nil)
+    @sdks << [path, url, md5]
   end
 
   private
 
-  def define_rule(path, url)
+  def define_rule(path, url, md5 = nil)
     download_task = file(path) do
       mkdir_p path
-      next if offline? || !platform_matches?(path)
+      next if offline?# || !platform_matches?(path)
 
       begin
-        sdk = Downloader.fetch url
-        destination = File.dirname path
+        if md5
+          sdk = CachingDownloader.fetch(url, md5)
+        else
+          sdk = Downloader.fetch(url)
+        end
 
+        destination = File.dirname path
         unpack sdk, destination
 
         # The directory was created - but for the move to replace it,
@@ -40,7 +44,7 @@ def define_rule(path, url)
 
         next
       ensure
-        rm_rf sdk if sdk && File.exist?(sdk)
+        rm_rf sdk if !md5 && sdk && File.exist?(sdk)
       end
     end