[rb] distinguish specs using firefox esr, nightly and production

.travis.yml

@@ -4,7 +4,8 @@ env:
   global:
     - DISPLAY=:99.0
     - PYTEST_ADDOPTS="--verbose --instafail"
-    - FF_LEGACY_BINARY=/usr/local/bin/firefox
+    - FF_ESR_BINARY=$HOME/firefox-latest-esr/firefox/firefox-bin
+    - FF_NIGHTLY_BINARY=$HOME/firefox-latest-nightly/firefox/firefox-bin
 matrix:
   include:
     - language: java
@@ -199,12 +200,18 @@ matrix:
       rvm: 2.3.3
       jdk: oraclejdk8
       env: MARIONETTE=1 TASK=//rb:firefox-test
+      addons:
+        firefox: latest
+    - language: ruby
+      rvm: 2.3.3
+      jdk: oraclejdk8
+      env: MARIONETTE=1 TASK=//rb:ff-nightly-test
       addons:
         firefox: latest-nightly
     - language: ruby
       rvm: 2.3.3
       jdk: oraclejdk8
-      env: TASK=//rb:ff-legacy-test
+      env: TASK=//rb:ff-esr-test
       addons:
         firefox: latest-esr
     - language: ruby
@@ -223,12 +230,18 @@ matrix:
       rvm: 2.3.3
       jdk: oraclejdk8
       env: MARIONETTE=1 TASK=//rb:remote-firefox-test
+      addons:
+        firefox: latest
+    - language: ruby
+      rvm: 2.3.3
+      jdk: oraclejdk8
+      env: MARIONETTE=1 TASK=//rb:remote-ff-nightly-test
       addons:
         firefox: latest-nightly
     - language: ruby
       rvm: 2.3.3
       jdk: oraclejdk8
-      env: TASK=//rb:remote-ff-legacy-test
+      env: TASK=//rb:remote-ff-esr-test
       addons:
         firefox: latest-esr
     - language: ruby

Rakefile

@@ -272,18 +272,20 @@ task :test_rb_local => [
   "//rb:chrome-test",
   "//rb:firefox-test",
   "//rb:phantomjs-test",
-  ("//rb:ff-legacy-test" if ENV['FF_LEGACY_BINARY']),
+  ("//rb:ff-esr-test" if ENV['FF_ESR_BINARY']),
+  ("//rb:ff-nightly-test" if ENV['FF_NIGHTLY_BINARY']),
   ("//rb:safari-preview-test" if mac?),
   ("//rb:safari-test" if mac?),
   ("//rb:ie-test" if windows?),
-  ("//rb:edge-test" if windows?),
+  ("//rb:edge-test" if windows?)
 ].compact
 
 task :test_rb_remote => [
   "//rb:remote-chrome-test",
   "//rb:remote-firefox-test",
   "//rb:remote-phantomjs-test",
-  ("//rb:remote-ff-legacy-test" if ENV['FF_LEGACY_BINARY']),
+  ("//rb:remote-ff-esr-test" if ENV['FF_ESR_BINARY']),
+  ("//rb:remote-ff-nightly-test" if ENV['FF_NIGHTLY_BINARY']),
   ("//rb:remote-safari-preview-test" if mac?),
   ("//rb:remote-safari-test" if mac?),
   ("//rb:remote-ie-test" if windows?),

rb/build.desc

@@ -6,7 +6,7 @@ rubygem(
     "//rb:support",
     "//rb:edge",
     "//rb:firefox",
-    "//rb:ff-legacy",
+    "//rb:ff-esr",
     "//rb:ie",
     "//rb:phantomjs",
     "//rb:remote",
@@ -88,7 +88,7 @@ ruby_test(name = "edge",
   deps = [ ":edge" ]
 )
 
-ruby_library(name = "ff-legacy",
+ruby_library(name = "ff-esr",
   srcs = [
     "lib/selenium/webdriver/firefox/**/*.rb",
     "lib/selenium/webdriver/firefox.rb"
@@ -102,13 +102,13 @@ ruby_library(name = "ff-legacy",
   deps = [":common"]
 )
 
-ruby_test(name = "ff-legacy",
+ruby_test(name = "ff-esr",
   srcs = [
     "spec/integration/selenium/webdriver/*_spec.rb",
     "spec/integration/selenium/webdriver/firefox/**/*_spec.rb"
   ],
   include = ["rb/spec/integration", "build/rb/lib"],
-  deps = [":ff-legacy"]
+  deps = [":ff-esr"]
 )
 
 ruby_library(name = "firefox",
@@ -123,6 +123,15 @@ ruby_library(name = "firefox",
   deps = [":common"]
 )
 
+ruby_test(name = "ff-nightly",
+  srcs = [
+    "spec/integration/selenium/webdriver/*_spec.rb",
+    "spec/integration/selenium/webdriver/firefox/**/*_spec.rb"
+  ],
+  include = ["rb/spec/integration", "build/rb/lib"],
+  deps = [":firefox"]
+)
+
 ruby_test(name = "firefox",
   srcs = [
     "spec/integration/selenium/webdriver/*_spec.rb",
@@ -197,7 +206,20 @@ ruby_test(name = "remote-chrome",
     ":chrome"]
 )
 
-ruby_test(name = "remote-ff-legacy",
+ruby_test(name = "remote-ff-esr",
+  srcs = [
+    "spec/integration/selenium/webdriver/*_spec.rb",
+    "spec/integration/selenium/webdriver/firefox/**/*_spec.rb",
+    "spec/integration/selenium/webdriver/remote/**/*_spec.rb"
+  ],
+  include = ["rb/spec/integration", "build/rb/lib"],
+  deps = [
+    ":remote",
+    ":ff-esr"
+  ]
+)
+
+ruby_test(name = "remote-ff-nightly",
   srcs = [
     "spec/integration/selenium/webdriver/*_spec.rb",
     "spec/integration/selenium/webdriver/firefox/**/*_spec.rb",
@@ -206,7 +228,7 @@ ruby_test(name = "remote-ff-legacy",
   include = ["rb/spec/integration", "build/rb/lib"],
   deps = [
     ":remote",
-    ":ff-legacy"
+    ":firefox"
   ]
 )
 

rb/spec/integration/selenium/webdriver/driver_spec.rb

@@ -273,16 +273,13 @@ module WebDriver
           end
 
           # Edge BUG - https://connect.microsoft.com/IE/feedback/details/1849991/
-          # https://bugzilla.mozilla.org/show_bug.cgi?id=1329559
           not_compliant_on browser: :edge do
-            not_compliant_on browser: :firefox do
-              not_compliant_on driver: :remote, platform: :macosx do
-                it 'times out if the callback is not invoked' do
-                  expect do
-                    # Script is expected to be async and explicitly callback, so this should timeout.
-                    driver.execute_async_script 'return 1 + 2;'
-                  end.to raise_error(Selenium::WebDriver::Error::ScriptTimeoutError)
-                end
+            not_compliant_on driver: :remote, platform: :macosx do
+              it 'times out if the callback is not invoked' do
+                expect do
+                  # Script is expected to be async and explicitly callback, so this should timeout.
+                  driver.execute_async_script 'return 1 + 2;'
+                end.to raise_error(Selenium::WebDriver::Error::ScriptTimeoutError)
               end
             end
           end

rb/spec/integration/selenium/webdriver/element_spec.rb

@@ -27,7 +27,7 @@ module WebDriver
         driver.find_element(id: 'imageButton').click
       end
 
-      compliant_on browser: [:chrome, :ff_legacy] do
+      compliant_on browser: [:chrome, :ff_esr] do
         not_compliant_on driver: :remote, platform: :macosx do
           it 'should raise if different element receives click' do
             driver.navigate.to url_for('click_tests/overlapping_elements.html')
@@ -39,7 +39,7 @@ module WebDriver
         end
       end
 
-      compliant_on browser: [:firefox, :ff_legacy] do
+      compliant_on browser: [:firefox, :ff_esr, :ff_nightly] do
         it 'should not raise if element is only partially covered' do
           driver.navigate.to url_for('click_tests/overlapping_elements.html')
           expect { driver.find_element(id: 'other_contents').click }.not_to raise_error
@@ -169,7 +169,7 @@ module WebDriver
       end
 
       # Firefox - "Actions Endpoint Not Yet Implemented"
-      not_compliant_on browser: [:safari, :firefox] do
+      not_compliant_on browser: [:safari, :firefox, :ff_nightly] do
         it 'should drag and drop' do
           driver.navigate.to url_for('dragAndDropTest.html')
 

rb/spec/integration/selenium/webdriver/error_spec.rb

@@ -32,7 +32,7 @@ module WebDriver
           end.to raise_error(WebDriver::Error::NoSuchElementError)
         end
 
-        compliant_on browser: :ff_legacy do
+        compliant_on browser: :ff_esr do
           it 'should show stack trace information' do
             driver.navigate.to url_for('xhtmlTest.html')
 

rb/spec/integration/selenium/webdriver/firefox/driver_spec.rb

@@ -21,7 +21,7 @@
 
 module Selenium
   module WebDriver
-    compliant_on browser: :firefox do
+    compliant_on browser: [:firefox, :ff_nightly] do
       describe Firefox do
         def restart_remote_server
           server = GlobalTestEnv.reset_remote_server
@@ -36,26 +36,35 @@ def restart_remote_server
 
         before(:each) do
           @opt = {}
-          @opt[:url] = restart_remote_server if GlobalTestEnv.driver == :remote
+          @browser = if GlobalTestEnv.driver == :remote
+                       @opt[:url] = restart_remote_server
+                       :remote
+                     else
+                       :firefox
+                     end
         end
 
         not_compliant_on driver: :remote do
           it 'creates default capabilities' do
             begin
-              driver1 = Selenium::WebDriver.for(GlobalTestEnv.driver, @opt)
+              driver1 = Selenium::WebDriver.for(@browser, @opt)
               caps = driver1.capabilities
               expect(caps.proxy).to be_nil
               expect(caps.platform_name).to_not be_nil
               expect(caps.browser_version).to match(/^\d\d\./)
               expect(caps.platform_version).to_not be_nil
-              expect(caps.accept_insecure_certs).to be == false
+
+              compliant_on browser: :ff_nightly do
+                expect(caps.accept_insecure_certs).to be == false
+                expect(caps.page_load_strategy).to be == 'normal'
+                expect(caps.accessibility_checks).to be == false
+                expect(caps.implicit_timeout).to be == 0
+                expect(caps.page_load_timeout).to be == 300000
+                expect(caps.script_timeout).to be == 30000
+              end
+
               expect(caps.remote_session_id).to be_nil
-              expect(caps.page_load_strategy).to be == 'normal'
-              expect(caps.accessibility_checks).to be == false
               expect(caps.rotatable).to be == false
-              expect(caps.implicit_timeout).to be == 0
-              expect(caps.page_load_timeout).to be == 300000
-              expect(caps.script_timeout).to be == 30000
             ensure
               driver1.quit
             end
@@ -68,15 +77,15 @@ def restart_remote_server
             pending "Set ENV['ALT_FIREFOX_BINARY'] to test this" unless ENV['ALT_FIREFOX_BINARY']
             begin
               @path = Firefox::Binary.path
-              driver1 = Selenium::WebDriver.for GlobalTestEnv.driver, @opt.dup
+              driver1 = Selenium::WebDriver.for @browser, @opt.dup
 
               default_version = driver1.capabilities.version
               expect { driver1.capabilities.browser_version }.to_not raise_exception
               driver1.quit
 
               caps = Remote::Capabilities.firefox(firefox_options: {binary: ENV['ALT_FIREFOX_BINARY']})
               @opt[:desired_capabilities] = caps
-              driver2 = Selenium::WebDriver.for GlobalTestEnv.driver, @opt
+              driver2 = Selenium::WebDriver.for @browser, @opt
 
               expect(driver2.capabilities.version).to_not eql(default_version)
               expect { driver2.capabilities.browser_version }.to_not raise_exception
@@ -90,7 +99,7 @@ def restart_remote_server
             pending "Set ENV['ALT_FIREFOX_BINARY'] to test this" unless ENV['ALT_FIREFOX_BINARY']
             begin
               @path = Firefox::Binary.path
-              driver1 = Selenium::WebDriver.for GlobalTestEnv.driver, @opt.dup
+              driver1 = Selenium::WebDriver.for @browser, @opt.dup
 
               default_path = Firefox::Binary.path
               default_version = driver1.capabilities.version
@@ -99,7 +108,7 @@ def restart_remote_server
               caps = Remote::Capabilities.firefox(firefox_options: {binary: ENV['ALT_FIREFOX_BINARY']},
                                                   service_args: {binary: default_path})
               @opt[:desired_capabilities] = caps
-              driver2 = Selenium::WebDriver.for GlobalTestEnv.driver, @opt
+              driver2 = Selenium::WebDriver.for @browser, @opt
 
               expect(driver2.capabilities.version).to_not eql(default_version)
               expect { driver2.capabilities.browser_version }.to_not raise_exception

rb/spec/integration/selenium/webdriver/firefox/legacy_spec.rb

@@ -22,7 +22,7 @@
 module Selenium
   module WebDriver
     module Firefox
-      compliant_on browser: :ff_legacy do
+      compliant_on browser: :ff_esr do
         describe Driver do
           describe '.new' do
             before do
@@ -49,7 +49,7 @@ module Firefox
             end
           end
 
-          it_behaves_like 'driver that can be started concurrently', :ff_legacy
+          it_behaves_like 'driver that can be started concurrently', :ff_esr
         end
       end
     end # Firefox

rb/spec/integration/selenium/webdriver/firefox/profile_spec.rb

@@ -22,7 +22,7 @@
 module Selenium
   module WebDriver
     module Firefox
-      compliant_on browser: :firefox do
+      compliant_on browser: [:firefox, :ff_nightly] do
         describe Profile do
           let(:profile) { Profile.new }
           let(:browser) { GlobalTestEnv.driver == :remote ? :remote : :firefox }
@@ -41,8 +41,7 @@ def profile_opts
               opt[:url] = GlobalTestEnv.remote_server.webdriver_url if GlobalTestEnv.remote_server?
               opt
             else
-              {desired_capabilities: Remote::Capabilities.firefox(marionette: GlobalTestEnv.browser == :firefox),
-               profile: profile}
+              {profile: profile}
             end
           end
 
@@ -164,7 +163,7 @@ def profile_opts
 
               it 'should instantiate the browser with the correct profile' do
                 begin
-                  driver1 = WebDriver.for(browser, profile_opts.dup)
+                  driver1 = GlobalTestEnv.send(:create_driver, profile_opts.dup)
                   expect { wait(5).until { driver1.find_element(id: 'oneline') } }.to_not raise_error
                 ensure
                   driver1.quit
@@ -173,9 +172,9 @@ def profile_opts
 
               it 'should be able to use the same profile more than once' do
                 begin
-                  driver1 = WebDriver.for(browser, profile_opts.dup)
+                  driver1 = GlobalTestEnv.send(:create_driver, profile_opts.dup)
                   expect { wait(5).until { driver1.find_element(id: 'oneline') } }.to_not raise_error
-                  driver2 = WebDriver.for(browser, profile_opts.dup)
+                  driver2 = GlobalTestEnv.send(:create_driver, profile_opts.dup)
                   expect { wait(5).until { driver2.find_element(id: 'oneline') } }.to_not raise_error
                 ensure
                   driver1.quit if driver1

rb/spec/integration/selenium/webdriver/keyboard_spec.rb

@@ -22,7 +22,7 @@
 module Selenium
   module WebDriver
     # Firefox - "Actions Endpoint Not Yet Implemented"
-    not_compliant_on browser: [:safari, :ff_legacy, :firefox] do
+    not_compliant_on browser: [:safari, :ff_nightly, :firefox] do
       describe Keyboard do
         # Edge - https://developer.microsoft.com/en-us/microsoft-edge/platform/issues/8339952
         not_compliant_on browser: :edge do

rb/spec/integration/selenium/webdriver/mouse_spec.rb

@@ -22,7 +22,7 @@
 module Selenium
   module WebDriver
     # Firefox - "Actions Endpoint Not Yet Implemented"
-    not_compliant_on browser: [:safari, :firefox] do
+    not_compliant_on browser: [:safari, :firefox, :ff_nightly] do
       describe Mouse do
         it 'clicks an element' do
           driver.navigate.to url_for('formPage.html')