[Android] Add an out-of-app instrumentation driver APK.

BUG=444049

Review URL: https://codereview.chromium.org/1034053002

Cr-Commit-Position: refs/heads/master@{#326900}

android_webview/android_webview_tests.gypi

@@ -94,6 +94,8 @@
         '../base/base.gyp:base_java_test_support',
         '../content/content_shell_and_tests.gyp:content_java_test_support',
         '../net/net.gyp:net_java_test_support',
+        '../testing/android/on_device_instrumentation.gyp:broker_java',
+        '../testing/android/on_device_instrumentation.gyp:require_driver_apk',
         'android_webview_apk_java',
       ],
       'variables': {

android_webview/javatests/AndroidManifest.xml

@@ -7,7 +7,7 @@
     <!-- TODO(boliu): Change minSdkVersion to 19 when bots no longer try
          to install webview apks on < K devices. crbug.com/474374 -->
     <uses-sdk android:minSdkVersion="16" android:targetSdkVersion="21" />
-    <instrumentation android:name="android.test.InstrumentationTestRunner"
+    <instrumentation android:name="org.chromium.base.test.BaseInstrumentationTestRunner"
         android:targetPackage="org.chromium.android_webview.shell"
         android:label="Tests for org.chromium.android_webview"/>
     <uses-permission android:name="android.permission.RUN_INSTRUMENTATION" />
@@ -19,5 +19,7 @@
          needed when building test cases. -->
     <application android:hardwareAccelerated="false">
         <uses-library android:name="android.test.runner" />
+        <activity android:name="org.chromium.test.broker.OnDeviceInstrumentationBroker"
+            android:exported="true"/>
     </application>
 </manifest>

base/BUILD.gn

@@ -1459,6 +1459,7 @@ if (is_android) {
   android_library("base_java_test_support") {
     deps = [
       ":base_java",
+      "//testing/android/reporter:reporter_java",
     ]
     DEPRECATED_java_in_dir = "test/android/javatests/src"
   }

base/base.gyp

@@ -1494,6 +1494,7 @@
           'type': 'none',
           'dependencies': [
             'base_java',
+            '../testing/android/on_device_instrumentation.gyp:reporter_java',
           ],
           'variables': {
             'java_in_dir': '../base/test/android/javatests',

base/test/android/javatests/src/org/chromium/base/test/BaseInstrumentationTestRunner.java

@@ -14,10 +14,12 @@
 import junit.framework.TestResult;
 
 import org.chromium.base.test.util.MinAndroidSdkLevel;
+import org.chromium.test.reporter.TestStatusListener;
 
 import java.util.ArrayList;
 import java.util.List;
 
+// TODO(jbudorick): Add support for on-device handling of timeouts.
 /**
  *  An Instrumentation test runner that checks SDK level for tests with specific requirements.
  */
@@ -88,14 +90,16 @@ protected void run(final TestCase test) {
 
     @Override
     protected AndroidTestRunner getAndroidTestRunner() {
-        return new AndroidTestRunner() {
+        AndroidTestRunner runner = new AndroidTestRunner() {
             @Override
             protected TestResult createTestResult() {
                 SkippingTestResult r = new SkippingTestResult();
                 r.addSkipCheck(new MinAndroidSdkLevelSkipCheck());
                 return r;
             }
         };
+        runner.addTestListener(new TestStatusListener(getContext()));
+        return runner;
     }
 
     /**

build/android/pylib/instrumentation/instrumentation_test_instance.py

@@ -30,6 +30,17 @@
 _DEFAULT_ANNOTATIONS = [
     'Smoke', 'SmallTest', 'MediumTest', 'LargeTest',
     'EnormousTest', 'IntegrationTest']
+_EXTRA_ENABLE_HTTP_SERVER = (
+    'org.chromium.chrome.test.ChromeInstrumentationTestRunner.'
+        + 'EnableTestHttpServer')
+_EXTRA_DRIVER_TEST_LIST = (
+    'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestList')
+_EXTRA_DRIVER_TEST_LIST_FILE = (
+    'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestListFile')
+_EXTRA_DRIVER_TARGET_PACKAGE = (
+    'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetPackage')
+_EXTRA_DRIVER_TARGET_CLASS = (
+    'org.chromium.test.driver.OnDeviceInstrumentationDriver.TargetClass')
 _NATIVE_CRASH_RE = re.compile('native crash', re.IGNORECASE)
 _PICKLE_FORMAT_VERSION = 10
 
@@ -130,29 +141,35 @@ def __init__(self, args, isolate_delegate, error_func):
 
     self._apk_under_test = None
     self._package_info = None
+    self._suite = None
     self._test_apk = None
     self._test_jar = None
     self._test_package = None
     self._test_runner = None
     self._test_support_apk = None
-    self.__initializeApkAttributes(args, error_func)
+    self._initializeApkAttributes(args, error_func)
 
     self._data_deps = None
     self._isolate_abs_path = None
     self._isolate_delegate = None
     self._isolated_abs_path = None
     self._test_data = None
-    self.__initializeDataDependencyAttributes(args, isolate_delegate)
+    self._initializeDataDependencyAttributes(args, isolate_delegate)
 
     self._annotations = None
     self._excluded_annotations = None
     self._test_filter = None
-    self.__initializeTestFilterAttributes(args)
+    self._initializeTestFilterAttributes(args)
 
     self._flags = None
-    self.__initializeFlagAttributes(args)
+    self._initializeFlagAttributes(args)
 
-  def __initializeApkAttributes(self, args, error_func):
+    self._driver_apk = None
+    self._driver_package = None
+    self._driver_name = None
+    self._initializeDriverAttributes()
+
+  def _initializeApkAttributes(self, args, error_func):
     if args.apk_under_test.endswith('.apk'):
       self._apk_under_test = args.apk_under_test
     else:
@@ -164,20 +181,20 @@ def __initializeApkAttributes(self, args, error_func):
       error_func('Unable to find APK under test: %s' % self._apk_under_test)
 
     if args.test_apk.endswith('.apk'):
-      test_apk_root = os.path.splitext(os.path.basename(args.test_apk))[0]
+      self._suite = os.path.splitext(os.path.basename(args.test_apk))[0]
       self._test_apk = args.test_apk
     else:
-      test_apk_root = args.test_apk
+      self._suite = args.test_apk
       self._test_apk = os.path.join(
           constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR,
           '%s.apk' % args.test_apk)
 
     self._test_jar = os.path.join(
         constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR,
-        '%s.jar' % test_apk_root)
+        '%s.jar' % self._suite)
     self._test_support_apk = os.path.join(
         constants.GetOutDirectory(), constants.SDK_BUILD_TEST_JAVALIB_DIR,
-        '%sSupport.apk' % test_apk_root)
+        '%sSupport.apk' % self._suite)
 
     if not os.path.exists(self._test_apk):
       error_func('Unable to find test APK: %s' % self._test_apk)
@@ -194,7 +211,7 @@ def __initializeApkAttributes(self, args, error_func):
     if not self._package_info:
       logging.warning('Unable to find package info for %s', self._test_package)
 
-  def __initializeDataDependencyAttributes(self, args, isolate_delegate):
+  def _initializeDataDependencyAttributes(self, args, isolate_delegate):
     self._data_deps = []
     if args.isolate_file_path:
       self._isolate_abs_path = os.path.abspath(args.isolate_file_path)
@@ -215,7 +232,7 @@ def __initializeDataDependencyAttributes(self, args, isolate_delegate):
     if not self._isolate_delegate and not self._test_data:
       logging.warning('No data dependencies will be pushed.')
 
-  def __initializeTestFilterAttributes(self, args):
+  def _initializeTestFilterAttributes(self, args):
     self._test_filter = args.test_filter
 
     def annotation_dict_element(a):
@@ -240,7 +257,7 @@ def annotation_dict_element(a):
     else:
       self._excluded_annotations = {}
 
-  def __initializeFlagAttributes(self, args):
+  def _initializeFlagAttributes(self, args):
     self._flags = ['--disable-fre', '--enable-test-intents']
     # TODO(jbudorick): Transition "--device-flags" to "--device-flags-file"
     if hasattr(args, 'device_flags') and args.device_flags:
@@ -252,9 +269,17 @@ def __initializeFlagAttributes(self, args):
         stripped_lines = (l.strip() for l in device_flags_file)
         self._flags.extend([flag for flag in stripped_lines if flag])
 
-  @property
-  def suite(self):
-    return 'instrumentation'
+  def _initializeDriverAttributes(self):
+    self._driver_apk = os.path.join(
+        constants.GetOutDirectory(), constants.SDK_BUILD_APKS_DIR,
+        'OnDeviceInstrumentationDriver.apk')
+    if os.path.exists(self._driver_apk):
+      self._driver_package = apk_helper.GetPackageName(
+          self._driver_apk)
+      self._driver_name = apk_helper.GetInstrumentationName(
+          self._driver_apk)
+    else:
+      self._driver_apk = None
 
   @property
   def apk_under_test(self):
@@ -264,10 +289,26 @@ def apk_under_test(self):
   def flags(self):
     return self._flags
 
+  @property
+  def driver_apk(self):
+    return self._driver_apk
+
+  @property
+  def driver_package(self):
+    return self._driver_package
+
+  @property
+  def driver_name(self):
+    return self._driver_name
+
   @property
   def package_info(self):
     return self._package_info
 
+  @property
+  def suite(self):
+    return self._suite
+
   @property
   def test_apk(self):
     return self._test_apk
@@ -445,6 +486,28 @@ def _InflateTests(self, tests):
         })
     return inflated_tests
 
+  @staticmethod
+  def GetHttpServerEnvironmentVars():
+    return {
+      _EXTRA_ENABLE_HTTP_SERVER: None,
+    }
+
+  def GetDriverEnvironmentVars(
+      self, test_list=None, test_list_file_path=None):
+    env = {
+      _EXTRA_DRIVER_TARGET_PACKAGE: self.test_package,
+      _EXTRA_DRIVER_TARGET_CLASS: self.test_runner,
+    }
+
+    if test_list:
+      env[_EXTRA_DRIVER_TEST_LIST] = ','.join(test_list)
+
+    if test_list_file_path:
+      env[_EXTRA_DRIVER_TEST_LIST_FILE] = (
+          os.path.basename(test_list_file_path))
+
+    return env
+
   @staticmethod
   def ParseAmInstrumentRawOutput(raw_output):
     return ParseAmInstrumentRawOutput(raw_output)

build/android/pylib/local/device/local_device_instrumentation_test_run.py

@@ -130,22 +130,41 @@ def _GetTestName(self, test):
 
   #override
   def _RunTest(self, device, test):
-    test_name = self._GetTestName(test)
-    logging.info('preparing to run %s: %s' % (test_name, test))
+    extras = self._test_instance.GetHttpServerEnvironmentVars()
+
+    if isinstance(test, list):
+      if not self._test_instance.driver_apk:
+        raise Exception('driver_apk does not exist. '
+                        'Please build it and try again.')
+
+      def name_and_timeout(t):
+        n = self._GetTestName(t)
+        i = self._GetTimeoutFromAnnotations(t['annotations'], n)
+        return (n, i)
+
+      test_names, timeouts = zip(*(name_and_timeout(t) for t in test))
+
+      test_name = ','.join(test_names)
+      target = '%s/%s' % (
+          self._test_instance.driver_package,
+          self._test_instance.driver_name)
+      extras.update(
+          self._test_instance.GetDriverEnvironmentVars(
+              test_list=test_names))
+      timeout = sum(timeouts)
+    else:
+      test_name = self._GetTestName(test)
+      target = '%s/%s' % (
+          self._test_instance.test_package, self._test_instance.test_runner)
+      extras['class'] = test_name
+      timeout = self._GetTimeoutFromAnnotations(test['annotations'], test_name)
 
-    extras = {
-      'class': test_name,
-      'org.chromium.chrome.test.ChromeInstrumentationTestRunner'
-          '.EnableTestHttpServer': '',
-    }
-    timeout = self._GetTimeoutFromAnnotations(test['annotations'], test_name)
+    logging.info('preparing to run %s: %s' % (test_name, test))
 
     time_ms = lambda: int(time.time() * 1e3)
     start_ms = time_ms()
     output = device.StartInstrumentation(
-        '%s/%s' % (self._test_instance.test_package,
-                   self._test_instance.test_runner),
-        raw=True, extras=extras, timeout=timeout, retries=0)
+        target, raw=True, extras=extras, timeout=timeout, retries=0)
     duration_ms = time_ms() - start_ms
 
     # TODO(jbudorick): Make instrumentation tests output a JSON so this

build/android/pylib/remote/device/remote_device_helper.py

@@ -20,4 +20,5 @@ def TestHttpResponse(response, error_msg):
       error_msg: Error message to display if bad response is seen.
   """
   if response.status_code != 200:
-    raise RemoteDeviceError(error_msg)
+    raise RemoteDeviceError(
+        '%s (%d: %s)' % (error_msg, response.status_code, response.reason))

build/android/pylib/remote/device/remote_device_instrumentation_test_run.py

@@ -8,6 +8,7 @@
 import os
 import tempfile
 
+from pylib import constants
 from pylib.base import base_test_result
 from pylib.remote.device import remote_device_test_run
 from pylib.utils import apk_helper
@@ -25,9 +26,33 @@ def TestPackage(self):
   def _TriggerSetUp(self):
     """Set up the triggering of a test run."""
     logging.info('Triggering test run.')
-    self._AmInstrumentTestSetup(
-        self._test_instance._apk_under_test, self._test_instance.test_apk,
-        self._test_instance.test_runner, environment_variables={})
+
+    with tempfile.NamedTemporaryFile(suffix='.txt') as test_list_file:
+      tests = self._test_instance.GetTests()
+      logging.debug('preparing to run %d instrumentation tests remotely:',
+                    len(tests))
+      for t in tests:
+        test_name = '%s#%s' % (t['class'], t['method'])
+        logging.debug('  %s', test_name)
+        test_list_file.write('%s\n' % test_name)
+      test_list_file.flush()
+      self._test_instance._data_deps.append(
+          (os.path.abspath(test_list_file.name), None))
+
+      env_vars = self._test_instance.GetDriverEnvironmentVars(
+          test_list_file_path=test_list_file.name)
+      env_vars.update(self._test_instance.GetHttpServerEnvironmentVars())
+
+      logging.debug('extras:')
+      for k, v in env_vars.iteritems():
+        logging.debug('  %s: %s', k, v)
+
+      self._AmInstrumentTestSetup(
+          self._test_instance.apk_under_test,
+          self._test_instance.driver_apk,
+          self._test_instance.driver_name,
+          environment_variables=env_vars,
+          extra_apks=[self._test_instance.test_apk])
 
   #override
   def _ParseTestResults(self):