Revert "Refractor manage_render_test_goldens.py and add manage_wpr_ar…

…chives.py"

This reverts commit 41dddd7.

Reason for revert: Break gclient runhooks for someone
https://bugs.chromium.org/p/chromium/issues/detail?id=1090465

Original change's description:
> Refractor manage_render_test_goldens.py and add manage_wpr_archives.py
> 
> R=​bsheedy@chromium.org,harringtond@chromium.org
> 
> Bug: 1060399
> Change-Id: Ifb06c6a94a51a6b1a2815bf94fa8f5a53dec1db8
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2219525
> Reviewed-by: Haiyang Pan <hypan@google.com>
> Reviewed-by: Robbie Iannucci <iannucci@chromium.org>
> Reviewed-by: Dan H <harringtond@chromium.org>
> Reviewed-by: Brian Sheedy <bsheedy@chromium.org>
> Commit-Queue: Arthur Wang <wuwang@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#774253}

TBR=iannucci@chromium.org,bsheedy@chromium.org,hypan@google.com,harringtond@chromium.org,wuwang@chromium.org

Change-Id: I3597b11b7ae43b93747ae4ee2821c15d5072d50c
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: 1060399
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2225632
Reviewed-by: Haiyang Pan <hypan@google.com>
Commit-Queue: Arthur Wang <wuwang@chromium.org>
Cr-Commit-Position: refs/heads/master@{#774349}

.gitignore

@@ -315,9 +315,6 @@ vs-chromium-project.txt
 # Ignore any Android RenderTest goldens
 **/render_tests/*.png
 
-# Ignore any WPR archive files
-**/wpr_tests/*.wprgo
-
 # Ignore IntelliJ files.
 .idea/
 

.vpython

@@ -112,13 +112,6 @@ wheel: <
   >
 >
 
-# Used by:
-#   chrome/test/data/android/upload_download_utils_test.py
-wheel: <
-  name: "infra/python/wheels/pyfakefs-py2_py3"
-  version: "version:3.7.2"
->
-
 # Used by:
 #   tools/perf/fetch_benchmark_deps.py
 wheel: <

DEPS

@@ -117,10 +117,6 @@ vars = {
   'checkout_traffic_annotation_tools': 'checkout_configuration != "small"',
   'checkout_instrumented_libraries': 'checkout_linux and checkout_configuration != "small"',
 
-  # By default bot checkouts the WPR archive files only when this
-  # flag is set True.
-  'checkout_wpr_archives': False,
-
   # By default, do not check out WebKit for iOS, as it is not needed unless
   # running against ToT WebKit rather than system WebKit. This can be overridden
   # e.g. with custom_vars.
@@ -4555,16 +4551,6 @@ hooks = [
                 'download',
     ],
   },
-  # Pull down WPR Archive files
-  {
-    'name': 'Fetch WPR archive files',
-    'pattern': '.',
-    'condition': 'checkout_android and (checkout_wpr_archives or checkout_src_internal)',
-    'action': [ 'python',
-                'src/chrome/test/data/android/manage_wpr_archives.py',
-                'download',
-    ],
-  },
   {
     'name': 'Fetch Android AFDO profile',
     'pattern': '.',

chrome/test/data/android/manage_render_test_goldens.py

@@ -8,9 +8,11 @@
 # render tests or upload any newly generated ones.
 
 import argparse
+import hashlib
+import multiprocessing
 import os
-from upload_download_utils import download
-from upload_download_utils import upload
+import subprocess
+
 
 STORAGE_BUCKET = 'chromium-android-render-test-goldens'
 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
@@ -44,8 +46,19 @@
 ]
 
 
-def _is_file_of_interest(f):
-  """Filter through png files with right device sdk combo in the names."""
+# Assume a quad core if we can't get the actual core count.
+try:
+  THREAD_COUNT = multiprocessing.cpu_count()
+  # cpu_count only gets the physical core count. There doesn't appear to be a
+  # simple way of determining whether a CPU supports simultaneous multithreading
+  # in Python, so assume that anything with 6 or more cores supports it.
+  if THREAD_COUNT >= 6:
+    THREAD_COUNT *= 2
+except NotImplementedError:
+  THREAD_COUNT = 4
+
+
+def is_file_of_interest(f):
   if not f.endswith('.png'):
     return False
   for combo in ALLOWED_DEVICE_SDK_COMBINATIONS:
@@ -54,22 +67,81 @@ def _is_file_of_interest(f):
   return False
 
 
+def download(directory):
+  # If someone removes a SHA1 file, we want to remove the associated PNG file
+  # the next time images are updated.
+  images_to_delete = []
+  for f in os.listdir(directory):
+    if not is_file_of_interest(f):
+      continue
+    sha1_path = os.path.join(directory, f + '.sha1')
+    if not os.path.exists(sha1_path):
+      images_to_delete.append(os.path.join(directory, f))
+  for image_path in images_to_delete:
+    os.remove(image_path)
+
+  # Downloading the files can be very spammy, so only show the output if
+  # something actually goes wrong.
+  try:
+    subprocess.check_output([
+        'download_from_google_storage',
+        '--bucket', STORAGE_BUCKET,
+        '-d', directory,
+        '-t', str(THREAD_COUNT),
+    ], stderr=subprocess.STDOUT)
+  except subprocess.CalledProcessError as e:
+    print ('Downloading RenderTest goldens in directory %s failed with error '
+           '%d: %s') % (directory, e.returncode, e.output)
+
+
+def upload(directory, dry_run):
+  files_to_upload = []
+  for f in os.listdir(directory):
+    # Skip any files that we don't care about.
+    if not is_file_of_interest(f):
+      continue
+
+    png_path = os.path.join(directory, f)
+    # upload_to_google_storage will upload a file even if it already exists
+    # in the bucket. As an optimization, hash locally and only pass files to
+    # the upload script if they don't have a matching .sha1 file already.
+    sha_path = png_path + '.sha1'
+    if os.path.isfile(sha_path):
+      with open(sha_path) as sha_file:
+        with open(png_path, 'rb') as png_file:
+          h = hashlib.sha1()
+          h.update(png_file.read())
+          if sha_file.read() == h.hexdigest():
+            continue
+    files_to_upload.append(png_path)
+
+  if len(files_to_upload):
+    if dry_run:
+      print ('Will upload the following files:')
+      for f in files_to_upload:
+        print ('  ' + f)
+      return
+    subprocess.check_call([
+        'upload_to_google_storage.py',
+        '--bucket', STORAGE_BUCKET,
+        '-t', str(THREAD_COUNT),
+    ] + files_to_upload)
+
+
 def main():
   parser = argparse.ArgumentParser()
   parser.add_argument('action', choices=['download', 'upload'],
                       help='Which action to perform')
   parser.add_argument('--dry_run', action='store_true',
-                      default=False, help='Dry run for uploading')
+                      help='Dry run for uploading')
   args = parser.parse_args()
 
   if args.action == 'download':
     for d in GOLDEN_DIRECTORIES:
-      download(d, _is_file_of_interest,
-               'RenderTest Goldens', STORAGE_BUCKET)
+      download(d)
   else:
     for d in GOLDEN_DIRECTORIES:
-      upload(d, _is_file_of_interest,
-             'RenderTest Goldens', STORAGE_BUCKET, args.dry_run)
+      upload(d, args.dry_run)
 
 
 if __name__ == '__main__':