Skip to content

Commit

Permalink
[wptrunner] Generate image diff for failing reftests
Browse files Browse the repository at this point in the history 
Change-Id: I2afb2ef2fd0c208efd787b51bd80d22cccdee1d5
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2760937
Reviewed-by: Stephen McGruer <smcgruer@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@google.com>
Commit-Queue: Luke Z <lpz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#863787}
  • Loading branch information
@LukeZielinski
LukeZielinski authored and Chromium LUCI CQ committed Mar 17, 2021
1 parent 344ab1c commit 5b0bcde
Show file tree
Hide file tree
Showing 4 changed files with 76 additions and 27 deletions.
5 changes: 4 additions & 1 deletion .vpython3
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,4 +124,7 @@ wheel: <
name: "infra/python/wheels/distro-py2_py3"
version: "version:1.4.0"
>

wheel: <
name: "infra/python/wheels/pillow/linux-amd64_cp38_cp38"
version: "version:8.1.2"
>
1 change: 1 addition & 0 deletions BUILD.gn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1039,6 +1039,7 @@ if (!is_ios) {
"//chrome:chrome",
"//chrome/test/chromedriver",
"//third_party/blink/tools:wpt_tests_isolate",
"//tools/imagediff",
]
}
}
Expand Down
37 changes: 34 additions & 3 deletions testing/scripts/wpt_common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
BLINK_TOOLS_DIR = os.path.join(common.SRC_DIR, 'third_party', 'blink', 'tools')
WEB_TESTS_DIR = os.path.join(BLINK_TOOLS_DIR, os.pardir, 'web_tests')
EXTERNAL_WPT_TESTS_DIR = os.path.join(WEB_TESTS_DIR, 'external', 'wpt')
LAYOUT_TEST_RESULTS_SUBDIR = 'layout-test-results'

if BLINK_TOOLS_DIR not in sys.path:
sys.path.append(BLINK_TOOLS_DIR)
Expand Down Expand Up @@ -60,7 +61,8 @@ def do_post_test_run_tasks(self):

# Move json results into layout-test-results directory
results_dir = os.path.dirname(self.wpt_output)
layout_test_results = os.path.join(results_dir, 'layout-test-results')
layout_test_results = os.path.join(results_dir,
LAYOUT_TEST_RESULTS_SUBDIR)
if self.fs.exists(layout_test_results):
self.fs.rmtree(layout_test_results)
self.fs.maybe_make_directory(layout_test_results)
Expand Down Expand Up @@ -320,7 +322,7 @@ def _write_text_artifact(self, suffix, results_dir, test_name, artifact,
to the |results_dir|.
"""
log_artifact_sub_path = (
os.path.join("layout-test-results",
os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
self.port.output_filename(
test_name, suffix, extension))
)
Expand Down Expand Up @@ -353,6 +355,9 @@ def _write_screenshot_artifact(self, results_dir, test_name,
path of the file for that screenshot
"""
result={}
# Remember the two images so we can diff them later.
actual_image_bytes = None
expected_image_bytes = None
for screenshot_pair in screenshot_artifact:
screenshot_split = screenshot_pair.split(":")
url = screenshot_split[0]
Expand All @@ -368,8 +373,13 @@ def _write_screenshot_artifact(self, results_dir, test_name,
screenshot_key = "actual_image"
file_suffix = test_failures.FILENAME_SUFFIX_ACTUAL

if screenshot_key == "expected_image":
expected_image_bytes = image_bytes
else:
actual_image_bytes = image_bytes

screenshot_sub_path = (
os.path.join("layout-test-results",
os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
self.port.output_filename(
test_name, file_suffix, ".png"))
)
Expand All @@ -381,4 +391,25 @@ def _write_screenshot_artifact(self, results_dir, test_name,
os.path.dirname(screenshot_full_path))
# Note: we are writing raw bytes to this file
self.fs.write_binary_file(screenshot_full_path, image_bytes)

# Diff the two images and output the diff file.
diff_bytes, error = self.port.diff_image(expected_image_bytes,
actual_image_bytes)
if diff_bytes and not error:
diff_sub_path = (
os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
self.port.output_filename(
test_name, test_failures.FILENAME_SUFFIX_DIFF,
".png")))
result["image_diff"] = diff_sub_path
diff_full_path = os.path.join(results_dir, diff_sub_path)
if not self.fs.exists(os.path.dirname(diff_full_path)):
self.fs.maybe_make_directory(os.path.dirname(diff_full_path))
# Note: we are writing raw bytes to this file
self.fs.write_binary_file(diff_full_path, diff_bytes)
else:
print("Error creating diff image for test %s. "
"Error=%s, diff_bytes is None? %s\n"
% (test_name, error, diff_bytes is None))

return result
60 changes: 37 additions & 23 deletions testing/scripts/wpt_common_unittest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
import unittest

from wpt_common import (
BaseWptScriptAdapter, EXTERNAL_WPT_TESTS_DIR, WEB_TESTS_DIR
BaseWptScriptAdapter, EXTERNAL_WPT_TESTS_DIR, WEB_TESTS_DIR,
LAYOUT_TEST_RESULTS_SUBDIR
)

from blinkpy.common.host_mock import MockHost
Expand Down Expand Up @@ -121,15 +122,15 @@ def test_write_jsons(self):
written_files = self.wpt_adapter.fs.written_files
self.assertEqual(written_files[OUTPUT_JSON_FILENAME],
written_files[os.path.join(
'layout-test-results', 'full_results.json')])
LAYOUT_TEST_RESULTS_SUBDIR, 'full_results.json')])
# Verify JSONP
full_results_jsonp = written_files[os.path.join(
'layout-test-results', 'full_results_jsonp.js')]
LAYOUT_TEST_RESULTS_SUBDIR, 'full_results_jsonp.js')]
match = re.match(r'ADD_FULL_RESULTS\((.*)\);$', full_results_jsonp)
self.assertIsNotNone(match)
self.assertEqual(match.group(1), written_files[OUTPUT_JSON_FILENAME])
failing_results_jsonp = written_files[os.path.join(
'layout-test-results', 'failing_results.json')]
LAYOUT_TEST_RESULTS_SUBDIR, 'failing_results.json')]
match = re.match(r'ADD_RESULTS\((.*)\);$', failing_results_jsonp)
self.assertIsNotNone(match)
failing_results = json.loads(match.group(1))
Expand Down Expand Up @@ -160,9 +161,10 @@ def test_write_text_outputs(self):
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_path = os.path.join("layout-test-results", "test-actual.txt")
diff_path = os.path.join("layout-test-results", "test-diff.txt")
pretty_diff_path = os.path.join("layout-test-results",
actual_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"test-actual.txt")
diff_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, "test-diff.txt")
pretty_diff_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"test-pretty-diff.html")
self.assertNotIn(actual_path, written_files)
self.assertNotIn(diff_path, written_files)
Expand All @@ -174,7 +176,8 @@ def test_write_text_outputs(self):
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_path = os.path.join("layout-test-results", "test-actual.txt")
actual_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"test-actual.txt")
self.assertEqual("test.html actual text", written_files[actual_path])
# Ensure the artifact in the json was replaced with the location of
# the newly-created file.
Expand All @@ -188,12 +191,12 @@ def test_write_text_outputs(self):
# Ensure that a diff was also generated. Since there's no expected
# output, the actual text is all new. We don't validate the entire diff
# files to avoid checking line numbers/markup.
diff_path = os.path.join("layout-test-results", "test-diff.txt")
diff_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, "test-diff.txt")
self.assertIn("+test.html actual text", written_files[diff_path])
self.assertEqual(
[diff_path],
updated_json["tests"]["test.html"]["artifacts"]["text_diff"])
pretty_diff_path = os.path.join("layout-test-results",
pretty_diff_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"test-pretty-diff.html")
self.assertIn("test.html actual text", written_files[pretty_diff_path])
self.assertEqual(
Expand All @@ -218,7 +221,8 @@ def test_write_log_artifact(self):
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
stderr_path = os.path.join("layout-test-results", "test-stderr.txt")
stderr_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"test-stderr.txt")
self.assertEqual("test.html exceptions", written_files[stderr_path])
# Ensure the artifact in the json was replaced with the location of
# the newly-created file.
Expand Down Expand Up @@ -248,7 +252,7 @@ def test_write_crashlog_artifact(self):
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
crash_log_path = os.path.join("layout-test-results",
crash_log_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"test-crash-log.txt")
self.assertEqual("test.html crashed!", written_files[crash_log_path])
# Ensure the artifact in the json was replaced with the location of
Expand All @@ -262,7 +266,7 @@ def test_write_crashlog_artifact(self):

def test_write_screenshot_artifacts(self):
# Ensure that screenshots are written to the correct filenames and
# their bytes are base64 decoded.
# their bytes are base64 decoded. The image diff should also be created.
json_dict = {
'tests': {
'reftest.html': {
Expand All @@ -282,14 +286,19 @@ def test_write_screenshot_artifacts(self):
self._create_json_output(json_dict)
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_image_path = os.path.join("layout-test-results",
actual_image_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"reftest-actual.png")
self.assertEqual(base64.b64decode('abcd'),
written_files[actual_image_path])
expected_image_path = os.path.join("layout-test-results",
expected_image_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"reftest-expected.png")
self.assertEqual(base64.b64decode('bcde'),
written_files[expected_image_path])
diff_image_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"reftest-diff.png")
# Make sure the diff image was written but don't check the contents,
# assume that diff_image does the right thing.
self.assertIsNotNone(written_files[diff_image_path])
# Ensure the artifacts in the json were replaced with the location of
# the newly-created files.
updated_json = self._load_json_output()
Expand All @@ -302,6 +311,9 @@ def test_write_screenshot_artifacts(self):
[expected_image_path],
updated_json["tests"]["reftest.html"]["artifacts"]
["expected_image"])
self.assertEqual([diff_image_path],
updated_json["tests"]["reftest.html"]["artifacts"]
["image_diff"])

def test_copy_expected_output(self):
# Check that an -expected.txt file is created from a checked-in metadata
Expand All @@ -326,10 +338,12 @@ def test_copy_expected_output(self):
"test.html checked-in metadata")
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_path = os.path.join("layout-test-results", "test-actual.txt")
actual_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"test-actual.txt")
self.assertEqual("test.html actual text", written_files[actual_path])
# The checked-in metadata file gets renamed from .ini to -expected.txt
expected_path = os.path.join("layout-test-results", "test-expected.txt")
expected_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"test-expected.txt")
self.assertEqual("test.html checked-in metadata",
written_files[expected_path])
# Ensure the artifacts in the json were replaced with the locations of
Expand All @@ -347,14 +361,14 @@ def test_copy_expected_output(self):
# Ensure that a diff was also generated. There should be both additions
# and deletions for this test since we have expected output. We don't
# validate the entire diff files to avoid checking line numbers/markup.
diff_path = os.path.join("layout-test-results", "test-diff.txt")
diff_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR, "test-diff.txt")
self.assertIn("-test.html checked-in metadata",
written_files[diff_path])
self.assertIn("+test.html actual text", written_files[diff_path])
self.assertEqual(
[diff_path],
updated_json["tests"]["test.html"]["artifacts"]["text_diff"])
pretty_diff_path = os.path.join("layout-test-results",
pretty_diff_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"test-pretty-diff.html")
self.assertIn("test.html checked-in metadata",
written_files[pretty_diff_path])
Expand Down Expand Up @@ -392,12 +406,12 @@ def test_expected_output_for_variant(self):
"variant.html checked-in metadata")
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_path = os.path.join("layout-test-results",
actual_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"variant_foo=bar_abc-actual.txt")
self.assertEqual("variant bar/abc actual text",
written_files[actual_path])
# The checked-in metadata file gets renamed from .ini to -expected.txt
expected_path = os.path.join("layout-test-results",
expected_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"variant_foo=bar_abc-expected.txt")
self.assertEqual("variant.html checked-in metadata",
written_files[expected_path])
Expand Down Expand Up @@ -448,13 +462,13 @@ def test_expected_output_for_multiglob(self):
"dir/multiglob checked-in metadata")
self.wpt_adapter.do_post_test_run_tasks()
written_files = self.wpt_adapter.fs.written_files
actual_path = os.path.join("layout-test-results",
actual_path = os.path.join(LAYOUT_TEST_RESULTS_SUBDIR,
"dir/multiglob.https.any.worker-actual.txt")
self.assertEqual("dir/multiglob worker actual text",
written_files[actual_path])
# The checked-in metadata file gets renamed from .ini to -expected.txt
expected_path = os.path.join(
"layout-test-results",
LAYOUT_TEST_RESULTS_SUBDIR,
"dir/multiglob.https.any.worker-expected.txt")
self.assertEqual("dir/multiglob checked-in metadata",
written_files[expected_path])
Expand Down

0 comments on commit 5b0bcde

Please sign in to comment.