<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[285213] trunk</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.webkit.org/projects/webkit/changeset/285213">285213</a></dd>
<dt>Author</dt> <dd>simon.fraser@apple.com</dd>
<dt>Date</dt> <dd>2021-11-03 10:05:50 -0700 (Wed, 03 Nov 2021)</dd>
</dl>
<h3>Log Message</h3>
<pre>Expose fuzzy match data in layout test results
https://bugs.webkit.org/show_bug.cgi?id=232523
Reviewed by Jonathan Bedard.
Tools:
For a failing ref (or image) test, include the fuzzy matching data in full_results.json
in the form "image_difference": { "max_difference": 13, "total_pixels": 167 }, and
show that in the image comparison page that's written out for each test result.
Fix some tests that relied on JSON property serialization ordering (we don't serialize
with `sort_keys=True`).
* Scripts/webkitpy/layout_tests/controllers/manager.py:
(Manager._save_json_files):
* Scripts/webkitpy/layout_tests/controllers/test_result_writer.py:
(TestResultWriter.write_image_diff_files):
* Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py:
(JSONResultsGenerator.generate_times_ms_file):
* Scripts/webkitpy/layout_tests/models/test_failures.py:
(FailureImageHashMismatch.formatted_diff_percent):
(FailureImageHashMismatch):
(FailureImageHashMismatch.formatted_fuzzy_data):
(FailureImageHashMismatch.write_failure):
(FailureReftestMismatch.message):
(FailureReftestMismatch.formatted_diff_percent):
(FailureReftestMismatch):
(FailureReftestMismatch.formatted_fuzzy_data):
(FailureReftestMismatch.write_failure):
* Scripts/webkitpy/layout_tests/models/test_run_results.py:
(_interpret_test_failures):
* Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py:
(InterpretTestFailuresTest.test_interpret_test_failures):
* Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
(RunTest.test_retrying_force_pixel_tests):
LayoutTests:
Add some "image_difference" values to the test JSON and update the resulting
files.
* fast/harness/full_results.json:
* fast/harness/image-diff-template-expected.txt:
* fast/harness/image-diff-template.html:
* fast/harness/results.html:</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#trunkLayoutTestsChangeLog">trunk/LayoutTests/ChangeLog</a></li>
<li><a href="#trunkLayoutTestsfastharnessfull_resultsjson">trunk/LayoutTests/fast/harness/full_results.json</a></li>
<li><a href="#trunkLayoutTestsfastharnessimagedifftemplateexpectedtxt">trunk/LayoutTests/fast/harness/image-diff-template-expected.txt</a></li>
<li><a href="#trunkLayoutTestsfastharnessimagedifftemplatehtml">trunk/LayoutTests/fast/harness/image-diff-template.html</a></li>
<li><a href="#trunkLayoutTestsfastharnessresultshtml">trunk/LayoutTests/fast/harness/results.html</a></li>
<li><a href="#trunkToolsChangeLog">trunk/Tools/ChangeLog</a></li>
<li><a href="#trunkToolsScriptswebkitpylayout_testscontrollersmanagerpy">trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py</a></li>
<li><a href="#trunkToolsScriptswebkitpylayout_testscontrollerstest_result_writerpy">trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py</a></li>
<li><a href="#trunkToolsScriptswebkitpylayout_testslayout_packagejson_results_generatorpy">trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py</a></li>
<li><a href="#trunkToolsScriptswebkitpylayout_testsmodelstest_failurespy">trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py</a></li>
<li><a href="#trunkToolsScriptswebkitpylayout_testsmodelstest_run_resultspy">trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py</a></li>
<li><a href="#trunkToolsScriptswebkitpylayout_testsmodelstest_run_results_unittestpy">trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py</a></li>
<li><a href="#trunkToolsScriptswebkitpylayout_testsrun_webkit_tests_integrationtestpy">trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="trunkLayoutTestsChangeLog"></a>
<div class="modfile"><h4>Modified: trunk/LayoutTests/ChangeLog (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/LayoutTests/ChangeLog 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/LayoutTests/ChangeLog 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -1,3 +1,18 @@
</span><ins>+2021-11-03 Simon Fraser <simon.fraser@apple.com>
+
+ Expose fuzzy match data in layout test results
+ https://bugs.webkit.org/show_bug.cgi?id=232523
+
+ Reviewed by Jonathan Bedard.
+
+ Add some "image_difference" values to the test JSON and update the resulting
+ files.
+
+ * fast/harness/full_results.json:
+ * fast/harness/image-diff-template-expected.txt:
+ * fast/harness/image-diff-template.html:
+ * fast/harness/results.html:
+
</ins><span class="cx"> 2021-11-03 Jer Noble <jer.noble@apple.com>
</span><span class="cx">
</span><span class="cx"> [iOS] AVAssetResourceLoadingRequest.request does not include a Range: header on iOS 15.
</span></span></pre></div>
<a id="trunkLayoutTestsfastharnessfull_resultsjson"></a>
<div class="modfile"><h4>Modified: trunk/LayoutTests/fast/harness/full_results.json (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/LayoutTests/fast/harness/full_results.json 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/LayoutTests/fast/harness/full_results.json 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -25,13 +25,21 @@
</span><span class="cx"> "report": "REGRESSION",
</span><span class="cx"> "expected": "PASS",
</span><span class="cx"> "actual": "IMAGE",
</span><del>- "image_diff_percent": 86.262301
</del><ins>+ "image_diff_percent": 86.262301,
+ "image_difference": {
+ "max_difference": 13,
+ "total_pixels": 167
+ }
</ins><span class="cx"> },
</span><span class="cx"> "tiny-image-fail.html": {
</span><span class="cx"> "report": "REGRESSION",
</span><span class="cx"> "expected": "PASS",
</span><span class="cx"> "actual": "IMAGE",
</span><del>- "image_diff_percent": 0.002
</del><ins>+ "image_diff_percent": 0.002,
+ "image_difference": {
+ "max_difference": 3,
+ "total_pixels": 24
+ }
</ins><span class="cx"> },
</span><span class="cx"> "leaky-worker.html": {
</span><span class="cx"> "report": "REGRESSION",
</span></span></pre></div>
<a id="trunkLayoutTestsfastharnessimagedifftemplateexpectedtxt"></a>
<div class="modfile"><h4>Modified: trunk/LayoutTests/fast/harness/image-diff-template-expected.txt (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/LayoutTests/fast/harness/image-diff-template-expected.txt 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/LayoutTests/fast/harness/image-diff-template-expected.txt 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -1,3 +1,5 @@
</span><ins>+Pixel difference: __PIXEL_DIFF__
+Fuzzy match: __FUZZY_DATA__
</ins><span class="cx"> Expected Image Actual Image Diff Image
</span><span class="cx"> Animate: Expected Actual Diff
</span><span class="cx"> Loading...
</span></span></pre></div>
<a id="trunkLayoutTestsfastharnessimagedifftemplatehtml"></a>
<div class="modfile"><h4>Modified: trunk/LayoutTests/fast/harness/image-diff-template.html (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/LayoutTests/fast/harness/image-diff-template.html 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/LayoutTests/fast/harness/image-diff-template.html 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -3,10 +3,17 @@
</span><span class="cx"> <head>
</span><span class="cx"> <title>__TITLE__</title>
</span><span class="cx"> <style>
</span><ins>+ body {
+ font-family: Helvetica, sans-serif;
+ font-size: 11pt;
+ }
+
</ins><span class="cx"> .imageContainer {
</span><span class="cx"> position: absolute;
</span><span class="cx"> margin: 10px;
</span><span class="cx"> outline: 10px solid silver;
</span><ins>+ min-width: 300px;
+ min-height: 150px;
</ins><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> .imageContainer > img {
</span><span class="lines">@@ -13,32 +20,38 @@
</span><span class="cx"> display: block;
</span><span class="cx"> }
</span><span class="cx">
</span><del>-
</del><span class="cx"> .controls {
</span><del>- margin-bottom: 20px;
</del><ins>+ margin-bottom: 15px;
+ padding: 10px;
</ins><span class="cx"> }
</span><span class="cx">
</span><ins>+ .pixel-diff {
+ font-size: 10pt;
+ font-family: -apple-system;
+ }
+
+ .pixel-diff > td:first-child {
+ text-align: right;
+ }
+
+ .hidden {
+ display: none;
+ }
+
</ins><span class="cx"> .controls button {
</span><span class="cx"> width: 12em;
</span><ins>+ padding: 4px;
</ins><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> .controls td {
</span><del>- padding: 0 10px;
</del><ins>+ padding: 5px 10px;
</ins><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> .controls label {
</span><del>- font-family: -webkit-system-font;
- font-size: 11px;
</del><ins>+ font-family: -apple-system;
+ font-size: 12px;
</ins><span class="cx"> }
</span><span class="cx">
</span><del>- .buttons {
- }
- .controls > .indicator {
- display: table-cell;
- -webkit-column-span: 1;
- border-top: 2px solid black;
- }
-
</del><span class="cx"> button.selected {
</span><span class="cx"> text-decoration: underline;
</span><span class="cx"> }
</span><span class="lines">@@ -46,6 +59,14 @@
</span><span class="cx"> </head>
</span><span class="cx"> <body>
</span><span class="cx"> <table class="controls">
</span><ins>+ <tr class="pixel-diff __HIDE_DIFF_CLASS__">
+ <td>Pixel difference:</td>
+ <td>__PIXEL_DIFF__</td>
+ </tr>
+ <tr class="pixel-diff __HIDE_FUZZY_CLASS__">
+ <td>Fuzzy match:</td>
+ <td>__FUZZY_DATA__</td>
+ </tr>
</ins><span class="cx"> <tr>
</span><span class="cx"> <td></td>
</span><span class="cx"> <td><button id="expected-label" data-type="expected" onclick="switchToImage(this)">Expected Image</button></td>
</span><span class="lines">@@ -53,10 +74,10 @@
</span><span class="cx"> <td><button id="diff-label" data-type="diff" onclick="switchToImage(this)">Diff Image</button></td>
</span><span class="cx"> </tr>
</span><span class="cx"> <tr>
</span><del>- <td><input type="checkbox" id="animate" onchange="toggleAnimate()" checked></input><label for="animate">Animate:</label></td>
- <td><input type="checkbox" id="cycle-expected" onchange="updateImageCycle()" checked></input><label for="cycle-expected">Expected</label></td>
- <td><input type="checkbox" id="cycle-actual" onchange="updateImageCycle()" checked></input><label for="cycle-actual">Actual</label></td>
- <td><input type="checkbox" id="cycle-diff" onchange="updateImageCycle()"></input><label for="cycle-diff">Diff</label></td>
</del><ins>+ <td><input type="checkbox" id="animate" onchange="toggleAnimate()" checked><label for="animate">Animate:</label></td>
+ <td><input type="checkbox" id="cycle-expected" onchange="updateImageCycle()" checked><label for="cycle-expected">Expected</label></td>
+ <td><input type="checkbox" id="cycle-actual" onchange="updateImageCycle()" checked><label for="cycle-actual">Actual</label></td>
+ <td><input type="checkbox" id="cycle-diff" onchange="updateImageCycle()"><label for="cycle-diff">Diff</label></td>
</ins><span class="cx"> </tr>
</span><span class="cx"> </table>
</span><span class="cx">
</span></span></pre></div>
<a id="trunkLayoutTestsfastharnessresultshtml"></a>
<div class="modfile"><h4>Modified: trunk/LayoutTests/fast/harness/results.html (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/LayoutTests/fast/harness/results.html 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/LayoutTests/fast/harness/results.html 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -1415,8 +1415,8 @@
</span><span class="cx"> result += TestResultsController.resultLink(testPrefix, '-actual.png', 'actual');
</span><span class="cx"> }
</span><span class="cx">
</span><del>- let diff = Math.max(testResult.info.image_diff_percent, 0.01).toFixed(2);
- result += TestResultsController.resultLink(testPrefix, '-diff.png', 'diff (' + diff + '%)');
</del><ins>+ const diff = Math.max(testResult.info.image_diff_percent, 0.01).toFixed(2);
+ result += TestResultsController.resultLink(testPrefix, '-diff.png', `diff (${diff}%)`);
</ins><span class="cx"> }
</span><span class="cx"> }
</span><span class="cx">
</span></span></pre></div>
<a id="trunkToolsChangeLog"></a>
<div class="modfile"><h4>Modified: trunk/Tools/ChangeLog (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Tools/ChangeLog 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/ChangeLog 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -1,3 +1,40 @@
</span><ins>+2021-11-03 Simon Fraser <simon.fraser@apple.com>
+
+ Expose fuzzy match data in layout test results
+ https://bugs.webkit.org/show_bug.cgi?id=232523
+
+ Reviewed by Jonathan Bedard.
+
+ For a failing ref (or image) test, include the fuzzy matching data in full_results.json
+ in the form "image_difference": { "max_difference": 13, "total_pixels": 167 }, and
+ show that in the image comparison page that's written out for each test result.
+
+ Fix some tests that relied on JSON property serialization ordering (we don't serialize
+ with `sort_keys=True`).
+
+ * Scripts/webkitpy/layout_tests/controllers/manager.py:
+ (Manager._save_json_files):
+ * Scripts/webkitpy/layout_tests/controllers/test_result_writer.py:
+ (TestResultWriter.write_image_diff_files):
+ * Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py:
+ (JSONResultsGenerator.generate_times_ms_file):
+ * Scripts/webkitpy/layout_tests/models/test_failures.py:
+ (FailureImageHashMismatch.formatted_diff_percent):
+ (FailureImageHashMismatch):
+ (FailureImageHashMismatch.formatted_fuzzy_data):
+ (FailureImageHashMismatch.write_failure):
+ (FailureReftestMismatch.message):
+ (FailureReftestMismatch.formatted_diff_percent):
+ (FailureReftestMismatch):
+ (FailureReftestMismatch.formatted_fuzzy_data):
+ (FailureReftestMismatch.write_failure):
+ * Scripts/webkitpy/layout_tests/models/test_run_results.py:
+ (_interpret_test_failures):
+ * Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py:
+ (InterpretTestFailuresTest.test_interpret_test_failures):
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+ (RunTest.test_retrying_force_pixel_tests):
+
</ins><span class="cx"> 2021-11-03 Kate Cheney <katherine_cheney@apple.com>
</span><span class="cx">
</span><span class="cx"> [iOS] Need API for marking file requests as non-app-initiated
</span></span></pre></div>
<a id="trunkToolsScriptswebkitpylayout_testscontrollersmanagerpy"></a>
<div class="modfile"><h4>Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -692,7 +692,7 @@
</span><span class="cx"> self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
</span><span class="cx">
</span><span class="cx"> full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
</span><del>- # We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
</del><ins>+ # We write full_results.json out as jsonp because we need to load it from a file url and WebKit doesn't allow that.
</ins><span class="cx"> json_results_generator.write_json(self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS")
</span><span class="cx">
</span><span class="cx"> generator = json_layout_results_generator.JSONLayoutResultsGenerator(
</span></span></pre></div>
<a id="trunkToolsScriptswebkitpylayout_testscontrollerstest_result_writerpy"></a>
<div class="modfile"><h4>Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -184,7 +184,7 @@
</span><span class="cx"> def write_image_files(self, actual_image, expected_image):
</span><span class="cx"> self.write_output_files('.png', actual_image, expected_image)
</span><span class="cx">
</span><del>- def write_image_diff_files(self, image_diff):
</del><ins>+ def write_image_diff_files(self, image_diff, diff_percent_text=None, fuzzy_data_text=None):
</ins><span class="cx"> diff_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFF)
</span><span class="cx"> self._write_binary_file(diff_filename, image_diff)
</span><span class="cx">
</span><span class="lines">@@ -198,6 +198,18 @@
</span><span class="cx"> html = image_diff_file.replace('__TITLE__', self._test_name)
</span><span class="cx"> html = html.replace('__PREFIX__', self._output_testname(''))
</span><span class="cx">
</span><ins>+ if not diff_percent_text:
+ html = html.replace('__HIDE_DIFF_CLASS__', 'hidden')
+ else:
+ html = html.replace('__HIDE_DIFF_CLASS__', '')
+ html = html.replace('__PIXEL_DIFF__', diff_percent_text)
+
+ if not fuzzy_data_text:
+ html = html.replace('__HIDE_FUZZY_CLASS__', 'hidden')
+ else:
+ html = html.replace('__HIDE_FUZZY_CLASS__', '')
+ html = html.replace('__FUZZY_DATA__', fuzzy_data_text)
+
</ins><span class="cx"> diffs_html_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFFS_HTML)
</span><span class="cx"> self._filesystem.write_text_file(diffs_html_filename, html)
</span><span class="cx">
</span></span></pre></div>
<a id="trunkToolsScriptswebkitpylayout_testslayout_packagejson_results_generatorpy"></a>
<div class="modfile"><h4>Modified: trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -285,8 +285,6 @@
</span><span class="cx"> return True
</span><span class="cx">
</span><span class="cx"> def generate_times_ms_file(self):
</span><del>- # FIXME: rename to generate_times_ms_file. This needs to be coordinated with
- # changing the calls to this on the chromium build workers.
</del><span class="cx"> times = test_timings_trie(self._port, self._test_results_map.values())
</span><span class="cx"> file_path = self._filesystem.join(self._results_directory, self.TIMES_MS_FILENAME)
</span><span class="cx"> write_json(self._filesystem, times, file_path)
</span></span></pre></div>
<a id="trunkToolsScriptswebkitpylayout_testsmodelstest_failurespy"></a>
<div class="modfile"><h4>Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -208,9 +208,18 @@
</span><span class="cx"> def message(self):
</span><span class="cx"> return "image diff"
</span><span class="cx">
</span><ins>+ def formatted_diff_percent(self):
+ return '{:.2f}%'.format(max(self.image_diff_result.diff_percent, 0.01))
+
+ def formatted_fuzzy_data(self):
+ fuzzy_data = self.image_diff_result.fuzzy_data if self.image_diff_result else None
+ if fuzzy_data:
+ return 'maxDifference={}; totalPixels={}'.format(fuzzy_data['max_difference'], fuzzy_data['total_pixels'])
+ return None
+
</ins><span class="cx"> def write_failure(self, writer, driver_output, expected_driver_output, port):
</span><span class="cx"> writer.write_image_files(driver_output.image, expected_driver_output.image)
</span><del>- writer.write_image_diff_files(driver_output.image_diff)
</del><ins>+ writer.write_image_diff_files(driver_output.image_diff, self.formatted_diff_percent(), self.formatted_fuzzy_data())
</ins><span class="cx">
</span><span class="cx">
</span><span class="cx"> class FailureImageHashIncorrect(TestFailure):
</span><span class="lines">@@ -225,8 +234,20 @@
</span><span class="cx"> self.image_diff_result = image_diff_result
</span><span class="cx">
</span><span class="cx"> def message(self):
</span><ins>+ fuzzy_data = self.image_diff_result.fuzzy_data if self.image_diff_result else None
+ if fuzzy_data:
+ return "reference mismatch maxDifference={}; totalPixels={}".format(fuzzy_data['max_difference'], fuzzy_data['total_pixels'])
</ins><span class="cx"> return "reference mismatch"
</span><span class="cx">
</span><ins>+ def formatted_diff_percent(self):
+ return '{:.2f}%'.format(max(self.image_diff_result.diff_percent, 0.01))
+
+ def formatted_fuzzy_data(self):
+ fuzzy_data = self.image_diff_result.fuzzy_data if self.image_diff_result else None
+ if fuzzy_data:
+ return 'maxDifference={}; totalPixels={}'.format(fuzzy_data['max_difference'], fuzzy_data['total_pixels'])
+ return None
+
</ins><span class="cx"> def write_failure(self, writer, driver_output, expected_driver_output, port):
</span><span class="cx"> writer.write_image_files(driver_output.image, expected_driver_output.image)
</span><span class="cx"> if self.image_diff_result:
</span><span class="lines">@@ -236,7 +257,7 @@
</span><span class="cx"> else:
</span><span class="cx"> diff_image = self.image_diff_result.diff_image
</span><span class="cx">
</span><del>- writer.write_image_diff_files(diff_image)
</del><ins>+ writer.write_image_diff_files(diff_image, self.formatted_diff_percent(), self.formatted_fuzzy_data())
</ins><span class="cx"> else:
</span><span class="cx"> _log.warn('ref test mismatch did not produce an image diff.')
</span><span class="cx"> writer.write_reftest(self.reference_filename)
</span></span></pre></div>
<a id="trunkToolsScriptswebkitpylayout_testsmodelstest_run_resultspy"></a>
<div class="modfile"><h4>Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -215,6 +215,11 @@
</span><span class="cx"> if isinstance(failure, test_failures.FailureImageHashMismatch) or isinstance(failure, test_failures.FailureReftestMismatch):
</span><span class="cx"> test_dict['image_diff_percent'] = failure.image_diff_result.diff_percent
</span><span class="cx">
</span><ins>+ if 'image_difference' not in test_dict:
+ for failure in failures:
+ if (isinstance(failure, test_failures.FailureImageHashMismatch) or isinstance(failure, test_failures.FailureReftestMismatch)) and failure.image_diff_result.fuzzy_data:
+ test_dict['image_difference'] = failure.image_diff_result.fuzzy_data
+
</ins><span class="cx"> return test_dict
</span><span class="cx">
</span><span class="cx">
</span></span></pre></div>
<a id="trunkToolsScriptswebkitpylayout_testsmodelstest_run_results_unittestpy"></a>
<div class="modfile"><h4>Modified: trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -128,8 +128,10 @@
</span><span class="cx"> test_dict = test_run_results._interpret_test_failures([test_failures.FailureImageHashMismatch(ImageDiffResult(passed=False, diff_image=b'', difference=0.42))])
</span><span class="cx"> self.assertEqual(test_dict['image_diff_percent'], 0.42)
</span><span class="cx">
</span><del>- test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/reftest-expected.html'), ImageDiffResult(passed=False, diff_image=b'', difference=100.0))])
</del><ins>+ result_fuzzy_data = {'max_difference': 6, 'total_pixels': 50}
+ test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/reftest-expected.html'), ImageDiffResult(passed=False, diff_image=b'', difference=100.0, fuzzy_data=result_fuzzy_data))])
</ins><span class="cx"> self.assertIn('image_diff_percent', test_dict)
</span><ins>+ self.assertIn('image_difference', test_dict)
</ins><span class="cx">
</span><span class="cx"> test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))])
</span><span class="cx"> self.assertEqual(len(test_dict), 0)
</span></span></pre></div>
<a id="trunkToolsScriptswebkitpylayout_testsrun_webkit_tests_integrationtestpy"></a>
<div class="modfile"><h4>Modified: trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (285212 => 285213)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2021-11-03 17:05:02 UTC (rev 285212)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py 2021-11-03 17:05:50 UTC (rev 285213)
</span><span class="lines">@@ -517,7 +517,7 @@
</span><span class="cx"> def test_pixel_test_directories(self):
</span><span class="cx"> host = MockHost()
</span><span class="cx">
</span><del>- """Both tests have faling checksum. We include only the first in pixel tests so only that should fail."""
</del><ins>+ """Both tests have failing checksum. We include only the first in pixel tests so only that should fail."""
</ins><span class="cx"> args = ['--pixel-tests', '--pixel-test-directory', 'failures/unexpected/pixeldir',
</span><span class="cx"> 'failures/unexpected/pixeldir/image_in_pixeldir.html',
</span><span class="cx"> 'failures/unexpected/image_not_in_pixeldir.html']
</span><span class="lines">@@ -524,9 +524,13 @@
</span><span class="cx"> details, err, _ = logging_run(extra_args=args, host=host, tests_included=True)
</span><span class="cx">
</span><span class="cx"> self.assertEqual(details.exit_code, 1)
</span><del>- expected_token = '"unexpected":{"pixeldir":{"image_in_pixeldir.html":{"report":"REGRESSION","expected":"PASS","actual":"IMAGE"'
</del><ins>+
</ins><span class="cx"> json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
</span><del>- self.assertTrue(json_string.find(expected_token) != -1)
</del><ins>+ json = parse_full_results(json_string)
+ test_data = json["tests"]["failures"]["unexpected"]["pixeldir"]["image_in_pixeldir.html"]
+ self.assertEqual(test_data["expected"], "PASS")
+ self.assertEqual(test_data["actual"], "IMAGE")
+ self.assertEqual(test_data["report"], "REGRESSION")
</ins><span class="cx">
</span><span class="cx"> def test_missing_and_unexpected_results_with_custom_exit_code(self):
</span><span class="cx"> # Test that we update expectations in place. If the expectation
</span><span class="lines">@@ -710,7 +714,7 @@
</span><span class="cx"> json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
</span><span class="cx"> json = parse_full_results(json_string)
</span><span class="cx"> self.assertEqual(json["tests"]["failures"]["unexpected"]["text-image-checksum.html"],
</span><del>- {"expected": "PASS", "actual": "TEXT IMAGE+TEXT", "image_diff_percent": 1, "report": "REGRESSION"})
</del><ins>+ {"expected": "PASS", "actual": "TEXT IMAGE+TEXT", "image_diff_percent": 1, 'image_difference': {'max_difference': 10, 'total_pixels': 20}, "report": "REGRESSION"})
</ins><span class="cx"> self.assertFalse(json["pixel_tests_enabled"])
</span><span class="cx"> self.assertEqual(details.enabled_pixel_tests_in_retry, True)
</span><span class="cx">
</span><span class="lines">@@ -723,9 +727,15 @@
</span><span class="cx"> tests_included=True, host=host)
</span><span class="cx"> file_list = host.filesystem.written_files.keys()
</span><span class="cx"> self.assertEqual(details.exit_code, 1)
</span><del>- expected_token = '"unexpected":{"text-image-missing.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT MISSING","is_missing_image":true}}'
</del><span class="cx"> json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
</span><del>- self.assertTrue(json_string.find(expected_token) != -1)
</del><ins>+
+ json = parse_full_results(json_string)
+ test_data = json["tests"]["failures"]["unexpected"]["text-image-missing.html"]
+ self.assertEqual(test_data["expected"], "PASS")
+ self.assertEqual(test_data["actual"], "TEXT MISSING")
+ self.assertEqual(test_data["report"], "REGRESSION")
+ self.assertEqual(test_data["is_missing_image"], True)
+
</ins><span class="cx"> self.assertTrue(json_string.find('"num_regressions":1') != -1)
</span><span class="cx"> self.assertTrue(json_string.find('"num_flaky":0') != -1)
</span><span class="cx"> self.assertTrue(json_string.find('"num_missing":1') != -1)
</span></span></pre>
</div>
</div>
</body>
</html>