run_rendering_benchmark_with_gated_performance.py

#!/usr/bin/env python
# Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Runs telemetry benchmarks on representative story tag.

This script is a wrapper around run_performance_tests.py to capture the
values of performance metrics and compare them with the acceptable limits
in order to prevent regressions.

Arguments used for this script are the same as run_performance_tests.py.

The name and some functionalities of this script should be adjusted for
use with other benchmarks.
"""

from __future__ import print_function

import argparse
import csv
import json
import numpy as np
import os
import sys
import time

import common
import run_performance_tests

# AVG_ERROR_MARGIN determines how much more the value of frame times can be
# compared to the recorded value (multiplier of upper limit).
AVG_ERROR_MARGIN = 1.1
# CI stands for confidence intervals. "ci_095"s recorded in the data is the
# recorded range between upper and lower CIs. CI_ERROR_MARGIN is the maximum
# acceptable ratio of calculated ci_095 to the recorded ones.
# TODO(behdadb) crbug.com/1052054
CI_ERROR_MARGIN = 1.5

METRIC_NAME = 'frame_times'

class ResultRecorder(object):
  def __init__(self):
    self.fails = 0
    self.tests = 0
    self.start_time = time.time()
    self.output = {}
    self.return_code = 0
    self._failed_stories = set()
    self._noisy_control_stories = set()
    # Set of _noisy_control_stories keeps track of control tests which failed
    # because of high noise values.

  def set_tests(self, output):
    self.output = output
    self.fails = output['num_failures_by_type'].get('FAIL', 0)
    self.tests = self.fails + output['num_failures_by_type'].get('PASS', 0)

  def add_failure(self, name, benchmark, is_control=False):
    self.output['tests'][benchmark][name]['actual'] = 'FAIL'
    self.output['tests'][benchmark][name]['is_unexpected'] = True
    self._failed_stories.add(name)
    self.fails += 1
    if is_control:
      self._noisy_control_stories.add(name)

  def remove_failure(self, name, benchmark, is_control=False,
                      invalidation_reason=None):
    self.output['tests'][benchmark][name]['actual'] = 'PASS'
    self.output['tests'][benchmark][name]['is_unexpected'] = False
    self._failed_stories.remove(name)
    self.fails -= 1
    if is_control:
      self._noisy_control_stories.remove(name)
    if invalidation_reason:
      self.add_invalidation_reason(name, benchmark, invalidation_reason)

  def invalidate_failures(self, benchmark):
    # The method is for invalidating the failures in case of noisy control test
    for story in self._failed_stories.copy():
      print(story + ' [Invalidated Failure]: The story failed but was ' +
        'invalidated as a result of noisy control test.')
      self.remove_failure(story, benchmark, False, 'Noisy control test')

  def add_invalidation_reason(self, name, benchmark, reason):
    self.output['tests'][benchmark][name]['invalidation_reason'] = reason

  @property
  def failed_stories(self):
    return self._failed_stories

  @property
  def is_control_stories_noisy(self):
    return len(self._noisy_control_stories) > 0

  def get_output(self, return_code):
    self.output['seconds_since_epoch'] = time.time() - self.start_time
    self.output['num_failures_by_type']['PASS'] = self.tests - self.fails
    self.output['num_failures_by_type']['FAIL'] = self.fails
    if return_code == 1:
      self.output['interrupted'] = True

    plural = lambda n, s, p: '%d %s' % (n, p if n != 1 else s)
    tests = lambda n: plural(n, 'test', 'tests')

    print('[  PASSED  ] ' + tests(self.tests - self.fails) + '.')
    if self.fails > 0:
      print('[  FAILED  ] ' + tests(self.fails) + '.')
      self.return_code = 1

    return (self.output, self.return_code)

class RenderingRepresentativePerfTest(object):
  def __init__(self, initialization_for_tests=False):
    self.return_code = 0
    # result_recorder for rerun, and non rerun
    self.result_recorder = {
      True: ResultRecorder(),
      False: ResultRecorder()
    }

    if initialization_for_tests is True:
      return

    self.options = parse_arguments()
    print (self.options)

    self.benchmark = self.options.benchmarks
    out_dir_path = os.path.dirname(self.options.isolated_script_test_output)
    re_run_output_dir = os.path.join(out_dir_path, 're_run_failures')

    self.output_path = {
      True: os.path.join(
        re_run_output_dir, self.benchmark, 'test_results.json'),
      False: os.path.join(out_dir_path, self.benchmark, 'test_results.json')
    }
    self.results_path = {
      True: os.path.join(
        re_run_output_dir, self.benchmark, 'perf_results.csv'),
      False: os.path.join(out_dir_path, self.benchmark, 'perf_results.csv')
    }

    re_run_test_output = os.path.join(re_run_output_dir,
      os.path.basename(self.options.isolated_script_test_output))

    self.set_platform_specific_attributes()

    # The values used as the upper limit are the 99th percentile of the
    # avg and ci_095 frame_times recorded by dashboard in the past 200
    # revisions. If the value measured here would be higher than this value at
    # least by 10 [AVG_ERROR_MARGIN] percent of upper limit, that would be
    # considered a failure. crbug.com/953895
    with open(
      os.path.join(os.path.dirname(__file__),
      'representative_perf_test_data',
      'representatives_frame_times_upper_limit.json')
    ) as bound_data:
      self.upper_limit_data = json.load(bound_data)[self.platform]

    self.args = list(sys.argv)
    # The first run uses all stories in the representative story tag, but for
    # rerun we use only the failed stories.
    self.args.extend(['--story-tag-filter', self.story_tag])

    self.re_run_args = replace_arg_values(list(sys.argv), [
      ('--isolated-script-test-output', re_run_test_output)])

  def parse_csv_results(self, csv_obj):
    """ Parses the raw CSV data
    Convers the csv_obj into an array of valid values for averages and
    confidence intervals based on the described upper_limits.

    Args:
      csv_obj: An array of rows (dict) describing the CSV results

    Raturns:
      A dictionary which has the stories as keys and an array of confidence
      intervals and valid averages as data.
    """
    values_per_story = {}
    for row in csv_obj:
      # For now only frame_times is used for testing representatives'
      # performance and cpu_wall_time_ratio is used for validation.
      if row['name'] != METRIC_NAME and row['name'] != 'cpu_wall_time_ratio':
        continue
      story_name = row['stories']
      if (story_name not in self.upper_limit_data):
        continue
      if story_name not in values_per_story:
        values_per_story[story_name] = {
          'averages': [],
          'ci_095': [],
          'cpu_wall_time_ratio': []
        }

      if row['name'] == METRIC_NAME and row['avg'] != '' and row['count'] != 0:
        values_per_story[story_name]['ci_095'].append(float(row['ci_095']))
        values_per_story[story_name]['averages'].append(float(row['avg']))
      elif row['name'] == 'cpu_wall_time_ratio' and row['avg'] != '':
        values_per_story[story_name]['cpu_wall_time_ratio'].append(
          float(row['avg']))

    return values_per_story

  def compare_values(self, values_per_story, rerun=False):
    """ Parses the raw CSV data
    Compares the values in values_per_story with the upper_limit_data and
    determines if the story passes or fails and updates the ResultRecorder.

    Args:
      values_per_story: An array of rows (dict) descriving the CSV results
      rerun: Is this a rerun or initial run
    """
    for story_name in values_per_story:
      # The experimental stories will not be considered for failing the tests
      if (self.is_experimental_story(story_name)):
        continue
      if len(values_per_story[story_name]['ci_095']) == 0:
        print(('[  FAILED  ] {}/{} has no valid values for {}. Check ' +
          'run_benchmark logs for more information.').format(
            self.benchmark, story_name, METRIC_NAME))
        self.result_recorder[rerun].add_failure(story_name, self.benchmark)
        continue

      upper_limits = self.upper_limit_data
      upper_limit_avg = upper_limits[story_name]['avg']
      upper_limit_ci = upper_limits[story_name]['ci_095']
      lower_limit_cpu_ratio = upper_limits[story_name]['cpu_wall_time_ratio']
      measured_avg = np.mean(np.array(values_per_story[story_name]['averages']))
      measured_ci = np.mean(np.array(values_per_story[story_name]['ci_095']))
      measured_cpu_ratio = np.mean(np.array(
        values_per_story[story_name]['cpu_wall_time_ratio']))

      if (measured_ci > upper_limit_ci * CI_ERROR_MARGIN and
        self.is_control_story(story_name)):
        print(('[  FAILED  ] {}/{} {} has higher noise ({:.3f}) ' +
          'compared to upper limit ({:.3f})').format(
            self.benchmark, story_name, METRIC_NAME, measured_ci,
          upper_limit_ci))
        self.result_recorder[rerun].add_failure(
          story_name, self.benchmark, True)
      elif (measured_avg > upper_limit_avg * AVG_ERROR_MARGIN):
        if (measured_cpu_ratio >= lower_limit_cpu_ratio):
          print(('[  FAILED  ] {}/{} higher average {}({:.3f}) compared' +
            ' to upper limit ({:.3f})').format(self.benchmark, story_name,
            METRIC_NAME, measured_avg, upper_limit_avg))
          self.result_recorder[rerun].add_failure(story_name, self.benchmark)
        else:
          print(('[       OK ] {}/{} higher average {}({:.3f}) compared ' +
            'to upper limit({:.3f}). Invalidated for low cpu_wall_time_ratio'
            ).format(self.benchmark, story_name, METRIC_NAME, measured_avg,
                      upper_limit_avg))
          self.result_recorder[rerun].add_invalidation_reason(
            story_name, self.benchmark, 'Low cpu_wall_time_ratio')
      else:
        print(('[       OK ] {}/{} lower average {}({:.3f}) compared ' +
          'to upper limit({:.3f}).').format(self.benchmark, story_name,
          METRIC_NAME, measured_avg, upper_limit_avg))

  def interpret_run_benchmark_results(self, rerun=False):
    with open(self.output_path[rerun], 'r+') as resultsFile:
      initialOut = json.load(resultsFile)
      self.result_recorder[rerun].set_tests(initialOut)

      with open(self.results_path[rerun]) as csv_file:
        csv_obj = csv.DictReader(csv_file)
        values_per_story = self.parse_csv_results(csv_obj)

      if not rerun:
        # Clearing the result of run_benchmark and write the gated perf results
        resultsFile.seek(0)
        resultsFile.truncate(0)

    self.compare_values(values_per_story, rerun)

  def run_perf_tests(self):
    self.return_code |= run_performance_tests.main(self.args)
    self.interpret_run_benchmark_results(False)

    if len(self.result_recorder[False].failed_stories) > 0:
      # For failed stories we run_tests again to make sure it's not a false
      # positive.
      print('============ Re_run the failed tests ============')
      all_failed_stories = '('+'|'.join(
        self.result_recorder[False].failed_stories)+')'
      # TODO(crbug.com/1055893): Remove the extra chrome categories after
      # investigation of flakes in representative perf tests.
      self.re_run_args.extend(
        ['--story-filter', all_failed_stories, '--pageset-repeat=3',
         '--extra-chrome-categories=blink,blink_gc,gpu,v8,viz'])
      self.return_code |= run_performance_tests.main(self.re_run_args)
      self.interpret_run_benchmark_results(True)

      for story_name in self.result_recorder[False].failed_stories.copy():
        if story_name not in self.result_recorder[True].failed_stories:
          self.result_recorder[False].remove_failure(story_name,
            self.benchmark, self.is_control_story(story_name))

    if self.result_recorder[False].is_control_stories_noisy:
      # In this case all failures are reported as expected, and the number of
      # Failed stories in output.json will be zero.
      self.result_recorder[False].invalidate_failures(self.benchmark)

    (
      finalOut,
      self.return_code
    ) = self.result_recorder[False].get_output(self.return_code)

    with open(self.output_path[False], 'r+') as resultsFile:
      json.dump(finalOut, resultsFile, indent=4)
    with open(self.options.isolated_script_test_output, 'w') as outputFile:
      json.dump(finalOut, outputFile, indent=4)

    if self.result_recorder[False].is_control_stories_noisy:
      assert self.return_code == 0
      print('Control story has high noise. These runs are not reliable!')

    return self.return_code

  def is_control_story(self, story_name):
    # The story tagged as control story in upper_limit_data, will be used to
    # identify possible flake and invalidates the results.
    return self.story_has_attribute_enabled(story_name, 'control')

  def is_experimental_story(self, story_name):
    # The story tagged as experimental story in upper_limit_data, will be used
    # to gather the performance results, but the test would not be failed as
    # a result of.
    return self.story_has_attribute_enabled(story_name, 'experimental')

  def story_has_attribute_enabled(self, story_name, attribute):
    return (attribute in self.upper_limit_data[story_name] and
      self.upper_limit_data[story_name][attribute] == True)


  def set_platform_specific_attributes(self):
    if self.benchmark == 'rendering.desktop':
      # Linux does not have it's own specific representatives
      # and uses the representatives chosen for windows.
      if sys.platform == 'win32' or sys.platform.startswith('linux'):
        self.platform = 'win'
        self.story_tag = 'representative_win_desktop'
      elif sys.platform == 'darwin':
        self.platform = 'mac'
        self.story_tag = 'representative_mac_desktop'
      else:
        self.return_code = 1
    elif self.benchmark == 'rendering.mobile':
      self.platform = 'android'
      self.story_tag = 'representative_mobile'
    else:
      self.return_code = 1

def replace_arg_values(args, key_value_pairs):
  for index in range(0, len(args)):
    for (key, value) in key_value_pairs:
      if args[index].startswith(key):
        if '=' in args[index]:
          args[index] = key + '=' + value
        else:
          args[index+1] = value
  return args

def main():
  test_runner = RenderingRepresentativePerfTest()
  if test_runner.return_code == 1:
    return 1

  return test_runner.run_perf_tests()

def parse_arguments():
  parser = argparse.ArgumentParser()
  parser.add_argument('executable', help='The name of the executable to run.')
  parser.add_argument(
      '--benchmarks', required=True)
  parser.add_argument(
      '--isolated-script-test-output', required=True)
  parser.add_argument(
      '--isolated-script-test-perf-output', required=False)
  return parser.parse_known_args()[0]

def main_compile_targets(args):
  json.dump([], args.output)

if __name__ == '__main__':
  # Conform minimally to the protocol defined by ScriptTest.
  if 'compile_targets' in sys.argv:
    funcs = {
      'run': None,
      'compile_targets': main_compile_targets,
    }
    sys.exit(common.run_script(sys.argv[1:], funcs))
  sys.exit(main())