# Copyright (c) 2013~2015 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""SuiteRunner defines the interface from crosperf to test script."""
from __future__ import print_function
import os
import time
import shlex
from cros_utils import command_executer
import test_flag
TEST_THAT_PATH = '/usr/bin/test_that'
AUTOTEST_DIR = '~/trunk/src/third_party/autotest/files'
CHROME_MOUNT_DIR = '/tmp/chrome_root'
def GetProfilerArgs(profiler_args):
# Remove "--" from in front of profiler args.
args_list = shlex.split(profiler_args)
new_list = []
for arg in args_list:
if arg[0:2] == '--':
arg = arg[2:]
new_list.append(arg)
args_list = new_list
# Remove "perf_options=" from middle of profiler args.
new_list = []
for arg in args_list:
idx = arg.find('perf_options=')
if idx != -1:
prefix = arg[0:idx]
suffix = arg[idx + len('perf_options=') + 1:-1]
new_arg = prefix + "'" + suffix + "'"
new_list.append(new_arg)
else:
new_list.append(arg)
args_list = new_list
return ' '.join(args_list)
class SuiteRunner(object):
"""This defines the interface from crosperf to test script."""
def __init__(self,
logger_to_use=None,
log_level='verbose',
cmd_exec=None,
cmd_term=None):
self.logger = logger_to_use
self.log_level = log_level
self._ce = cmd_exec or command_executer.GetCommandExecuter(
self.logger, log_level=self.log_level)
self._ct = cmd_term or command_executer.CommandTerminator()
def Run(self, machine, label, benchmark, test_args, profiler_args):
for i in range(0, benchmark.retries + 1):
self.PinGovernorExecutionFrequencies(machine, label.chromeos_root)
if benchmark.suite == 'telemetry':
self.DecreaseWaitTime(machine, label.chromeos_root)
ret_tup = self.Telemetry_Run(machine, label, benchmark, profiler_args)
elif benchmark.suite == 'telemetry_Crosperf':
self.DecreaseWaitTime(machine, label.chromeos_root)
ret_tup = self.Telemetry_Crosperf_Run(machine, label, benchmark,
test_args, profiler_args)
else:
ret_tup = self.Test_That_Run(machine, label, benchmark, test_args,
profiler_args)
if ret_tup[0] != 0:
self.logger.LogOutput('benchmark %s failed. Retries left: %s' %
(benchmark.name, benchmark.retries - i))
elif i > 0:
self.logger.LogOutput('benchmark %s succeded after %s retries' %
(benchmark.name, i))
break
else:
self.logger.LogOutput('benchmark %s succeded on first try' %
benchmark.name)
break
return ret_tup
def PinGovernorExecutionFrequencies(self, machine_name, chromeos_root):
"""Set min and max frequencies to max static frequency."""
# pyformat: disable
set_cpu_freq = (
'set -e && '
'for f in /sys/devices/system/cpu/cpu*/cpufreq; do '
'cd $f; '
'val=0; '
'if [[ -e scaling_available_frequencies ]]; then '
# pylint: disable=line-too-long
' val=`cat scaling_available_frequencies | tr " " "\\n" | sort -n -b -r`; '
'else '
' val=`cat scaling_max_freq | tr " " "\\n" | sort -n -b -r`; fi ; '
'set -- $val; '
'highest=$1; '
'if [[ $# -gt 1 ]]; then '
' case $highest in *1000) highest=$2;; esac; '
'fi ;'
'echo $highest > scaling_max_freq; '
'echo $highest > scaling_min_freq; '
'echo performance > scaling_governor; '
'done'
)
# pyformat: enable
if self.log_level == 'average':
self.logger.LogOutput('Pinning governor execution frequencies for %s' %
machine_name)
ret = self._ce.CrosRunCommand(
set_cpu_freq, machine=machine_name, chromeos_root=chromeos_root)
self.logger.LogFatalIf(ret, 'Could not pin frequencies on machine: %s' %
machine_name)
def DecreaseWaitTime(self, machine_name, chromeos_root):
"""Change the ten seconds wait time for pagecycler to two seconds."""
FILE = '/usr/local/telemetry/src/tools/perf/page_sets/page_cycler_story.py'
ret = self._ce.CrosRunCommand(
'ls ' + FILE, machine=machine_name, chromeos_root=chromeos_root)
self.logger.LogFatalIf(ret, 'Could not find {} on machine: {}'.format(
FILE, machine_name))
if not ret:
sed_command = 'sed -i "s/_TTI_WAIT_TIME = 10/_TTI_WAIT_TIME = 2/g" '
ret = self._ce.CrosRunCommand(
sed_command + FILE, machine=machine_name, chromeos_root=chromeos_root)
self.logger.LogFatalIf(ret, 'Could not modify {} on machine: {}'.format(
FILE, machine_name))
def RebootMachine(self, machine_name, chromeos_root):
command = 'reboot && exit'
self._ce.CrosRunCommand(
command, machine=machine_name, chromeos_root=chromeos_root)
time.sleep(60)
# Whenever we reboot the machine, we need to restore the governor settings.
self.PinGovernorExecutionFrequencies(machine_name, chromeos_root)
def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
"""Run the test_that test.."""
options = ''
if label.board:
options += ' --board=%s' % label.board
if test_args:
options += ' %s' % test_args
if profiler_args:
self.logger.LogFatal('test_that does not support profiler.')
command = 'rm -rf /usr/local/autotest/results/*'
self._ce.CrosRunCommand(
command, machine=machine, chromeos_root=label.chromeos_root)
# We do this because some tests leave the machine in weird states.
# Rebooting between iterations has proven to help with this.
self.RebootMachine(machine, label.chromeos_root)
autotest_dir = AUTOTEST_DIR
if label.autotest_path != '':
autotest_dir = label.autotest_path
autotest_dir_arg = '--autotest_dir %s' % autotest_dir
# For non-telemetry tests, specify an autotest directory only if the
# specified directory is different from default (crosbug.com/679001).
if autotest_dir == AUTOTEST_DIR:
autotest_dir_arg = ''
command = (('%s %s --fast '
'%s %s %s') % (TEST_THAT_PATH, autotest_dir_arg, options,
machine, benchmark.test_name))
if self.log_level != 'verbose':
self.logger.LogOutput('Running test.')
self.logger.LogOutput('CMD: %s' % command)
# Use --no-ns-pid so that cros_sdk does not create a different
# process namespace and we can kill process created easily by
# their process group.
return self._ce.ChrootRunCommandWOutput(
label.chromeos_root,
command,
command_terminator=self._ct,
cros_sdk_options='--no-ns-pid')
def RemoveTelemetryTempFile(self, machine, chromeos_root):
filename = 'telemetry@%s' % machine
fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename)
if os.path.exists(fullname):
os.remove(fullname)
def Telemetry_Crosperf_Run(self, machine, label, benchmark, test_args,
profiler_args):
if not os.path.isdir(label.chrome_src):
self.logger.LogFatal('Cannot find chrome src dir to'
' run telemetry: %s' % label.chrome_src)
# Check for and remove temporary file that may have been left by
# previous telemetry runs (and which might prevent this run from
# working).
self.RemoveTelemetryTempFile(machine, label.chromeos_root)
# For telemetry runs, we can use the autotest copy from the source
# location. No need to have one under /build/<board>.
autotest_dir_arg = '--autotest_dir %s' % AUTOTEST_DIR
if label.autotest_path != '':
autotest_dir_arg = '--autotest_dir %s' % label.autotest_path
profiler_args = GetProfilerArgs(profiler_args)
fast_arg = ''
if not profiler_args:
# --fast works unless we are doing profiling (autotest limitation).
# --fast avoids unnecessary copies of syslogs.
fast_arg = '--fast'
args_string = ''
if test_args:
# Strip double quotes off args (so we can wrap them in single
# quotes, to pass through to Telemetry).
if test_args[0] == '"' and test_args[-1] == '"':
test_args = test_args[1:-1]
args_string = "test_args='%s'" % test_args
cmd = ('{} {} {} --board={} --args="{} run_local={} test={} '
'{}" {} telemetry_Crosperf'.format(TEST_THAT_PATH, autotest_dir_arg,
fast_arg, label.board,
args_string, benchmark.run_local,
benchmark.test_name,
profiler_args, machine))
# Use --no-ns-pid so that cros_sdk does not create a different
# process namespace and we can kill process created easily by their
# process group.
chrome_root_options = ('--no-ns-pid '
'--chrome_root={} --chrome_root_mount={} '
"FEATURES=\"-usersandbox\" "
'CHROME_ROOT={}'.format(label.chrome_src,
CHROME_MOUNT_DIR,
CHROME_MOUNT_DIR))
if self.log_level != 'verbose':
self.logger.LogOutput('Running test.')
self.logger.LogOutput('CMD: %s' % cmd)
return self._ce.ChrootRunCommandWOutput(
label.chromeos_root,
cmd,
command_terminator=self._ct,
cros_sdk_options=chrome_root_options)
def Telemetry_Run(self, machine, label, benchmark, profiler_args):
telemetry_run_path = ''
if not os.path.isdir(label.chrome_src):
self.logger.LogFatal('Cannot find chrome src dir to' ' run telemetry.')
else:
telemetry_run_path = os.path.join(label.chrome_src, 'src/tools/perf')
if not os.path.exists(telemetry_run_path):
self.logger.LogFatal('Cannot find %s directory.' % telemetry_run_path)
if profiler_args:
self.logger.LogFatal('Telemetry does not support the perf profiler.')
# Check for and remove temporary file that may have been left by
# previous telemetry runs (and which might prevent this run from
# working).
if not test_flag.GetTestMode():
self.RemoveTelemetryTempFile(machine, label.chromeos_root)
rsa_key = os.path.join(
label.chromeos_root,
'src/scripts/mod_for_test_scripts/ssh_keys/testing_rsa')
cmd = ('cd {0} && '
'./run_measurement '
'--browser=cros-chrome '
'--output-format=csv '
'--remote={1} '
'--identity {2} '
'{3} {4}'.format(telemetry_run_path, machine, rsa_key,
benchmark.test_name, benchmark.test_args))
if self.log_level != 'verbose':
self.logger.LogOutput('Running test.')
self.logger.LogOutput('CMD: %s' % cmd)
return self._ce.RunCommandWOutput(cmd, print_to_console=False)
def CommandTerminator(self):
return self._ct
def Terminate(self):
self._ct.Terminate()
class MockSuiteRunner(object):
"""Mock suite runner for test."""
def __init__(self):
self._true = True
def Run(self, *_args):
if self._true:
return [0, '', '']
else:
return [0, '', '']