#!/usr/bin/python2.4
"""Run reliability tests using Android instrumentation.
A test file consists of list web sites to test is needed as a parameter
Usage:
run_reliability_tests.py path/to/url/list
"""
import logging
import optparse
import os
import subprocess
import sys
import time
from Numeric import *
TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt"
TEST_LOAD_TIME_FILE = "/sdcard/android/reliability_load_time.txt"
HTTP_URL_FILE = "urllist_http"
HTTPS_URL_FILE = "urllist_https"
NUM_URLS = 25
def DumpRenderTreeFinished(adb_cmd):
"""Check if DumpRenderTree finished running.
Args:
adb_cmd: adb command string
Returns:
True if DumpRenderTree has finished, False otherwise
"""
# pull test status file and look for "#DONE"
shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
adb_output = subprocess.Popen(shell_cmd_str,
shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()[0]
return adb_output.strip() == "#DONE"
def RemoveDeviceFile(adb_cmd, file_name):
shell_cmd_str = adb_cmd + " shell rm " + file_name
subprocess.Popen(shell_cmd_str,
shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
def Bugreport(url, bugreport_dir, adb_cmd):
"""Pull a bugreport from the device."""
bugreport_filename = "%s/reliability_bugreport_%d.txt" % (bugreport_dir,
int(time.time()))
# prepend the report with url
handle = open(bugreport_filename, "w")
handle.writelines("Bugreport for crash in url - %s\n\n" % url)
handle.close()
cmd = "%s bugreport >> %s" % (adb_cmd, bugreport_filename)
os.system(cmd)
def ProcessPageLoadTime(raw_log):
"""Processes the raw page load time logged by test app."""
log_handle = open(raw_log, "r")
load_times = {}
for line in log_handle:
line = line.strip()
pair = line.split("|")
if len(pair) != 2:
logging.info("Line has more than one '|': " + line)
continue
if pair[0] not in load_times:
load_times[pair[0]] = []
try:
pair[1] = int(pair[1])
except ValueError:
logging.info("Lins has non-numeric load time: " + line)
continue
load_times[pair[0]].append(pair[1])
log_handle.close()
# rewrite the average time to file
log_handle = open(raw_log, "w")
for url, times in load_times.iteritems():
# calculate std
arr = array(times)
avg = average(arr)
d = arr - avg
std = sqrt(sum(d * d) / len(arr))
output = ("%-70s%-10d%-10d%-12.2f%-12.2f%s\n" %
(url, min(arr), max(arr), avg, std,
array2string(arr)))
log_handle.write(output)
log_handle.close()
def main(options, args):
"""Send the url list to device and start testing, restart if crashed."""
# Set up logging format.
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(level=log_level,
format="%(message)s")
# Include all tests if none are specified.
if not args:
print "Missing URL list file"
sys.exit(1)
else:
path = args[0]
if not options.crash_file:
print "Missing crash file name, use --crash-file to specify"
sys.exit(1)
else:
crashed_file = options.crash_file
if not options.timeout_file:
print "Missing timeout file, use --timeout-file to specify"
sys.exit(1)
else:
timedout_file = options.timeout_file
if not options.delay:
manual_delay = 0
else:
manual_delay = options.delay
if not options.bugreport:
bugreport_dir = "."
else:
bugreport_dir = options.bugreport
if not os.path.exists(bugreport_dir):
os.makedirs(bugreport_dir)
if not os.path.isdir(bugreport_dir):
logging.error("Cannot create results dir: " + bugreport_dir)
sys.exit(1)
adb_cmd = "adb "
if options.adb_options:
adb_cmd += options.adb_options + " "
# push url list to device
test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\""
proc = subprocess.Popen(test_cmd, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(adb_output, adb_error) = proc.communicate()
if proc.returncode != 0:
logging.error("failed to push url list to device.")
logging.error(adb_output)
logging.error(adb_error)
sys.exit(1)
# clean up previous results
RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE)
RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE)
RemoveDeviceFile(adb_cmd, TEST_LOAD_TIME_FILE)
logging.info("Running the test ...")
# Count crashed tests.
crashed_tests = []
if options.time_out_ms:
timeout_ms = options.time_out_ms
# Run test until it's done
test_cmd_prefix = adb_cmd + " shell am instrument"
test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
# Call ReliabilityTestsAutoTest#startReliabilityTests
test_cmd = (test_cmd_prefix + " -e class "
"com.android.dumprendertree.ReliabilityTest#"
"runReliabilityTest -e timeout %s -e delay %s" %
(str(timeout_ms), str(manual_delay)))
if options.logtime:
test_cmd += " -e logtime true"
test_cmd += test_cmd_postfix
adb_output = subprocess.Popen(test_cmd, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()[0]
while not DumpRenderTreeFinished(adb_cmd):
logging.error("DumpRenderTree exited before all URLs are visited.")
shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
crashed_test = ""
while not crashed_test:
(crashed_test, err) = subprocess.Popen(
shell_cmd_str, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
crashed_test = crashed_test.strip()
if not crashed_test:
logging.error('Cannot get crashed test name, device offline?')
logging.error('stderr: ' + err)
logging.error('retrying in 10s...')
time.sleep(10)
logging.info(crashed_test + " CRASHED")
crashed_tests.append(crashed_test)
Bugreport(crashed_test, bugreport_dir, adb_cmd)
logging.info("Resuming reliability test runner...")
adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()[0]
if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
adb_output.find("Process crashed.") != -1):
logging.error("Error happened : " + adb_output)
sys.exit(1)
logging.info(adb_output)
logging.info("Done\n")
if crashed_tests:
file_handle = open(crashed_file, "w")
file_handle.writelines("\n".join(crashed_tests))
logging.info("Crashed URL list stored in: " + crashed_file)
file_handle.close()
else:
logging.info("No crash found.")
# get timeout file from sdcard
test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \""
+ timedout_file + "\"")
subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
if options.logtime:
# get logged page load times from sdcard
test_cmd = (adb_cmd + "pull \"" + TEST_LOAD_TIME_FILE + "\" \""
+ options.logtime + "\"")
subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
ProcessPageLoadTime(options.logtime)
if "__main__" == __name__:
option_parser = optparse.OptionParser()
option_parser.add_option("-t", "--time-out-ms",
default=60000,
help="set the timeout for each test")
option_parser.add_option("-v", "--verbose", action="store_true",
default=False,
help="include debug-level logging")
option_parser.add_option("-a", "--adb-options",
default=None,
help="pass options to adb, such as -d -e, etc")
option_parser.add_option("-c", "--crash-file",
default="reliability_crashed_sites.txt",
help="the list of sites that cause browser to crash")
option_parser.add_option("-f", "--timeout-file",
default="reliability_timedout_sites.txt",
help="the list of sites that timedout during test")
option_parser.add_option("-d", "--delay",
default=0,
help="add a manual delay between pages (in ms)")
option_parser.add_option("-b", "--bugreport",
default=".",
help="the directory to store bugreport for crashes")
option_parser.add_option("-l", "--logtime",
default=None,
help="Logs page load time for each url to the file")
opts, arguments = option_parser.parse_args()
main(opts, arguments)