#!/usr/bin/python
"""Run layout tests using Android emulator and instrumentation.
First, you need to get an SD card or sdcard image that has layout tests on it.
Layout tests are in following directory:
/sdcard/webkit/layout_tests
For example, /sdcard/webkit/layout_tests/fast
Usage:
Run all tests under fast/ directory:
run_layout_tests.py, or
run_layout_tests.py fast
Run all tests under a sub directory:
run_layout_tests.py fast/dom
Run a single test:
run_layout_tests.py fast/dom/
After a merge, if there are changes of layout tests in SD card, you need to
use --refresh-test-list option *once* to re-generate test list on the card.
Some other options are:
--rebaseline generates expected layout tests results under /sdcard/webkit/expected_result/
--time-out-ms (default is 8000 millis) for each test
--adb-options="-e" passes option string to adb
--results-directory=..., (default is ./layout-test-results) directory name under which results are stored.
--js-engine the JavaScript engine currently in use, determines which set of Android-specific expected results we should use, should be 'jsc' or 'v8'
"""
import logging
import optparse
import os
import subprocess
import sys
import time
def CountLineNumber(filename):
"""Compute the number of lines in a given file.
Args:
filename: a file name related to the current directory.
"""
fp = open(os.path.abspath(filename), "r");
lines = 0
for line in fp.readlines():
lines = lines + 1
fp.close()
return lines
def DumpRenderTreeFinished(adb_cmd):
""" Check if DumpRenderTree finished running tests
Args:
output: adb_cmd string
"""
# pull /sdcard/webkit/running_test.txt, if the content is "#DONE", it's done
shell_cmd_str = adb_cmd + " shell cat /sdcard/webkit/running_test.txt"
adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
return adb_output.strip() == "#DONE"
def DiffResults(marker, new_results, old_results, diff_results, strip_reason,
new_count_first=True):
""" Given two result files, generate diff and
write to diff_results file. All arguments are absolute paths
to files.
"""
old_file = open(old_results, "r")
new_file = open(new_results, "r")
diff_file = open(diff_results, "a")
# Read lines from each file
ndict = new_file.readlines()
cdict = old_file.readlines()
# Write marker to diff file
diff_file.writelines(marker + "\n")
diff_file.writelines("###############\n")
# Strip reason from result lines
if strip_reason is True:
for i in range(0, len(ndict)):
ndict[i] = ndict[i].split(' ')[0] + "\n"
for i in range(0, len(cdict)):
cdict[i] = cdict[i].split(' ')[0] + "\n"
params = {
"new": [0, ndict, cdict, "+"],
"miss": [0, cdict, ndict, "-"]
}
if new_count_first:
order = ["new", "miss"]
else:
order = ["miss", "new"]
for key in order:
for line in params[key][1]:
if line not in params[key][2]:
if line[-1] != "\n":
line += "\n";
diff_file.writelines(params[key][3] + line)
params[key][0] += 1
logging.info(marker + " >>> " + str(params["new"][0]) + " new, " +
str(params["miss"][0]) + " misses")
diff_file.writelines("\n\n")
old_file.close()
new_file.close()
diff_file.close()
return
def CompareResults(ref_dir, results_dir):
"""Compare results in two directories
Args:
ref_dir: the reference directory having layout results as references
results_dir: the results directory
"""
logging.info("Comparing results to " + ref_dir)
diff_result = os.path.join(results_dir, "layout_tests_diff.txt")
if os.path.exists(diff_result):
os.remove(diff_result)
files=["crashed", "failed", "passed", "nontext"]
for f in files:
result_file_name = "layout_tests_" + f + ".txt"
DiffResults(f, os.path.join(results_dir, result_file_name),
os.path.join(ref_dir, result_file_name), diff_result,
False, f != "passed")
logging.info("Detailed diffs are in " + diff_result)
def main(options, args):
"""Run the tests. Will call sys.exit when complete.
Args:
options: a dictionary of command line options
args: a list of sub directories or files to test
"""
# Set up logging format.
log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(level=log_level,
format='%(message)s')
# Include all tests if none are specified.
if not args:
path = '/';
else:
path = ' '.join(args);
adb_cmd = "adb ";
if options.adb_options:
adb_cmd += options.adb_options
# Re-generate the test list if --refresh-test-list is on
if options.refresh_test_list:
logging.info("Generating test list.");
generate_test_list_cmd_str = adb_cmd + " shell am instrument -e class com.android.dumprendertree.LayoutTestsAutoTest#generateTestList -e path \"" + path + "\" -w com.android.dumprendertree/.LayoutTestsAutoRunner"
adb_output = subprocess.Popen(generate_test_list_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
if adb_output.find('Process crashed') != -1:
logging.info("Aborting because cannot generate test list.\n" + adb_output)
sys.exit(1)
logging.info("Running tests")
# Count crashed tests.
crashed_tests = []
timeout_ms = '15000'
if options.time_out_ms:
timeout_ms = options.time_out_ms
# Run test until it's done
run_layout_test_cmd_prefix = adb_cmd + " shell am instrument"
run_layout_test_cmd_postfix = " -e path \"" + path + "\" -e timeout " + timeout_ms
if options.rebaseline:
run_layout_test_cmd_postfix += " -e rebaseline true"
# If the JS engine is not specified on the command line, try reading the
# JS_ENGINE environment variable, which is used by the build system in
# external/webkit/Android.mk.
js_engine = options.js_engine
if not js_engine and os.environ.has_key('JS_ENGINE'):
js_engine = os.environ['JS_ENGINE']
if js_engine:
run_layout_test_cmd_postfix += " -e jsengine " + js_engine
run_layout_test_cmd_postfix += " -w com.android.dumprendertree/.LayoutTestsAutoRunner"
# Call LayoutTestsAutoTest::startLayoutTests.
run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#startLayoutTests" + run_layout_test_cmd_postfix
adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
while not DumpRenderTreeFinished(adb_cmd):
# Get the running_test.txt
logging.error("DumpRenderTree crashed, output:\n" + adb_output)
shell_cmd_str = adb_cmd + " shell cat /sdcard/webkit/running_test.txt"
crashed_test = ""
while not crashed_test:
(crashed_test, err) = subprocess.Popen(
shell_cmd_str, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
crashed_test = crashed_test.strip()
if not crashed_test:
logging.error('Cannot get crashed test name, device offline?')
logging.error('stderr: ' + err)
logging.error('retrying in 10s...')
time.sleep(10)
logging.info(crashed_test + " CRASHED");
crashed_tests.append(crashed_test);
logging.info("Resuming layout test runner...");
# Call LayoutTestsAutoTest::resumeLayoutTests
run_layout_test_cmd = run_layout_test_cmd_prefix + " -e class com.android.dumprendertree.LayoutTestsAutoTest#resumeLayoutTests" + run_layout_test_cmd_postfix
adb_output = subprocess.Popen(run_layout_test_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
if adb_output.find('INSTRUMENTATION_FAILED') != -1:
logging.error("Error happened : " + adb_output)
sys.exit(1)
logging.debug(adb_output);
logging.info("Done\n");
# Pull results from /sdcard
results_dir = options.results_directory
if not os.path.exists(results_dir):
os.makedirs(results_dir)
if not os.path.isdir(results_dir):
logging.error("Cannot create results dir: " + results_dir);
sys.exit(1);
result_files = ["/sdcard/layout_tests_passed.txt",
"/sdcard/layout_tests_failed.txt",
"/sdcard/layout_tests_ignored.txt",
"/sdcard/layout_tests_nontext.txt"]
for file in result_files:
shell_cmd_str = adb_cmd + " pull " + file + " " + results_dir
adb_output = subprocess.Popen(shell_cmd_str, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
logging.debug(adb_output)
# Create the crash list.
fp = open(results_dir + "/layout_tests_crashed.txt", "w");
for crashed_test in crashed_tests:
fp.writelines(crashed_test + '\n')
fp.close()
# Count the number of tests in each category.
passed_tests = CountLineNumber(results_dir + "/layout_tests_passed.txt")
logging.info(str(passed_tests) + " passed")
failed_tests = CountLineNumber(results_dir + "/layout_tests_failed.txt")
logging.info(str(failed_tests) + " failed")
ignored_tests = CountLineNumber(results_dir + "/layout_tests_ignored.txt")
logging.info(str(ignored_tests) + " ignored results")
crashed_tests = CountLineNumber(results_dir + "/layout_tests_crashed.txt")
logging.info(str(crashed_tests) + " crashed")
nontext_tests = CountLineNumber(results_dir + "/layout_tests_nontext.txt")
logging.info(str(nontext_tests) + " no dumpAsText")
logging.info(str(passed_tests + failed_tests + ignored_tests + crashed_tests + nontext_tests) + " TOTAL")
logging.info("Results are stored under: " + results_dir + "\n")
# Comparing results to references to find new fixes and regressions.
results_dir = os.path.abspath(options.results_directory)
ref_dir = options.ref_directory
# if ref_dir is null, cannonify ref_dir to the script dir.
if not ref_dir:
script_self = sys.argv[0]
script_dir = os.path.dirname(script_self)
ref_dir = os.path.join(script_dir, "results")
ref_dir = os.path.abspath(ref_dir)
CompareResults(ref_dir, results_dir)
if '__main__' == __name__:
option_parser = optparse.OptionParser()
option_parser.add_option("", "--rebaseline", action="store_true",
default=False,
help="generate expected results for those tests not having one")
option_parser.add_option("", "--time-out-ms",
default=None,
help="set the timeout for each test")
option_parser.add_option("", "--verbose", action="store_true",
default=False,
help="include debug-level logging")
option_parser.add_option("", "--refresh-test-list", action="store_true",
default=False,
help="re-generate test list, it may take some time.")
option_parser.add_option("", "--adb-options",
default=None,
help="pass options to adb, such as -d -e, etc");
option_parser.add_option("", "--results-directory",
default="layout-test-results",
help="directory which results are stored.")
option_parser.add_option("", "--ref-directory",
default=None,
dest="ref_directory",
help="directory where reference results are stored.")
option_parser.add_option("", "--js-engine",
default=None,
help="The JavaScript engine currently in use, which determines which set of Android-specific expected results we should use. Should be 'jsc' or 'v8'.");
options, args = option_parser.parse_args();
main(options, args)