#!/usr/bin/python2.4
#
# Copyright 2008, The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Command line utility for running Android tests
runtest helps automate the instructions for building and running tests
- It builds the corresponding test package for the code you want to test
- It pushes the test package to your device or emulator
- It launches InstrumentationTestRunner (or similar) to run the tests you
specify.
runtest supports running tests whose attributes have been pre-defined in
_TEST_FILE_NAME files, (runtest <testname>), or by specifying the file
system path to the test to run (runtest --path <path>).
Do runtest --help to see full list of options.
"""
# Python imports
import glob
import optparse
import os
from sets import Set
import sys
# local imports
import adb_interface
import android_build
import coverage
import errors
import logger
import run_command
from test_defs import test_defs
from test_defs import test_walker
class TestRunner(object):
"""Command line utility class for running pre-defined Android test(s)."""
_TEST_FILE_NAME = "test_defs.xml"
# file path to android core platform tests, relative to android build root
# TODO move these test data files to another directory
_CORE_TEST_PATH = os.path.join("development", "testrunner",
_TEST_FILE_NAME)
# vendor glob file path patterns to tests, relative to android
# build root
_VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo",
_TEST_FILE_NAME)
_RUNTEST_USAGE = (
"usage: runtest.py [options] short-test-name[s]\n\n"
"The runtest script works in two ways. You can query it "
"for a list of tests, or you can launch one or more tests.")
# default value for make -jX
_DEFAULT_JOBS = 4
def __init__(self):
# disable logging of timestamp
self._root_path = android_build.GetTop()
logger.SetTimestampLogging(False)
self._adb = None
self._known_tests = None
self._options = None
self._test_args = None
self._tests_to_run = None
def _ProcessOptions(self):
"""Processes command-line options."""
# TODO error messages on once-only or mutually-exclusive options.
user_test_default = os.path.join(os.environ.get("HOME"), ".android",
self._TEST_FILE_NAME)
parser = optparse.OptionParser(usage=self._RUNTEST_USAGE)
parser.add_option("-l", "--list-tests", dest="only_list_tests",
default=False, action="store_true",
help="To view the list of tests")
parser.add_option("-b", "--skip-build", dest="skip_build", default=False,
action="store_true", help="Skip build - just launch")
parser.add_option("-j", "--jobs", dest="make_jobs",
metavar="X", default=self._DEFAULT_JOBS,
help="Number of make jobs to use when building")
parser.add_option("-n", "--skip_execute", dest="preview", default=False,
action="store_true",
help="Do not execute, just preview commands")
parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False,
action="store_true",
help="Raw mode (for output to other tools)")
parser.add_option("-a", "--suite-assign", dest="suite_assign_mode",
default=False, action="store_true",
help="Suite assignment (for details & usage see "
"InstrumentationTestRunner)")
parser.add_option("-v", "--verbose", dest="verbose", default=False,
action="store_true",
help="Increase verbosity of %s" % sys.argv[0])
parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger",
default=False, action="store_true",
help="Wait for debugger before launching tests")
parser.add_option("-c", "--test-class", dest="test_class",
help="Restrict test to a specific class")
parser.add_option("-m", "--test-method", dest="test_method",
help="Restrict test to a specific method")
parser.add_option("-p", "--test-package", dest="test_package",
help="Restrict test to a specific java package")
parser.add_option("-z", "--size", dest="test_size",
help="Restrict test to a specific test size")
parser.add_option("-u", "--user-tests-file", dest="user_tests_file",
metavar="FILE", default=user_test_default,
help="Alternate source of user test definitions")
parser.add_option("-o", "--coverage", dest="coverage",
default=False, action="store_true",
help="Generate code coverage metrics for test(s)")
parser.add_option("-x", "--path", dest="test_path",
help="Run test(s) at given file system path")
parser.add_option("-t", "--all-tests", dest="all_tests",
default=False, action="store_true",
help="Run all defined tests")
parser.add_option("--continuous", dest="continuous_tests",
default=False, action="store_true",
help="Run all tests defined as part of the continuous "
"test set")
parser.add_option("--timeout", dest="timeout",
default=300, help="Set a timeout limit (in sec) for "
"running native tests on a device (default: 300 secs)")
parser.add_option("--cts", dest="cts_tests",
default=False, action="store_true",
help="Run all tests defined as part of the "
"compatibility test suite")
group = optparse.OptionGroup(
parser, "Targets", "Use these options to direct tests to a specific "
"Android target")
group.add_option("-e", "--emulator", dest="emulator", default=False,
action="store_true", help="use emulator")
group.add_option("-d", "--device", dest="device", default=False,
action="store_true", help="use device")
group.add_option("-s", "--serial", dest="serial",
help="use specific serial")
parser.add_option_group(group)
self._options, self._test_args = parser.parse_args()
if (not self._options.only_list_tests
and not self._options.all_tests
and not self._options.continuous_tests
and not self._options.cts_tests
and not self._options.test_path
and len(self._test_args) < 1):
parser.print_help()
logger.SilentLog("at least one test name must be specified")
raise errors.AbortError
self._adb = adb_interface.AdbInterface()
if self._options.emulator:
self._adb.SetEmulatorTarget()
elif self._options.device:
self._adb.SetDeviceTarget()
elif self._options.serial is not None:
self._adb.SetTargetSerial(self._options.serial)
if self._options.verbose:
logger.SetVerbose(True)
self._known_tests = self._ReadTests()
self._options.host_lib_path = android_build.GetHostLibraryPath()
self._options.test_data_path = android_build.GetTestAppPath()
def _ReadTests(self):
"""Parses the set of test definition data.
Returns:
A TestDefinitions object that contains the set of parsed tests.
Raises:
AbortError: If a fatal error occurred when parsing the tests.
"""
core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH)
try:
known_tests = test_defs.TestDefinitions()
known_tests.Parse(core_test_path)
# read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths
vendor_tests_pattern = os.path.join(self._root_path,
self._VENDOR_TEST_PATH)
test_file_paths = glob.glob(vendor_tests_pattern)
for test_file_path in test_file_paths:
known_tests.Parse(test_file_path)
if os.path.isfile(self._options.user_tests_file):
known_tests.Parse(self._options.user_tests_file)
return known_tests
except errors.ParseError:
raise errors.AbortError
def _DumpTests(self):
"""Prints out set of defined tests."""
print "The following tests are currently defined:\n"
print "%-25s %-40s %s" % ("name", "build path", "description")
print "-" * 80
for test in self._known_tests:
print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(),
test.GetDescription())
print "\nSee %s for more information" % self._TEST_FILE_NAME
def _DoBuild(self):
logger.SilentLog("Building tests...")
target_set = Set()
extra_args_set = Set()
tests = self._GetTestsToRun()
for test_suite in tests:
self._AddBuildTarget(test_suite, target_set, extra_args_set)
if target_set:
if self._options.coverage:
coverage.EnableCoverageBuild()
# hack to build cts dependencies
# TODO: remove this when build dependency support added to runtest or
# cts dependencies are removed
if self._IsCtsTests(tests):
# need to use make since these fail building with ONE_SHOT_MAKEFILE
cmd = ('make -j%s CtsTestStubs android.core.tests.runner' %
self._options.make_jobs)
logger.Log(cmd)
if not self._options.preview:
old_dir = os.getcwd()
os.chdir(self._root_path)
run_command.RunCommand(cmd, return_output=False)
os.chdir(old_dir)
target_build_string = " ".join(list(target_set))
extra_args_string = " ".join(list(extra_args_set))
# mmm cannot be used from python, so perform a similar operation using
# ONE_SHOT_MAKEFILE
cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" files %s' % (
target_build_string, self._options.make_jobs, self._root_path,
extra_args_string)
logger.Log(cmd)
if self._options.preview:
# in preview mode, just display to the user what command would have been
# run
logger.Log("adb sync")
else:
run_command.RunCommand(cmd, return_output=False)
logger.Log("Syncing to device...")
self._adb.Sync()
def _AddBuildTarget(self, test_suite, target_set, extra_args_set):
build_dir = test_suite.GetBuildPath()
if self._AddBuildTargetPath(build_dir, target_set):
extra_args_set.add(test_suite.GetExtraBuildArgs())
for path in test_suite.GetBuildDependencies(self._options):
self._AddBuildTargetPath(path, target_set)
def _AddBuildTargetPath(self, build_dir, target_set):
if build_dir is not None:
build_file_path = os.path.join(build_dir, "Android.mk")
if os.path.isfile(os.path.join(self._root_path, build_file_path)):
target_set.add(build_file_path)
return True
else:
logger.Log("%s has no Android.mk, skipping" % build_dir)
return False
def _GetTestsToRun(self):
"""Get a list of TestSuite objects to run, based on command line args."""
if self._tests_to_run:
return self._tests_to_run
self._tests_to_run = []
if self._options.all_tests:
self._tests_to_run = self._known_tests.GetTests()
elif self._options.continuous_tests:
self._tests_to_run = self._known_tests.GetContinuousTests()
elif self._options.cts_tests:
self._tests_to_run = self._known_tests.GetCtsTests()
elif self._options.test_path:
walker = test_walker.TestWalker()
self._tests_to_run = walker.FindTests(self._options.test_path)
for name in self._test_args:
test = self._known_tests.GetTest(name)
if test is None:
logger.Log("Error: Could not find test %s" % name)
self._DumpTests()
raise errors.AbortError
self._tests_to_run.append(test)
return self._tests_to_run
def _IsCtsTests(self, test_list):
"""Check if any cts tests are included in given list of tests to run."""
for test in test_list:
if test.IsCts():
return True
return False
def RunTests(self):
"""Main entry method - executes the tests according to command line args."""
try:
run_command.SetAbortOnError()
self._ProcessOptions()
if self._options.only_list_tests:
self._DumpTests()
return
if not self._options.skip_build:
self._DoBuild()
for test_suite in self._GetTestsToRun():
test_suite.Run(self._options, self._adb)
except KeyboardInterrupt:
logger.Log("Exiting...")
except errors.AbortError, error:
logger.Log(error.msg)
logger.SilentLog("Exiting due to AbortError...")
except errors.WaitForResponseTimedOutError:
logger.Log("Timed out waiting for response")
def RunTests():
runner = TestRunner()
runner.RunTests()
if __name__ == "__main__":
RunTests()