#!/usr/bin/python # Copyright (C) 2009 The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Module for generating CTS test descriptions and test plans.""" import glob import os import re import shutil import subprocess import sys import xml.dom.minidom as dom from cts import tools from multiprocessing import Pool def GetSubDirectories(root): """Return all directories under the given root directory.""" return [x for x in os.listdir(root) if os.path.isdir(os.path.join(root, x))] def ReadFileLines(filePath): """Reads a file and returns its contents as a line list.""" f = open(filePath, 'r'); lines = [line.strip() for line in f.readlines()] f.close() return lines def GetMakeFileVars(makefile_path): """Extracts variable definitions from the given make file. Args: makefile_path: Path to the make file. Returns: A dictionary mapping variable names to their assigned value. """ result = {} pattern = re.compile(r'^\s*([^:#=\s]+)\s*:=\s*(.*?[^\\])$', re.MULTILINE + re.DOTALL) stream = open(makefile_path, 'r') content = stream.read() for match in pattern.finditer(content): result[match.group(1)] = match.group(2) stream.close() return result class CtsBuilder(object): """Main class for generating test descriptions and test plans.""" def __init__(self, argv): """Initialize the CtsBuilder from command line arguments.""" if len(argv) != 6: print 'Usage: %s <testRoot> <ctsOutputDir> <tempDir> <androidRootDir> <docletPath>' % argv[0] print '' print 'testRoot: Directory under which to search for CTS tests.' print 'ctsOutputDir: Directory in which the CTS repository should be created.' print 'tempDir: Directory to use for storing temporary files.' print 'androidRootDir: Root directory of the Android source tree.' print 'docletPath: Class path where the DescriptionGenerator doclet can be found.' sys.exit(1) self.test_root = sys.argv[1] self.out_dir = sys.argv[2] self.temp_dir = sys.argv[3] self.android_root = sys.argv[4] self.doclet_path = sys.argv[5] self.test_repository = os.path.join(self.out_dir, 'repository/testcases') self.plan_repository = os.path.join(self.out_dir, 'repository/plans') self.definedplans_repository = os.path.join(self.android_root, 'cts/tests/plans') def GenerateTestDescriptions(self): """Generate test descriptions for all packages.""" pool = Pool(processes=2) # generate test descriptions for android tests results = [] pool.close() pool.join() return sum(map(lambda result: result.get(), results)) def __WritePlan(self, plan, plan_name): print 'Generating test plan %s' % plan_name plan.Write(os.path.join(self.plan_repository, plan_name + '.xml')) def GenerateTestPlans(self): """Generate default test plans.""" # TODO: Instead of hard-coding the plans here, use a configuration file, # such as test_defs.xml packages = [] descriptions = sorted(glob.glob(os.path.join(self.test_repository, '*.xml'))) for description in descriptions: doc = tools.XmlFile(description) packages.append(doc.GetAttr('TestPackage', 'appPackageName')) # sort the list to give the same sequence based on name packages.sort() plan = tools.TestPlan(packages) plan.Exclude('android\.performance.*') self.__WritePlan(plan, 'CTS') self.__WritePlan(plan, 'CTS-TF') plan = tools.TestPlan(packages) plan.Exclude('android\.performance.*') plan.Exclude('android\.media\.cts\.StreamingMediaPlayerTest.*') # Test plan to not include media streaming tests self.__WritePlan(plan, 'CTS-No-Media-Stream') plan = tools.TestPlan(packages) plan.Exclude('android\.performance.*') self.__WritePlan(plan, 'SDK') plan.Exclude(r'android\.signature') plan.Exclude(r'android\.core.*') self.__WritePlan(plan, 'Android') plan = tools.TestPlan(packages) plan.Include(r'android\.core\.tests.*') plan.Exclude(r'android\.core\.tests\.libcore.\package.\harmony*') self.__WritePlan(plan, 'Java') # TODO: remove this once the tests are fixed and merged into Java plan above. plan = tools.TestPlan(packages) plan.Include(r'android\.core\.tests\.libcore.\package.\harmony*') self.__WritePlan(plan, 'Harmony') plan = tools.TestPlan(packages) plan.Include(r'android\.core\.vm-tests-tf') self.__WritePlan(plan, 'VM-TF') plan = tools.TestPlan(packages) plan.Include(r'android\.tests\.appsecurity') self.__WritePlan(plan, 'AppSecurity') # hard-coded white list for PDK plan plan.Exclude('.*') plan.Include('android\.aadb') plan.Include('android\.bluetooth') plan.Include('android\.graphics.*') plan.Include('android\.hardware') plan.Include('android\.media') plan.Exclude('android\.mediastress') plan.Include('android\.net') plan.Include('android\.opengl.*') plan.Include('android\.renderscript') plan.Include('android\.telephony') plan.Include('android\.nativemedia.*') plan.Include('com\.android\.cts\..*')#TODO(stuartscott): Should PDK have all these? self.__WritePlan(plan, 'PDK') temporarily_known_failure_tests = BuildCtsTemporarilyKnownFailureList(); flaky_tests = BuildCtsFlakyTestList() releasekey_tests = BuildListForReleaseBuildTest() # CTS Stable plan plan = tools.TestPlan(packages) plan.Exclude(r'com\.android\.cts\.browserbench') for package, test_list in flaky_tests.iteritems(): plan.ExcludeTests(package, test_list) for package, test_list in releasekey_tests.iteritems(): plan.ExcludeTests(package, test_list) self.__WritePlan(plan, 'CTS-stable') # CTS Flaky plan - list of tests known to be flaky in lab environment plan = tools.TestPlan(packages) plan.Exclude('.*') plan.Include(r'com\.android\.cts\.browserbench') for package, test_list in flaky_tests.iteritems(): plan.Include(package+'$') plan.IncludeTests(package, test_list) self.__WritePlan(plan, 'CTS-flaky') small_tests = BuildAospSmallSizeTestList() medium_tests = BuildAospMediumSizeTestList() new_test_packages = BuildCtsVettedNewPackagesList() # CTS - sub plan for public, small size tests plan = tools.TestPlan(packages) plan.Exclude('.*') for package, test_list in small_tests.iteritems(): plan.Include(package+'$') plan.Exclude(r'com\.android\.cts\.browserbench') for package, test_list in flaky_tests.iteritems(): plan.ExcludeTests(package, test_list) for package, test_list in releasekey_tests.iteritems(): plan.ExcludeTests(package, test_list) self.__WritePlan(plan, 'CTS-kitkat-small') self.__WritePlan(plan, 'CTS-public-small') # CTS - sub plan for public, medium size tests plan = tools.TestPlan(packages) plan.Exclude('.*') for package, test_list in medium_tests.iteritems(): plan.Include(package+'$') plan.Exclude(r'com\.android\.cts\.browserbench') for package, test_list in flaky_tests.iteritems(): plan.ExcludeTests(package, test_list) for package, test_list in releasekey_tests.iteritems(): plan.ExcludeTests(package, test_list) self.__WritePlan(plan, 'CTS-kitkat-medium') self.__WritePlan(plan, 'CTS-public-medium') # CTS - sub plan for hardware tests which is public, large plan = tools.TestPlan(packages) plan.Exclude('.*') plan.Include(r'android\.hardware$') plan.Exclude(r'com\.android\.cts\.browserbench') for package, test_list in flaky_tests.iteritems(): plan.ExcludeTests(package, test_list) for package, test_list in releasekey_tests.iteritems(): plan.ExcludeTests(package, test_list) self.__WritePlan(plan, 'CTS-hardware') # CTS - sub plan for media tests which is public, large plan = tools.TestPlan(packages) plan.Exclude('.*') plan.Include(r'android\.media$') plan.Include(r'android\.view$') plan.Exclude(r'com\.android\.cts\.browserbench') for package, test_list in flaky_tests.iteritems(): plan.ExcludeTests(package, test_list) for package, test_list in releasekey_tests.iteritems(): plan.ExcludeTests(package, test_list) self.__WritePlan(plan, 'CTS-media') # CTS - sub plan for mediastress tests which is public, large plan = tools.TestPlan(packages) plan.Exclude('.*') plan.Include(r'android\.mediastress$') plan.Exclude(r'com\.android\.cts\.browserbench') for package, test_list in flaky_tests.iteritems(): plan.ExcludeTests(package, test_list) for package, test_list in releasekey_tests.iteritems(): plan.ExcludeTests(package, test_list) self.__WritePlan(plan, 'CTS-mediastress') # CTS - sub plan for new tests that is vetted for L launch plan = tools.TestPlan(packages) plan.Exclude('.*') for package, test_list in new_test_packages.iteritems(): plan.Include(package+'$') plan.Exclude(r'com\.android\.cts\.browserbench') for package, test_list in flaky_tests.iteritems(): plan.ExcludeTests(package, test_list) for package, test_list in releasekey_tests.iteritems(): plan.ExcludeTests(package, test_list) self.__WritePlan(plan, 'CTS-l-tests') # CTS - sub plan for tests in drawelement packages plan = tools.TestPlan(packages) plan.Exclude('.*') plan.Include(r'com\.drawelements\.') self.__WritePlan(plan, 'CTS-DEQP') # CTS - sub plan for new test packages added for staging plan = tools.TestPlan(packages) for package, test_list in small_tests.iteritems(): plan.Exclude(package+'$') for package, test_list in medium_tests.iteritems(): plan.Exclude(package+'$') for package, tests_list in new_test_packages.iteritems(): plan.Exclude(package+'$') plan.Exclude(r'com\.drawelements\.') plan.Exclude(r'android\.hardware$') plan.Exclude(r'android\.media$') plan.Exclude(r'android\.view$') plan.Exclude(r'android\.mediastress$') plan.Exclude(r'com\.android\.cts\.browserbench') for package, test_list in flaky_tests.iteritems(): plan.ExcludeTests(package, test_list) for package, test_list in releasekey_tests.iteritems(): plan.ExcludeTests(package, test_list) self.__WritePlan(plan, 'CTS-m-tests') # CTS - sub plan for new test packages added for staging plan = tools.TestPlan(packages) plan.Exclude('.*') for package, test_list in temporarily_known_failure_tests.iteritems(): plan.Include(package+'$') plan.IncludeTests(package, test_list) self.__WritePlan(plan, 'CTS-staging') plan = tools.TestPlan(packages) plan.Exclude('.*') self.__WritePlan(plan, 'CTS-webview') def BuildAospMediumSizeTestList(): """ Construct a defaultdic that lists package names of medium tests already published to aosp. """ return { 'android.app' : [], 'android.core.tests.libcore.package.libcore' : [], 'android.core.tests.libcore.package.org' : [], 'android.core.vm-tests-tf' : [], 'android.dpi' : [], 'android.host.security' : [], 'android.net' : [], 'android.os' : [], 'android.permission2' : [], 'android.security' : [], 'android.telephony' : [], 'android.webkit' : [], 'android.widget' : [], 'com.android.cts.browserbench' : []} def BuildAospSmallSizeTestList(): """ Construct a default dict that lists packages names of small tests already published to aosp. """ return { 'android.aadb' : [], 'android.acceleration' : [], 'android.accessibility' : [], 'android.accessibilityservice' : [], 'android.accounts' : [], 'android.admin' : [], 'android.animation' : [], 'android.bionic' : [], 'android.bluetooth' : [], 'android.calendarcommon' : [], 'android.content' : [], 'android.core.tests.libcore.package.com' : [], 'android.core.tests.libcore.package.conscrypt' : [], 'android.core.tests.libcore.package.dalvik' : [], 'android.core.tests.libcore.package.sun' : [], 'android.core.tests.libcore.package.tests' : [], 'android.database' : [], 'android.dreams' : [], 'android.drm' : [], 'android.effect' : [], 'android.gesture' : [], 'android.graphics' : [], 'android.graphics2' : [], 'android.jni' : [], 'android.keystore' : [], 'android.location' : [], 'android.nativemedia.sl' : [], 'android.nativemedia.xa' : [], 'android.nativeopengl' : [], 'android.ndef' : [], 'android.opengl' : [], 'android.openglperf' : [], 'android.permission' : [], 'android.preference' : [], 'android.preference2' : [], 'android.provider' : [], 'android.renderscript' : [], 'android.rscpp' : [], 'android.rsg' : [], 'android.sax' : [], 'android.signature' : [], 'android.speech' : [], 'android.tests.appsecurity' : [], 'android.text' : [], 'android.textureview' : [], 'android.theme' : [], 'android.usb' : [], 'android.util' : [], 'com.android.cts.dram' : [], 'com.android.cts.filesystemperf' : [], 'com.android.cts.jank' : [], 'com.android.cts.jank2' : [], 'com.android.cts.opengl' : [], 'com.android.cts.simplecpu' : [], 'com.android.cts.ui' : [], 'com.android.cts.uihost' : [], 'com.android.cts.videoperf' : [], 'zzz.android.monkey' : []} def BuildCtsVettedNewPackagesList(): """ Construct a defaultdict that maps package names that is vetted for L. """ return { 'android.JobScheduler' : [], 'android.core.tests.libcore.package.harmony_annotation' : [], 'android.core.tests.libcore.package.harmony_beans' : [], 'android.core.tests.libcore.package.harmony_java_io' : [], 'android.core.tests.libcore.package.harmony_java_lang' : [], 'android.core.tests.libcore.package.harmony_java_math' : [], 'android.core.tests.libcore.package.harmony_java_net' : [], 'android.core.tests.libcore.package.harmony_java_nio' : [], 'android.core.tests.libcore.package.harmony_java_util' : [], 'android.core.tests.libcore.package.harmony_java_text' : [], 'android.core.tests.libcore.package.harmony_javax_security' : [], 'android.core.tests.libcore.package.harmony_logging' : [], 'android.core.tests.libcore.package.harmony_prefs' : [], 'android.core.tests.libcore.package.harmony_sql' : [], 'android.core.tests.libcore.package.jsr166' : [], 'android.core.tests.libcore.package.okhttp' : [], 'android.display' : [], 'android.host.theme' : [], 'android.jdwp' : [], 'android.location2' : [], 'android.print' : [], 'android.renderscriptlegacy' : [], 'android.signature' : [], 'android.tv' : [], 'android.uiautomation' : [], 'android.uirendering' : []} def BuildListForReleaseBuildTest(): """ Construct a defaultdict that maps package name to a list of tests that are expected to pass only when running against a user/release-key build. """ return { 'android.app' : [ 'android.app.cts.ActivityManagerTest#testIsRunningInTestHarness',], 'android.dpi' : [ 'android.dpi.cts.DefaultManifestAttributesSdkTest#testPackageHasExpectedSdkVersion',], 'android.host.security' : [ 'android.cts.security.SELinuxHostTest#testAllEnforcing', 'android.cts.security.SELinuxHostTest#testSuDomain',], 'android.os' : [ 'android.os.cts.BuildVersionTest#testReleaseVersion', 'android.os.cts.BuildTest#testIsSecureUserBuild',], 'android.security' : [ 'android.security.cts.BannedFilesTest#testNoSu', 'android.security.cts.BannedFilesTest#testNoSuInPath', 'android.security.cts.PackageSignatureTest#testPackageSignatures', 'android.security.cts.SELinuxDomainTest#testSuDomain',], '' : []} def BuildCtsFlakyTestList(): """ Construct a defaultdict that maps package name to a list of tests that flaky during dev cycle and cause other subsequent tests to fail. """ return { 'android.hardware' : [ 'android.hardware.cts.CameraTest#testVideoSnapshot', 'android.hardware.cts.CameraGLTest#testCameraToSurfaceTextureMetadata', 'android.hardware.cts.CameraGLTest#testSetPreviewTextureBothCallbacks', 'android.hardware.cts.CameraGLTest#testSetPreviewTexturePreviewCallback',], 'android.media' : [ 'android.media.cts.DecoderTest#testCodecResetsH264WithSurface', 'android.media.cts.StreamingMediaPlayerTest#testHLS',], 'android.net' : [ 'android.net.cts.ConnectivityManagerTest#testStartUsingNetworkFeature_enableHipri', 'android.net.cts.DnsTest#testDnsWorks', 'android.net.cts.SSLCertificateSocketFactoryTest#testCreateSocket', 'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_bind', 'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_simple', 'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_wrapping', 'android.net.cts.TrafficStatsTest#testTrafficStatsForLocalhost', 'android.net.wifi.cts.NsdManagerTest#testAndroidTestCaseSetupProperly',], 'android.security' : [ 'android.security.cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdp6Ports', 'android.security.cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdpPorts',], 'android.webkit' : [ 'android.webkit.cts.WebViewClientTest#testOnUnhandledKeyEvent',], 'com.android.cts.filesystemperf' : [ 'com.android.cts.filesystemperf.RandomRWTest#testRandomRead', 'com.android.cts.filesystemperf.RandomRWTest#testRandomUpdate',], '' : []} def BuildCtsTemporarilyKnownFailureList(): """ Construct a defaultdict that maps package name to a list of tests that are known failures during dev cycle but expected to be fixed before launch """ return { 'android.alarmclock' : [ 'android.alarmclock.cts.DismissAlarmTest#testAll', 'android.alarmclock.cts.SetAlarmTest#testAll', 'android.alarmclock.cts.SnoozeAlarmTest#testAll', ], 'android.assist' : [ 'android.assist.cts.AssistantContentViewTest', 'android.assist.cts.ExtraAssistDataTest', 'android.assist.cts.FocusChangeTest', 'android.assist.cts.LargeViewHierarchyTest', 'android.assist.cts.ScreenshotTest', 'android.assist.cts.TextViewTest', 'android.assist.cts.WebViewTest', ], 'android.calllog' : [ 'android.calllog.cts.CallLogBackupTest#testSingleCallBackup', ], 'android.dumpsys' : [ 'android.dumpsys.cts.DumpsysHostTest#testBatterystatsOutput', 'android.dumpsys.cts.DumpsysHostTest#testGfxinfoFramestats', ], 'android.telecom' : [ 'android.telecom.cts.ExtendedInCallServiceTest#testAddNewOutgoingCallAndThenDisconnect', 'android.telecom.cts.RemoteConferenceTest#testRemoteConferenceCallbacks_ConferenceableConnections', ], 'android.transition' : [ 'android.transition.cts.ChangeScrollTest#testChangeScroll', ], 'android.voicesettings' : [ 'android.voicesettings.cts.ZenModeTest#testAll', ], 'com.android.cts.systemui' : [ 'com.android.cts.systemui.LightStatusBarTests#testLightStatusBarIcons', ], 'com.android.cts.app.os' : [ 'com.android.cts.app.os.OsHostTests#testNonExportedActivities', ], 'com.android.cts.devicepolicy' : [ 'com.android.cts.devicepolicy.MixedDeviceOwnerTest#testPackageInstallUserRestrictions', 'com.android.cts.devicepolicy.MixedProfileOwnerTest#testPackageInstallUserRestrictions', ], '' : []} def LogGenerateDescription(name): print 'Generating test description for package %s' % name if __name__ == '__main__': builder = CtsBuilder(sys.argv) result = builder.GenerateTestDescriptions() if result != 0: sys.exit(result) builder.GenerateTestPlans()