testrunner/runtest.py

#!/usr/bin/env python
#
# Copyright 2008, The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Command line utility for running Android tests

runtest helps automate the instructions for building and running tests
- It builds the corresponding test package for the code you want to test
- It pushes the test package to your device or emulator
- It launches InstrumentationTestRunner (or similar) to run the tests you
specify.

runtest supports running tests whose attributes have been pre-defined in
_TEST_FILE_NAME files, (runtest <testname>), or by specifying the file
system path to the test to run (runtest --path <path>).

Do runtest --help to see full list of options.
"""

# Python imports
import glob
import optparse
import os
import re
from sets import Set
import sys
import time

# local imports
import adb_interface
import android_build
from coverage import coverage
import errors
import logger
import make_tree
import run_command
from test_defs import test_defs
from test_defs import test_walker


class TestRunner(object):
  """Command line utility class for running pre-defined Android test(s)."""

  _TEST_FILE_NAME = "test_defs.xml"

  # file path to android core platform tests, relative to android build root
  # TODO move these test data files to another directory
  _CORE_TEST_PATH = os.path.join("development", "testrunner",
                                 _TEST_FILE_NAME)

  # vendor glob file path patterns to tests, relative to android
  # build root
  _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo",
                                   _TEST_FILE_NAME)

  _RUNTEST_USAGE = (
      "usage: runtest.py [options] short-test-name[s]\n\n"
      "The runtest script works in two ways.  You can query it "
      "for a list of tests, or you can launch one or more tests.")

  # default value for make -jX
  _DEFAULT_JOBS = 16

  _DALVIK_VERIFIER_PROP = "dalvik.vm.dexopt-flags"
  _DALVIK_VERIFIER_OFF_VALUE = "v=n"
  _DALVIK_VERIFIER_OFF_PROP = "%s = %s" %(_DALVIK_VERIFIER_PROP, _DALVIK_VERIFIER_OFF_VALUE)

  # regular expression to match path to artifacts to install in make output
  _RE_MAKE_INSTALL = re.compile(r'INSTALL-PATH:\s([^\s]+)\s(.*)$')

  def __init__(self):
    # disable logging of timestamp
    self._root_path = android_build.GetTop()
    out_base_name = os.path.basename(android_build.GetOutDir())
    # regular expression to find remote device path from a file path relative
    # to build root
    pattern = r'' + out_base_name + r'\/target\/product\/\w+\/(.+)$'
    self._re_make_install_path = re.compile(pattern)
    logger.SetTimestampLogging(False)
    self._adb = None
    self._known_tests = None
    self._options = None
    self._test_args = None
    self._tests_to_run = None

  def _ProcessOptions(self):
    """Processes command-line options."""
    # TODO error messages on once-only or mutually-exclusive options.
    user_test_default = os.path.join(os.environ.get("HOME"), ".android",
                                     self._TEST_FILE_NAME)

    parser = optparse.OptionParser(usage=self._RUNTEST_USAGE)

    parser.add_option("-l", "--list-tests", dest="only_list_tests",
                      default=False, action="store_true",
                      help="To view the list of tests")
    parser.add_option("-b", "--skip-build", dest="skip_build", default=False,
                      action="store_true", help="Skip build - just launch")
    parser.add_option("-j", "--jobs", dest="make_jobs",
                      metavar="X", default=self._DEFAULT_JOBS,
                      help="Number of make jobs to use when building")
    parser.add_option("-n", "--skip_execute", dest="preview", default=False,
                      action="store_true",
                      help="Do not execute, just preview commands")
    parser.add_option("-i", "--build-install-only", dest="build_install_only", default=False,
                      action="store_true",
                      help="Do not execute, build tests and install to device only")
    parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False,
                      action="store_true",
                      help="Raw mode (for output to other tools)")
    parser.add_option("-a", "--suite-assign", dest="suite_assign_mode",
                      default=False, action="store_true",
                      help="Suite assignment (for details & usage see "
                      "InstrumentationTestRunner)")
    parser.add_option("-v", "--verbose", dest="verbose", default=False,
                      action="store_true",
                      help="Increase verbosity of %s" % sys.argv[0])
    parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger",
                      default=False, action="store_true",
                      help="Wait for debugger before launching tests")
    parser.add_option("-c", "--test-class", dest="test_class",
                      help="Restrict test to a specific class")
    parser.add_option("-m", "--test-method", dest="test_method",
                      help="Restrict test to a specific method")
    parser.add_option("-p", "--test-package", dest="test_package",
                      help="Restrict test to a specific java package")
    parser.add_option("-z", "--size", dest="test_size",
                      help="Restrict test to a specific test size")
    parser.add_option("--annotation", dest="test_annotation",
                      help="Include only those tests tagged with a specific"
                      " annotation")
    parser.add_option("--not-annotation", dest="test_not_annotation",
                      help="Exclude any tests tagged with a specific"
                      " annotation")
    parser.add_option("-u", "--user-tests-file", dest="user_tests_file",
                      metavar="FILE", default=user_test_default,
                      help="Alternate source of user test definitions")
    parser.add_option("-o", "--coverage", dest="coverage",
                      default=False, action="store_true",
                      help="Generate code coverage metrics for test(s)")
    parser.add_option("--coverage-target", dest="coverage_target_path",
                      default=None,
                      help="Path to app to collect code coverage target data for.")
    parser.add_option("-k", "--skip-permissions", dest="skip_permissions",
                      default=False, action="store_true",
                      help="Do not grant runtime permissions during test package"
                      " installation.")
    parser.add_option("-x", "--path", dest="test_path",
                      help="Run test(s) at given file system path")
    parser.add_option("-t", "--all-tests", dest="all_tests",
                      default=False, action="store_true",
                      help="Run all defined tests")
    parser.add_option("--continuous", dest="continuous_tests",
                      default=False, action="store_true",
                      help="Run all tests defined as part of the continuous "
                      "test set")
    parser.add_option("--timeout", dest="timeout",
                      default=300, help="Set a timeout limit (in sec) for "
                      "running native tests on a device (default: 300 secs)")
    parser.add_option("--suite", dest="suite",
                      help="Run all tests defined as part of the "
                      "the given test suite")
    parser.add_option("--user", dest="user",
                      help="The user that test apks are installing to."
                      " This is the integer user id, e.g. 0 or 10."
                      " If no user is specified, apk will be installed with"
                      " adb's default behavior, which is currently all users.")
    parser.add_option("--install-filter", dest="filter_re",
                      help="Regular expression which generated apks have to"
                      " match to be installed to target device. Default is None"
                      " and will install all packages built.  This is"
                      " useful when the test path has a lot of apks but you"
                      " only care about one.")
    group = optparse.OptionGroup(
        parser, "Targets", "Use these options to direct tests to a specific "
        "Android target")
    group.add_option("-e", "--emulator", dest="emulator", default=False,
                     action="store_true", help="use emulator")
    group.add_option("-d", "--device", dest="device", default=False,
                     action="store_true", help="use device")
    group.add_option("-s", "--serial", dest="serial",
                     help="use specific serial")
    parser.add_option_group(group)
    self._options, self._test_args = parser.parse_args()

    if (not self._options.only_list_tests
        and not self._options.all_tests
        and not self._options.continuous_tests
        and not self._options.suite
        and not self._options.test_path
        and len(self._test_args) < 1):
      parser.print_help()
      logger.SilentLog("at least one test name must be specified")
      raise errors.AbortError

    self._adb = adb_interface.AdbInterface()
    if self._options.emulator:
      self._adb.SetEmulatorTarget()
    elif self._options.device:
      self._adb.SetDeviceTarget()
    elif self._options.serial is not None:
      self._adb.SetTargetSerial(self._options.serial)
    if self._options.verbose:
      logger.SetVerbose(True)

    if self._options.coverage_target_path:
      self._options.coverage = True

    self._known_tests = self._ReadTests()

    self._options.host_lib_path = android_build.GetHostLibraryPath()
    self._options.test_data_path = android_build.GetTestAppPath()

  def _ReadTests(self):
    """Parses the set of test definition data.

    Returns:
      A TestDefinitions object that contains the set of parsed tests.
    Raises:
      AbortError: If a fatal error occurred when parsing the tests.
    """
    try:
      known_tests = test_defs.TestDefinitions()
      # only read tests when not in path mode
      if not self._options.test_path:
        core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH)
        if os.path.isfile(core_test_path):
          known_tests.Parse(core_test_path)
        # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths
        vendor_tests_pattern = os.path.join(self._root_path,
                                            self._VENDOR_TEST_PATH)
        test_file_paths = glob.glob(vendor_tests_pattern)
        for test_file_path in test_file_paths:
          known_tests.Parse(test_file_path)
        if os.path.isfile(self._options.user_tests_file):
          known_tests.Parse(self._options.user_tests_file)
      return known_tests
    except errors.ParseError:
      raise errors.AbortError

  def _DumpTests(self):
    """Prints out set of defined tests."""
    print "The following tests are currently defined:\n"
    print "%-25s %-40s %s" % ("name", "build path", "description")
    print "-" * 80
    for test in self._known_tests:
      print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(),
                                test.GetDescription())
    print "\nSee %s for more information" % self._TEST_FILE_NAME

  def _DoBuild(self):
    logger.SilentLog("Building tests...")
    tests = self._GetTestsToRun()

    # Build and install tests that do not get granted permissions
    self._DoPermissionAwareBuild(tests, False)

    # Build and install tests that require granted permissions
    self._DoPermissionAwareBuild(tests, True)

  def _DoPermissionAwareBuild(self, tests, test_requires_permissions):
    # turn off dalvik verifier if necessary
    # TODO: skip turning off verifier for now, since it puts device in bad
    # state b/14088982
    #self._TurnOffVerifier(tests)
    self._DoFullBuild(tests, test_requires_permissions)

    target_tree = make_tree.MakeTree()

    extra_args_set = []
    for test_suite in tests:
      if test_suite.IsGrantedPermissions() == test_requires_permissions:
        self._AddBuildTarget(test_suite, target_tree, extra_args_set)

    if not self._options.preview:
      self._adb.EnableAdbRoot()
    else:
      logger.Log("adb root")

    if not target_tree.IsEmpty():
      if self._options.coverage:
        coverage.EnableCoverageBuild()
        target_tree.AddPath("external/emma")

      target_list = target_tree.GetPrunedMakeList()
      target_dir_list = [re.sub(r'Android[.]mk$', r'', i) for i in target_list]
      target_build_string = " ".join(target_list)
      target_dir_build_string = " ".join(target_dir_list)
      extra_args_string = " ".join(extra_args_set)

      install_path_goals = []
      mmma_goals = []
      for d in target_dir_list:
        if d.startswith("./"):
          d = d[2:]
        if d.endswith("/"):
          d = d[:-1]
        install_path_goals.append("GET-INSTALL-PATH-IN-" + d.replace("/","-"))
        mmma_goals.append("MODULES-IN-" + d.replace("/","-"))
      # mmm cannot be used from python, so perform a similar operation using
      # ONE_SHOT_MAKEFILE
      cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" %s %s %s' % (
          target_build_string, self._options.make_jobs, self._root_path,
          " ".join(install_path_goals), " ".join(mmma_goals), extra_args_string)
      # mmma cannot be used from python, so perform a similar operation
      alt_cmd = 'make -j%s -C "%s" -f build/core/main.mk %s %s' % (
              self._options.make_jobs, self._root_path, extra_args_string, " ".join(mmma_goals))

      logger.Log(cmd)
      if not self._options.preview:
        run_command.SetAbortOnError()
        try:
          output = run_command.RunCommand(cmd, return_output=True, timeout_time=600)
          ## Chances are this failed because it didn't build the dependencies
        except errors.AbortError:
          logger.Log("make failed. Trying to rebuild all dependencies.")
          logger.Log("mmma -j%s %s" %(self._options.make_jobs, target_dir_build_string))
          # Try again with mma equivalent, which will build the dependencies
          run_command.RunCommand(alt_cmd, return_output=False, timeout_time=600)
          # Run mmm again to get the install paths only
          output = run_command.RunCommand(cmd, return_output=True, timeout_time=600)
        run_command.SetAbortOnError(False)
        logger.SilentLog(output)
        filter_re = re.compile(self._options.filter_re) if self._options.filter_re else None

        self._DoInstall(output, test_requires_permissions, filter_re=filter_re)

  def _DoInstall(self, make_output, test_requires_permissions, filter_re=None):
    """Install artifacts from build onto device.

    Looks for 'install:' text from make output to find artifacts to install.

    Files with the .apk extension get 'adb install'ed, all other files
    get 'adb push'ed onto the device.

    Args:
      make_output: stdout from make command
    """
    for line in make_output.split("\n"):
      m = self._RE_MAKE_INSTALL.match(line)
      if m:
        # strip the 'INSTALL: <name>' from the left hand side
        # the remaining string is a space-separated list of build-generated files
        install_paths = m.group(2)
        for install_path in re.split(r'\s+', install_paths):
          if filter_re and not filter_re.match(install_path):
            continue
          if install_path.endswith(".apk"):
            abs_install_path = os.path.join(self._root_path, install_path)
            extra_flags = ""
            if test_requires_permissions and not self._options.skip_permissions:
              extra_flags = "-g"
            if self._options.user:
              extra_flags += " --user " + self._options.user
            logger.Log("adb install -r %s %s" % (extra_flags, abs_install_path))
            logger.Log(self._adb.Install(abs_install_path, extra_flags))
          else:
            self._PushInstallFileToDevice(install_path)

  def _PushInstallFileToDevice(self, install_path):
    m = self._re_make_install_path.match(install_path)
    if m:
      remote_path = m.group(1)
      remote_dir = os.path.dirname(remote_path)
      logger.Log("adb shell mkdir -p %s" % remote_dir)
      self._adb.SendShellCommand("mkdir -p %s" % remote_dir)
      abs_install_path = os.path.join(self._root_path, install_path)
      logger.Log("adb push %s %s" % (abs_install_path, remote_path))
      self._adb.Push(abs_install_path, remote_path)
    else:
      logger.Log("Error: Failed to recognize path of file to install %s" % install_path)

  def _DoFullBuild(self, tests, test_requires_permissions):
    """If necessary, run a full 'make' command for the tests that need it."""
    extra_args_set = Set()

    for test in tests:
      if test.IsFullMake() and test.IsGrantedPermissions() == test_requires_permissions:
        if test.GetExtraBuildArgs():
          # extra args contains the args to pass to 'make'
          extra_args_set.add(test.GetExtraBuildArgs())
        else:
          logger.Log("Warning: test %s needs a full build but does not specify"
                     " extra_build_args" % test.GetName())

    # check if there is actually any tests that required a full build
    if extra_args_set:
      cmd = ('make -j%s %s' % (self._options.make_jobs,
                               ' '.join(list(extra_args_set))))
      logger.Log(cmd)
      if not self._options.preview:
        old_dir = os.getcwd()
        os.chdir(self._root_path)
        output = run_command.RunCommand(cmd, return_output=True)
        logger.SilentLog(output)
        os.chdir(old_dir)
        self._DoInstall(output, test_requires_permissions)

  def _AddBuildTarget(self, test_suite, target_tree, extra_args_set):
    if not test_suite.IsFullMake():
      build_dir = test_suite.GetBuildPath()
      if self._AddBuildTargetPath(build_dir, target_tree):
        extra_args_set.append(test_suite.GetExtraBuildArgs())
      for path in test_suite.GetBuildDependencies(self._options):
        self._AddBuildTargetPath(path, target_tree)

  def _AddBuildTargetPath(self, build_dir, target_tree):
    if build_dir is not None:
      target_tree.AddPath(build_dir)
      return True
    return False

  def _GetTestsToRun(self):
    """Get a list of TestSuite objects to run, based on command line args."""
    if self._tests_to_run:
      return self._tests_to_run

    self._tests_to_run = []
    if self._options.all_tests:
      self._tests_to_run = self._known_tests.GetTests()
    elif self._options.continuous_tests:
      self._tests_to_run = self._known_tests.GetContinuousTests()
    elif self._options.suite:
      self._tests_to_run = \
          self._known_tests.GetTestsInSuite(self._options.suite)
    elif self._options.test_path:
      walker = test_walker.TestWalker()
      self._tests_to_run = walker.FindTests(self._options.test_path)

    for name in self._test_args:
      test = self._known_tests.GetTest(name)
      if test is None:
        logger.Log("Error: Could not find test %s" % name)
        self._DumpTests()
        raise errors.AbortError
      self._tests_to_run.append(test)
    return self._tests_to_run

  def _TurnOffVerifier(self, test_list):
    """Turn off the dalvik verifier if needed by given tests.

    If one or more tests needs dalvik verifier off, and it is not already off,
    turns off verifier and reboots device to allow change to take effect.
    """
    # hack to check if these are frameworks/base tests. If so, turn off verifier
    # to allow framework tests to access private/protected/package-private framework api
    framework_test = False
    for test in test_list:
      if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]):
        framework_test = True
    if framework_test:
      # check if verifier is off already - to avoid the reboot if not
      # necessary
      output = self._adb.SendShellCommand("cat /data/local.prop")
      if not self._DALVIK_VERIFIER_OFF_PROP in output:

        # Read the existing dalvik verifier flags.
        old_prop_value = self._adb.SendShellCommand("getprop %s" \
            %(self._DALVIK_VERIFIER_PROP))
        old_prop_value = old_prop_value.strip() if old_prop_value else ""

        # Append our verifier flags to existing flags
        new_prop_value = "%s %s" %(self._DALVIK_VERIFIER_OFF_VALUE, old_prop_value)

        # Update property now, as /data/local.prop is not read until reboot
        logger.Log("adb shell setprop %s '%s'" \
            %(self._DALVIK_VERIFIER_PROP, new_prop_value))
        if not self._options.preview:
          self._adb.SendShellCommand("setprop %s '%s'" \
            %(self._DALVIK_VERIFIER_PROP, new_prop_value))

        # Write prop to /data/local.prop
        # Every time device is booted, it will pick up this value
        new_prop_assignment = "%s = %s" %(self._DALVIK_VERIFIER_PROP, new_prop_value)
        if self._options.preview:
          logger.Log("adb shell \"echo %s >> /data/local.prop\""
                     % new_prop_assignment)
          logger.Log("adb shell chmod 644 /data/local.prop")
        else:
          logger.Log("Turning off dalvik verifier and rebooting")
          self._adb.SendShellCommand("\"echo %s >> /data/local.prop\""
                                     % new_prop_assignment)

        # Reset runtime so that dalvik picks up new verifier flags from prop
        self._ChmodRuntimeReset()
      elif not self._options.preview:
        # check the permissions on the file
        permout = self._adb.SendShellCommand("ls -l /data/local.prop")
        if not "-rw-r--r--" in permout:
          logger.Log("Fixing permissions on /data/local.prop and rebooting")
          self._ChmodRuntimeReset()

  def _ChmodRuntimeReset(self):
    """Perform a chmod of /data/local.prop and reset the runtime.
    """
    logger.Log("adb shell chmod 644 /data/local.prop ## u+w,a+r")
    if not self._options.preview:
      self._adb.SendShellCommand("chmod 644 /data/local.prop")

    self._adb.RuntimeReset(preview_only=self._options.preview)

    if not self._options.preview:
      self._adb.EnableAdbRoot()


  def RunTests(self):
    """Main entry method - executes the tests according to command line args."""
    try:
      run_command.SetAbortOnError()
      self._ProcessOptions()
      if self._options.only_list_tests:
        self._DumpTests()
        return

      if not self._options.skip_build:
        self._DoBuild()

      if self._options.build_install_only:
        logger.Log("Skipping test execution (due to --build-install-only flag)")
        return

      for test_suite in self._GetTestsToRun():
        try:
          test_suite.Run(self._options, self._adb)
        except errors.WaitForResponseTimedOutError:
          logger.Log("Timed out waiting for response")

    except KeyboardInterrupt:
      logger.Log("Exiting...")
    except errors.AbortError, error:
      logger.Log(error.msg)
      logger.SilentLog("Exiting due to AbortError...")
    except errors.WaitForResponseTimedOutError:
      logger.Log("Timed out waiting for response")


def RunTests():
  runner = TestRunner()
  runner.RunTests()

if __name__ == "__main__":
  RunTests()