dx/tests/run-all-tests

#!/bin/bash
#
# Copyright (C) 2007 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Set up prog to be the path of this script, including following symlinks,
# and set up progdir to be the fully-qualified pathname of its directory.
prog="$0"
while [ -h "${prog}" ]; do
    newProg=`/bin/ls -ld "${prog}"`
    newProg=`expr "${newProg}" : ".* -> \(.*\)$"`
    if expr "x${newProg}" : 'x/' >/dev/null; then
        prog="${newProg}"
    else
        progdir=`dirname "${prog}"`
        prog="${progdir}/${newProg}"
    fi
done
oldwd=`pwd`
progdir=`dirname "${prog}"`
cd "${progdir}"
progdir=`pwd`
prog="${progdir}"/`basename "${prog}"`

skip_tests=""

# Command-line options
sequential="no"
usage="no"
while [[ "$1" == "-"* ]]; do
  case $1 in
    --seq) sequential="yes" ;;
    --skip) skip_tests="$2 $skip_tests"
            shift ;;
    *) usage="yes" ;;
  esac
  shift
done

if [ $usage = "yes" ]; then
    prog=`basename $prog`
    cat 1>&2 <<END_USAGE
Usage:
  $prog [options]   Run all tests with given options.
Options:
  --seq             Run tests sequentially (default: parallel)
  --skip <test>     Skip running specified test
END_USAGE
    exit 1
fi

# Globals for tracking numbers of successes and failures and their names.
passed=()
surprised=()
ignored=()
failed=()
skipped=()

# Tests failing and require attention (e.g. 115-merge)
known_bad="100-local-mismatch 115-merge 119-merge-conflict"

function display_results {
  printf    "\n\nTest Results\n"
  printf -- "----------------------------\n"
  printf    "Pass:                   % 4d\n" ${#passed[@]}
  printf    "Surprise pass:          % 4d\n" ${#surprised[@]}
  printf    "Known failures:         % 4d\n" ${#ignored[@]}
  printf    "Failures:               % 4d\n" ${#failed[@]}
  printf    "Skipped:                % 4d\n" ${#skipped[@]}
  printf -- "----------------------------\n"
  printf    "Elapsed time(s):        % 4d\n" $SECONDS

  list_files "Unexpected successes" ${surprised[@]}
  list_files "Known failures" ${ignored[@]}
  list_files "Failures" ${failed[@]}
  list_files "Skipped" ${skipped[@]}

  needing_attention=$(( ${#failed[@]} + ${#surprised[@]} ))
  exit ${needing_attention}
}

function list_files {
  # Arguments: Title test_name0 test_name1 ... test_nameN
  echo "$1:"
  shift
  if [[ "$1" = "" ]]; then
    echo "  NONE"
    return
  fi
  while [[ "$1" != "" ]]; do
    echo "  $1"
    shift
  done
}

function update_result {
  local -r test_name=$1
  local -r output=$2
  local -r result=$3
  local expectFail

  if [[ "$known_bad" == *"$test_name"* ]]; then
    expectFail=1
  else
    expectFail=0
  fi
  if [ $result = 0 ]; then
    if [[ $expectFail = 0 ]]; then
      passed+=(${test_name})
    else
      echo "Failing on unexpected success of $test_name"
      surprised+=(${test_name})
    fi
  else
    if [[ $expectFail = 0 ]]; then
      failed+=(${test_name})
    else
      echo "Ignoring expected failure of $test_name"
      ignored+=(${test_name})
      # Clean up when we expect a test to fail.
      # run-test only does this on success.
      rm -rf "$output"
    fi
  fi
}

function run_one_test_with_flock {
  local -r output_dir=$1
  local -r test_name=$2
  local -r lock_file=$3

  # Wait for the lock and run the test when acquired
  flock "${lock_file}" ./run-test --output_dir "${output_dir}" "${test_name}"
}

function run_tests {
  readonly test_root=$(mktemp -d)
  trap "rm -rf ${test_root}" EXIT
  if [[ "$sequential" = "yes" ]]; then
    for test_name in *; do
      if [[ "$skip_tests" = *"$test_name"* ]]; then
        skipped+=(${test_name})
        continue
      fi
      if [ -d "$test_name" -a -r "$test_name" ]; then
        output="${test_root}/${test_name}"
        ./run-test --output_dir "${output}" "${test_name}"
        update_result "${test_name}" "${output}" $?
      fi
    done
  else
    readonly num_workers=4
    local i=0
    for test_name in *; do
      if [[ "$skip_tests" = *"$test_name"* ]]; then
        skipped+=(${test_name})
        continue
      fi
      local lock_file=${test_root}/lock.$((i % num_workers))
      if [ -d "${test_name}" -a -r "${test_name}" ]; then
        output="${test_root}/${test_name}"
        run_one_test_with_flock "$output" "$test_name" "$lock_file" &
        test_pids[i]=$!
        test_names[test_pids[i]]="$test_name"
        test_outputs[test_pids[i]]="output"
        let i+=1
      fi
    done

    for pid in ${test_pids[@]}; do
      wait $pid
      update_result ${test_names[$pid]} ${test_outputs[$pid]} $?
    done
  fi
}

function handle_interrupt {
  trap INT
  display_results
  if [ ! -z "${test_pids}" ]; then
    killall ${test_pids}
  fi
}

trap handle_interrupt INT
run_tests
display_results