Renaming tool: jfuzz
Test: run_jfuzz_test.py
Change-Id: I0efa31c3f1c30344c796a9077488e7e467d2456e
diff --git a/tools/jfuzz/Android.mk b/tools/jfuzz/Android.mk
new file mode 100644
index 0000000..c7002d6
--- /dev/null
+++ b/tools/jfuzz/Android.mk
@@ -0,0 +1,25 @@
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Fuzzer tool.
+
+LOCAL_PATH:= $(call my-dir)
+
+include $(CLEAR_VARS)
+LOCAL_CPP_EXTENSION := cc
+LOCAL_SRC_FILES := jfuzz.cc
+LOCAL_CFLAGS += -O0 -g -Wall
+LOCAL_MODULE_HOST_OS := darwin linux windows
+LOCAL_MODULE := jfuzz
+include $(BUILD_HOST_EXECUTABLE)
diff --git a/tools/jfuzz/README.md b/tools/jfuzz/README.md
new file mode 100644
index 0000000..1d566a9
--- /dev/null
+++ b/tools/jfuzz/README.md
@@ -0,0 +1,98 @@
+JFuzz
+=====
+
+JFuzz is a tool for generating random programs with the objective
+of fuzz testing the ART infrastructure. Each randomly generated program
+can be run under various modes of execution, such as using the interpreter,
+using the optimizing compiler, using an external reference implementation,
+or using various target architectures. Any difference between the outputs
+(**divergence**) may indicate a bug in one of the execution modes.
+
+JFuzz can be combined with DexFuzz to get multi-layered fuzz testing.
+
+How to run JFuzz
+===================
+
+ jfuzz [-s seed] [-d expr-depth] [-l stmt-length]
+ [-i if-nest] [-n loop-nest]
+
+where
+
+ -s : defines a deterministic random seed
+ (randomized using time by default)
+ -d : defines a fuzzing depth for expressions
+ (higher values yield deeper expressions)
+ -l : defines a fuzzing length for statement lists
+ (higher values yield longer statement sequences)
+ -i : defines a fuzzing nest for if/switch statements
+ (higher values yield deeper nested conditionals)
+ -n : defines a fuzzing nest for for/while/do-while loops
+ (higher values yield deeper nested loops)
+
+The current version of JFuzz sends all output to stdout, and uses
+a fixed testing class named Test. So a typical test run looks as follows.
+
+ jfuzz > Test.java
+ jack -cp ${JACK_CLASSPATH} --output-dex . Test.java
+ art -classpath classes.dex Test
+
+How to start JFuzz testing
+=============================
+
+ run_jfuzz_test.py
+ [--num_tests=NUM_TESTS]
+ [--device=DEVICE]
+ [--mode1=MODE] [--mode2=MODE]
+
+where
+
+ --num_tests : number of tests to run (10000 by default)
+ --device : target device serial number (passed to adb -s)
+ --mode1 : m1
+ --mode2 : m2, with m1 != m2, and values one of
+ ri = reference implementation on host (default for m1)
+ hint = Art interpreter on host
+ hopt = Art optimizing on host (default for m2)
+ tint = Art interpreter on target
+ topt = Art optimizing on target
+
+How to start J/DexFuzz testing (multi-layered)
+==============================================
+
+ run_dex_fuzz_test.py
+ [--num_tests=NUM_TESTS]
+ [--num_inputs=NUM_INPUTS]
+ [--device=DEVICE]
+
+where
+
+ --num_tests : number of tests to run (10000 by default)
+ --num_inputs: number of JFuzz programs to generate
+ --device : target device serial number (passed to adb -s)
+
+Background
+==========
+
+Although test suites are extremely useful to validate the correctness of a
+system and to ensure that no regressions occur, any test suite is necessarily
+finite in size and scope. Tests typically focus on validating particular
+features by means of code sequences most programmers would expect. Regression
+tests often use slightly less idiomatic code sequences, since they reflect
+problems that were not anticipated originally, but occurred “in the field”.
+Still, any test suite leaves the developer wondering whether undetected bugs
+and flaws still linger in the system.
+
+Over the years, fuzz testing has gained popularity as a testing technique for
+discovering such lingering bugs, including bugs that can bring down a system
+in an unexpected way. Fuzzing refers to feeding a large amount of random data
+as input to a system in an attempt to find bugs or make it crash. Generation-
+based fuzz testing constructs random, but properly formatted input data.
+Mutation-based fuzz testing applies small random changes to existing inputs
+in order to detect shortcomings in a system. Profile-guided or coverage-guided
+fuzzing adds a direction to the way these random changes are applied. Multi-
+layered approaches generate random inputs that are subsequently mutated at
+various stages of execution.
+
+The randomness of fuzz testing implies that the size and scope of testing is no
+longer bounded. Every new run can potentially discover bugs and crashes that were
+hereto undetected.
diff --git a/tools/jfuzz/__init__.py b/tools/jfuzz/__init__.py
new file mode 100644
index 0000000..3955c71
--- /dev/null
+++ b/tools/jfuzz/__init__.py
@@ -0,0 +1,17 @@
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This file is intentionally left empty. It indicates that the directory is a Python package.
\ No newline at end of file
diff --git a/tools/jfuzz/jfuzz.cc b/tools/jfuzz/jfuzz.cc
new file mode 100644
index 0000000..125b56a
--- /dev/null
+++ b/tools/jfuzz/jfuzz.cc
@@ -0,0 +1,1108 @@
+/*
+ * Copyright 2016, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <random>
+
+#include <inttypes.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <time.h>
+
+namespace {
+
+/*
+ * Operators.
+ */
+
+#define EMIT(x) fputs((x)[random0(sizeof(x)/sizeof(const char*))], out_);
+
+static constexpr const char* kIncDecOps[] = { "++", "--" };
+static constexpr const char* kIntUnaryOps[] = { "+", "-", "~" };
+static constexpr const char* kFpUnaryOps[] = { "+", "-" };
+
+static constexpr const char* kBoolBinOps[] = { "&&", "||", "&", "|", "^" }; // few less common
+static constexpr const char* kIntBinOps[] = { "+", "-", "*", "/", "%",
+ ">>", ">>>", "<<", "&", "|", "^" };
+static constexpr const char* kFpBinOps[] = { "+", "-", "*", "/" };
+
+static constexpr const char* kBoolAssignOps[] = { "=", "&=" , "|=", "^=" }; // few less common
+static constexpr const char* kIntAssignOps[] = { "=", "+=", "-=", "*=", "/=", "%=",
+ ">>=", ">>>=", "<<=", "&=", "|=", "^=" };
+static constexpr const char* kFpAssignOps[] = { "=", "+=", "-=", "*=", "/=" };
+
+static constexpr const char* kBoolRelOps[] = { "==", "!=" };
+static constexpr const char* kRelOps[] = { "==", "!=", ">", ">=", "<", "<=" };
+
+/*
+ * Version of JFuzz. Increase this each time changes are made to the program
+ * to preserve the property that a given version of JFuzz yields the same
+ * fuzzed program for a deterministic random seed.
+ */
+const char* VERSION = "1.1";
+
+static const uint32_t MAX_DIMS[11] = { 0, 1000, 32, 10, 6, 4, 3, 3, 2, 2, 2 };
+
+/**
+ * A class that generates a random program that compiles correctly. The program
+ * is generated using rules that generate various programming constructs. Each rule
+ * has a fixed probability to "fire". Running a generated program yields deterministic
+ * output, making it suited to test various modes of execution (e.g an interpreter vs.
+ * an compiler or two different run times) for divergences.
+ *
+ * TODO: Due to the original scope of this project, the generated program is heavy
+ * on loops, arrays, and basic operations; fuzzing other aspects, like elaborate
+ * typing, class hierarchies, and interfaces is still TBD.
+ */
+class JFuzz {
+ public:
+ JFuzz(FILE* out,
+ uint32_t seed,
+ uint32_t expr_depth,
+ uint32_t stmt_length,
+ uint32_t if_nest,
+ uint32_t loop_nest)
+ : out_(out),
+ fuzz_random_engine_(seed),
+ fuzz_seed_(seed),
+ fuzz_expr_depth_(expr_depth),
+ fuzz_stmt_length_(stmt_length),
+ fuzz_if_nest_(if_nest),
+ fuzz_loop_nest_(loop_nest),
+ return_type_(randomType()),
+ array_type_(randomType()),
+ array_dim_(random1(10)),
+ array_size_(random1(MAX_DIMS[array_dim_])),
+ indentation_(0),
+ expr_depth_(0),
+ stmt_length_(0),
+ if_nest_(0),
+ loop_nest_(0),
+ switch_nest_(0),
+ do_nest_(0),
+ boolean_local_(0),
+ int_local_(0),
+ long_local_(0),
+ float_local_(0),
+ double_local_(0) { }
+
+ ~JFuzz() { }
+
+ void emitProgram() {
+ emitHeader();
+ emitTestClassWithMain();
+ }
+
+ private:
+ //
+ // Types.
+ //
+
+ // Current type of each expression during generation.
+ enum Type {
+ kBoolean,
+ kInt,
+ kLong,
+ kFloat,
+ kDouble
+ };
+
+ // Test for an integral type.
+ static bool isInteger(Type tp) {
+ return tp == kInt || tp == kLong;
+ }
+
+ // Test for a floating-point type.
+ static bool isFP(Type tp) {
+ return tp == kFloat || tp == kDouble;
+ }
+
+ // Emit type.
+ void emitType(Type tp) const {
+ switch (tp) {
+ case kBoolean: fputs("boolean", out_); break;
+ case kInt: fputs("int", out_); break;
+ case kLong: fputs("long", out_); break;
+ case kFloat: fputs("float", out_); break;
+ case kDouble: fputs("double", out_); break;
+ }
+ }
+
+ // Emit type class.
+ void emitTypeClass(Type tp) const {
+ switch (tp) {
+ case kBoolean: fputs("Boolean", out_); break;
+ case kInt: fputs("Integer", out_); break;
+ case kLong: fputs("Long", out_); break;
+ case kFloat: fputs("Float", out_); break;
+ case kDouble: fputs("Double", out_); break;
+ }
+ }
+
+ // Return a random type.
+ Type randomType() {
+ switch (random1(5)) {
+ case 1: return kBoolean;
+ case 2: return kInt;
+ case 3: return kLong;
+ case 4: return kFloat;
+ default: return kDouble;
+ }
+ }
+
+ //
+ // Expressions.
+ //
+
+ // Emit an unary operator (same type in-out).
+ void emitUnaryOp(Type tp) {
+ if (tp == kBoolean) {
+ fputc('!', out_);
+ } else if (isInteger(tp)) {
+ EMIT(kIntUnaryOps);
+ } else { // isFP(tp)
+ EMIT(kFpUnaryOps);
+ }
+ }
+
+ // Emit a pre/post-increment/decrement operator (same type in-out).
+ void emitIncDecOp(Type tp) {
+ if (tp == kBoolean) {
+ // Not applicable, just leave "as is".
+ } else { // isInteger(tp) || isFP(tp)
+ EMIT(kIncDecOps);
+ }
+ }
+
+ // Emit a binary operator (same type in-out).
+ void emitBinaryOp(Type tp) {
+ if (tp == kBoolean) {
+ EMIT(kBoolBinOps);
+ } else if (isInteger(tp)) {
+ EMIT(kIntBinOps);
+ } else { // isFP(tp)
+ EMIT(kFpBinOps);
+ }
+ }
+
+ // Emit an assignment operator (same type in-out).
+ void emitAssignmentOp(Type tp) {
+ if (tp == kBoolean) {
+ EMIT(kBoolAssignOps);
+ } else if (isInteger(tp)) {
+ EMIT(kIntAssignOps);
+ } else { // isFP(tp)
+ EMIT(kFpAssignOps);
+ }
+ }
+
+ // Emit a relational operator (one type in, boolean out).
+ void emitRelationalOp(Type tp) {
+ if (tp == kBoolean) {
+ EMIT(kBoolRelOps);
+ } else { // isInteger(tp) || isFP(tp)
+ EMIT(kRelOps);
+ }
+ }
+
+ // Emit a type conversion operator sequence (out type given, new suitable in type picked).
+ Type emitTypeConversionOp(Type tp) {
+ if (tp == kInt) {
+ switch (random1(5)) {
+ case 1: fputs("(int)", out_); return kLong;
+ case 2: fputs("(int)", out_); return kFloat;
+ case 3: fputs("(int)", out_); return kDouble;
+ // Narrowing-widening.
+ case 4: fputs("(int)(byte)(int)", out_); return kInt;
+ case 5: fputs("(int)(short)(int)", out_); return kInt;
+ }
+ } else if (tp == kLong) {
+ switch (random1(6)) {
+ case 1: /* implicit */ return kInt;
+ case 2: fputs("(long)", out_); return kFloat;
+ case 3: fputs("(long)", out_); return kDouble;
+ // Narrowing-widening.
+ case 4: fputs("(long)(byte)(long)", out_); return kLong;
+ case 5: fputs("(long)(short)(long)", out_); return kLong;
+ case 6: fputs("(long)(int)(long)", out_); return kLong;
+ }
+ } else if (tp == kFloat) {
+ switch (random1(4)) {
+ case 1: fputs("(float)", out_); return kInt;
+ case 2: fputs("(float)", out_); return kLong;
+ case 3: fputs("(float)", out_); return kDouble;
+ // Narrowing-widening.
+ case 4: fputs("(float)(int)(float)", out_); return kFloat;
+ }
+ } else if (tp == kDouble) {
+ switch (random1(5)) {
+ case 1: fputs("(double)", out_); return kInt;
+ case 2: fputs("(double)", out_); return kLong;
+ case 3: fputs("(double)", out_); return kFloat;
+ // Narrowing-widening.
+ case 4: fputs("(double)(int)(double)", out_); return kDouble;
+ case 5: fputs("(double)(float)(double)", out_); return kDouble;
+ }
+ }
+ return tp; // nothing suitable, just keep type
+ }
+
+ // Emit a type conversion (out type given, new suitable in type picked).
+ void emitTypeConversion(Type tp) {
+ if (tp == kBoolean) {
+ Type tp = randomType();
+ emitExpression(tp);
+ fputc(' ', out_);
+ emitRelationalOp(tp);
+ fputc(' ', out_);
+ emitExpression(tp);
+ } else {
+ tp = emitTypeConversionOp(tp);
+ fputc(' ', out_);
+ emitExpression(tp);
+ }
+ }
+
+ // Emit an unary intrinsic (out type given, new suitable in type picked).
+ Type emitIntrinsic1(Type tp) {
+ if (tp == kBoolean) {
+ switch (random1(6)) {
+ case 1: fputs("Float.isNaN", out_); return kFloat;
+ case 2: fputs("Float.isFinite", out_); return kFloat;
+ case 3: fputs("Float.isInfinite", out_); return kFloat;
+ case 4: fputs("Double.isNaN", out_); return kDouble;
+ case 5: fputs("Double.isFinite", out_); return kDouble;
+ case 6: fputs("Double.isInfinite", out_); return kDouble;
+ }
+ } else if (isInteger(tp)) {
+ const char* prefix = tp == kLong ? "Long" : "Integer";
+ switch (random1(13)) {
+ case 1: fprintf(out_, "%s.highestOneBit", prefix); break;
+ case 2: fprintf(out_, "%s.lowestOneBit", prefix); break;
+ case 3: fprintf(out_, "%s.numberOfLeadingZeros", prefix); break;
+ case 4: fprintf(out_, "%s.numberOfTrailingZeros", prefix); break;
+ case 5: fprintf(out_, "%s.bitCount", prefix); break;
+ case 6: fprintf(out_, "%s.signum", prefix); break;
+ case 7: fprintf(out_, "%s.reverse", prefix); break;
+ case 8: fprintf(out_, "%s.reverseBytes", prefix); break;
+ case 9: fputs("Math.incrementExact", out_); break;
+ case 10: fputs("Math.decrementExact", out_); break;
+ case 11: fputs("Math.negateExact", out_); break;
+ case 12: fputs("Math.abs", out_); break;
+ case 13: fputs("Math.round", out_);
+ return tp == kLong ? kDouble : kFloat;
+ }
+ } else { // isFP(tp)
+ switch (random1(6)) {
+ case 1: fputs("Math.abs", out_); break;
+ case 2: fputs("Math.ulp", out_); break;
+ case 3: fputs("Math.signum", out_); break;
+ case 4: fputs("Math.nextUp", out_); break;
+ case 5: fputs("Math.nextDown", out_); break;
+ case 6: if (tp == kDouble) {
+ fputs("Double.longBitsToDouble", out_);
+ return kLong;
+ } else {
+ fputs("Float.intBitsToFloat", out_);
+ return kInt;
+ }
+ }
+ }
+ return tp; // same type in-out
+ }
+
+ // Emit a binary intrinsic (out type given, new suitable in type picked).
+ Type emitIntrinsic2(Type tp) {
+ if (tp == kBoolean) {
+ switch (random1(3)) {
+ case 1: fputs("Boolean.logicalAnd", out_); break;
+ case 2: fputs("Boolean.logicalOr", out_); break;
+ case 3: fputs("Boolean.logicalXor", out_); break;
+ }
+ } else if (isInteger(tp)) {
+ const char* prefix = tp == kLong ? "Long" : "Integer";
+ switch (random1(11)) {
+ case 1: fprintf(out_, "%s.compare", prefix); break;
+ case 2: fprintf(out_, "%s.sum", prefix); break;
+ case 3: fprintf(out_, "%s.min", prefix); break;
+ case 4: fprintf(out_, "%s.max", prefix); break;
+ case 5: fputs("Math.min", out_); break;
+ case 6: fputs("Math.max", out_); break;
+ case 7: fputs("Math.floorDiv", out_); break;
+ case 8: fputs("Math.floorMod", out_); break;
+ case 9: fputs("Math.addExact", out_); break;
+ case 10: fputs("Math.subtractExact", out_); break;
+ case 11: fputs("Math.multiplyExact", out_); break;
+ }
+ } else { // isFP(tp)
+ const char* prefix = tp == kDouble ? "Double" : "Float";
+ switch (random1(5)) {
+ case 1: fprintf(out_, "%s.sum", prefix); break;
+ case 2: fprintf(out_, "%s.min", prefix); break;
+ case 3: fprintf(out_, "%s.max", prefix); break;
+ case 4: fputs("Math.min", out_); break;
+ case 5: fputs("Math.max", out_); break;
+ }
+ }
+ return tp; // same type in-out
+ }
+
+ // Emit an intrinsic (out type given, new suitable in type picked).
+ void emitIntrinsic(Type tp) {
+ if (random1(2) == 1) {
+ tp = emitIntrinsic1(tp);
+ fputc('(', out_);
+ emitExpression(tp);
+ fputc(')', out_);
+ } else {
+ tp = emitIntrinsic2(tp);
+ fputc('(', out_);
+ emitExpression(tp);
+ fputs(", ", out_);
+ emitExpression(tp);
+ fputc(')', out_);
+ }
+ }
+
+ // Emit unboxing boxed object.
+ void emitUnbox(Type tp) {
+ fputc('(', out_);
+ emitType(tp);
+ fputs(") new ", out_);
+ emitTypeClass(tp);
+ fputc('(', out_);
+ emitExpression(tp);
+ fputc(')', out_);
+ }
+
+ // Emit miscellaneous constructs.
+ void emitMisc(Type tp) {
+ if (tp == kBoolean) {
+ fputs("this instanceof Test", out_);
+ } else if (isInteger(tp)) {
+ const char* prefix = tp == kLong ? "Long" : "Integer";
+ switch (random1(2)) {
+ case 1: fprintf(out_, "%s.MIN_VALUE", prefix); break;
+ case 2: fprintf(out_, "%s.MAX_VALUE", prefix); break;
+ }
+ } else { // isFP(tp)
+ const char* prefix = tp == kDouble ? "Double" : "Float";
+ switch (random1(6)) {
+ case 1: fprintf(out_, "%s.MIN_NORMAL", prefix); break;
+ case 2: fprintf(out_, "%s.MIN_VALUE", prefix); break;
+ case 3: fprintf(out_, "%s.MAX_VALUE", prefix); break;
+ case 4: fprintf(out_, "%s.POSITIVE_INFINITY", prefix); break;
+ case 5: fprintf(out_, "%s.NEGATIVE_INFINITY", prefix); break;
+ case 6: fprintf(out_, "%s.NaN", prefix); break;
+ }
+ }
+ }
+
+ // Adjust local of given type and return adjusted value.
+ uint32_t adjustLocal(Type tp, int32_t a) {
+ switch (tp) {
+ case kBoolean: boolean_local_ += a; return boolean_local_;
+ case kInt: int_local_ += a; return int_local_;
+ case kLong: long_local_ += a; return long_local_;
+ case kFloat: float_local_ += a; return float_local_;
+ default: double_local_ += a; return double_local_;
+ }
+ }
+
+ // Emit an expression that is a strict upper bound for an array index.
+ void emitUpperBound() {
+ if (random1(8) == 1) {
+ fputs("mArray.length", out_);
+ } else if (random1(8) == 1) {
+ fprintf(out_, "%u", random1(array_size_)); // random in range
+ } else {
+ fprintf(out_, "%u", array_size_);
+ }
+ }
+
+ // Emit an array index, usually within proper range.
+ void emitArrayIndex() {
+ if (loop_nest_ > 0 && random1(2) == 1) {
+ fprintf(out_, "i%u", random0(loop_nest_));
+ } else if (random1(8) == 1) {
+ fputs("mArray.length - 1", out_);
+ } else {
+ fprintf(out_, "%u", random0(array_size_)); // random in range
+ }
+ // Introduce potential off by one errors with low probability.
+ if (random1(100) == 1) {
+ if (random1(2) == 1) {
+ fputs(" - 1", out_);
+ } else {
+ fputs(" + 1", out_);
+ }
+ }
+ }
+
+ // Emit a literal.
+ void emitLiteral(Type tp) {
+ switch (tp) {
+ case kBoolean: fputs(random1(2) == 1 ? "true" : "false", out_); break;
+ case kInt: fprintf(out_, "%d", random()); break;
+ case kLong: fprintf(out_, "%dL", random()); break;
+ case kFloat: fprintf(out_, "%d.0f", random()); break;
+ case kDouble: fprintf(out_, "%d.0", random()); break;
+ }
+ }
+
+ // Emit array variable, if available.
+ bool emitArrayVariable(Type tp) {
+ if (tp == array_type_) {
+ fputs("mArray", out_);
+ for (uint32_t i = 0; i < array_dim_; i++) {
+ fputc('[', out_);
+ emitArrayIndex();
+ fputc(']', out_);
+ }
+ return true;
+ }
+ return false;
+ }
+
+ // Emit a local variable, if available.
+ bool emitLocalVariable(Type tp) {
+ uint32_t locals = adjustLocal(tp, 0);
+ if (locals > 0) {
+ uint32_t local = random0(locals);
+ switch (tp) {
+ case kBoolean: fprintf(out_, "lZ%u", local); break;
+ case kInt: fprintf(out_, "lI%u", local); break;
+ case kLong: fprintf(out_, "lJ%u", local); break;
+ case kFloat: fprintf(out_, "lF%u", local); break;
+ case kDouble: fprintf(out_, "lD%u", local); break;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ // Emit a field variable.
+ void emitFieldVariable(Type tp) {
+ switch (tp) {
+ case kBoolean:fputs("mZ", out_); break;
+ case kInt: fputs("mI", out_); break;
+ case kLong: fputs("mJ", out_); break;
+ case kFloat: fputs("mF", out_); break;
+ case kDouble: fputs("mD", out_); break;
+ }
+ }
+
+ // Emit a variable.
+ void emitVariable(Type tp) {
+ switch (random1(4)) {
+ case 1:
+ if (emitArrayVariable(tp))
+ return;
+ // FALL-THROUGH
+ case 2:
+ if (emitLocalVariable(tp))
+ return;
+ // FALL-THROUGH
+ default:
+ emitFieldVariable(tp);
+ break;
+ }
+ }
+
+ // Emit an expression.
+ void emitExpression(Type tp) {
+ // Continuing expression becomes less likely as the depth grows.
+ if (random1(expr_depth_ + 1) > fuzz_expr_depth_) {
+ if (random1(2) == 1) {
+ emitLiteral(tp);
+ } else {
+ emitVariable(tp);
+ }
+ return;
+ }
+
+ expr_depth_++;
+
+ fputc('(', out_);
+ switch (random1(12)) { // favor binary operations
+ case 1:
+ // Unary operator: ~ x
+ emitUnaryOp(tp);
+ fputc(' ', out_);
+ emitExpression(tp);
+ break;
+ case 2:
+ // Pre-increment: ++x
+ emitIncDecOp(tp);
+ emitVariable(tp);
+ break;
+ case 3:
+ // Post-increment: x++
+ emitVariable(tp);
+ emitIncDecOp(tp);
+ break;
+ case 4:
+ // Ternary operator: b ? x : y
+ emitExpression(kBoolean);
+ fputs(" ? ", out_);
+ emitExpression(tp);
+ fputs(" : ", out_);
+ emitExpression(tp);
+ break;
+ case 5:
+ // Type conversion: (float) x
+ emitTypeConversion(tp);
+ break;
+ case 6:
+ // Intrinsic: foo(x)
+ emitIntrinsic(tp);
+ break;
+ case 7:
+ // Emit unboxing boxed value: (int) Integer(x)
+ emitUnbox(tp);
+ break;
+ case 8:
+ // Miscellaneous constructs: a.length
+ emitMisc(tp);
+ break;
+ default:
+ // Binary operator: x + y
+ emitExpression(tp);
+ fputc(' ', out_);
+ emitBinaryOp(tp);
+ fputc(' ', out_);
+ emitExpression(tp);
+ break;
+ }
+ fputc(')', out_);
+
+ --expr_depth_;
+ }
+
+ //
+ // Statements.
+ //
+
+ // Emit current indentation.
+ void emitIndentation() const {
+ for (uint32_t i = 0; i < indentation_; i++) {
+ fputc(' ', out_);
+ }
+ }
+
+ // Emit a return statement.
+ bool emitReturn(bool mustEmit) {
+ // Only emit when we must, or with low probability inside ifs/loops,
+ // but outside do-while to avoid confusing the may follow status.
+ if (mustEmit || ((if_nest_ + loop_nest_) > 0 && do_nest_ == 0 && random1(10) == 1)) {
+ fputs("return ", out_);
+ emitExpression(return_type_);
+ fputs(";\n", out_);
+ return false;
+ }
+ // Fall back to assignment.
+ return emitAssignment();
+ }
+
+ // Emit a continue statement.
+ bool emitContinue() {
+ // Only emit with low probability inside loops.
+ if (loop_nest_ > 0 && random1(10) == 1) {
+ fputs("continue;\n", out_);
+ return false;
+ }
+ // Fall back to assignment.
+ return emitAssignment();
+ }
+
+ // Emit a break statement.
+ bool emitBreak() {
+ // Only emit with low probability inside loops, but outside switches
+ // to avoid confusing the may follow status.
+ if (loop_nest_ > 0 && switch_nest_ == 0 && random1(10) == 1) {
+ fputs("break;\n", out_);
+ return false;
+ }
+ // Fall back to assignment.
+ return emitAssignment();
+ }
+
+ // Emit a new scope with a local variable declaration statement.
+ bool emitScope() {
+ Type tp = randomType();
+ fputs("{\n", out_);
+ indentation_ += 2;
+ emitIndentation();
+ emitType(tp);
+ switch (tp) {
+ case kBoolean: fprintf(out_, " lZ%u = ", boolean_local_); break;
+ case kInt: fprintf(out_, " lI%u = ", int_local_); break;
+ case kLong: fprintf(out_, " lJ%u = ", long_local_); break;
+ case kFloat: fprintf(out_, " lF%u = ", float_local_); break;
+ case kDouble: fprintf(out_, " lD%u = ", double_local_); break;
+ }
+ emitExpression(tp);
+ fputs(";\n", out_);
+
+ adjustLocal(tp, 1); // local now visible
+
+ bool mayFollow = emitStatementList();
+
+ adjustLocal(tp, -1); // local no longer visible
+
+ indentation_ -= 2;
+ emitIndentation();
+ fputs("}\n", out_);
+ return mayFollow;
+ }
+
+ // Emit a for loop.
+ bool emitForLoop() {
+ // Continuing loop nest becomes less likely as the depth grows.
+ if (random1(loop_nest_ + 1) > fuzz_loop_nest_) {
+ return emitAssignment(); // fall back
+ }
+
+ bool goesUp = random1(2) == 1;
+ fprintf(out_, "for (int i%u = ", loop_nest_);
+ if (goesUp) {
+ fprintf(out_, "0; i%u < ", loop_nest_);
+ emitUpperBound();
+ fprintf(out_, "; i%u++) {\n", loop_nest_);
+ } else {
+ emitUpperBound();
+ fprintf(out_, " - 1; i%d >= 0", loop_nest_);
+ fprintf(out_, "; i%d--) {\n", loop_nest_);
+ }
+
+ ++loop_nest_; // now in loop
+
+ indentation_ += 2;
+ emitStatementList();
+
+ --loop_nest_; // no longer in loop
+
+ indentation_ -= 2;
+ emitIndentation();
+ fprintf(out_, "}\n");
+ return true; // loop-body does not block flow
+ }
+
+ // Emit while or do-while loop.
+ bool emitDoLoop() {
+ // Continuing loop nest becomes less likely as the depth grows.
+ if (random1(loop_nest_ + 1) > fuzz_loop_nest_) {
+ return emitAssignment(); // fall back
+ }
+
+ // TODO: remove this
+ // The jack bug b/28862040 prevents generating while/do-while loops because otherwise
+ // we get dozens of reports on the same issue per nightly/ run.
+ if (true) {
+ return emitAssignment();
+ }
+
+ bool isWhile = random1(2) == 1;
+ fputs("{\n", out_);
+ indentation_ += 2;
+ emitIndentation();
+ fprintf(out_, "int i%u = %d;", loop_nest_, isWhile ? -1 : 0);
+ emitIndentation();
+ if (isWhile) {
+ fprintf(out_, "while (++i%u < ", loop_nest_);
+ emitUpperBound();
+ fputs(") {\n", out_);
+ } else {
+ fputs("do {\n", out_);
+ do_nest_++;
+ }
+
+ ++loop_nest_; // now in loop
+
+ indentation_ += 2;
+ emitStatementList();
+
+ --loop_nest_; // no longer in loop
+
+ indentation_ -= 2;
+ emitIndentation();
+ if (isWhile) {
+ fputs("}\n", out_);
+ } else {
+ fprintf(out_, "} while (++i%u < ", loop_nest_);
+ emitUpperBound();
+ fputs(");\n", out_);
+ --do_nest_;
+ }
+ indentation_ -= 2;
+ emitIndentation();
+ fputs("}\n", out_);
+ return true; // loop-body does not block flow
+ }
+
+ // Emit an if statement.
+ bool emitIfStmt() {
+ // Continuing if nest becomes less likely as the depth grows.
+ if (random1(if_nest_ + 1) > fuzz_if_nest_) {
+ return emitAssignment(); // fall back
+ }
+
+ fputs("if (", out_);
+ emitExpression(kBoolean);
+ fputs(") {\n", out_);
+
+ ++if_nest_; // now in if
+
+ indentation_ += 2;
+ bool mayFollowTrue = emitStatementList();
+ indentation_ -= 2;
+ emitIndentation();
+ fprintf(out_, "} else {\n");
+ indentation_ += 2;
+ bool mayFollowFalse = emitStatementList();
+
+ --if_nest_; // no longer in if
+
+ indentation_ -= 2;
+ emitIndentation();
+ fprintf(out_, "}\n");
+ return mayFollowTrue || mayFollowFalse;
+ }
+
+ // Emit a switch statement.
+ bool emitSwitch() {
+ // Continuing if nest becomes less likely as the depth grows.
+ if (random1(if_nest_ + 1) > fuzz_if_nest_) {
+ return emitAssignment(); // fall back
+ }
+
+ bool mayFollow = false;
+ fputs("switch (", out_);
+ emitArrayIndex(); // restrict its range
+ fputs(") {\n", out_);
+
+ ++if_nest_;
+ ++switch_nest_; // now in switch
+
+ indentation_ += 2;
+ for (uint32_t i = 0; i < 2; i++) {
+ emitIndentation();
+ if (i == 0) {
+ fprintf(out_, "case %u: {\n", random0(array_size_));
+ } else {
+ fprintf(out_, "default: {\n");
+ }
+ indentation_ += 2;
+ if (emitStatementList()) {
+ // Must end with break.
+ emitIndentation();
+ fputs("break;\n", out_);
+ mayFollow = true;
+ }
+ indentation_ -= 2;
+ emitIndentation();
+ fputs("}\n", out_);
+ }
+
+ --if_nest_;
+ --switch_nest_; // no longer in switch
+
+ indentation_ -= 2;
+ emitIndentation();
+ fprintf(out_, "}\n");
+ return mayFollow;
+ }
+
+ // Emit an assignment statement.
+ bool emitAssignment() {
+ Type tp = randomType();
+ emitVariable(tp);
+ fputc(' ', out_);
+ emitAssignmentOp(tp);
+ fputc(' ', out_);
+ emitExpression(tp);
+ fputs(";\n", out_);
+ return true;
+ }
+
+ // Emit a single statement. Returns true if statements may follow.
+ bool emitStatement() {
+ switch (random1(16)) { // favor assignments
+ case 1: return emitReturn(false); break;
+ case 2: return emitContinue(); break;
+ case 3: return emitBreak(); break;
+ case 4: return emitScope(); break;
+ case 5: return emitForLoop(); break;
+ case 6: return emitDoLoop(); break;
+ case 7: return emitIfStmt(); break;
+ case 8: return emitSwitch(); break;
+ default: return emitAssignment(); break;
+ }
+ }
+
+ // Emit a statement list. Returns true if statements may follow.
+ bool emitStatementList() {
+ while (stmt_length_ < 1000) { // avoid run-away
+ stmt_length_++;
+ emitIndentation();
+ if (!emitStatement()) {
+ return false; // rest would be dead code
+ }
+ // Continuing this list becomes less likely as the total statement list grows.
+ if (random1(stmt_length_) > fuzz_stmt_length_) {
+ break;
+ }
+ }
+ return true;
+ }
+
+ // Emit field declarations.
+ void emitFieldDecls() {
+ fputs(" private boolean mZ = false;\n", out_);
+ fputs(" private int mI = 0;\n", out_);
+ fputs(" private long mJ = 0;\n", out_);
+ fputs(" private float mF = 0;\n", out_);
+ fputs(" private double mD = 0;\n\n", out_);
+ }
+
+ // Emit array declaration.
+ void emitArrayDecl() {
+ fputs(" private ", out_);
+ emitType(array_type_);
+ for (uint32_t i = 0; i < array_dim_; i++) {
+ fputs("[]", out_);
+ }
+ fputs(" mArray = new ", out_);
+ emitType(array_type_);
+ for (uint32_t i = 0; i < array_dim_; i++) {
+ fprintf(out_, "[%d]", array_size_);
+ }
+ fputs(";\n\n", out_);
+ }
+
+ // Emit test constructor.
+ void emitTestConstructor() {
+ fputs(" private Test() {\n", out_);
+ indentation_ += 2;
+ emitIndentation();
+ emitType(array_type_);
+ fputs(" a = ", out_);
+ emitLiteral(array_type_);
+ fputs(";\n", out_);
+ for (uint32_t i = 0; i < array_dim_; i++) {
+ emitIndentation();
+ fprintf(out_, "for (int i%u = 0; i%u < %u; i%u++) {\n", i, i, array_size_, i);
+ indentation_ += 2;
+ }
+ emitIndentation();
+ fputs("mArray", out_);
+ for (uint32_t i = 0; i < array_dim_; i++) {
+ fprintf(out_, "[i%u]", i);
+ }
+ fputs(" = a;\n", out_);
+ emitIndentation();
+ if (array_type_ == kBoolean) {
+ fputs("a = !a;\n", out_);
+ } else {
+ fputs("a++;\n", out_);
+ }
+ for (uint32_t i = 0; i < array_dim_; i++) {
+ indentation_ -= 2;
+ emitIndentation();
+ fputs("}\n", out_);
+ }
+ indentation_ -= 2;
+ fputs(" }\n\n", out_);
+ }
+
+ // Emit test method.
+ void emitTestMethod() {
+ fputs(" private ", out_);
+ emitType(return_type_);
+ fputs(" testMethod() {\n", out_);
+ indentation_ += 2;
+ if (emitStatementList()) {
+ // Must end with return.
+ emitIndentation();
+ emitReturn(true);
+ }
+ indentation_ -= 2;
+ fputs(" }\n\n", out_);
+ }
+
+ // Emit main method driver.
+ void emitMainMethod() {
+ fputs(" public static void main(String[] args) {\n", out_);
+ indentation_ += 2;
+ fputs(" Test t = new Test();\n ", out_);
+ emitType(return_type_);
+ fputs(" r = ", out_);
+ emitLiteral(return_type_);
+ fputs(";\n", out_);
+ fputs(" try {\n", out_);
+ fputs(" r = t.testMethod();\n", out_);
+ fputs(" } catch (Exception e) {\n", out_);
+ fputs(" // Arithmetic, null pointer, index out of bounds, etc.\n", out_);
+ fputs(" System.out.println(\"An exception was caught.\");\n", out_);
+ fputs(" }\n", out_);
+ fputs(" System.out.println(\"r = \" + r);\n", out_);
+ fputs(" System.out.println(\"mZ = \" + t.mZ);\n", out_);
+ fputs(" System.out.println(\"mI = \" + t.mI);\n", out_);
+ fputs(" System.out.println(\"mJ = \" + t.mJ);\n", out_);
+ fputs(" System.out.println(\"mF = \" + t.mF);\n", out_);
+ fputs(" System.out.println(\"mD = \" + t.mD);\n", out_);
+ fputs(" System.out.println(\"mArray = \" + ", out_);
+ if (array_dim_ == 1) {
+ fputs("Arrays.toString(t.mArray)", out_);
+ } else {
+ fputs("Arrays.deepToString(t.mArray)", out_);
+ }
+ fputs(");\n", out_);
+ indentation_ -= 2;
+ fputs(" }\n", out_);
+ }
+
+ // Emit program header. Emit command line options in the comments.
+ void emitHeader() {
+ fputs("\n/**\n * AOSP JFuzz Tester.\n", out_);
+ fputs(" * Automatically generated program.\n", out_);
+ fprintf(out_,
+ " * jfuzz -s %u -d %u -l %u -i %u -n %u (version %s)\n */\n\n",
+ fuzz_seed_,
+ fuzz_expr_depth_,
+ fuzz_stmt_length_,
+ fuzz_if_nest_,
+ fuzz_loop_nest_,
+ VERSION);
+ fputs("import java.util.Arrays;\n\n", out_);
+ }
+
+ // Emit single test class with main driver.
+ void emitTestClassWithMain() {
+ fputs("public class Test {\n\n", out_);
+ indentation_ += 2;
+ emitFieldDecls();
+ emitArrayDecl();
+ emitTestConstructor();
+ emitTestMethod();
+ emitMainMethod();
+ indentation_ -= 2;
+ fputs("}\n\n", out_);
+ }
+
+ //
+ // Random integers.
+ //
+
+ // Return random integer.
+ int32_t random() {
+ return fuzz_random_engine_();
+ }
+
+ // Return random integer in range [0,max).
+ uint32_t random0(uint32_t max) {
+ std::uniform_int_distribution<uint32_t> gen(0, max - 1);
+ return gen(fuzz_random_engine_);
+ }
+
+ // Return random integer in range [1,max].
+ uint32_t random1(uint32_t max) {
+ std::uniform_int_distribution<uint32_t> gen(1, max);
+ return gen(fuzz_random_engine_);
+ }
+
+ // Fuzzing parameters.
+ FILE* out_;
+ std::mt19937 fuzz_random_engine_;
+ const uint32_t fuzz_seed_;
+ const uint32_t fuzz_expr_depth_;
+ const uint32_t fuzz_stmt_length_;
+ const uint32_t fuzz_if_nest_;
+ const uint32_t fuzz_loop_nest_;
+
+ // Return and array setup.
+ const Type return_type_;
+ const Type array_type_;
+ const uint32_t array_dim_;
+ const uint32_t array_size_;
+
+ // Current context.
+ uint32_t indentation_;
+ uint32_t expr_depth_;
+ uint32_t stmt_length_;
+ uint32_t if_nest_;
+ uint32_t loop_nest_;
+ uint32_t switch_nest_;
+ uint32_t do_nest_;
+ uint32_t boolean_local_;
+ uint32_t int_local_;
+ uint32_t long_local_;
+ uint32_t float_local_;
+ uint32_t double_local_;
+};
+
+} // anonymous namespace
+
+int32_t main(int32_t argc, char** argv) {
+ // Defaults.
+ uint32_t seed = time(NULL);
+ uint32_t expr_depth = 1;
+ uint32_t stmt_length = 8;
+ uint32_t if_nest = 2;
+ uint32_t loop_nest = 3;
+
+ // Parse options.
+ while (1) {
+ int32_t option = getopt(argc, argv, "s:d:l:i:n:h");
+ if (option < 0) {
+ break; // done
+ }
+ switch (option) {
+ case 's':
+ seed = strtoul(optarg, nullptr, 0); // deterministic seed
+ break;
+ case 'd':
+ expr_depth = strtoul(optarg, nullptr, 0);
+ break;
+ case 'l':
+ stmt_length = strtoul(optarg, nullptr, 0);
+ break;
+ case 'i':
+ if_nest = strtoul(optarg, nullptr, 0);
+ break;
+ case 'n':
+ loop_nest = strtoul(optarg, nullptr, 0);
+ break;
+ case 'h':
+ default:
+ fprintf(stderr,
+ "usage: %s [-s seed] "
+ "[-d expr-depth] [-l stmt-length] "
+ "[-i if-nest] [-n loop-nest] [-h]\n",
+ argv[0]);
+ return 1;
+ }
+ }
+
+ // Seed global random generator.
+ srand(seed);
+
+ // Generate fuzzed program.
+ JFuzz fuzz(stdout, seed, expr_depth, stmt_length, if_nest, loop_nest);
+ fuzz.emitProgram();
+ return 0;
+}
diff --git a/tools/jfuzz/run_dex_fuzz_test.py b/tools/jfuzz/run_dex_fuzz_test.py
new file mode 100755
index 0000000..56cdf02
--- /dev/null
+++ b/tools/jfuzz/run_dex_fuzz_test.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python3.4
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import shutil
+import sys
+
+from subprocess import check_call
+from tempfile import mkdtemp
+
+sys.path.append(os.path.dirname(os.path.dirname(
+ os.path.realpath(__file__))))
+
+from common.common import FatalError
+from common.common import GetJackClassPath
+from common.common import RetCode
+from common.common import RunCommand
+
+
+#
+# Tester class.
+#
+
+
+class DexFuzzTester(object):
+ """Tester that feeds JFuzz programs into DexFuzz testing."""
+
+ def __init__(self, num_tests, num_inputs, device):
+ """Constructor for the tester.
+
+ Args:
+ num_tests: int, number of tests to run
+ num_inputs: int, number of JFuzz programs to generate
+ device: string, target device serial number (or None)
+ """
+ self._num_tests = num_tests
+ self._num_inputs = num_inputs
+ self._device = device
+ self._save_dir = None
+ self._results_dir = None
+ self._dexfuzz_dir = None
+ self._inputs_dir = None
+
+ def __enter__(self):
+ """On entry, enters new temp directory after saving current directory.
+
+ Raises:
+ FatalError: error when temp directory cannot be constructed
+ """
+ self._save_dir = os.getcwd()
+ self._results_dir = mkdtemp(dir='/tmp/')
+ self._dexfuzz_dir = mkdtemp(dir=self._results_dir)
+ self._inputs_dir = mkdtemp(dir=self._dexfuzz_dir)
+ if self._results_dir is None or self._dexfuzz_dir is None or \
+ self._inputs_dir is None:
+ raise FatalError('Cannot obtain temp directory')
+ os.chdir(self._dexfuzz_dir)
+ return self
+
+ def __exit__(self, etype, evalue, etraceback):
+ """On exit, re-enters previously saved current directory and cleans up."""
+ os.chdir(self._save_dir)
+ # TODO: detect divergences or shutil.rmtree(self._results_dir)
+
+ def Run(self):
+ """Feeds JFuzz programs into DexFuzz testing."""
+ print()
+ print('**\n**** JFuzz Testing\n**')
+ print()
+ print('#Tests :', self._num_tests)
+ print('Device :', self._device)
+ print('Directory :', self._results_dir)
+ print()
+ self.GenerateJFuzzPrograms()
+ self.RunDexFuzz()
+
+
+ def GenerateJFuzzPrograms(self):
+ """Generates JFuzz programs.
+
+ Raises:
+ FatalError: error when generation fails
+ """
+ os.chdir(self._inputs_dir)
+ for i in range(1, self._num_inputs + 1):
+ jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.', 'Test.java']
+ if RunCommand(['jfuzz'], out='Test.java', err=None) != RetCode.SUCCESS:
+ raise FatalError('Unexpected error while running JFuzz')
+ if RunCommand(['jack'] + jack_args, out=None, err='jackerr.txt',
+ timeout=30) != RetCode.SUCCESS:
+ raise FatalError('Unexpected error while running Jack')
+ shutil.move('Test.java', '../Test' + str(i) + '.java')
+ shutil.move('classes.dex', 'classes' + str(i) + '.dex')
+ os.unlink('jackerr.txt')
+
+ def RunDexFuzz(self):
+ """Starts the DexFuzz testing."""
+ os.chdir(self._dexfuzz_dir)
+ os.environ['ANDROID_DATA'] = self._dexfuzz_dir
+ dexfuzz_args = ['--inputs=' + self._inputs_dir, '--execute',
+ '--execute-class=Test', '--repeat=' + str(self._num_tests),
+ '--dump-output', '--interpreter', '--optimizing']
+ if self._device is not None:
+ dexfuzz_args += ['--device=' + self._device, '--allarm']
+ else:
+ dexfuzz_args += ['--host'] # Assume host otherwise.
+ check_call(['dexfuzz'] + dexfuzz_args)
+ # TODO: summarize findings.
+
+
+def main():
+ # Handle arguments.
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--num_tests', default=10000,
+ type=int, help='number of tests to run')
+ parser.add_argument('--num_inputs', default=50,
+ type=int, help='number of JFuzz program to generate')
+ parser.add_argument('--device', help='target device serial number')
+ args = parser.parse_args()
+ # Run the DexFuzz tester.
+ with DexFuzzTester(args.num_tests, args.num_inputs, args.device) as fuzzer:
+ fuzzer.Run()
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/jfuzz/run_jfuzz_test.py b/tools/jfuzz/run_jfuzz_test.py
new file mode 100755
index 0000000..cf2364b
--- /dev/null
+++ b/tools/jfuzz/run_jfuzz_test.py
@@ -0,0 +1,507 @@
+#!/usr/bin/env python3.4
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+import argparse
+import filecmp
+import os
+import shlex
+import shutil
+import sys
+
+from glob import glob
+from tempfile import mkdtemp
+
+sys.path.append(os.path.dirname(os.path.dirname(
+ os.path.realpath(__file__))))
+
+from common.common import RetCode
+from common.common import CommandListToCommandString
+from common.common import FatalError
+from common.common import GetJackClassPath
+from common.common import GetEnvVariableOrError
+from common.common import RunCommand
+from common.common import DeviceTestEnv
+
+# Return codes supported by bisection bug search.
+BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT)
+
+
+def GetExecutionModeRunner(device, mode):
+ """Returns a runner for the given execution mode.
+
+ Args:
+ device: string, target device serial number (or None)
+ mode: string, execution mode
+ Returns:
+ TestRunner with given execution mode
+ Raises:
+ FatalError: error for unknown execution mode
+ """
+ if mode == 'ri':
+ return TestRunnerRIOnHost()
+ if mode == 'hint':
+ return TestRunnerArtIntOnHost()
+ if mode == 'hopt':
+ return TestRunnerArtOptOnHost()
+ if mode == 'tint':
+ return TestRunnerArtIntOnTarget(device)
+ if mode == 'topt':
+ return TestRunnerArtOptOnTarget(device)
+ raise FatalError('Unknown execution mode')
+
+
+#
+# Execution mode classes.
+#
+
+
+class TestRunner(object):
+ """Abstraction for running a test in a particular execution mode."""
+ __meta_class__ = abc.ABCMeta
+
+ @abc.abstractproperty
+ def description(self):
+ """Returns a description string of the execution mode."""
+
+ @abc.abstractproperty
+ def id(self):
+ """Returns a short string that uniquely identifies the execution mode."""
+
+ @property
+ def output_file(self):
+ return self.id + '_out.txt'
+
+ @abc.abstractmethod
+ def GetBisectionSearchArgs(self):
+ """Get arguments to pass to bisection search tool.
+
+ Returns:
+ list of strings - arguments for bisection search tool, or None if
+ runner is not bisectable
+ """
+
+ @abc.abstractmethod
+ def CompileAndRunTest(self):
+ """Compile and run the generated test.
+
+ Ensures that the current Test.java in the temporary directory is compiled
+ and executed under the current execution mode. On success, transfers the
+ generated output to the file self.output_file in the temporary directory.
+
+ Most nonzero return codes are assumed non-divergent, since systems may
+ exit in different ways. This is enforced by normalizing return codes.
+
+ Returns:
+ normalized return code
+ """
+
+
+class TestRunnerRIOnHost(TestRunner):
+ """Concrete test runner of the reference implementation on host."""
+
+ @property
+ def description(self):
+ return 'RI on host'
+
+ @property
+ def id(self):
+ return 'RI'
+
+ def CompileAndRunTest(self):
+ if RunCommand(['javac', 'Test.java'],
+ out=None, err=None, timeout=30) == RetCode.SUCCESS:
+ retc = RunCommand(['java', 'Test'], self.output_file, err=None)
+ else:
+ retc = RetCode.NOTCOMPILED
+ return retc
+
+ def GetBisectionSearchArgs(self):
+ return None
+
+
+class TestRunnerArtOnHost(TestRunner):
+ """Abstract test runner of Art on host."""
+
+ def __init__(self, extra_args=None):
+ """Constructor for the Art on host tester.
+
+ Args:
+ extra_args: list of strings, extra arguments for dalvikvm
+ """
+ self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex']
+ if extra_args is not None:
+ self._art_cmd += extra_args
+ self._art_cmd.append('Test')
+ self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.',
+ 'Test.java']
+
+ def CompileAndRunTest(self):
+ if RunCommand(['jack'] + self._jack_args, out=None, err='jackerr.txt',
+ timeout=30) == RetCode.SUCCESS:
+ retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt')
+ else:
+ retc = RetCode.NOTCOMPILED
+ return retc
+
+
+class TestRunnerArtIntOnHost(TestRunnerArtOnHost):
+ """Concrete test runner of interpreter mode Art on host."""
+
+ def __init__(self):
+ """Constructor."""
+ super().__init__(['-Xint'])
+
+ @property
+ def description(self):
+ return 'Art interpreter on host'
+
+ @property
+ def id(self):
+ return 'HInt'
+
+ def GetBisectionSearchArgs(self):
+ return None
+
+
+class TestRunnerArtOptOnHost(TestRunnerArtOnHost):
+ """Concrete test runner of optimizing compiler mode Art on host."""
+
+ def __init__(self):
+ """Constructor."""
+ super().__init__(None)
+
+ @property
+ def description(self):
+ return 'Art optimizing on host'
+
+ @property
+ def id(self):
+ return 'HOpt'
+
+ def GetBisectionSearchArgs(self):
+ cmd_str = CommandListToCommandString(
+ self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:])
+ return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
+
+
+class TestRunnerArtOnTarget(TestRunner):
+ """Abstract test runner of Art on target."""
+
+ def __init__(self, device, extra_args=None):
+ """Constructor for the Art on target tester.
+
+ Args:
+ device: string, target device serial number (or None)
+ extra_args: list of strings, extra arguments for dalvikvm
+ """
+ self._test_env = DeviceTestEnv('jfuzz_', specific_device=device)
+ self._dalvik_cmd = ['dalvikvm']
+ if extra_args is not None:
+ self._dalvik_cmd += extra_args
+ self._device = device
+ self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.',
+ 'Test.java']
+ self._device_classpath = None
+
+ def CompileAndRunTest(self):
+ if RunCommand(['jack'] + self._jack_args, out=None, err='jackerr.txt',
+ timeout=30) == RetCode.SUCCESS:
+ self._device_classpath = self._test_env.PushClasspath('classes.dex')
+ cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']
+ (output, retc) = self._test_env.RunCommand(
+ cmd, {'ANDROID_LOG_TAGS': '*:s'})
+ with open(self.output_file, 'w') as run_out:
+ run_out.write(output)
+ else:
+ retc = RetCode.NOTCOMPILED
+ return retc
+
+ def GetBisectionSearchArgs(self):
+ cmd_str = CommandListToCommandString(
+ self._dalvik_cmd + ['-cp',self._device_classpath, 'Test'])
+ cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
+ if self._device:
+ cmd += ['--device-serial', self._device]
+ else:
+ cmd.append('--device')
+ return cmd
+
+
+class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget):
+ """Concrete test runner of interpreter mode Art on target."""
+
+ def __init__(self, device):
+ """Constructor.
+
+ Args:
+ device: string, target device serial number (or None)
+ """
+ super().__init__(device, ['-Xint'])
+
+ @property
+ def description(self):
+ return 'Art interpreter on target'
+
+ @property
+ def id(self):
+ return 'TInt'
+
+ def GetBisectionSearchArgs(self):
+ return None
+
+
+class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget):
+ """Concrete test runner of optimizing compiler mode Art on target."""
+
+ def __init__(self, device):
+ """Constructor.
+
+ Args:
+ device: string, target device serial number (or None)
+ """
+ super().__init__(device, None)
+
+ @property
+ def description(self):
+ return 'Art optimizing on target'
+
+ @property
+ def id(self):
+ return 'TOpt'
+
+ def GetBisectionSearchArgs(self):
+ cmd_str = CommandListToCommandString(
+ self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'])
+ cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
+ if self._device:
+ cmd += ['--device-serial', self._device]
+ else:
+ cmd.append('--device')
+ return cmd
+
+
+#
+# Tester class.
+#
+
+
+class JFuzzTester(object):
+ """Tester that runs JFuzz many times and report divergences."""
+
+ def __init__(self, num_tests, device, mode1, mode2):
+ """Constructor for the tester.
+
+ Args:
+ num_tests: int, number of tests to run
+ device: string, target device serial number (or None)
+ mode1: string, execution mode for first runner
+ mode2: string, execution mode for second runner
+ """
+ self._num_tests = num_tests
+ self._device = device
+ self._runner1 = GetExecutionModeRunner(device, mode1)
+ self._runner2 = GetExecutionModeRunner(device, mode2)
+ self._save_dir = None
+ self._results_dir = None
+ self._jfuzz_dir = None
+ # Statistics.
+ self._test = 0
+ self._num_success = 0
+ self._num_not_compiled = 0
+ self._num_not_run = 0
+ self._num_timed_out = 0
+ self._num_divergences = 0
+
+ def __enter__(self):
+ """On entry, enters new temp directory after saving current directory.
+
+ Raises:
+ FatalError: error when temp directory cannot be constructed
+ """
+ self._save_dir = os.getcwd()
+ self._results_dir = mkdtemp(dir='/tmp/')
+ self._jfuzz_dir = mkdtemp(dir=self._results_dir)
+ if self._results_dir is None or self._jfuzz_dir is None:
+ raise FatalError('Cannot obtain temp directory')
+ os.chdir(self._jfuzz_dir)
+ return self
+
+ def __exit__(self, etype, evalue, etraceback):
+ """On exit, re-enters previously saved current directory and cleans up."""
+ os.chdir(self._save_dir)
+ shutil.rmtree(self._jfuzz_dir)
+ if self._num_divergences == 0:
+ shutil.rmtree(self._results_dir)
+
+ def Run(self):
+ """Runs JFuzz many times and report divergences."""
+ print()
+ print('**\n**** JFuzz Testing\n**')
+ print()
+ print('#Tests :', self._num_tests)
+ print('Device :', self._device)
+ print('Directory :', self._results_dir)
+ print('Exec-mode1:', self._runner1.description)
+ print('Exec-mode2:', self._runner2.description)
+ print()
+ self.ShowStats()
+ for self._test in range(1, self._num_tests + 1):
+ self.RunJFuzzTest()
+ self.ShowStats()
+ if self._num_divergences == 0:
+ print('\n\nsuccess (no divergences)\n')
+ else:
+ print('\n\nfailure (divergences)\n')
+
+ def ShowStats(self):
+ """Shows current statistics (on same line) while tester is running."""
+ print('\rTests:', self._test,
+ 'Success:', self._num_success,
+ 'Not-compiled:', self._num_not_compiled,
+ 'Not-run:', self._num_not_run,
+ 'Timed-out:', self._num_timed_out,
+ 'Divergences:', self._num_divergences,
+ end='')
+ sys.stdout.flush()
+
+ def RunJFuzzTest(self):
+ """Runs a single JFuzz test, comparing two execution modes."""
+ self.ConstructTest()
+ retc1 = self._runner1.CompileAndRunTest()
+ retc2 = self._runner2.CompileAndRunTest()
+ self.CheckForDivergence(retc1, retc2)
+ self.CleanupTest()
+
+ def ConstructTest(self):
+ """Use JFuzz to generate next Test.java test.
+
+ Raises:
+ FatalError: error when jfuzz fails
+ """
+ if RunCommand(['jfuzz'], out='Test.java', err=None) != RetCode.SUCCESS:
+ raise FatalError('Unexpected error while running JFuzz')
+
+ def CheckForDivergence(self, retc1, retc2):
+ """Checks for divergences and updates statistics.
+
+ Args:
+ retc1: int, normalized return code of first runner
+ retc2: int, normalized return code of second runner
+ """
+ if retc1 == retc2:
+ # Non-divergent in return code.
+ if retc1 == RetCode.SUCCESS:
+ # Both compilations and runs were successful, inspect generated output.
+ runner1_out = self._runner1.output_file
+ runner2_out = self._runner2.output_file
+ if not filecmp.cmp(runner1_out, runner2_out, shallow=False):
+ self.ReportDivergence(retc1, retc2, is_output_divergence=True)
+ else:
+ self._num_success += 1
+ elif retc1 == RetCode.TIMEOUT:
+ self._num_timed_out += 1
+ elif retc1 == RetCode.NOTCOMPILED:
+ self._num_not_compiled += 1
+ else:
+ self._num_not_run += 1
+ else:
+ # Divergent in return code.
+ self.ReportDivergence(retc1, retc2, is_output_divergence=False)
+
+ def GetCurrentDivergenceDir(self):
+ return self._results_dir + '/divergence' + str(self._num_divergences)
+
+ def ReportDivergence(self, retc1, retc2, is_output_divergence):
+ """Reports and saves a divergence."""
+ self._num_divergences += 1
+ print('\n' + str(self._num_divergences), end='')
+ if is_output_divergence:
+ print(' divergence in output')
+ else:
+ print(' divergence in return code: ' + retc1.name + ' vs. ' +
+ retc2.name)
+ # Save.
+ ddir = self.GetCurrentDivergenceDir()
+ os.mkdir(ddir)
+ for f in glob('*.txt') + ['Test.java']:
+ shutil.copy(f, ddir)
+ # Maybe run bisection bug search.
+ if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES:
+ self.MaybeBisectDivergence(retc1, retc2, is_output_divergence)
+
+ def RunBisectionSearch(self, args, expected_retcode, expected_output,
+ runner_id):
+ ddir = self.GetCurrentDivergenceDir()
+ outfile_path = ddir + '/' + runner_id + '_bisection_out.txt'
+ logfile_path = ddir + '/' + runner_id + '_bisection_log.txt'
+ errfile_path = ddir + '/' + runner_id + '_bisection_err.txt'
+ args = list(args) + ['--logfile', logfile_path, '--cleanup']
+ args += ['--expected-retcode', expected_retcode.name]
+ if expected_output:
+ args += ['--expected-output', expected_output]
+ bisection_search_path = os.path.join(
+ GetEnvVariableOrError('ANDROID_BUILD_TOP'),
+ 'art/tools/bisection_search/bisection_search.py')
+ if RunCommand([bisection_search_path] + args, out=outfile_path,
+ err=errfile_path, timeout=300) == RetCode.TIMEOUT:
+ print('Bisection search TIMEOUT')
+
+ def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence):
+ bisection_args1 = self._runner1.GetBisectionSearchArgs()
+ bisection_args2 = self._runner2.GetBisectionSearchArgs()
+ if is_output_divergence:
+ maybe_output1 = self._runner1.output_file
+ maybe_output2 = self._runner2.output_file
+ else:
+ maybe_output1 = maybe_output2 = None
+ if bisection_args1 is not None:
+ self.RunBisectionSearch(bisection_args1, retc2, maybe_output2,
+ self._runner1.id)
+ if bisection_args2 is not None:
+ self.RunBisectionSearch(bisection_args2, retc1, maybe_output1,
+ self._runner2.id)
+
+ def CleanupTest(self):
+ """Cleans up after a single test run."""
+ for file_name in os.listdir(self._jfuzz_dir):
+ file_path = os.path.join(self._jfuzz_dir, file_name)
+ if os.path.isfile(file_path):
+ os.unlink(file_path)
+ elif os.path.isdir(file_path):
+ shutil.rmtree(file_path)
+
+
+def main():
+ # Handle arguments.
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--num_tests', default=10000,
+ type=int, help='number of tests to run')
+ parser.add_argument('--device', help='target device serial number')
+ parser.add_argument('--mode1', default='ri',
+ help='execution mode 1 (default: ri)')
+ parser.add_argument('--mode2', default='hopt',
+ help='execution mode 2 (default: hopt)')
+ args = parser.parse_args()
+ if args.mode1 == args.mode2:
+ raise FatalError('Identical execution modes given')
+ # Run the JFuzz tester.
+ with JFuzzTester(args.num_tests, args.device,
+ args.mode1, args.mode2) as fuzzer:
+ fuzzer.Run()
+
+if __name__ == '__main__':
+ main()