Ongoing improvements in java fuzz testing
Added --device option to runner to define target in case
multiple devices are attached
Added more cases to JavaFuzz, removed a few cases that
are likely to hang the program (such as changing loop
variables) since throughput is more important for fuzzing
Test: run_java_fuzz_test.py
BUG=30610121
Change-Id: I0cd6f5d3b7fb35ddb91d91c0c0c00afaf83218d8
diff --git a/tools/javafuzz/README.md b/tools/javafuzz/README.md
index 35c057c..68fc171 100644
--- a/tools/javafuzz/README.md
+++ b/tools/javafuzz/README.md
@@ -8,7 +8,7 @@
or using various target architectures. Any difference between the outputs
(**divergence**) may indicate a bug in one of the execution modes.
-JavaFuzz can be combined with dexfuzz to get multilayered fuzz testing.
+JavaFuzz can be combined with dexfuzz to get multi-layered fuzz testing.
How to run JavaFuzz
===================
@@ -40,19 +40,20 @@
===============================
run_java_fuzz_test.py [--num_tests]
+ [--device]
[--mode1=mode] [--mode2=mode]
where
- --num_tests: number of tests to run (10000 by default)
- --mode1:m1
- --mode2:m2
- with m1 != m2, and one of
- ri : reference implementation on host (default for m1)
- hint : Art interpreter on host
- hopt : Art optimizing on host (default for m2)
- tint : Art interpreter on target
- topt : Art optimizing on target
+ --num_tests : number of tests to run (10000 by default)
+ --device : target device serial number (passed to adb -s)
+ --mode1 : m1
+ --mode2 : m2, with m1 != m2, and values one of
+ ri = reference implementation on host (default for m1)
+ hint = Art interpreter on host
+ hopt = Art optimizing on host (default for m2)
+ tint = Art interpreter on target
+ topt = Art optimizing on target
Background
==========
@@ -67,14 +68,15 @@
and flaws still linger in the system.
Over the years, fuzz testing has gained popularity as a testing technique for
-discovering such lingering bugs, including bugs that can bring down a system in
-an unexpected way. Fuzzing refers to feeding a large amount of random data as
-input to a system in an attempt to find bugs or make it crash. Mutation-based
-fuzz testing is a special form of fuzzing that applies small random changes to
-existing inputs in order to detect shortcomings in a system. Profile-guided or
-coverage-guided fuzzing adds a direction to the way these random changes are
-applied. Multilayer approaches generate random inputs that are subsequently
-mutated at various stages of execution.
+discovering such lingering bugs, including bugs that can bring down a system
+in an unexpected way. Fuzzing refers to feeding a large amount of random data
+as input to a system in an attempt to find bugs or make it crash. Generation-
+based fuzz testing constructs random, but properly formatted input data.
+Mutation-based fuzz testing applies small random changes to existing inputs
+in order to detect shortcomings in a system. Profile-guided or coverage-guided
+fuzzing adds a direction to the way these random changes are applied. Multi-
+layered approaches generate random inputs that are subsequently mutated at
+various stages of execution.
The randomness of fuzz testing implies that the size and scope of testing is no
longer bounded. Every new run can potentially discover bugs and crashes that were
diff --git a/tools/javafuzz/javafuzz.cc b/tools/javafuzz/javafuzz.cc
index 4e6e978..161ae0a 100644
--- a/tools/javafuzz/javafuzz.cc
+++ b/tools/javafuzz/javafuzz.cc
@@ -53,7 +53,9 @@
* to preserve the property that a given version of JavaFuzz yields the same
* fuzzed Java program for a deterministic random seed.
*/
-const char* VERSION = "1.0";
+const char* VERSION = "1.1";
+
+static const uint32_t MAX_DIMS[11] = { 0, 1000, 32, 10, 6, 4, 3, 3, 2, 2, 2 };
/**
* A class that generates a random Java program that compiles correctly. The program
@@ -83,8 +85,8 @@
fuzz_loop_nest_(loop_nest),
return_type_(randomType()),
array_type_(randomType()),
- array_dim_(random1(3)),
- array_size_(random1(10)),
+ array_dim_(random1(10)),
+ array_size_(random1(MAX_DIMS[array_dim_])),
indentation_(0),
expr_depth_(0),
stmt_length_(0),
@@ -169,7 +171,7 @@
// Emit an unary operator (same type in-out).
void emitUnaryOp(Type tp) {
if (tp == kBoolean) {
- fputs("!", out_);
+ fputc('!', out_);
} else if (isInteger(tp)) {
EMIT(kIntUnaryOps);
} else { // isFP(tp)
@@ -239,16 +241,21 @@
case 6: fputs("(long)(int)(long)", out_); return kLong;
}
} else if (tp == kFloat) {
- switch (random1(3)) {
+ switch (random1(4)) {
case 1: fputs("(float)", out_); return kInt;
case 2: fputs("(float)", out_); return kLong;
case 3: fputs("(float)", out_); return kDouble;
+ // Narrowing-widening.
+ case 4: fputs("(float)(int)(float)", out_); return kFloat;
}
} else if (tp == kDouble) {
- switch (random1(3)) {
+ switch (random1(5)) {
case 1: fputs("(double)", out_); return kInt;
case 2: fputs("(double)", out_); return kLong;
case 3: fputs("(double)", out_); return kFloat;
+ // Narrowing-widening.
+ case 4: fputs("(double)(int)(double)", out_); return kDouble;
+ case 5: fputs("(double)(float)(double)", out_); return kDouble;
}
}
return tp; // nothing suitable, just keep type
@@ -273,15 +280,17 @@
// Emit an unary intrinsic (out type given, new suitable in type picked).
Type emitIntrinsic1(Type tp) {
if (tp == kBoolean) {
- switch (random1(4)) {
+ switch (random1(6)) {
case 1: fputs("Float.isNaN", out_); return kFloat;
- case 2: fputs("Float.isInfinite", out_); return kFloat;
- case 3: fputs("Double.isNaN", out_); return kDouble;
- case 4: fputs("Double.isInfinite", out_); return kDouble;
+ case 2: fputs("Float.isFinite", out_); return kFloat;
+ case 3: fputs("Float.isInfinite", out_); return kFloat;
+ case 4: fputs("Double.isNaN", out_); return kDouble;
+ case 5: fputs("Double.isFinite", out_); return kDouble;
+ case 6: fputs("Double.isInfinite", out_); return kDouble;
}
} else if (isInteger(tp)) {
const char* prefix = tp == kLong ? "Long" : "Integer";
- switch (random1(9)) {
+ switch (random1(13)) {
case 1: fprintf(out_, "%s.highestOneBit", prefix); break;
case 2: fprintf(out_, "%s.lowestOneBit", prefix); break;
case 3: fprintf(out_, "%s.numberOfLeadingZeros", prefix); break;
@@ -290,15 +299,27 @@
case 6: fprintf(out_, "%s.signum", prefix); break;
case 7: fprintf(out_, "%s.reverse", prefix); break;
case 8: fprintf(out_, "%s.reverseBytes", prefix); break;
- case 9: fputs("Math.abs", out_); break;
+ case 9: fputs("Math.incrementExact", out_); break;
+ case 10: fputs("Math.decrementExact", out_); break;
+ case 11: fputs("Math.negateExact", out_); break;
+ case 12: fputs("Math.abs", out_); break;
+ case 13: fputs("Math.round", out_);
+ return tp == kLong ? kDouble : kFloat;
}
} else { // isFP(tp)
- switch (random1(5)) {
+ switch (random1(6)) {
case 1: fputs("Math.abs", out_); break;
case 2: fputs("Math.ulp", out_); break;
case 3: fputs("Math.signum", out_); break;
case 4: fputs("Math.nextUp", out_); break;
case 5: fputs("Math.nextDown", out_); break;
+ case 6: if (tp == kDouble) {
+ fputs("Double.longBitsToDouble", out_);
+ return kLong;
+ } else {
+ fputs("Float.intBitsToFloat", out_);
+ return kInt;
+ }
}
}
return tp; // same type in-out
@@ -314,15 +335,27 @@
}
} else if (isInteger(tp)) {
const char* prefix = tp == kLong ? "Long" : "Integer";
- switch (random1(3)) {
+ switch (random1(11)) {
case 1: fprintf(out_, "%s.compare", prefix); break;
- case 2: fputs("Math.min", out_); break;
- case 3: fputs("Math.max", out_); break;
+ case 2: fprintf(out_, "%s.sum", prefix); break;
+ case 3: fprintf(out_, "%s.min", prefix); break;
+ case 4: fprintf(out_, "%s.max", prefix); break;
+ case 5: fputs("Math.min", out_); break;
+ case 6: fputs("Math.max", out_); break;
+ case 7: fputs("Math.floorDiv", out_); break;
+ case 8: fputs("Math.floorMod", out_); break;
+ case 9: fputs("Math.addExact", out_); break;
+ case 10: fputs("Math.subtractExact", out_); break;
+ case 11: fputs("Math.multiplyExact", out_); break;
}
} else { // isFP(tp)
- switch (random1(2)) {
- case 1: fputs("Math.min", out_); break;
- case 2: fputs("Math.max", out_); break;
+ const char* prefix = tp == kDouble ? "Double" : "Float";
+ switch (random1(5)) {
+ case 1: fprintf(out_, "%s.sum", prefix); break;
+ case 2: fprintf(out_, "%s.min", prefix); break;
+ case 3: fprintf(out_, "%s.max", prefix); break;
+ case 4: fputs("Math.min", out_); break;
+ case 5: fputs("Math.max", out_); break;
}
}
return tp; // same type in-out
@@ -358,12 +391,24 @@
// Emit miscellaneous constructs.
void emitMisc(Type tp) {
- switch (tp) {
- case kBoolean: fputs("this instanceof Test", out_); break;
- case kInt: fputs("mArray.length", out_); break;
- case kLong: fputs("Long.MAX_VALUE", out_); break;
- case kFloat: fputs("Float.MAX_VALUE", out_); break;
- case kDouble: fputs("Double.MAX_VALUE", out_); break;
+ if (tp == kBoolean) {
+ fputs("this instanceof Test", out_);
+ } else if (isInteger(tp)) {
+ const char* prefix = tp == kLong ? "Long" : "Integer";
+ switch (random1(2)) {
+ case 1: fprintf(out_, "%s.MIN_VALUE", prefix); break;
+ case 2: fprintf(out_, "%s.MAX_VALUE", prefix); break;
+ }
+ } else { // isFP(tp)
+ const char* prefix = tp == kDouble ? "Double" : "Float";
+ switch (random1(6)) {
+ case 1: fprintf(out_, "%s.MIN_NORMAL", prefix); break;
+ case 2: fprintf(out_, "%s.MIN_VALUE", prefix); break;
+ case 3: fprintf(out_, "%s.MAX_VALUE", prefix); break;
+ case 4: fprintf(out_, "%s.POSITIVE_INFINITY", prefix); break;
+ case 5: fprintf(out_, "%s.NEGATIVE_INFINITY", prefix); break;
+ case 6: fprintf(out_, "%s.NaN", prefix); break;
+ }
}
}
@@ -412,10 +457,10 @@
void emitLiteral(Type tp) {
switch (tp) {
case kBoolean: fputs(random1(2) == 1 ? "true" : "false", out_); break;
- case kInt: fprintf(out_, "%d", random0(100)); break;
- case kLong: fprintf(out_, "%dL", random0(100)); break;
- case kFloat: fprintf(out_, "%d.0f", random0(100)); break;
- case kDouble: fprintf(out_, "%d.0", random0(100)); break;
+ case kInt: fprintf(out_, "%d", random()); break;
+ case kLong: fprintf(out_, "%dL", random()); break;
+ case kFloat: fprintf(out_, "%d.0f", random()); break;
+ case kDouble: fprintf(out_, "%d.0", random()); break;
}
}
@@ -433,17 +478,6 @@
return false;
}
- // Emit a loop variable, if available.
- bool emitLoopVariable(Type tp) {
- if (tp == kInt) {
- if (loop_nest_ > 0) {
- fprintf(out_, "i%u", random0(loop_nest_));
- return true;
- }
- }
- return false;
- }
-
// Emit a local variable, if available.
bool emitLocalVariable(Type tp) {
uint32_t locals = adjustLocal(tp, 0);
@@ -483,10 +517,6 @@
if (emitLocalVariable(tp))
return;
// FALL-THROUGH
- case 3:
- if (emitLoopVariable(tp))
- return;
- // FALL-THROUGH
default:
emitFieldVariable(tp);
break;
@@ -510,8 +540,9 @@
fputc('(', out_);
switch (random1(12)) { // favor binary operations
case 1:
- // Unary operator: ~x
+ // Unary operator: ~ x
emitUnaryOp(tp);
+ fputc(' ', out_);
emitExpression(tp);
break;
case 2:
@@ -761,7 +792,7 @@
bool mayFollow = false;
fputs("switch (", out_);
- emitExpression(kInt);
+ emitArrayIndex(); // restrict its range
fputs(") {\n", out_);
++if_nest_;
@@ -771,7 +802,7 @@
for (uint32_t i = 0; i < 2; i++) {
emitIndentation();
if (i == 0) {
- fprintf(out_, "case %d: {\n", random0(100));
+ fprintf(out_, "case %u: {\n", random0(array_size_));
} else {
fprintf(out_, "default: {\n");
}
@@ -977,6 +1008,11 @@
// Random integers.
//
+ // Return random integer.
+ int32_t random() {
+ return fuzz_random_engine_();
+ }
+
// Return random integer in range [0,max).
uint32_t random0(uint32_t max) {
std::uniform_int_distribution<uint32_t> gen(0, max - 1);
@@ -1025,7 +1061,7 @@
// Defaults.
uint32_t seed = time(NULL);
uint32_t expr_depth = 1;
- uint32_t stmt_length = 4;
+ uint32_t stmt_length = 8;
uint32_t if_nest = 2;
uint32_t loop_nest = 3;
diff --git a/tools/javafuzz/run_java_fuzz_test.py b/tools/javafuzz/run_java_fuzz_test.py
index 4f192e7..5f527b8 100755
--- a/tools/javafuzz/run_java_fuzz_test.py
+++ b/tools/javafuzz/run_java_fuzz_test.py
@@ -78,10 +78,11 @@
return libdir + '/core-libart-hostdex_intermediates/classes.jack:' \
+ libdir + '/core-oj-hostdex_intermediates/classes.jack'
-def GetExecutionModeRunner(mode):
+def GetExecutionModeRunner(device, mode):
"""Returns a runner for the given execution mode.
Args:
+ device: string, target device serial number (or None)
mode: string, execution mode
Returns:
TestRunner with given execution mode
@@ -95,9 +96,9 @@
if mode == 'hopt':
return TestRunnerArtOnHost(False)
if mode == 'tint':
- return TestRunnerArtOnTarget(True)
+ return TestRunnerArtOnTarget(device, True)
if mode == 'topt':
- return TestRunnerArtOnTarget(False)
+ return TestRunnerArtOnTarget(device, False)
raise FatalError('Unknown execution mode')
def GetReturnCode(retc):
@@ -210,13 +211,14 @@
class TestRunnerArtOnTarget(TestRunner):
"""Concrete test runner of Art on target (interpreter or optimizing)."""
- def __init__(self, interpreter):
+ def __init__(self, device, interpreter):
"""Constructor for the Art on target tester.
Args:
+ device: string, target device serial number (or None)
interpreter: boolean, selects between interpreter or optimizing
"""
- self._dalvik_args = '-cp /data/local/tmp/classes.dex Test'
+ self._dalvik_args = 'shell dalvikvm -cp /data/local/tmp/classes.dex Test'
if interpreter:
self._description = 'Art interpreter on target'
self._id = 'TInt'
@@ -224,16 +226,19 @@
else:
self._description = 'Art optimizing on target'
self._id = 'TOpt'
+ self._adb = 'adb'
+ if device != None:
+ self._adb = self._adb + ' -s ' + device
self._jack_args = '-cp ' + GetJackClassPath() + ' --output-dex . Test.java'
def CompileAndRunTest(self):
if RunCommand('jack', self._jack_args,
out=None, err='jackerr.txt', timeout=30) == EXIT_SUCCESS:
- if RunCommand('adb push', 'classes.dex /data/local/tmp/',
+ if RunCommand(self._adb, 'push classes.dex /data/local/tmp/',
'adb.txt', err=None) != EXIT_SUCCESS:
raise FatalError('Cannot push to target device')
out = self.GetId() + '_run_out.txt'
- retc = RunCommand('adb shell dalvikvm', self._dalvik_args, out, err=None)
+ retc = RunCommand(self._adb, self._dalvik_args, out, err=None)
if retc != EXIT_SUCCESS and retc != EXIT_TIMEOUT:
retc = EXIT_NOTRUN
else:
@@ -241,7 +246,7 @@
# Cleanup and return.
RunCommand('rm', '-f classes.dex jackerr.txt adb.txt',
out=None, err=None)
- RunCommand('adb shell', 'rm -f /data/local/tmp/classes.dex',
+ RunCommand(self._adb, 'shell rm -f /data/local/tmp/classes.dex',
out=None, err=None)
return retc
@@ -256,17 +261,19 @@
class JavaFuzzTester(object):
"""Tester that runs JavaFuzz many times and report divergences."""
- def __init__(self, num_tests, mode1, mode2):
+ def __init__(self, num_tests, device, mode1, mode2):
"""Constructor for the tester.
Args:
num_tests: int, number of tests to run
+ device: string, target device serial number (or None)
mode1: string, execution mode for first runner
mode2: string, execution mode for second runner
"""
self._num_tests = num_tests
- self._runner1 = GetExecutionModeRunner(mode1)
- self._runner2 = GetExecutionModeRunner(mode2)
+ self._device = device
+ self._runner1 = GetExecutionModeRunner(device, mode1)
+ self._runner2 = GetExecutionModeRunner(device, mode2)
self._save_dir = None
self._tmp_dir = None
# Statistics.
@@ -302,6 +309,7 @@
print '**\n**** JavaFuzz Testing\n**'
print
print '#Tests :', self._num_tests
+ print 'Device :', self._device
print 'Directory :', self._tmp_dir
print 'Exec-mode1:', self._runner1.GetDescription()
print 'Exec-mode2:', self._runner2.GetDescription()
@@ -391,6 +399,7 @@
parser = argparse.ArgumentParser()
parser.add_argument('--num_tests', default=10000,
type=int, help='number of tests to run')
+ parser.add_argument('--device', help='target device serial number')
parser.add_argument('--mode1', default='ri',
help='execution mode 1 (default: ri)')
parser.add_argument('--mode2', default='hopt',
@@ -399,7 +408,8 @@
if args.mode1 == args.mode2:
raise FatalError("Identical execution modes given")
# Run the JavaFuzz tester.
- with JavaFuzzTester(args.num_tests, args.mode1, args.mode2) as fuzzer:
+ with JavaFuzzTester(args.num_tests, args.device,
+ args.mode1, args.mode2) as fuzzer:
fuzzer.Run()
if __name__ == "__main__":