scripts/native_heapdump_viewer.py

#!/usr/bin/env python
#
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Generates a human-interpretable view of a native heap dump from 'am dumpheap -n'."""

import os
import os.path
import re
import subprocess
import sys

usage = """
Usage:
1. Collect a native heap dump from the device. For example:
   $ adb shell stop
   $ adb shell setprop libc.debug.malloc.program app_process
   $ adb shell setprop libc.debug.malloc.options backtrace=64
   $ adb shell start
    (launch and use app)
   $ adb shell am dumpheap -n <pid> /data/local/tmp/native_heap.txt
   $ adb pull /data/local/tmp/native_heap.txt

2. Run the viewer:
   $ python native_heapdump_viewer.py [options] native_heap.txt
      [--verbose]: verbose output
      [--html]: interactive html output
      [--reverse]: reverse the backtraces (start the tree from the leaves)
      [--symbols SYMBOL_DIR] SYMBOL_DIR is the directory containing the .so files with symbols.
                 Defaults to $ANDROID_PRODUCT_OUT/symbols
   This outputs a file with lines of the form:

      5831776  29.09% 100.00%    10532     71b07bc0b0 /system/lib64/libandroid_runtime.so Typeface_createFromArray frameworks/base/core/jni/android/graphics/Typeface.cpp:68

   5831776 is the total number of bytes allocated at this stack frame, which
   is 29.09% of the total number of bytes allocated and 100.00% of the parent
   frame's bytes allocated. 10532 is the total number of allocations at this
   stack frame. 71b07bc0b0 is the address of the stack frame.
"""

verbose = False
html_output = False
reverse_frames = False
product_out = os.getenv("ANDROID_PRODUCT_OUT")
if product_out:
    symboldir = product_out + "/symbols"
else:
    symboldir = "./symbols"

args = sys.argv[1:]
while len(args) > 1:
    if args[0] == "--symbols":
        symboldir = args[1]
        args = args[2:]
    elif args[0] == "--verbose":
        verbose = True
        args = args[1:]
    elif args[0] == "--html":
        html_output = True
        args = args[1:]
    elif args[0] == "--reverse":
        reverse_frames = True
        args = args[1:]
    else:
        print "Invalid option "+args[0]
        break

if len(args) != 1:
    print usage
    exit(0)

native_heap = args[0]

re_map = re.compile("(?P<start>[0-9a-f]+)-(?P<end>[0-9a-f]+) .... (?P<offset>[0-9a-f]+) [0-9a-f]+:[0-9a-f]+ [0-9]+ +(?P<name>.*)")

class Backtrace:
    def __init__(self, is_zygote, size, frames):
        self.is_zygote = is_zygote
        self.size = size
        self.frames = frames

class Mapping:
    def __init__(self, start, end, offset, name):
        self.start = start
        self.end = end
        self.offset = offset
        self.name = name

class FrameDescription:
    def __init__(self, function, location, library):
        self.function = function
        self.location = location
        self.library = library


backtraces = []
mappings = []

for line in open(native_heap, "r"):
    parts = line.split()
    if len(parts) > 7 and parts[0] == "z" and parts[2] == "sz":
        is_zygote = parts[1] != "1"
        size = int(parts[3])
        frames = map(lambda x: int(x, 16), parts[7:])
        if reverse_frames:
            frames = list(reversed(frames))
        backtraces.append(Backtrace(is_zygote, size, frames))
        continue

    m = re_map.match(line)
    if m:
        start = int(m.group('start'), 16)
        end = int(m.group('end'), 16)
        offset = int(m.group('offset'), 16)
        name = m.group('name')
        mappings.append(Mapping(start, end, offset, name))
        continue

# Return the mapping that contains the given address.
# Returns None if there is no such mapping.
def find_mapping(addr):
    min = 0
    max = len(mappings) - 1
    while True:
        if max < min:
            return None
        mid = (min + max) // 2
        if mappings[mid].end <= addr:
            min = mid + 1
        elif mappings[mid].start > addr:
            max = mid - 1
        else:
            return mappings[mid]

# Resolve address libraries and offsets.
# addr_offsets maps addr to .so file offset
# addrs_by_lib maps library to list of addrs from that library
# Resolved addrs maps addr to FrameDescription
addr_offsets = {}
addrs_by_lib = {}
resolved_addrs = {}
EMPTY_FRAME_DESCRIPTION = FrameDescription("???", "???", "???")
for backtrace in backtraces:
    for addr in backtrace.frames:
        if addr in addr_offsets:
            continue
        mapping = find_mapping(addr)
        if mapping:
            addr_offsets[addr] = addr - mapping.start + mapping.offset
            if not (mapping.name in addrs_by_lib):
                addrs_by_lib[mapping.name] = []
            addrs_by_lib[mapping.name].append(addr)
        else:
            resolved_addrs[addr] = EMPTY_FRAME_DESCRIPTION


# Resolve functions and line numbers
if html_output == False:
  print "Resolving symbols using directory %s..." % symboldir
for lib in addrs_by_lib:
    sofile = symboldir + lib
    if os.path.isfile(sofile):
        file_offset = 0
        result = subprocess.check_output(["objdump", "-w", "-j", ".text", "-h", sofile])
        for line in result.split("\n"):
            splitted = line.split()
            if len(splitted) > 5 and splitted[1] == ".text":
                file_offset = int(splitted[5], 16)
                break

        input_addrs = ""
        for addr in addrs_by_lib[lib]:
            input_addrs += "%s\n" % hex(addr_offsets[addr] - file_offset)
        p = subprocess.Popen(["addr2line", "-C", "-j", ".text", "-e", sofile, "-f"], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
        result = p.communicate(input_addrs)[0]
        splitted = result.split("\n")
        for x in range(0, len(addrs_by_lib[lib])):
            function = splitted[2*x];
            location = splitted[2*x+1];
            resolved_addrs[addrs_by_lib[lib][x]] = FrameDescription(function, location, lib)

    else:
        if html_output == False:
            print "%s not found for symbol resolution" % lib
        fd = FrameDescription("???", "???", lib)
        for addr in addrs_by_lib[lib]:
            resolved_addrs[addr] = fd

def addr2line(addr):
    if addr == "ZYGOTE" or addr == "APP":
        return FrameDescription("", "", "")

    return resolved_addrs[int(addr, 16)]

class AddrInfo:
    def __init__(self, addr):
        self.addr = addr
        self.size = 0
        self.number = 0
        self.children = {}

    def addStack(self, size, stack):
        self.size += size
        self.number += 1
        if len(stack) > 0:
            child = stack[0]
            if not (child.addr in self.children):
                self.children[child.addr] = child
            self.children[child.addr].addStack(size, stack[1:])

zygote = AddrInfo("ZYGOTE")
app = AddrInfo("APP")

def display(indent, total, parent_total, node):
    fd = addr2line(node.addr)
    total_percent = 0
    if total != 0:
      total_percent = 100 * node.size / float(total)
    parent_percent = 0
    if parent_total != 0:
      parent_percent = 100 * node.size / float(parent_total)
    print "%9d %6.2f%% %6.2f%% %8d %s%s %s %s %s" % (node.size, total_percent, parent_percent, node.number, indent, node.addr, fd.library, fd.function, fd.location)
    children = sorted(node.children.values(), key=lambda x: x.size, reverse=True)
    for child in children:
        display(indent + "  ", total, node.size, child)

label_count=0
def display_html(total, node, extra):
    global label_count
    fd = addr2line(node.addr)
    if verbose:
        lib = fd.library
    else:
        lib = os.path.basename(fd.library)
    total_percent = 0
    if total != 0:
        total_percent = 100 * node.size / float(total)
    label = "%d %6.2f%% %6d %s%s %s %s" % (node.size, total_percent, node.number, extra, lib, fd.function, fd.location)
    label = label.replace("&", "&amp;")
    label = label.replace("'", "&apos;")
    label = label.replace('"', "&quot;")
    label = label.replace("<", "&lt;")
    label = label.replace(">", "&gt;")
    children = sorted(node.children.values(), key=lambda x: x.size, reverse=True)
    print '<li>'
    if len(children) > 0:
        print '<label for="' + str(label_count) + '">' + label + '</label>'
        print '<input type="checkbox" id="' + str(label_count) + '"/>'
        print '<ol>'
        label_count+=1
        for child in children:
            display_html(total, child, "")
        print '</ol>'
    else:
        print label
    print '</li>'
for backtrace in backtraces:
    stack = []
    for addr in backtrace.frames:
        stack.append(AddrInfo("%x" % addr))
    stack.reverse()
    if backtrace.is_zygote:
        zygote.addStack(backtrace.size, stack)
    else:
        app.addStack(backtrace.size, stack)

html_header = """
<!DOCTYPE html>
<html><head><style>
li input {
    display: none;
}
li input:checked + ol > li {
    display: block;
}
li input + ol > li {
    display: none;
}
li {
    font-family: Roboto Mono,monospace;
}
label {
    font-family: Roboto Mono,monospace;
    cursor: pointer
}
</style></head><body>Native allocation HTML viewer<ol>
"""
html_footer = "</ol></body></html>"

if html_output:
    print html_header
    display_html(app.size, app, "app ")
    if zygote.size>0:
        display_html(zygote.size, zygote, "zygote ")
    print html_footer
else:
    print ""
    print "%9s %6s %6s %8s    %s %s %s %s" % ("BYTES", "%TOTAL", "%PARENT", "COUNT", "ADDR", "LIBRARY", "FUNCTION", "LOCATION")
    display("", app.size, app.size + zygote.size, app)
    print ""
    display("", zygote.size, app.size + zygote.size, zygote)
    print ""