simpleperf/scripts/annotate.py

#!/usr/bin/env python
#
# Copyright (C) 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""annotate.py: annotate source files based on perf.data.
"""


import argparse
import os
import os.path
import shutil
import subprocess
import sys

from simpleperf_report_lib import *
from utils import *

class SourceLine(object):
    def __init__(self, file, function, line):
        self.file = file
        self.function = function
        self.line = line

    @property
    def file_key(self):
        return self.file

    @property
    def function_key(self):
        return (self.file, self.function)

    @property
    def line_key(self):
        return (self.file, self.line)


# TODO: using addr2line can't convert from function_start_address to
# source_file:line very well for java code. Because in .debug_line section,
# there is some distance between function_start_address and the address
# of the first instruction which can be mapped to source line.
class Addr2Line(object):
    """collect information of how to map [dso_name,vaddr] to [source_file:line].
    """
    def __init__(self, addr2line_path, symfs_dir=None):
        self.dso_dict = dict()
        self.addr2line_path = addr2line_path
        self.symfs_dir = symfs_dir


    def add_addr(self, dso_name, addr):
        dso = self.dso_dict.get(dso_name)
        if dso is None:
            self.dso_dict[dso_name] = dso = dict()
        if addr not in dso:
            dso[addr] = None


    def convert_addrs_to_lines(self):
        # store a list of source files
        self.file_list = []
        # map from file to id with file_list[id] == file
        self.file_dict = {}
        self.file_list.append('')
        self.file_dict[''] = 0

        for dso_name in self.dso_dict.keys():
            self._convert_addrs_to_lines(dso_name, self.dso_dict[dso_name])
        self._combine_source_files()


    def _convert_addrs_to_lines(self, dso_name, dso):
        dso_path = self._find_dso_path(dso_name)
        if dso_path is None:
            log_warning("can't find dso '%s'" % dso_name)
            dso.clear()
            return
        addrs = sorted(dso.keys())
        addr_str = []
        for addr in addrs:
            addr_str.append('0x%x' % addr)
        addr_str = '\n'.join(addr_str)
        subproc = subprocess.Popen([self.addr2line_path, '-e', dso_path, '-aifC'],
                                   stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        (stdoutdata, _) = subproc.communicate(str_to_bytes(addr_str))
        stdoutdata = bytes_to_str(stdoutdata)
        stdoutdata = stdoutdata.strip().split('\n')
        if len(stdoutdata) < len(addrs):
            log_fatal("addr2line didn't output enough lines")
        addr_pos = 0
        out_pos = 0
        while addr_pos < len(addrs) and out_pos < len(stdoutdata):
            addr_line = stdoutdata[out_pos]
            out_pos += 1
            assert addr_line[:2] == "0x"
            assert out_pos < len(stdoutdata)
            source_lines = []
            while out_pos < len(stdoutdata) and stdoutdata[out_pos][:2] != "0x":
                function = stdoutdata[out_pos]
                out_pos += 1
                assert out_pos < len(stdoutdata)
                # Handle lines like "C:\Users\...\file:32".
                items = stdoutdata[out_pos].rsplit(':', 1)
                if len(items) != 2:
                    continue
                (file, line) = items
                line = line.split()[0]  # Remove comments after line number
                out_pos += 1
                if file.find('?') != -1:
                    file = 0
                else:
                    file = self._get_file_id(file)
                if line.find('?') != -1:
                    line = 0
                else:
                    line = int(line)
                source_lines.append(SourceLine(file, function, line))
            dso[addrs[addr_pos]] = source_lines
            addr_pos += 1
        assert addr_pos == len(addrs)


    def _get_file_id(self, file):
        id = self.file_dict.get(file)
        if id is None:
            id = len(self.file_list)
            self.file_list.append(file)
            self.file_dict[file] = id
        return id

    def _combine_source_files(self):
        """It is possible that addr2line gives us different names for the same
           file, like:
            /usr/local/.../src/main/jni/sudo-game-jni.cpp
            sudo-game-jni.cpp
           We'd better combine these two files. We can do it by combining
           source files with no conflicts in path.
        """
        # Collect files having the same filename.
        filename_dict = dict()
        for file in self.file_list:
            index = max(file.rfind('/'), file.rfind(os.sep))
            filename = file[index+1:]
            entry = filename_dict.get(filename)
            if entry is None:
                filename_dict[filename] = entry = []
            entry.append(file)

        # Combine files having the same filename and having no conflicts in path.
        for filename in filename_dict.keys():
            files = filename_dict[filename]
            if len(files) == 1:
                continue
            for file in files:
                to_file = file
                # Test if we can merge files[i] with another file having longer
                # path.
                for f in files:
                    if len(f) > len(to_file) and f.find(file) != -1:
                        to_file = f
                if to_file != file:
                    from_id = self.file_dict[file]
                    to_id = self.file_dict[to_file]
                    self.file_list[from_id] = self.file_list[to_id]


    def get_sources(self, dso_name, addr):
        dso = self.dso_dict.get(dso_name)
        if dso is None:
            return []
        item = dso.get(addr, [])
        source_lines = []
        for source in item:
            source_lines.append(SourceLine(self.file_list[source.file],
                                           source.function, source.line))
        return source_lines


    def _find_dso_path(self, dso):
        if dso[0] != '/' or dso == '//anon':
            return None
        if self.symfs_dir:
            dso_path = os.path.join(self.symfs_dir, dso[1:])
            if os.path.isfile(dso_path):
                return dso_path
        if os.path.isfile(dso):
            return dso
        return None


class Period(object):
    """event count information. It can be used to represent event count
       of a line, a function, a source file, or a binary. It contains two
       parts: period and acc_period.
       When used for a line, period is the event count occurred when running
       that line, acc_period is the accumulated event count occurred when
       running that line and functions called by that line. Same thing applies
       when it is used for a function, a source file, or a binary.
    """
    def __init__(self, period=0, acc_period=0):
        self.period = period
        self.acc_period = acc_period


    def __iadd__(self, other):
        self.period += other.period
        self.acc_period += other.acc_period
        return self


class DsoPeriod(object):
    """Period for each shared library"""
    def __init__(self, dso_name):
        self.dso_name = dso_name
        self.period = Period()


    def add_period(self, period):
        self.period += period


class FilePeriod(object):
    """Period for each source file"""
    def __init__(self, file):
        self.file = file
        self.period = Period()
        # Period for each line in the file.
        self.line_dict = {}
        # Period for each function in the source file.
        self.function_dict = {}


    def add_period(self, period):
        self.period += period


    def add_line_period(self, line, period):
        a = self.line_dict.get(line)
        if a is None:
            self.line_dict[line] = a = Period()
        a += period


    def add_function_period(self, function_name, function_start_line, period):
        a = self.function_dict.get(function_name)
        if not a:
            if function_start_line is None:
                function_start_line = -1
            self.function_dict[function_name] = a = [function_start_line, Period()]
        a[1] += period


class SourceFileAnnotator(object):
    """group code for annotating source files"""
    def __init__(self, config):
        # check config variables
        config_names = ['perf_data_list', 'symfs_dir', 'source_dirs',
                        'annotate_dest_dir', 'comm_filters', 'pid_filters',
                        'tid_filters', 'dso_filters', 'addr2line_path']
        for name in config_names:
            if name not in config:
                log_fatal('config [%s] is missing' % name)
        symfs_dir = config['symfs_dir']
        if symfs_dir and not os.path.isdir(symfs_dir):
            log_fatal('[symfs_dir] "%s" is not a dir' % symfs_dir)
        kallsyms = config['kallsyms']
        if kallsyms and not os.path.isfile(kallsyms):
            log_fatal('[kallsyms] "%s" is not a file' % kallsyms)
        source_dirs = config['source_dirs']
        for dir in source_dirs:
            if not os.path.isdir(dir):
                log_fatal('[source_dirs] "%s" is not a dir' % dir)

        # init member variables
        self.config = config
        self.symfs_dir = config.get('symfs_dir')
        self.kallsyms = config.get('kallsyms')
        self.comm_filter = set(config['comm_filters']) if config.get('comm_filters') else None
        if config.get('pid_filters'):
            self.pid_filter = {int(x) for x in config['pid_filters']}
        else:
            self.pid_filter = None
        if config.get('tid_filters'):
            self.tid_filter = {int(x) for x in config['tid_filters']}
        else:
            self.tid_filter = None
        self.dso_filter = set(config['dso_filters']) if config.get('dso_filters') else None

        output_dir = config['annotate_dest_dir']
        if os.path.isdir(output_dir):
            shutil.rmtree(output_dir)
        os.makedirs(output_dir)

        self.addr2line = Addr2Line(self.config['addr2line_path'], symfs_dir)


    def annotate(self):
        self._collect_addrs()
        self._convert_addrs_to_lines()
        self._generate_periods()
        self._write_summary()
        self._collect_source_files()
        self._annotate_files()


    def _collect_addrs(self):
        """Read perf.data, collect all addresses we need to convert to
           source file:line.
        """
        for perf_data in self.config['perf_data_list']:
            lib = ReportLib()
            lib.SetRecordFile(perf_data)
            if self.symfs_dir:
                lib.SetSymfs(self.symfs_dir)
            if self.kallsyms:
                lib.SetKallsymsFile(self.kallsyms)
            while True:
                sample = lib.GetNextSample()
                if sample is None:
                    lib.Close()
                    break
                if not self._filter_sample(sample):
                    continue
                symbols = []
                symbols.append(lib.GetSymbolOfCurrentSample())
                callchain = lib.GetCallChainOfCurrentSample()
                for i in range(callchain.nr):
                    symbols.append(callchain.entries[i].symbol)
                for symbol in symbols:
                    if self._filter_symbol(symbol):
                        self.addr2line.add_addr(symbol.dso_name, symbol.vaddr_in_file)
                        self.addr2line.add_addr(symbol.dso_name, symbol.symbol_addr)


    def _filter_sample(self, sample):
        """Return true if the sample can be used."""
        if self.comm_filter:
            if sample.thread_comm not in self.comm_filter:
                return False
        if self.pid_filter:
            if sample.pid not in self.pid_filter:
                return False
        if self.tid_filter:
            if sample.tid not in self.tid_filter:
                return False
        return True


    def _filter_symbol(self, symbol):
        if not self.dso_filter or symbol.dso_name in self.dso_filter:
            return True
        return False


    def _convert_addrs_to_lines(self):
        self.addr2line.convert_addrs_to_lines()


    def _generate_periods(self):
        """read perf.data, collect Period for all types:
            binaries, source files, functions, lines.
        """
        self.period = 0
        self.dso_periods = dict()
        self.file_periods = dict()
        for perf_data in self.config['perf_data_list']:
            lib = ReportLib()
            lib.SetRecordFile(perf_data)
            if self.symfs_dir:
                lib.SetSymfs(self.symfs_dir)
            if self.kallsyms:
                lib.SetKallsymsFile(self.kallsyms)
            while True:
                sample = lib.GetNextSample()
                if sample is None:
                    lib.Close()
                    break
                if not self._filter_sample(sample):
                    continue
                symbols = []
                symbols.append(lib.GetSymbolOfCurrentSample())
                callchain = lib.GetCallChainOfCurrentSample()
                for i in range(callchain.nr):
                    symbols.append(callchain.entries[i].symbol)
                # Each sample has a callchain, but its period is only used once
                # to add period for each function/source_line/source_file/binary.
                # For example, if more than one entry in the callchain hits a
                # function, the event count of that function is only increased once.
                # Otherwise, we may get periods > 100%.
                is_sample_used = False
                used_dso_dict = dict()
                used_file_dict = dict()
                used_function_dict = dict()
                used_line_dict = dict()
                period = Period(sample.period, sample.period)
                for i in range(len(symbols)):
                    symbol = symbols[i]
                    if i == 1:
                        period = Period(0, sample.period)
                    if not self._filter_symbol(symbol):
                        continue
                    is_sample_used = True
                    # Add period to dso.
                    self._add_dso_period(symbol.dso_name, period, used_dso_dict)
                    # Add period to source file.
                    sources = self.addr2line.get_sources(symbol.dso_name, symbol.vaddr_in_file)
                    for source in sources:
                        if source.file:
                            self._add_file_period(source, period, used_file_dict)
                            # Add period to line.
                            if source.line:
                                self._add_line_period(source, period, used_line_dict)
                    # Add period to function.
                    sources = self.addr2line.get_sources(symbol.dso_name, symbol.symbol_addr)
                    for source in sources:
                        if source.file:
                            self._add_file_period(source, period, used_file_dict)
                            if source.function:
                                self._add_function_period(source, period, used_function_dict)

                if is_sample_used:
                    self.period += sample.period


    def _add_dso_period(self, dso_name, period, used_dso_dict):
        if dso_name not in used_dso_dict:
            used_dso_dict[dso_name] = True
            dso_period = self.dso_periods.get(dso_name)
            if dso_period is None:
                dso_period = self.dso_periods[dso_name] = DsoPeriod(dso_name)
            dso_period.add_period(period)


    def _add_file_period(self, source, period, used_file_dict):
        if source.file_key not in used_file_dict:
            used_file_dict[source.file_key] = True
            file_period = self.file_periods.get(source.file)
            if file_period is None:
                file_period = self.file_periods[source.file] = FilePeriod(source.file)
            file_period.add_period(period)


    def _add_line_period(self, source, period, used_line_dict):
        if source.line_key not in used_line_dict:
            used_line_dict[source.line_key] = True
            file_period = self.file_periods[source.file]
            file_period.add_line_period(source.line, period)


    def _add_function_period(self, source, period, used_function_dict):
        if source.function_key not in used_function_dict:
            used_function_dict[source.function_key] = True
            file_period = self.file_periods[source.file]
            file_period.add_function_period(source.function, source.line, period)


    def _write_summary(self):
        summary = os.path.join(self.config['annotate_dest_dir'], 'summary')
        with open(summary, 'w') as f:
            f.write('total period: %d\n\n' % self.period)
            dso_periods = sorted(self.dso_periods.values(),
                                 key=lambda x: x.period.acc_period, reverse=True)
            for dso_period in dso_periods:
                f.write('dso %s: %s\n' % (dso_period.dso_name,
                                          self._get_percentage_str(dso_period.period)))
            f.write('\n')

            file_periods = sorted(self.file_periods.values(),
                                  key=lambda x: x.period.acc_period, reverse=True)
            for file_period in file_periods:
                f.write('file %s: %s\n' % (file_period.file,
                                           self._get_percentage_str(file_period.period)))
            for file_period in file_periods:
                f.write('\n\n%s: %s\n' % (file_period.file,
                                          self._get_percentage_str(file_period.period)))
                values = []
                for func_name in file_period.function_dict.keys():
                    func_start_line, period = file_period.function_dict[func_name]
                    values.append((func_name, func_start_line, period))
                values = sorted(values, key=lambda x: x[2].acc_period, reverse=True)
                for value in values:
                    f.write('\tfunction (%s): line %d, %s\n' % (
                        value[0], value[1], self._get_percentage_str(value[2])))
                f.write('\n')
                for line in sorted(file_period.line_dict.keys()):
                    f.write('\tline %d: %s\n' % (
                        line, self._get_percentage_str(file_period.line_dict[line])))


    def _get_percentage_str(self, period, short=False):
        s = 'acc_p: %f%%, p: %f%%' if short else 'accumulated_period: %f%%, period: %f%%'
        return s % self._get_percentage(period)


    def _get_percentage(self, period):
        if self.period == 0:
            return (0, 0)
        acc_p = 100.0 * period.acc_period / self.period
        p = 100.0 * period.period / self.period
        return (acc_p, p)


    def _collect_source_files(self):
        self.source_file_dict = dict()
        source_file_suffix = ['h', 'c', 'cpp', 'cc', 'java']
        for source_dir in self.config['source_dirs']:
            for root, _, files in os.walk(source_dir):
                for file in files:
                    if file[file.rfind('.')+1:] in source_file_suffix:
                        entry = self.source_file_dict.get(file)
                        if entry is None:
                            entry = self.source_file_dict[file] = []
                        entry.append(os.path.join(root, file))


    def _find_source_file(self, file):
        filename = file[file.rfind(os.sep)+1:]
        source_files = self.source_file_dict.get(filename)
        if source_files is None:
            return None
        match_count = 0
        result = None
        for path in source_files:
            if path.find(file) != -1:
                match_count += 1
                result = path
        if match_count > 1:
            log_warning('multiple source for %s, select %s' % (file, result))
        return result


    def _annotate_files(self):
        """Annotate Source files: add acc_period/period for each source file.
           1. Annotate java source files, which have $JAVA_SRC_ROOT prefix.
           2. Annotate c++ source files.
        """
        dest_dir = self.config['annotate_dest_dir']
        for key in self.file_periods.keys():
            is_java = False
            if key.startswith('$JAVA_SRC_ROOT/'):
                path = key[len('$JAVA_SRC_ROOT/'):]
                items = path.split('/')
                path = os.sep.join(items)
                from_path = self._find_source_file(path)
                to_path = os.path.join(dest_dir, 'java', path)
                is_java = True
            elif key.startswith('/') and os.path.isfile(key):
                path = key
                from_path = path
                to_path = os.path.join(dest_dir, path[1:])
            elif is_windows() and key.find(':\\') != -1 and os.path.isfile(key):
                from_path = key
                to_path = os.path.join(dest_dir, key.replace(':\\', '\\'))
            else:
                path = key[1:] if key.startswith('/') else key
                # Change path on device to path on host
                path = os.sep.join(path.split('/'))
                from_path = self._find_source_file(path)
                to_path = os.path.join(dest_dir, path)
            if from_path is None:
                log_warning("can't find source file for path %s" % key)
                continue
            self._annotate_file(from_path, to_path, self.file_periods[key], is_java)


    def _annotate_file(self, from_path, to_path, file_period, is_java):
        """Annotate a source file.

        Annotate a source file in three steps:
          1. In the first line, show periods of this file.
          2. For each function, show periods of this function.
          3. For each line not hitting the same line as functions, show
             line periods.
        """
        log_info('annotate file %s' % from_path)
        with open(from_path, 'r') as rf:
            lines = rf.readlines()

        annotates = dict()
        for line in file_period.line_dict.keys():
            annotates[line] = self._get_percentage_str(file_period.line_dict[line], True)
        for func_name in file_period.function_dict.keys():
            func_start_line, period = file_period.function_dict[func_name]
            if func_start_line == -1:
                continue
            line = func_start_line - 1 if is_java else func_start_line
            annotates[line] = '[func] ' + self._get_percentage_str(period, True)
        annotates[1] = '[file] ' + self._get_percentage_str(file_period.period, True)

        max_annotate_cols = 0
        for key in annotates.keys():
            max_annotate_cols = max(max_annotate_cols, len(annotates[key]))

        empty_annotate = ' ' * (max_annotate_cols + 6)

        dirname = os.path.dirname(to_path)
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        with open(to_path, 'w') as wf:
            for line in range(1, len(lines) + 1):
                annotate = annotates.get(line)
                if annotate is None:
                    if not lines[line-1].strip():
                        annotate = ''
                    else:
                        annotate = empty_annotate
                else:
                    annotate = '/* ' + annotate + (
                        ' ' * (max_annotate_cols - len(annotate))) + ' */'
                wf.write(annotate)
                wf.write(lines[line-1])


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Annotate based on perf.data. See configurations in annotate.config.')
    parser.add_argument('--config', default='annotate.config',
                        help='Set configuration file. Default is annotate.config.')
    args = parser.parse_args()
    config = load_config(args.config)
    annotator = SourceFileAnnotator(config)
    annotator.annotate()
    log_info('annotate finish successfully, please check result in annotated_files/.')