blob: f6582506311cca9fdf4695fceab364ea9921497c [file] [log] [blame]
Serge Guelton6da47be2019-01-03 15:44:24 +00001#!/usr/bin/env python
Adam Nemetdb695f42016-10-07 17:06:34 +00002
3from __future__ import print_function
4
Brian Gesiakfb669912017-06-29 18:56:25 +00005import argparse
6import cgi
Adam Nemet747f7a42017-12-14 18:42:42 +00007import codecs
Brian Gesiakfb669912017-06-29 18:56:25 +00008import errno
9import functools
10from multiprocessing import cpu_count
11import os.path
12import re
13import shutil
Roman Lebedevcbdd3892017-10-10 19:34:15 +000014import sys
Brian Gesiakfb669912017-06-29 18:56:25 +000015
16from pygments import highlight
17from pygments.lexers.c_cpp import CppLexer
18from pygments.formatters import HtmlFormatter
19
20import optpmap
21import optrecord
22
23
Adam Nemetdb695f42016-10-07 17:06:34 +000024desc = '''Generate HTML output to visualize optimization records from the YAML files
25generated with -fsave-optimization-record and -fdiagnostics-show-hotness.
26
Adam Nemeta3f7c6a2017-03-01 00:54:40 +000027The tools requires PyYAML and Pygments Python packages.'''
Adam Nemetdb695f42016-10-07 17:06:34 +000028
Adam Nemetdb695f42016-10-07 17:06:34 +000029
Adam Nemet2276fd32017-01-27 06:39:09 +000030# This allows passing the global context to the child processes.
31class Context:
Adam Nemetaa64e902017-03-01 21:35:00 +000032 def __init__(self, caller_loc = dict()):
Adam Nemet2276fd32017-01-27 06:39:09 +000033 # Map function names to their source location for function where inlining happened
34 self.caller_loc = caller_loc
Adam Nemet29e16232016-11-11 06:11:56 +000035
Adam Nemet2276fd32017-01-27 06:39:09 +000036context = Context()
37
Adam Nemetaf12c3e2017-12-06 16:50:50 +000038def suppress(remark):
39 if remark.Name == 'sil.Specialized':
40 return remark.getArgDict()['Function'][0].startswith('\"Swift.')
41 elif remark.Name == 'sil.Inlined':
42 return remark.getArgDict()['Callee'][0].startswith(('\"Swift.', '\"specialized Swift.'))
43 return False
44
Adam Nemetdb695f42016-10-07 17:06:34 +000045class SourceFileRenderer:
Zachary Turner135e9422018-01-05 22:05:13 +000046 def __init__(self, source_dir, output_dir, filename, no_highlight):
Adam Nemet2ee19a32018-02-26 21:15:50 +000047 self.filename = filename
Adam Nemet97813432016-11-11 01:08:02 +000048 existing_filename = None
49 if os.path.exists(filename):
50 existing_filename = filename
51 else:
Adam Nemet9d0307b2017-01-27 06:38:31 +000052 fn = os.path.join(source_dir, filename)
Adam Nemet97813432016-11-11 01:08:02 +000053 if os.path.exists(fn):
54 existing_filename = fn
55
Zachary Turner135e9422018-01-05 22:05:13 +000056 self.no_highlight = no_highlight
Adam Nemet747f7a42017-12-14 18:42:42 +000057 self.stream = codecs.open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w', encoding='utf-8')
Adam Nemet97813432016-11-11 01:08:02 +000058 if existing_filename:
59 self.source_stream = open(existing_filename)
60 else:
61 self.source_stream = None
62 print('''
63<html>
64<h1>Unable to locate file {}</h1>
65</html>
66 '''.format(filename), file=self.stream)
67
Adam Nemet20a63f12017-02-10 04:50:18 +000068 self.html_formatter = HtmlFormatter(encoding='utf-8')
Brian Cain34b49082017-02-18 15:13:58 +000069 self.cpp_lexer = CppLexer(stripnl=False)
Adam Nemetdb695f42016-10-07 17:06:34 +000070
Brian Cain34b49082017-02-18 15:13:58 +000071 def render_source_lines(self, stream, line_remarks):
72 file_text = stream.read()
Adam Nemetd3ee0652017-11-29 17:07:41 +000073
Zachary Turner135e9422018-01-05 22:05:13 +000074 if self.no_highlight:
Serge Guelton76339062019-01-03 14:40:34 +000075 if sys.version_info.major >= 3:
76 html_highlighted = file_text
77 else:
78 html_highlighted = file_text.decode('utf-8')
Adam Nemetd3ee0652017-11-29 17:07:41 +000079 else:
80 html_highlighted = highlight(
Brian Gesiak6ab32972017-08-11 18:05:26 +000081 file_text,
Adam Nemetd3ee0652017-11-29 17:07:41 +000082 self.cpp_lexer,
83 self.html_formatter)
Roman Lebedevcbdd3892017-10-10 19:34:15 +000084
Adam Nemet747f7a42017-12-14 18:42:42 +000085 # Note that the API is different between Python 2 and 3. On
86 # Python 3, pygments.highlight() returns a bytes object, so we
87 # have to decode. On Python 2, the output is str but since we
88 # support unicode characters and the output streams is unicode we
89 # decode too.
90 html_highlighted = html_highlighted.decode('utf-8')
Brian Cain34b49082017-02-18 15:13:58 +000091
Adam Nemetd3ee0652017-11-29 17:07:41 +000092 # Take off the header and footer, these must be
93 # reapplied line-wise, within the page structure
94 html_highlighted = html_highlighted.replace('<div class="highlight"><pre>', '')
95 html_highlighted = html_highlighted.replace('</pre></div>', '')
Brian Cain34b49082017-02-18 15:13:58 +000096
97 for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1):
Adam Nemet747f7a42017-12-14 18:42:42 +000098 print(u'''
Adam Nemetdb695f42016-10-07 17:06:34 +000099<tr>
100<td><a name=\"L{linenum}\">{linenum}</a></td>
101<td></td>
102<td></td>
Brian Cain34b49082017-02-18 15:13:58 +0000103<td><div class="highlight"><pre>{html_line}</pre></div></td>
Adam Nemetdb695f42016-10-07 17:06:34 +0000104</tr>'''.format(**locals()), file=self.stream)
105
Brian Cain34b49082017-02-18 15:13:58 +0000106 for remark in line_remarks.get(linenum, []):
Adam Nemetaf12c3e2017-12-06 16:50:50 +0000107 if not suppress(remark):
108 self.render_inline_remarks(remark, html_line)
Brian Cain34b49082017-02-18 15:13:58 +0000109
Adam Nemet58da3942016-11-11 01:51:34 +0000110 def render_inline_remarks(self, r, line):
Adam Nemet92957962016-11-11 01:25:04 +0000111 inlining_context = r.DemangledFunctionName
Adam Nemet2276fd32017-01-27 06:39:09 +0000112 dl = context.caller_loc.get(r.Function)
Adam Nemet92957962016-11-11 01:25:04 +0000113 if dl:
Adam Nemetb2613732017-07-19 22:04:59 +0000114 dl_dict = dict(list(dl))
115 link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2)
Adam Nemet92957962016-11-11 01:25:04 +0000116 inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals())
117
Adam Nemet58da3942016-11-11 01:51:34 +0000118 # Column is the number of characters *including* tabs, keep those and
119 # replace everything else with spaces.
Adam Nemet2f28d0c2017-02-28 23:59:46 +0000120 indent = line[:max(r.Column, 1) - 1]
Adam Nemet58da3942016-11-11 01:51:34 +0000121 indent = re.sub('\S', ' ', indent)
Adam Nemet77d2e862017-02-02 05:49:02 +0000122
Adam Nemet747f7a42017-12-14 18:42:42 +0000123 print(u'''
Adam Nemetdb695f42016-10-07 17:06:34 +0000124<tr>
125<td></td>
Adam Nemet29e16232016-11-11 06:11:56 +0000126<td>{r.RelativeHotness}</td>
Adam Nemet6a466e72017-03-02 17:00:59 +0000127<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
Adam Nemet58da3942016-11-11 01:51:34 +0000128<td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\"> {r.message}&nbsp;</span></td>
Adam Nemet92957962016-11-11 01:25:04 +0000129<td class=\"column-entry-yellow\">{inlining_context}</td>
Adam Nemetdb695f42016-10-07 17:06:34 +0000130</tr>'''.format(**locals()), file=self.stream)
131
132 def render(self, line_remarks):
Adam Nemet97813432016-11-11 01:08:02 +0000133 if not self.source_stream:
134 return
135
Adam Nemetdb695f42016-10-07 17:06:34 +0000136 print('''
137<html>
Adam Nemet2ee19a32018-02-26 21:15:50 +0000138<title>{}</title>
Adam Nemet71f21a22017-12-14 18:55:33 +0000139<meta charset="utf-8" />
Adam Nemetdb695f42016-10-07 17:06:34 +0000140<head>
141<link rel='stylesheet' type='text/css' href='style.css'>
142</head>
143<body>
144<div class="centered">
Adam Nemet47a36152017-11-14 04:48:18 +0000145<table class="source">
146<thead>
Adam Nemetdb695f42016-10-07 17:06:34 +0000147<tr>
Adam Nemet47a36152017-11-14 04:48:18 +0000148<th style="width: 2%">Line</td>
149<th style="width: 3%">Hotness</td>
150<th style="width: 10%">Optimization</td>
151<th style="width: 70%">Source</td>
152<th style="width: 15%">Inline Context</td>
153</tr>
154</thead>
Adam Nemet2ee19a32018-02-26 21:15:50 +0000155<tbody>'''.format(os.path.basename(self.filename)), file=self.stream)
Brian Cain34b49082017-02-18 15:13:58 +0000156 self.render_source_lines(self.source_stream, line_remarks)
157
Adam Nemetdb695f42016-10-07 17:06:34 +0000158 print('''
Adam Nemet47a36152017-11-14 04:48:18 +0000159</tbody>
Adam Nemetdb695f42016-10-07 17:06:34 +0000160</table>
161</body>
162</html>''', file=self.stream)
163
Mandeep Singh Grangc75e3c22016-11-11 04:51:27 +0000164
Adam Nemetdb695f42016-10-07 17:06:34 +0000165class IndexRenderer:
Zachary Turner135e9422018-01-05 22:05:13 +0000166 def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_index):
Adam Nemet747f7a42017-12-14 18:42:42 +0000167 self.stream = codecs.open(os.path.join(output_dir, 'index.html'), 'w', encoding='utf-8')
Adam Nemet0c383f62017-11-14 04:37:32 +0000168 self.should_display_hotness = should_display_hotness
Zachary Turner135e9422018-01-05 22:05:13 +0000169 self.max_hottest_remarks_on_index = max_hottest_remarks_on_index
Adam Nemetdb695f42016-10-07 17:06:34 +0000170
Adam Nemet73efe1f2017-02-02 05:49:00 +0000171 def render_entry(self, r, odd):
Adam Nemet77d2e862017-02-02 05:49:02 +0000172 escaped_name = cgi.escape(r.DemangledFunctionName)
Adam Nemet747f7a42017-12-14 18:42:42 +0000173 print(u'''
Adam Nemet43573fb2016-10-11 16:19:06 +0000174<tr>
Adam Nemet73efe1f2017-02-02 05:49:00 +0000175<td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td>
176<td class=\"column-entry-{odd}\">{r.RelativeHotness}</td>
Adam Nemet77d2e862017-02-02 05:49:02 +0000177<td class=\"column-entry-{odd}\">{escaped_name}</td>
Adam Nemet6a466e72017-03-02 17:00:59 +0000178<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
Adam Nemet43573fb2016-10-11 16:19:06 +0000179</tr>'''.format(**locals()), file=self.stream)
Adam Nemetdb695f42016-10-07 17:06:34 +0000180
181 def render(self, all_remarks):
182 print('''
183<html>
Adam Nemet71f21a22017-12-14 18:55:33 +0000184<meta charset="utf-8" />
Adam Nemetdb695f42016-10-07 17:06:34 +0000185<head>
186<link rel='stylesheet' type='text/css' href='style.css'>
187</head>
188<body>
189<div class="centered">
190<table>
191<tr>
192<td>Source Location</td>
193<td>Hotness</td>
194<td>Function</td>
195<td>Pass</td>
196</tr>''', file=self.stream)
Adam Nemet0c383f62017-11-14 04:37:32 +0000197
198 max_entries = None
Zachary Turner135e9422018-01-05 22:05:13 +0000199 if self.should_display_hotness:
200 max_entries = self.max_hottest_remarks_on_index
Adam Nemet0c383f62017-11-14 04:37:32 +0000201
202 for i, remark in enumerate(all_remarks[:max_entries]):
Adam Nemetaf12c3e2017-12-06 16:50:50 +0000203 if not suppress(remark):
204 self.render_entry(remark, i % 2)
Adam Nemetdb695f42016-10-07 17:06:34 +0000205 print('''
206</table>
207</body>
208</html>''', file=self.stream)
209
210
Zachary Turner135e9422018-01-05 22:05:13 +0000211def _render_file(source_dir, output_dir, ctx, no_highlight, entry):
Adam Nemet2276fd32017-01-27 06:39:09 +0000212 global context
213 context = ctx
Adam Nemet9d0307b2017-01-27 06:38:31 +0000214 filename, remarks = entry
Zachary Turner135e9422018-01-05 22:05:13 +0000215 SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks)
Adam Nemet9d0307b2017-01-27 06:38:31 +0000216
217
Adam Nemet9d0307b2017-01-27 06:38:31 +0000218def map_remarks(all_remarks):
219 # Set up a map between function names and their source location for
220 # function where inlining happened
Brian Gesiak11d48a52017-06-29 18:47:31 +0000221 for remark in optrecord.itervalues(all_remarks):
Adam Nemetaa64e902017-03-01 21:35:00 +0000222 if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined":
Adam Nemet9d0307b2017-01-27 06:38:31 +0000223 for arg in remark.Args:
Adam Nemetb2613732017-07-19 22:04:59 +0000224 arg_dict = dict(list(arg))
225 caller = arg_dict.get('Caller')
Adam Nemet9d0307b2017-01-27 06:38:31 +0000226 if caller:
Davide Italiano54df34c2017-07-13 04:19:13 +0000227 try:
Adam Nemetb2613732017-07-19 22:04:59 +0000228 context.caller_loc[caller] = arg_dict['DebugLoc']
Davide Italiano54df34c2017-07-13 04:19:13 +0000229 except KeyError:
230 pass
Adam Nemet92957962016-11-11 01:25:04 +0000231
Adam Nemetdb695f42016-10-07 17:06:34 +0000232
Brian Gesiakfb669912017-06-29 18:56:25 +0000233def generate_report(all_remarks,
234 file_remarks,
235 source_dir,
236 output_dir,
Zachary Turner135e9422018-01-05 22:05:13 +0000237 no_highlight,
Brian Gesiakfb669912017-06-29 18:56:25 +0000238 should_display_hotness,
Zachary Turner135e9422018-01-05 22:05:13 +0000239 max_hottest_remarks_on_index,
Brian Gesiakfb669912017-06-29 18:56:25 +0000240 num_jobs,
241 should_print_progress):
Adam Nemet9d0307b2017-01-27 06:38:31 +0000242 try:
243 os.makedirs(output_dir)
244 except OSError as e:
245 if e.errno == errno.EEXIST and os.path.isdir(output_dir):
246 pass
247 else:
248 raise
Adam Nemetdb695f42016-10-07 17:06:34 +0000249
Brian Gesiakfb669912017-06-29 18:56:25 +0000250 if should_print_progress:
Adam Nemet26256312018-02-26 21:15:47 +0000251 print('Rendering index page...')
Adam Nemetaa64e902017-03-01 21:35:00 +0000252 if should_display_hotness:
Brian Gesiak11d48a52017-06-29 18:47:31 +0000253 sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True)
Adam Nemet9d0307b2017-01-27 06:38:31 +0000254 else:
Brian Gesiak11d48a52017-06-29 18:47:31 +0000255 sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function))
Zachary Turner135e9422018-01-05 22:05:13 +0000256 IndexRenderer(output_dir, should_display_hotness, max_hottest_remarks_on_index).render(sorted_remarks)
Adam Nemetdb695f42016-10-07 17:06:34 +0000257
Adam Nemet9d0307b2017-01-27 06:38:31 +0000258 shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)),
259 "style.css"), output_dir)
260
Adam Nemet26256312018-02-26 21:15:47 +0000261 _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context, no_highlight)
262 if should_print_progress:
263 print('Rendering HTML files...')
264 optpmap.pmap(_render_file_bound,
265 file_remarks.items(),
266 num_jobs,
267 should_print_progress)
268
Adam Nemet9d0307b2017-01-27 06:38:31 +0000269
Zachary Turner135e9422018-01-05 22:05:13 +0000270def main():
Adam Nemet9d0307b2017-01-27 06:38:31 +0000271 parser = argparse.ArgumentParser(description=desc)
Adam Nemetef314682017-07-17 18:00:41 +0000272 parser.add_argument(
273 'yaml_dirs_or_files',
274 nargs='+',
275 help='List of optimization record files or directories searched '
276 'for optimization record files.')
Brian Gesiak11eef882017-06-29 16:20:31 +0000277 parser.add_argument(
278 '--output-dir',
279 '-o',
280 default='html',
281 help='Path to a directory where generated HTML files will be output. '
282 'If the directory does not already exist, it will be created. '
283 '"%(default)s" by default.')
Adam Nemet9d0307b2017-01-27 06:38:31 +0000284 parser.add_argument(
285 '--jobs',
286 '-j',
Zachary Turner135e9422018-01-05 22:05:13 +0000287 default=None,
Adam Nemet9d0307b2017-01-27 06:38:31 +0000288 type=int,
Brian Gesiak23ff0952017-06-10 21:33:27 +0000289 help='Max job count (defaults to %(default)s, the current CPU count)')
Adam Nemet9d0307b2017-01-27 06:38:31 +0000290 parser.add_argument(
Adam Nemetaa1bb202017-11-27 19:00:22 +0000291 '--source-dir',
Adam Nemet9d0307b2017-01-27 06:38:31 +0000292 '-s',
293 default='',
294 help='set source directory')
Brian Gesiakfb669912017-06-29 18:56:25 +0000295 parser.add_argument(
296 '--no-progress-indicator',
297 '-n',
298 action='store_true',
299 default=False,
300 help='Do not display any indicator of how many YAML files were read '
301 'or rendered into HTML.')
Adam Nemet0c383f62017-11-14 04:37:32 +0000302 parser.add_argument(
303 '--max-hottest-remarks-on-index',
304 default=1000,
305 type=int,
306 help='Maximum number of the hottest remarks to appear on the index page')
Adam Nemetd3ee0652017-11-29 17:07:41 +0000307 parser.add_argument(
308 '--no-highlight',
309 action='store_true',
310 default=False,
311 help='Do not use a syntax highlighter when rendering the source code')
312 parser.add_argument(
313 '--demangler',
314 help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler)
Zachary Turner135e9422018-01-05 22:05:13 +0000315
316 # Do not make this a global variable. Values needed to be propagated through
317 # to individual classes and functions to be portable with multiprocessing across
318 # Windows and non-Windows.
Adam Nemet9d0307b2017-01-27 06:38:31 +0000319 args = parser.parse_args()
320
Brian Gesiakfb669912017-06-29 18:56:25 +0000321 print_progress = not args.no_progress_indicator
Adam Nemetd3ee0652017-11-29 17:07:41 +0000322 if args.demangler:
323 optrecord.Remark.set_demangler(args.demangler)
Adam Nemetef314682017-07-17 18:00:41 +0000324
Adam Nemet41247e12017-09-29 05:20:53 +0000325 files = optrecord.find_opt_files(*args.yaml_dirs_or_files)
Adam Nemetef314682017-07-17 18:00:41 +0000326 if not files:
327 parser.error("No *.opt.yaml files found")
328 sys.exit(1)
329
Brian Gesiakfb669912017-06-29 18:56:25 +0000330 all_remarks, file_remarks, should_display_hotness = \
Adam Nemetef314682017-07-17 18:00:41 +0000331 optrecord.gather_results(files, args.jobs, print_progress)
Adam Nemet9d0307b2017-01-27 06:38:31 +0000332
333 map_remarks(all_remarks)
334
Brian Gesiakfb669912017-06-29 18:56:25 +0000335 generate_report(all_remarks,
336 file_remarks,
337 args.source_dir,
338 args.output_dir,
Zachary Turner135e9422018-01-05 22:05:13 +0000339 args.no_highlight,
Brian Gesiakfb669912017-06-29 18:56:25 +0000340 should_display_hotness,
Zachary Turner135e9422018-01-05 22:05:13 +0000341 args.max_hottest_remarks_on_index,
Brian Gesiakfb669912017-06-29 18:56:25 +0000342 args.jobs,
343 print_progress)
Zachary Turner135e9422018-01-05 22:05:13 +0000344
345if __name__ == '__main__':
346 main()