Gitiles
Code Review Sign In
review.blissroms.org / platform_system_bt / refs/heads/q / . / test / gen_coverage.py
blob: 3697c37cc7fcd2937c05d04368326de0d4688aff [file] [log] [blame]
Ajay Panickerff48ce82018-09-20 14:39:00 -0700[diff] [blame]1#!/usr/bin/env python
2#
3# Copyright 2018, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import argparse
18import datetime
19import logging
20import json
21import os
22import shutil
23import subprocess
24import sys
25import webbrowser
26
27from run_host_unit_tests import *
28
29"""
30This script is used to generate code coverage results host supported libraries.
31The script by default will generate an html report that summarizes the coverage
32results of the specified tests. The results can also be browsed to provide a
33report of which lines have been traveled upon execution of the binary.
34
35NOTE: Code that is compiled out or hidden by a #DEFINE will be listed as
36having been executed 0 times, thus reducing overall coverage.
37
38The steps in order to add coverage support to a new library and its
39corrisponding host test are as follows.
40
411. Add "clang_file_coverage" (defined in //build/Android.bp) as a default to the
42 source library(s) you want statistics for.
43 NOTE: Forgoing this default will cause no coverage data to be generated for
44 the source files in the library.
45
462. Add "clang_coverage_bin" as a default to the host supported test binary that
47 excercises the libraries that you covered in step 1.
48 NOTE: Forgetting to add this will cause there to be *NO* coverage data
49 generated when the binary is run.
50
513. Add the host test binary name and the files/directories you want coverage
52 statistics for to the COVERAGE_TESTS variable defined below. You may add
53 individual filenames or a directory to be tested.
54 NOTE: Avoid using a / at the beginning of a covered_files entry as this
55 breaks how the coverage generator resolves filenames.
56
57TODO: Support generating XML data and printing results to standard out.
58"""
59
60COVERAGE_TESTS = [
61 {
62 "test_name": "net_test_avrcp",
63 "covered_files": [
64 "system/bt/profile/avrcp",
65 ],
66 }, {
67 "test_name": "bluetooth_test_sdp",
68 "covered_files": [
69 "system/bt/profile/sdp",
70 ],
Myles Watson43b70242018-11-07 12:10:46 -0800[diff] [blame]71 }, {
72 "test_name": "test-vendor_test_host",
73 "covered_files": [
74 "system/bt/vendor_libs/test_vendor_lib/include",
75 "system/bt/vendor_libs/test_vendor_lib/src",
76 ],
77 }, {
78 "test_name": "rootcanal-packets_test_host",
79 "covered_files": [
80 "system/bt/vendor_libs/test_vendor_lib/packets",
81 ],
Hansong Zhang7972cd52018-12-12 14:52:00 -0800[diff] [blame]82 }, {
83 "test_name": "bluetooth_test_common",
84 "covered_files": [
85 "system/bt/common",
86 ],
Ajay Panickerff48ce82018-09-20 14:39:00 -0700[diff] [blame]87 },
88]
89
90WORKING_DIR = '/tmp/coverage'
91SOONG_UI_BASH = 'build/soong/soong_ui.bash'
92LLVM_DIR = 'prebuilts/clang/host/linux-x86/clang-r328903/bin'
93LLVM_MERGE = LLVM_DIR + '/llvm-profdata'
94LLVM_COV = LLVM_DIR + '/llvm-cov'
95
96def write_root_html_head(f):
97 # Write the header part of the root html file. This was pulled from the
98 # page source of one of the generated html files.
99 f.write("<!doctype html><html><head>" \
100 "<meta name='viewport' content='width=device-width,initial-scale=1'><met" \
101 "a charset='UTF-8'><link rel='stylesheet' type='text/css' href='style.cs" \
102 "s'></head><body><h2>Coverage Report</h2><h4>Created: " +
103 str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M')) +
104 "</h4><p>Click <a href='http://clang.llvm.org/docs/SourceBasedCodeCovera" \
105 "ge.html#interpreting-reports'>here</a> for information about interpreti" \
106 "ng this report.</p><div class='centered'><table><tr><td class='column-e" \
107 "ntry-bold'>Filename</td><td class='column-entry-bold'>Function Coverage" \
108 "</td><td class='column-entry-bold'>Instantiation Coverage</td><td class" \
109 "='column-entry-bold'>Line Coverage</td><td class='column-entry-bold'>Re" \
110 "gion Coverage</td></tr>"
111 )
112
113
114def write_root_html_column(f, covered, count):
115 percent = covered * 100.0 / count
116 value = "%.2f%% (%d/%d) " % (percent, covered, count)
117 color = 'column-entry-yellow'
118 if percent == 100:
119 color = 'column-entry-green'
120 if percent < 80.0:
121 color = 'column-entry-red'
122 f.write("<td class=\'" + color + "\'><pre>" + value + "</pre></td>")
123
124
125def write_root_html_rows(f, tests):
126 totals = {
127 "functions":{
128 "covered": 0,
129 "count": 0
130 },
131 "instantiations":{
132 "covered": 0,
133 "count": 0
134 },
135 "lines":{
136 "covered": 0,
137 "count": 0
138 },
139 "regions":{
140 "covered": 0,
141 "count": 0
142 }
143 }
144
145 # Write the tests with their coverage summaries.
146 for test in tests:
147 test_name = test['test_name']
148 covered_files = test['covered_files']
149 json_results = generate_coverage_json(test)
150 test_totals = json_results['data'][0]['totals']
151
152 f.write("<tr class='light-row'><td><pre><a href=\'" +
153 os.path.join(test_name, "index.html") + "\'>" + test_name +
154 "</a></pre></td>")
155 for field_name in ['functions', 'instantiations', 'lines', 'regions']:
156 field = test_totals[field_name]
157 totals[field_name]['covered'] += field['covered']
158 totals[field_name]['count'] += field['count']
159 write_root_html_column(f, field['covered'], field['count'])
160 f.write("</tr>");
161
162 #Write the totals row.
163 f.write("<tr class='light-row-bold'><td><pre>Totals</a></pre></td>")
164 for field_name in ['functions', 'instantiations', 'lines', 'regions']:
165 field = totals[field_name]
166 write_root_html_column(f, field['covered'], field['count'])
167 f.write("</tr>");
168
169
170def write_root_html_tail(f):
171 # Pulled from the generated html coverage report.
172 f.write("</table></div><h5>Generated by llvm-cov -- llvm version 7.0.2svn<" \
173 "/h5></body></html>")
174
175
176def generate_root_html(tests):
177 # Copy the css file from one of the coverage reports.
178 source_file = os.path.join(os.path.join(WORKING_DIR, tests[0]['test_name']), "style.css")
179 dest_file = os.path.join(WORKING_DIR, "style.css")
180 shutil.copy2(source_file, dest_file)
181
182 # Write the root index.html file that sumarizes all the tests.
183 f = open(os.path.join(WORKING_DIR, "index.html"), "w")
184 write_root_html_head(f)
185 write_root_html_rows(f, tests)
186 write_root_html_tail(f)
187
188
189def get_profraw_for_test(test_name):
190 test_root = get_native_test_root_or_die()
191 test_cmd = os.path.join(os.path.join(test_root, test_name), test_name)
192 if not os.path.isfile(test_cmd):
193 logging.error('The test ' + test_name + ' does not exist, please compile first')
194 sys.exit(1)
195
196 profraw_file_name = test_name + ".profraw"
197 profraw_path = os.path.join(WORKING_DIR, os.path.join(test_name, profraw_file_name))
198 llvm_env_var = "LLVM_PROFILE_FILE=\"" + profraw_path + "\""
199
200 test_cmd = llvm_env_var + " " + test_cmd
201 logging.info('Generating profraw data for ' + test_name)
202 logging.debug('cmd: ' + test_cmd)
203 if subprocess.call(test_cmd, shell=True) != 0:
204 logging.error('Test ' + test_name + ' failed. Please fix the test before generating coverage.')
205 sys.exit(1)
206
207 if not os.path.isfile(profraw_path):
208 logging.error('Generating the profraw file failed. Did you remember to add the proper compiler flags to your build?')
209 sys.exit(1)
210
211 return profraw_file_name
212
213
214def merge_profraw_data(test_name):
215 cmd = []
216 cmd.append(os.path.join(get_android_root_or_die(), LLVM_MERGE + " merge "))
217
218 test_working_dir = os.path.join(WORKING_DIR, test_name);
219 cmd.append(os.path.join(test_working_dir, test_name + ".profraw"))
220 profdata_file = os.path.join(test_working_dir, test_name + ".profdata")
221
222 cmd.append('-o ' + profdata_file)
223 logging.info('Combining profraw files into profdata for ' + test_name)
224 logging.debug('cmd: ' + " ".join(cmd))
225 if subprocess.call(" ".join(cmd), shell=True) != 0:
226 logging.error('Failed to merge profraw files for ' + test_name)
227 sys.exit(1)
228
229
230def generate_coverage_html(test):
231 COVERAGE_ROOT = '/proc/self/cwd'
232
233 test_name = test['test_name']
234 file_list = test['covered_files']
235
236 test_working_dir = os.path.join(WORKING_DIR, test_name)
237 test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata")
238
239 cmd = [
240 os.path.join(get_android_root_or_die(), LLVM_COV),
241 "show",
242 "-format=html",
243 "-summary-only",
244 "-show-line-counts-or-regions",
245 "-show-instantiation-summary",
246 "-instr-profile=" + test_profdata_file,
247 "-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" +
248 get_android_root_or_die() + "\"",
249 "-output-dir=" + test_working_dir
250 ]
251
252 # We have to have one object file not as an argument otherwise we can't specify source files.
253 test_cmd = os.path.join(os.path.join(get_native_test_root_or_die(), test_name), test_name)
254 cmd.append(test_cmd)
255
256 # Filter out the specific files we want coverage for
257 for filename in file_list:
258 cmd.append(os.path.join(get_android_root_or_die(), filename))
259
260 logging.info('Generating coverage report for ' + test['test_name'])
261 logging.debug('cmd: ' + " ".join(cmd))
262 if subprocess.call(" ".join(cmd), shell=True) != 0:
263 logging.error('Failed to generate coverage for ' + test['test_name'])
264 sys.exit(1)
265
266
267def generate_coverage_json(test):
268 COVERAGE_ROOT = '/proc/self/cwd'
269 test_name = test['test_name']
270 file_list = test['covered_files']
271
272 test_working_dir = os.path.join(WORKING_DIR, test_name)
273 test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata")
274
275 cmd = [
276 os.path.join(get_android_root_or_die(), LLVM_COV),
277 "export",
278 "-summary-only",
279 "-show-region-summary",
280 "-instr-profile=" + test_profdata_file,
281 "-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" + get_android_root_or_die() + "\"",
282 ]
283
284 test_cmd = os.path.join(os.path.join(get_native_test_root_or_die(), test_name), test_name)
285 cmd.append(test_cmd)
286
287 # Filter out the specific files we want coverage for
288 for filename in file_list:
289 cmd.append(os.path.join(get_android_root_or_die(), filename))
290
291 logging.info('Generating coverage json for ' + test['test_name'])
292 logging.debug('cmd: ' + " ".join(cmd))
293
294 json_str = subprocess.check_output(" ".join(cmd), shell=True)
295 return json.loads(json_str)
296
297
298def write_json_summary(test):
299 test_name = test['test_name']
300 test_working_dir = os.path.join(WORKING_DIR, test_name)
301 test_json_summary_file = os.path.join(test_working_dir, test_name + '.json')
302 logging.debug('Writing json summary file: ' + test_json_summary_file)
303 json_file = open(test_json_summary_file, 'w')
304 json.dump(generate_coverage_json(test), json_file)
305 json_file.close()
306
307
308def list_tests():
309 for test in COVERAGE_TESTS:
310 print "Test Name: " + test['test_name']
311 print "Covered Files: "
312 for covered_file in test['covered_files']:
313 print " " + covered_file
314 print
315
316
317def main():
318 parser = argparse.ArgumentParser(description='Generate code coverage for enabled tests.')
319 parser.add_argument(
320 '-l', '--list-tests',
321 action='store_true',
322 dest='list_tests',
323 help='List all the available tests to be run as well as covered files.')
324 parser.add_argument(
325 '-a', '--all',
326 action='store_true',
327 help='Runs all available tests and prints their outputs. If no tests ' \
328 'are specified via the -t option all tests will be run.')
329 parser.add_argument(
330 '-t', '--test',
331 dest='tests',
332 action='append',
333 type=str,
334 metavar='TESTNAME',
335 default=[],
336 help='Specifies a test to be run. Multiple tests can be specified by ' \
337 'using this option multiple times. ' \
338 'Example: \"gen_coverage.py -t test1 -t test2\"')
339 parser.add_argument(
340 '-o', '--output',
341 type=str,
342 metavar='DIRECTORY',
343 default='/tmp/coverage',
344 help='Specifies the directory to store all files. The directory will be ' \
345 'created if it does not exist. Default is \"/tmp/coverage\"')
346 parser.add_argument(
347 '-s', '--skip-html',
348 dest='skip_html',
349 action='store_true',
350 help='Skip opening up the results of the coverage report in a browser.')
351 parser.add_argument(
352 '-j', '--json-file',
353 dest='json_file',
354 action='store_true',
355 help='Write out summary results to json file in test directory.')
356
357 logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(levelname)s %(message)s')
358 logging.addLevelName(logging.DEBUG, "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.DEBUG))
359 logging.addLevelName(logging.INFO, "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.INFO))
360 logging.addLevelName(logging.WARNING, "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.WARNING))
361 logging.addLevelName(logging.ERROR, "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.ERROR))
362
363 args = parser.parse_args()
364 logging.debug("Args: " + str(args))
365
366 # Set the working directory
367 global WORKING_DIR
368 WORKING_DIR = os.path.abspath(args.output)
369 logging.debug("Working Dir: " + WORKING_DIR)
370
371 # Print out the list of tests then exit
372 if args.list_tests:
373 list_tests()
374 sys.exit(0)
375
376 # Check to see if a test was specified and if so only generate coverage for
377 # that test.
378 if len(args.tests) == 0:
379 args.all = True
380
381 tests_to_run = []
382 for test in COVERAGE_TESTS:
383 if args.all or test['test_name'] in args.tests:
384 tests_to_run.append(test)
385 if test['test_name'] in args.tests:
386 args.tests.remove(test['test_name'])
387
388 # Error if a test was specified but doesn't exist.
389 if len(args.tests) != 0:
390 for test_name in args.tests:
391 logging.error('\"' + test_name + '\" was not found in the list of available tests.')
392 sys.exit(1)
393
394 # Generate the info for the tests
395 for test in tests_to_run:
396 logging.info('Getting coverage for ' + test['test_name'])
397 get_profraw_for_test(test['test_name'])
398 merge_profraw_data(test['test_name'])
399 if args.json_file:
400 write_json_summary(test)
401 generate_coverage_html(test)
402
403 # Generate the root index.html page that sumarizes all of the coverage reports.
404 generate_root_html(tests_to_run)
405
406 # Open the results in a browser.
407 if not args.skip_html:
408 webbrowser.open('file://' + os.path.join(WORKING_DIR, 'index.html'))
409
410
411if __name__ == '__main__':
412 main()