Skip to content

Commit 640af58

Browse files
authored
Add support for branch coverage (google#1375)
* instroduce branch coverage metric for llvm source-based coverage * add pylint: disable=too-many-arguments * use single quote and update argument description * use branch coverage as default and fix test * format code * format code * change unique covered regions to unique covered branches * remove comment * add final newline * address pull request feedbacks * set region coverage as an optional coverage metric
1 parent 0905238 commit 640af58

14 files changed

Lines changed: 235 additions & 172 deletions

analysis/benchmark_results.py

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -95,30 +95,30 @@ def _benchmark_snapshot_df(self):
9595
@property
9696
@functools.lru_cache()
9797
def _benchmark_coverage_dict(self):
98-
"""Covered regions of each fuzzer on this benchmark."""
98+
"""Covered branches of each fuzzer on this benchmark."""
9999
return coverage_data_utils.get_benchmark_cov_dict(
100100
self._coverage_dict, self.name)
101101

102102
@property
103103
@functools.lru_cache()
104104
def _benchmark_aggregated_coverage_df(self):
105-
"""Aggregated covered regions of each fuzzer on this benchmark."""
105+
"""Aggregated covered branches of each fuzzer on this benchmark."""
106106
return coverage_data_utils.get_benchmark_aggregated_cov_df(
107107
self._coverage_dict, self.name)
108108

109109
@property
110110
@functools.lru_cache()
111-
def _unique_region_dict(self):
112-
"""Unique regions with the fuzzers that cover it."""
113-
return coverage_data_utils.get_unique_region_dict(
111+
def _unique_branch_dict(self):
112+
"""Unique branches with the fuzzers that cover it."""
113+
return coverage_data_utils.get_unique_branch_dict(
114114
self._benchmark_coverage_dict)
115115

116116
@property
117117
@functools.lru_cache()
118-
def unique_region_cov_df(self):
119-
"""Fuzzers with the number of covered unique regions."""
120-
return coverage_data_utils.get_unique_region_cov_df(
121-
self._unique_region_dict, self.fuzzer_names)
118+
def unique_branch_cov_df(self):
119+
"""Fuzzers with the number of covered unique branches."""
120+
return coverage_data_utils.get_unique_branch_cov_df(
121+
self._unique_branch_dict, self.fuzzer_names)
122122

123123
@property
124124
def fuzzers_with_not_enough_samples(self):
@@ -344,12 +344,12 @@ def _generic_violin_plot(self, filename, bugs=False):
344344

345345
@property
346346
def violin_plot(self):
347-
"""Region coverage violin plot."""
347+
"""Branch coverage violin plot."""
348348
return self._generic_violin_plot('violin.svg')
349349

350350
@property
351351
def bug_violin_plot(self):
352-
"""Region coverage violin plot."""
352+
"""Branch coverage violin plot."""
353353
return self._generic_violin_plot('bug_violin.svg', bugs=True)
354354

355355
def _generic_box_plot(self, filename, bugs=False):
@@ -362,7 +362,7 @@ def _generic_box_plot(self, filename, bugs=False):
362362

363363
@property
364364
def box_plot(self):
365-
"""Region coverage boxplot."""
365+
"""Branch coverage boxplot."""
366366
return self._generic_box_plot('boxplot.svg')
367367

368368
@property
@@ -399,19 +399,19 @@ def better_than_plot(self):
399399
@property
400400
def unique_coverage_ranking_plot(self):
401401
"""Ranking plot for unique coverage."""
402-
plot_filename = self._prefix_with_benchmark('ranking_unique_region.svg')
403-
unique_region_cov_df_combined = self.unique_region_cov_df.merge(
402+
plot_filename = self._prefix_with_benchmark('ranking_unique_branch.svg')
403+
unique_branch_cov_df_combined = self.unique_branch_cov_df.merge(
404404
self._benchmark_aggregated_coverage_df, on='fuzzer')
405405
self._plotter.write_unique_coverage_ranking_plot(
406-
unique_region_cov_df_combined, self._get_full_path(plot_filename))
406+
unique_branch_cov_df_combined, self._get_full_path(plot_filename))
407407
return plot_filename
408408

409409
@property
410410
@functools.lru_cache()
411411
def pairwise_unique_coverage_table(self):
412412
"""Pairwise unique coverage table for each pair of fuzzers."""
413-
fuzzers = self.unique_region_cov_df.sort_values(
414-
by='unique_regions_covered', ascending=False).fuzzer
413+
fuzzers = self.unique_branch_cov_df.sort_values(
414+
by='unique_branches_covered', ascending=False).fuzzer
415415
return coverage_data_utils.get_pairwise_unique_coverage_table(
416416
self._benchmark_coverage_dict, fuzzers)
417417

analysis/coverage_data_utils.py

Lines changed: 49 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -74,109 +74,109 @@ def get_coverage_report_filestore_path(fuzzer: str, benchmark: str,
7474
fuzzer, 'index.html')
7575

7676

77-
def get_covered_regions_dict(experiment_df: pd.DataFrame) -> Dict:
77+
def get_covered_branches_dict(experiment_df: pd.DataFrame) -> Dict:
7878
"""Combines json files for different fuzzer-benchmark pair in
79-
|experiment_df| and returns a dictionary of the covered regions."""
79+
|experiment_df| and returns a dictionary of the covered branches."""
8080
fuzzers_and_benchmarks = set(
8181
zip(experiment_df.fuzzer, experiment_df.benchmark))
8282
arguments = [(fuzzer, benchmark,
8383
get_experiment_filestore_path_for_fuzzer_benchmark(
8484
fuzzer, benchmark, experiment_df))
8585
for fuzzer, benchmark in fuzzers_and_benchmarks]
86-
result = itertools.starmap(get_fuzzer_benchmark_covered_regions_and_key,
86+
result = itertools.starmap(get_fuzzer_benchmark_covered_branches_and_key,
8787
arguments)
8888
return dict(result)
8989

9090

91-
def get_fuzzer_benchmark_covered_regions_filestore_path(
91+
def get_fuzzer_benchmark_covered_branches_filestore_path(
9292
fuzzer: str, benchmark: str, exp_filestore_path: str) -> str:
93-
"""Returns the path to the covered regions json file in the |filestore| for
93+
"""Returns the path to the covered branches json file in the |filestore| for
9494
|fuzzer| and |benchmark|."""
9595
return posixpath.join(exp_filestore_path, 'coverage', 'data', benchmark,
96-
fuzzer, 'covered_regions.json')
96+
fuzzer, 'covered_branches.json')
9797

9898

99-
def get_fuzzer_covered_regions(fuzzer: str, benchmark: str, filestore: str):
100-
"""Returns the covered regions dict for |fuzzer| from the json file in the
99+
def get_fuzzer_covered_branches(fuzzer: str, benchmark: str, filestore: str):
100+
"""Returns the covered branches dict for |fuzzer| from the json file in the
101101
filestore."""
102-
src_file = get_fuzzer_benchmark_covered_regions_filestore_path(
102+
src_file = get_fuzzer_benchmark_covered_branches_filestore_path(
103103
fuzzer, benchmark, filestore)
104104
with tempfile.NamedTemporaryFile() as dst_file:
105105
if filestore_utils.cp(src_file, dst_file.name,
106106
expect_zero=False).retcode:
107-
logger.warning('covered_regions.json file: %s could not be copied.',
108-
src_file)
107+
logger.warning(
108+
'covered_branches.json file: %s could not be copied.', src_file)
109109
return {}
110110
with open(dst_file.name) as json_file:
111111
return json.load(json_file)
112112

113113

114-
def get_fuzzer_benchmark_covered_regions_and_key(
114+
def get_fuzzer_benchmark_covered_branches_and_key(
115115
fuzzer: str, benchmark: str, filestore: str) -> Tuple[str, Dict]:
116116
"""Accepts |fuzzer|, |benchmark|, |filestore|.
117-
Returns a tuple containing the fuzzer benchmark key and the regions covered
117+
Returns a tuple containing the fuzzer benchmark key and the branches covered
118118
by the fuzzer on the benchmark."""
119-
fuzzer_benchmark_covered_regions = get_fuzzer_covered_regions(
119+
fuzzer_benchmark_covered_branches = get_fuzzer_covered_branches(
120120
fuzzer, benchmark, filestore)
121121
key = fuzzer_and_benchmark_to_key(fuzzer, benchmark)
122-
return key, fuzzer_benchmark_covered_regions
122+
return key, fuzzer_benchmark_covered_branches
123123

124124

125-
def get_unique_region_dict(benchmark_coverage_dict: Dict) -> Dict:
125+
def get_unique_branch_dict(benchmark_coverage_dict: Dict) -> Dict:
126126
"""Returns a dictionary containing the covering fuzzers for each unique
127-
region, where the |threshold| defines which regions are unique."""
128-
region_dict = collections.defaultdict(list)
129-
unique_region_dict = {}
127+
branch, where the |threshold| defines which branches are unique."""
128+
branch_dict = collections.defaultdict(list)
129+
unique_branch_dict = {}
130130
threshold_count = 1
131131
for fuzzer in benchmark_coverage_dict:
132-
for region in benchmark_coverage_dict[fuzzer]:
133-
region_dict[region].append(fuzzer)
134-
for region, fuzzers in region_dict.items():
132+
for branch in benchmark_coverage_dict[fuzzer]:
133+
branch_dict[branch].append(fuzzer)
134+
for branch, fuzzers in branch_dict.items():
135135
if len(fuzzers) <= threshold_count:
136-
unique_region_dict[region] = fuzzers
137-
return unique_region_dict
136+
unique_branch_dict[branch] = fuzzers
137+
return unique_branch_dict
138138

139139

140-
def get_unique_region_cov_df(unique_region_dict: Dict,
140+
def get_unique_branch_cov_df(unique_branch_dict: Dict,
141141
fuzzer_names: List[str]) -> pd.DataFrame:
142142
"""Returns a DataFrame where the two columns are fuzzers and the number of
143-
unique regions covered."""
143+
unique branches covered."""
144144
fuzzers = collections.defaultdict(int)
145-
for region in unique_region_dict:
146-
for fuzzer in unique_region_dict[region]:
145+
for branch in unique_branch_dict:
146+
for fuzzer in unique_branch_dict[branch]:
147147
fuzzers[fuzzer] += 1
148-
dict_to_transform = {'fuzzer': [], 'unique_regions_covered': []}
148+
dict_to_transform = {'fuzzer': [], 'unique_branches_covered': []}
149149
for fuzzer in fuzzer_names:
150150
covered_num = fuzzers[fuzzer]
151151
dict_to_transform['fuzzer'].append(fuzzer)
152-
dict_to_transform['unique_regions_covered'].append(covered_num)
152+
dict_to_transform['unique_branches_covered'].append(covered_num)
153153
return pd.DataFrame(dict_to_transform)
154154

155155

156156
def get_benchmark_cov_dict(coverage_dict, benchmark):
157-
"""Returns a dictionary to store the covered regions of each fuzzer. Uses a
158-
set of tuples to store the covered regions."""
157+
"""Returns a dictionary to store the covered branches of each fuzzer. Uses a
158+
set of tuples to store the covered branches."""
159159
benchmark_cov_dict = {}
160-
for key, covered_regions in coverage_dict.items():
160+
for key, covered_braches in coverage_dict.items():
161161
current_fuzzer, current_benchmark = key_to_fuzzer_and_benchmark(key)
162162
if current_benchmark == benchmark:
163-
covered_regions_in_set = set()
164-
for region in covered_regions:
165-
covered_regions_in_set.add(tuple(region))
166-
benchmark_cov_dict[current_fuzzer] = covered_regions_in_set
163+
covered_braches_in_set = set()
164+
for branch in covered_braches:
165+
covered_braches_in_set.add(tuple(branch))
166+
benchmark_cov_dict[current_fuzzer] = covered_braches_in_set
167167
return benchmark_cov_dict
168168

169169

170170
def get_benchmark_aggregated_cov_df(coverage_dict, benchmark):
171171
"""Returns a dataframe where each row represents a fuzzer and its aggregated
172172
coverage number."""
173173
dict_to_transform = {'fuzzer': [], 'aggregated_edges_covered': []}
174-
for key, covered_regions in coverage_dict.items():
174+
for key, covered_branches in coverage_dict.items():
175175
current_fuzzer, current_benchmark = key_to_fuzzer_and_benchmark(key)
176176
if current_benchmark == benchmark:
177177
dict_to_transform['fuzzer'].append(current_fuzzer)
178178
dict_to_transform['aggregated_edges_covered'].append(
179-
len(covered_regions))
179+
len(covered_branches))
180180
return pd.DataFrame(dict_to_transform)
181181

182182

@@ -186,7 +186,7 @@ def get_pairwise_unique_coverage_table(benchmark_coverage_dict, fuzzers):
186186
187187
The pairwise unique coverage table is a square matrix where each
188188
row and column represents a fuzzer, and each cell contains a number
189-
showing the regions covered by the fuzzer of the column but not by
189+
showing the branches covered by the fuzzer of the column but not by
190190
the fuzzer of the row."""
191191

192192
pairwise_unique_coverage_values = []
@@ -204,16 +204,16 @@ def get_pairwise_unique_coverage_table(benchmark_coverage_dict, fuzzers):
204204
columns=fuzzers)
205205

206206

207-
def get_unique_covered_percentage(fuzzer_row_covered_regions,
208-
fuzzer_col_covered_regions):
209-
"""Returns the number of regions covered by the fuzzer of the column but not
210-
by the fuzzer of the row."""
207+
def get_unique_covered_percentage(fuzzer_row_covered_branches,
208+
fuzzer_col_covered_branches):
209+
"""Returns the number of branches covered by the fuzzer of the
210+
column but not by the fuzzer of the row."""
211211

212-
unique_region_count = 0
213-
for region in fuzzer_col_covered_regions:
214-
if region not in fuzzer_row_covered_regions:
215-
unique_region_count += 1
216-
return unique_region_count
212+
unique_branch_count = 0
213+
for branch in fuzzer_col_covered_branches:
214+
if branch not in fuzzer_row_covered_branches:
215+
unique_branch_count += 1
216+
return unique_branch_count
217217

218218

219219
def rank_by_average_normalized_score(benchmarks_unique_coverage_list):

analysis/experiment_results.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ def rank_by_unique_coverage_average_normalized_score(self):
264264
"""Rank fuzzers using average normalized score on unique code coverage
265265
across benchmarks."""
266266
benchmarks_unique_coverage_list = [
267-
benchmark.unique_region_cov_df for benchmark in self.benchmarks
267+
benchmark.unique_branch_cov_df for benchmark in self.benchmarks
268268
]
269269
return coverage_data_utils.rank_by_average_normalized_score(
270270
benchmarks_unique_coverage_list)

analysis/generate_report.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ def generate_report(experiment_names,
226226
coverage_dict = {}
227227
if coverage_report:
228228
logger.info('Generating coverage report info.')
229-
coverage_dict = coverage_data_utils.get_covered_regions_dict(
229+
coverage_dict = coverage_data_utils.get_covered_branches_dict(
230230
experiment_df)
231231
logger.info('Finished generating coverage report info.')
232232

analysis/plotting.py

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ def coverage_growth_plot(self,
181181
loc='upper left',
182182
frameon=False)
183183

184-
axes.set(ylabel='Bug coverage' if bugs else 'Code region coverage')
184+
axes.set(ylabel='Bug coverage' if bugs else 'Code branch coverage')
185185
axes.set(xlabel='Time (hour:minute)')
186186

187187
if self._logscale or logscale:
@@ -260,7 +260,7 @@ def box_or_violin_plot(self,
260260
sns.stripplot(**common_args, size=3, color="black", alpha=0.6)
261261

262262
axes.set_title(_formatted_title(benchmark_snapshot_df))
263-
ylabel = 'Reached {} coverage'.format('bug' if bugs else 'region')
263+
ylabel = 'Reached {} coverage'.format('bug' if bugs else 'branch')
264264
axes.set(ylabel=ylabel)
265265
axes.set(xlabel='Fuzzer (highest median coverage on the left)')
266266
axes.set_xticklabels(axes.get_xticklabels(),
@@ -307,7 +307,7 @@ def distribution_plot(self, benchmark_snapshot_df, axes=None, bugs=False):
307307
axes.set_title(_formatted_title(benchmark_snapshot_df))
308308
axes.legend(loc='upper right', frameon=False)
309309

310-
axes.set(xlabel='Bug coverage' if bugs else 'Code region coverage')
310+
axes.set(xlabel='Bug coverage' if bugs else 'Code branch coverage')
311311
axes.set(ylabel='Density')
312312
axes.set_xticklabels(axes.get_xticklabels(),
313313
rotation=_DEFAULT_LABEL_ROTATION,
@@ -339,7 +339,7 @@ def ranking_plot(self, benchmark_snapshot_df, axes=None, bugs=False):
339339
ax=axes)
340340

341341
axes.set_title(_formatted_title(benchmark_snapshot_df))
342-
ylabel = 'Reached {} coverage'.format('bug' if bugs else 'region')
342+
ylabel = 'Reached {} coverage'.format('bug' if bugs else 'branch')
343343
axes.set(ylabel=ylabel)
344344
axes.set(xlabel='Fuzzer (highest median coverage on the left)')
345345
axes.set_xticklabels(axes.get_xticklabels(),
@@ -491,17 +491,17 @@ def write_critical_difference_plot(self, average_ranks, num_of_benchmarks,
491491
plt.close(fig)
492492

493493
def unique_coverage_ranking_plot(self,
494-
unique_region_cov_df_combined,
494+
unique_branch_cov_df_combined,
495495
axes=None):
496496
"""Draws unique_coverage_ranking plot. The fuzzer labels will be in
497497
the order of their coverage."""
498498

499-
fuzzer_order = unique_region_cov_df_combined.sort_values(
500-
by='unique_regions_covered', ascending=False).fuzzer
499+
fuzzer_order = unique_branch_cov_df_combined.sort_values(
500+
by='unique_branches_covered', ascending=False).fuzzer
501501

502-
axes = sns.barplot(y='unique_regions_covered',
502+
axes = sns.barplot(y='unique_branches_covered',
503503
x='fuzzer',
504-
data=unique_region_cov_df_combined,
504+
data=unique_branch_cov_df_combined,
505505
order=fuzzer_order,
506506
palette=self._fuzzer_colors,
507507
ax=axes)
@@ -517,7 +517,7 @@ def unique_coverage_ranking_plot(self,
517517

518518
sns.barplot(y='aggregated_edges_covered',
519519
x='fuzzer',
520-
data=unique_region_cov_df_combined,
520+
data=unique_branch_cov_df_combined,
521521
order=fuzzer_order,
522522
facecolor=(1, 1, 1, 0),
523523
edgecolor='0.2',
@@ -531,11 +531,11 @@ def unique_coverage_ranking_plot(self,
531531

532532
sns.despine(ax=axes, trim=True)
533533

534-
def write_unique_coverage_ranking_plot(self, unique_region_cov_df_combined,
534+
def write_unique_coverage_ranking_plot(self, unique_branch_cov_df_combined,
535535
image_path):
536536
"""Writes ranking plot for unique coverage."""
537537
self._write_plot_to_image(self.unique_coverage_ranking_plot,
538-
unique_region_cov_df_combined,
538+
unique_branch_cov_df_combined,
539539
image_path,
540540
wide=True)
541541

analysis/report_templates/default.html

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -385,12 +385,12 @@ <h5 class="center-align">Mann-Whitney U test</h4>
385385

386386
<div class="row">
387387
<div class="col s6 offset-s3">
388-
<h5 class="center-align">Ranking by unique code regions covered</h4>
388+
<h5 class="center-align">Ranking by unique code branches covered</h4>
389389
<img class="responsive-img materialboxed"
390390
src="{{ benchmark.unique_coverage_ranking_plot }}">
391-
Each bar shows the total number of code regions found by a given fuzzer.
392-
The colored area shows the number of unique code regions
393-
(i.e., regions that were not covered by any other fuzzers).
391+
Each bar shows the total number of code branches found by a given fuzzer.
392+
The colored area shows the number of unique code branches
393+
(i.e., branches that were not covered by any other fuzzers).
394394
</div>
395395
</div> <!-- row -->
396396

@@ -399,7 +399,7 @@ <h5 class="center-align">Ranking by unique code regions covered</h4>
399399
<h5 class="center-align">Pairwise unique code coverage</h4>
400400
<img class="responsive-img materialboxed"
401401
src="{{ benchmark.pairwise_unique_coverage_plot }}">
402-
Each cell represents the number of code regions covered by the fuzzer
402+
Each cell represents the number of code branches covered by the fuzzer
403403
of the column but not by the fuzzer of the row
404404
</div>
405405
</div> <!-- row -->

0 commit comments

Comments
 (0)