-
Notifications
You must be signed in to change notification settings - Fork 2.2k
/
test_all.py
executable file
·224 lines (190 loc) · 7.84 KB
/
test_all.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
#!/usr/bin/env python3
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
"""Does bad_build_check on all fuzz targets in $OUT."""
import contextlib
import multiprocessing
import os
import re
import shutil
import subprocess
import stat
import sys
TMP_FUZZER_DIR = '/tmp/not-out'
EXECUTABLE = stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH
IGNORED_TARGETS = [
r'do_stuff_fuzzer', r'checksum_fuzzer', r'fuzz_dump', r'fuzz_keyring',
r'xmltest', r'fuzz_compression_sas_rle', r'ares_*_fuzzer'
]
IGNORED_TARGETS_RE = re.compile('^' + r'$|^'.join(IGNORED_TARGETS) + '$')
def recreate_directory(directory):
"""Creates |directory|. If it already exists than deletes it first before
creating."""
if os.path.exists(directory):
shutil.rmtree(directory)
os.mkdir(directory)
def move_directory_contents(src_directory, dst_directory):
"""Moves contents of |src_directory| to |dst_directory|."""
# Use mv because mv preserves file permissions. If we don't preserve file
# permissions that can mess up CheckFuzzerBuildTest in cifuzz_test.py and
# other cases where one is calling test_all on files not in OSS-Fuzz's real
# out directory.
src_contents = [
os.path.join(src_directory, filename)
for filename in os.listdir(src_directory)
]
command = ['mv'] + src_contents + [dst_directory]
subprocess.check_call(command)
def is_elf(filepath):
"""Returns True if |filepath| is an ELF file."""
result = subprocess.run(['file', filepath],
stdout=subprocess.PIPE,
check=False)
return b'ELF' in result.stdout
def find_fuzz_targets(directory, fuzzing_language):
"""Returns paths to fuzz targets in |directory|."""
# TODO(https://github.com/google/oss-fuzz/issues/4585): Use libClusterFuzz for
# this.
fuzz_targets = []
for filename in os.listdir(directory):
path = os.path.join(directory, filename)
if filename == 'llvm-symbolizer':
continue
if filename.startswith('afl-'):
continue
if filename.startswith('jazzer_'):
continue
if not os.path.isfile(path):
continue
if not os.stat(path).st_mode & EXECUTABLE:
continue
# Fuzz targets are expected to be ELF binaries for languages other than
# Python and Java.
if (fuzzing_language != 'python' and fuzzing_language != 'jvm' and
not is_elf(path)):
continue
if os.getenv('FUZZING_ENGINE') != 'none':
with open(path, 'rb') as file_handle:
binary_contents = file_handle.read()
if b'LLVMFuzzerTestOneInput' not in binary_contents:
continue
fuzz_targets.append(path)
return fuzz_targets
def do_bad_build_check(fuzz_target):
"""Runs bad_build_check on |fuzz_target|. Returns a
Subprocess.ProcessResult."""
print('INFO: performing bad build checks for', fuzz_target)
command = ['bad_build_check', fuzz_target]
return subprocess.run(command,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
check=False)
def get_broken_fuzz_targets(bad_build_results, fuzz_targets):
"""Returns a list of broken fuzz targets and their process results in
|fuzz_targets| where each item in |bad_build_results| is the result of
bad_build_check on the corresponding element in |fuzz_targets|."""
broken = []
for result, fuzz_target in zip(bad_build_results, fuzz_targets):
if result.returncode != 0:
broken.append((fuzz_target, result))
return broken
def has_ignored_targets(out_dir):
"""Returns True if |out_dir| has any fuzz targets we are supposed to ignore
bad build checks of."""
out_files = set(os.listdir(out_dir))
for filename in out_files:
if re.match(IGNORED_TARGETS_RE, filename):
return True
return False
@contextlib.contextmanager
def use_different_out_dir():
"""Context manager that moves OUT to TMP_FUZZER_DIR. This is useful for
catching hardcoding. Note that this sets the environment variable OUT and
therefore must be run before multiprocessing.Pool is created. Resets OUT at
the end."""
# Use a fake OUT directory to catch path hardcoding that breaks on
# ClusterFuzz.
out = os.getenv('OUT')
initial_out = out
recreate_directory(TMP_FUZZER_DIR)
out = TMP_FUZZER_DIR
# Set this so that run_fuzzer which is called by bad_build_check works
# properly.
os.environ['OUT'] = out
# We move the contents of the directory because we can't move the
# directory itself because it is a mount.
move_directory_contents(initial_out, out)
try:
yield out
finally:
move_directory_contents(out, initial_out)
shutil.rmtree(out)
os.environ['OUT'] = initial_out
def test_all_outside_out(fuzzing_language, allowed_broken_targets_percentage):
"""Wrapper around test_all that changes OUT and returns the result."""
with use_different_out_dir() as out:
return test_all(out, fuzzing_language, allowed_broken_targets_percentage)
def test_all(out, fuzzing_language, allowed_broken_targets_percentage):
"""Do bad_build_check on all fuzz targets."""
# TODO(metzman): Refactor so that we can convert test_one to python.
fuzz_targets = find_fuzz_targets(out, fuzzing_language)
if not fuzz_targets:
print('ERROR: No fuzz targets found.')
return False
pool = multiprocessing.Pool()
bad_build_results = pool.map(do_bad_build_check, fuzz_targets)
broken_targets = get_broken_fuzz_targets(bad_build_results, fuzz_targets)
broken_targets_count = len(broken_targets)
if not broken_targets_count:
return True
print('Broken fuzz targets', broken_targets_count)
total_targets_count = len(fuzz_targets)
broken_targets_percentage = 100 * broken_targets_count / total_targets_count
for broken_target, result in broken_targets:
print(broken_target)
# Use write because we can't print binary strings.
sys.stdout.buffer.write(result.stdout + result.stderr + b'\n')
if broken_targets_percentage > allowed_broken_targets_percentage:
print('ERROR: {broken_targets_percentage}% of fuzz targets seem to be '
'broken. See the list above for a detailed information.'.format(
broken_targets_percentage=broken_targets_percentage))
if has_ignored_targets(out):
print('Build check automatically passing because of ignored targets.')
return True
return False
print('{total_targets_count} fuzzers total, {broken_targets_count} '
'seem to be broken ({broken_targets_percentage}%).'.format(
total_targets_count=total_targets_count,
broken_targets_count=broken_targets_count,
broken_targets_percentage=broken_targets_percentage))
return True
def get_allowed_broken_targets_percentage():
"""Returns the value of the environment value
'ALLOWED_BROKEN_TARGETS_PERCENTAGE' as an int or returns a reasonable
default."""
return int(os.getenv('ALLOWED_BROKEN_TARGETS_PERCENTAGE') or '10')
def main():
"""Does bad_build_check on all fuzz targets in parallel. Returns 0 on success.
Returns 1 on failure."""
# Set these environment variables here so that stdout
fuzzing_language = os.getenv('FUZZING_LANGUAGE')
allowed_broken_targets_percentage = get_allowed_broken_targets_percentage()
if not test_all_outside_out(fuzzing_language,
allowed_broken_targets_percentage):
return 1
return 0
if __name__ == '__main__':
sys.exit(main())