/
mach
executable file
·302 lines (248 loc) · 9.89 KB
1
2
3
4
5
#!/usr/bin/env python
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
##########################################################################
7
8
9
10
11
12
#
# This is a collection of helper tools to get stuff done in NSS.
#
import sys
import argparse
13
import fnmatch
14
15
16
import subprocess
import os
import platform
17
18
import tempfile
19
from hashlib import sha256
20
from gtests.common.wycheproof.genTestVectors import generate_test_vectors
21
22
DEVNULL = open(os.devnull, 'wb')
23
24
cwd = os.path.dirname(os.path.abspath(__file__))
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
def run_tests(test, cycles="standard", env={}, silent=False):
domsuf = os.getenv('DOMSUF', "localdomain")
host = os.getenv('HOST', "localhost")
env = env.copy()
env.update({
"NSS_TESTS": test,
"NSS_CYCLES": cycles,
"DOMSUF": domsuf,
"HOST": host
})
os_env = os.environ
os_env.update(env)
command = cwd + "/tests/all.sh"
stdout = stderr = DEVNULL if silent else None
subprocess.check_call(command, env=os_env, stdout=stdout, stderr=stderr)
40
41
42
class cfAction(argparse.Action):
docker_command = ["docker"]
43
restorecon = None
44
45
def __call__(self, parser, args, values, option_string=None):
46
if not args.noroot:
47
self.setDockerCommand()
48
49
if values:
50
files = [os.path.relpath(os.path.abspath(x), start=cwd) for x in values]
51
else:
52
files = self.modifiedFiles()
53
files = [os.path.join('/home/worker/nss', x) for x in files]
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
# First check if we can run docker.
try:
with open(os.devnull, "w") as f:
subprocess.check_call(
self.docker_command + ["images"], stdout=f)
except:
print("Please install docker and start the docker daemon.")
sys.exit(1)
docker_image = 'clang-format-service:latest'
cf_docker_folder = cwd + "/automation/clang-format"
# Build the image if necessary.
if self.filesChanged(cf_docker_folder):
self.buildImage(docker_image, cf_docker_folder)
# Check if we have the docker image.
try:
command = self.docker_command + [
"image", "inspect", "clang-format-service:latest"
]
with open(os.devnull, "w") as f:
subprocess.check_call(command, stdout=f)
except:
print("I have to build the docker image first.")
self.buildImage(docker_image, cf_docker_folder)
command = self.docker_command + [
83
'run', '-v', cwd + ':/home/worker/nss:Z', '--rm', '-ti', docker_image
84
]
85
86
# The clang format script returns 1 if something's to do. We don't
# care.
87
88
89
subprocess.call(command + files)
if self.restorecon is not None:
subprocess.call([self.restorecon, '-R', cwd])
90
91
92
93
94
95
96
def filesChanged(self, path):
hash = sha256()
for dirname, dirnames, files in os.walk(path):
for file in files:
with open(os.path.join(dirname, file), "rb") as f:
hash.update(f.read())
97
chk_file = cwd + "/.chk"
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
old_chk = ""
new_chk = hash.hexdigest()
if os.path.exists(chk_file):
with open(chk_file) as f:
old_chk = f.readline()
if old_chk != new_chk:
with open(chk_file, "w+") as f:
f.write(new_chk)
return True
return False
def buildImage(self, docker_image, cf_docker_folder):
command = self.docker_command + [
"build", "-t", docker_image, cf_docker_folder
]
subprocess.check_call(command)
return
def setDockerCommand(self):
if platform.system() == "Linux":
118
119
from distutils.spawn import find_executable
self.restorecon = find_executable('restorecon')
120
121
self.docker_command = ["sudo"] + self.docker_command
122
123
124
125
def modifiedFiles(self):
files = []
if os.path.exists(os.path.join(cwd, '.hg')):
st = subprocess.Popen(['hg', 'status', '-m', '-a'],
126
cwd=cwd, stdout=subprocess.PIPE, universal_newlines=True)
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
for line in iter(st.stdout.readline, ''):
files += [line[2:].rstrip()]
elif os.path.exists(os.path.join(cwd, '.git')):
st = subprocess.Popen(['git', 'status', '--porcelain'],
cwd=cwd, stdout=subprocess.PIPE)
for line in iter(st.stdout.readline, ''):
if line[1] == 'M' or line[1] != 'D' and \
(line[0] == 'M' or line[0] == 'A' or
line[0] == 'C' or line[0] == 'U'):
files += [line[3:].rstrip()]
elif line[0] == 'R':
files += [line[line.index(' -> ', beg=4) + 4:]]
else:
print('Warning: neither mercurial nor git detected!')
def isFormatted(x):
return x[-2:] == '.c' or x[-3:] == '.cc' or x[-2:] == '.h'
return [x for x in files if isFormatted(x)]
146
147
class buildAction(argparse.Action):
148
149
150
151
152
153
def __call__(self, parser, args, values, option_string=None):
subprocess.check_call([cwd + "/build.sh"] + values)
class testAction(argparse.Action):
154
155
156
157
158
159
160
161
def __call__(self, parser, args, values, option_string=None):
run_tests(values)
class covAction(argparse.Action):
def runSslGtests(self, outdir):
162
env = {
163
164
"GTESTFILTER": "*", # Prevent parallel test runs.
"ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir
165
}
166
167
168
169
170
171
172
173
174
run_tests("ssl_gtests", env=env, silent=True)
def findSanCovFile(self, outdir):
for file in os.listdir(outdir):
if fnmatch.fnmatch(file, 'ssl_gtest.*.sancov'):
return os.path.join(outdir, file)
return None
175
176
def __call__(self, parser, args, values, option_string=None):
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
outdir = args.outdir
print("Output directory: " + outdir)
print("\nBuild with coverage sanitizers...\n")
sancov_args = "edge,no-prune,trace-pc-guard,trace-cmp"
subprocess.check_call([
os.path.join(cwd, "build.sh"), "-c", "--clang", "--asan",
"--sancov=" + sancov_args
])
print("\nRun ssl_gtests to get a coverage report...")
self.runSslGtests(outdir)
print("Done.")
sancov_file = self.findSanCovFile(outdir)
if not sancov_file:
print("Couldn't find .sancov file.")
sys.exit(1)
symcov_file = os.path.join(outdir, "ssl_gtest.symcov")
out = open(symcov_file, 'wb')
subprocess.check_call([
"sancov",
"-blacklist=" + os.path.join(cwd, ".sancov-blacklist"),
"-symbolize", sancov_file,
os.path.join(cwd, "../dist/Debug/bin/ssl_gtest")
], stdout=out)
out.close()
print("\nCoverage report: " + symcov_file)
207
208
209
210
class commandsAction(argparse.Action):
commands = []
211
212
213
214
215
def __call__(self, parser, args, values, option_string=None):
for c in commandsAction.commands:
print(c)
216
217
218
class wycheproofAction(argparse.Action):
def __call__(self, parser, args, values, option_string=None):
219
generate_test_vectors()
220
221
222
clangFormat = cfAction(None, None, None)
clangFormat(None, args, None)
223
224
225
226
227
228
229
230
231
232
233
234
235
236
def parse_arguments():
parser = argparse.ArgumentParser(
description='NSS helper script. ' +
'Make sure to separate sub-command arguments with --.')
subparsers = parser.add_subparsers()
parser_build = subparsers.add_parser(
'build', help='All arguments are passed to build.sh')
parser_build.add_argument(
'build_args', nargs='*', help="build arguments", action=buildAction)
parser_cf = subparsers.add_parser(
'clang-format',
237
238
239
240
241
242
243
244
245
246
help="""
Run clang-format.
By default this runs against any files that you have modified. If
there are no modified files, it checks everything.
""")
parser_cf.add_argument(
'--noroot',
help='On linux, suppress the use of \'sudo\' for running docker.',
action='store_true')
247
parser_cf.add_argument(
248
'<file/dir>',
249
nargs='*',
250
help="Specify files or directories to run clang-format on",
251
252
253
254
255
action=cfAction)
parser_test = subparsers.add_parser(
'tests', help='Run tests through tests/all.sh.')
tests = [
256
"cipher", "lowhash", "chains", "cert", "dbtests", "tools", "fips",
257
"sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec",
258
"gtests", "ssl_gtests", "bogo", "interop", "policy"
259
260
261
]
parser_test.add_argument(
'test', choices=tests, help="Available tests", action=testAction)
262
263
264
265
266
267
268
269
270
271
272
parser_cov = subparsers.add_parser(
'coverage', help='Generate coverage report')
cov_modules = ["ssl_gtests"]
parser_cov.add_argument(
'--outdir', help='Output directory for coverage report data.',
default=tempfile.mkdtemp())
parser_cov.add_argument(
'module', choices=cov_modules, help="Available coverage modules",
action=covAction)
273
274
275
276
277
278
279
280
parser_commands = subparsers.add_parser(
'mach-commands',
help="list commands")
parser_commands.add_argument(
'mach-commands',
nargs='*',
action=commandsAction)
281
282
283
284
285
286
287
288
289
290
291
292
parser_wycheproof = subparsers.add_parser(
'wycheproof',
help="generate wycheproof test headers")
parser_wycheproof.add_argument(
'--noroot',
help='On linux, suppress the use of \'sudo\' for running docker.',
action='store_true')
parser_wycheproof.add_argument(
'wycheproof',
nargs='*',
action=wycheproofAction)
293
commandsAction.commands = [c for c in subparsers.choices]
294
295
296
297
298
299
300
301
302
return parser.parse_args()
def main():
parse_arguments()
if __name__ == '__main__':
main()