#!/usr/bin/env python # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. ########################################################################## # # This is a collection of helper tools to get stuff done in NSS. # import sys import argparse import fnmatch import subprocess import os import platform import tempfile from hashlib import sha256 from gtests.common.wycheproof.genTestVectors import generate_test_vectors DEVNULL = open(os.devnull, 'wb') cwd = os.path.dirname(os.path.abspath(__file__)) def run_tests(test, cycles="standard", env={}, silent=False): domsuf = os.getenv('DOMSUF', "localdomain") host = os.getenv('HOST', "localhost") env = env.copy() env.update({ "NSS_TESTS": test, "NSS_CYCLES": cycles, "DOMSUF": domsuf, "HOST": host }) os_env = os.environ os_env.update(env) command = cwd + "/tests/all.sh" stdout = stderr = DEVNULL if silent else None subprocess.check_call(command, env=os_env, stdout=stdout, stderr=stderr) class cfAction(argparse.Action): docker_command = ["docker"] restorecon = None def __call__(self, parser, args, values, option_string=None): if not args.noroot: self.setDockerCommand() if values: files = [os.path.relpath(os.path.abspath(x), start=cwd) for x in values] else: files = self.modifiedFiles() files = [os.path.join('/home/worker/nss', x) for x in files] # First check if we can run docker. try: with open(os.devnull, "w") as f: subprocess.check_call( self.docker_command + ["images"], stdout=f) except: print("Please install docker and start the docker daemon.") sys.exit(1) docker_image = 'clang-format-service:latest' cf_docker_folder = cwd + "/automation/clang-format" # Build the image if necessary. if self.filesChanged(cf_docker_folder): self.buildImage(docker_image, cf_docker_folder) # Check if we have the docker image. try: command = self.docker_command + [ "image", "inspect", "clang-format-service:latest" ] with open(os.devnull, "w") as f: subprocess.check_call(command, stdout=f) except: print("I have to build the docker image first.") self.buildImage(docker_image, cf_docker_folder) command = self.docker_command + [ 'run', '-v', cwd + ':/home/worker/nss:Z', '--rm', '-ti', docker_image ] # The clang format script returns 1 if something's to do. We don't # care. subprocess.call(command + files) if self.restorecon is not None: subprocess.call([self.restorecon, '-R', cwd]) def filesChanged(self, path): hash = sha256() for dirname, dirnames, files in os.walk(path): for file in files: with open(os.path.join(dirname, file), "rb") as f: hash.update(f.read()) chk_file = cwd + "/.chk" old_chk = "" new_chk = hash.hexdigest() if os.path.exists(chk_file): with open(chk_file) as f: old_chk = f.readline() if old_chk != new_chk: with open(chk_file, "w+") as f: f.write(new_chk) return True return False def buildImage(self, docker_image, cf_docker_folder): command = self.docker_command + [ "build", "-t", docker_image, cf_docker_folder ] subprocess.check_call(command) return def setDockerCommand(self): if platform.system() == "Linux": from distutils.spawn import find_executable self.restorecon = find_executable('restorecon') self.docker_command = ["sudo"] + self.docker_command def modifiedFiles(self): files = [] if os.path.exists(os.path.join(cwd, '.hg')): st = subprocess.Popen(['hg', 'status', '-m', '-a'], cwd=cwd, stdout=subprocess.PIPE, universal_newlines=True) for line in iter(st.stdout.readline, ''): files += [line[2:].rstrip()] elif os.path.exists(os.path.join(cwd, '.git')): st = subprocess.Popen(['git', 'status', '--porcelain'], cwd=cwd, stdout=subprocess.PIPE) for line in iter(st.stdout.readline, ''): if line[1] == 'M' or line[1] != 'D' and \ (line[0] == 'M' or line[0] == 'A' or line[0] == 'C' or line[0] == 'U'): files += [line[3:].rstrip()] elif line[0] == 'R': files += [line[line.index(' -> ', beg=4) + 4:]] else: print('Warning: neither mercurial nor git detected!') def isFormatted(x): return x[-2:] == '.c' or x[-3:] == '.cc' or x[-2:] == '.h' return [x for x in files if isFormatted(x)] class buildAction(argparse.Action): def __call__(self, parser, args, values, option_string=None): subprocess.check_call([cwd + "/build.sh"] + values) class testAction(argparse.Action): def __call__(self, parser, args, values, option_string=None): run_tests(values) class covAction(argparse.Action): def runSslGtests(self, outdir): env = { "GTESTFILTER": "*", # Prevent parallel test runs. "ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir } run_tests("ssl_gtests", env=env, silent=True) def findSanCovFile(self, outdir): for file in os.listdir(outdir): if fnmatch.fnmatch(file, 'ssl_gtest.*.sancov'): return os.path.join(outdir, file) return None def __call__(self, parser, args, values, option_string=None): outdir = args.outdir print("Output directory: " + outdir) print("\nBuild with coverage sanitizers...\n") sancov_args = "edge,no-prune,trace-pc-guard,trace-cmp" subprocess.check_call([ os.path.join(cwd, "build.sh"), "-c", "--clang", "--asan", "--sancov=" + sancov_args ]) print("\nRun ssl_gtests to get a coverage report...") self.runSslGtests(outdir) print("Done.") sancov_file = self.findSanCovFile(outdir) if not sancov_file: print("Couldn't find .sancov file.") sys.exit(1) symcov_file = os.path.join(outdir, "ssl_gtest.symcov") out = open(symcov_file, 'wb') subprocess.check_call([ "sancov", "-blacklist=" + os.path.join(cwd, ".sancov-blacklist"), "-symbolize", sancov_file, os.path.join(cwd, "../dist/Debug/bin/ssl_gtest") ], stdout=out) out.close() print("\nCoverage report: " + symcov_file) class commandsAction(argparse.Action): commands = [] def __call__(self, parser, args, values, option_string=None): for c in commandsAction.commands: print(c) class wycheproofAction(argparse.Action): def __call__(self, parser, args, values, option_string=None): generate_test_vectors() clangFormat = cfAction(None, None, None) clangFormat(None, args, None) def parse_arguments(): parser = argparse.ArgumentParser( description='NSS helper script. ' + 'Make sure to separate sub-command arguments with --.') subparsers = parser.add_subparsers() parser_build = subparsers.add_parser( 'build', help='All arguments are passed to build.sh') parser_build.add_argument( 'build_args', nargs='*', help="build arguments", action=buildAction) parser_cf = subparsers.add_parser( 'clang-format', help=""" Run clang-format. By default this runs against any files that you have modified. If there are no modified files, it checks everything. """) parser_cf.add_argument( '--noroot', help='On linux, suppress the use of \'sudo\' for running docker.', action='store_true') parser_cf.add_argument( '', nargs='*', help="Specify files or directories to run clang-format on", action=cfAction) parser_test = subparsers.add_parser( 'tests', help='Run tests through tests/all.sh.') tests = [ "cipher", "lowhash", "chains", "cert", "dbtests", "tools", "fips", "sdr", "crmf", "smime", "ssl", "ocsp", "merge", "pkits", "ec", "gtests", "ssl_gtests", "bogo", "interop", "policy" ] parser_test.add_argument( 'test', choices=tests, help="Available tests", action=testAction) parser_cov = subparsers.add_parser( 'coverage', help='Generate coverage report') cov_modules = ["ssl_gtests"] parser_cov.add_argument( '--outdir', help='Output directory for coverage report data.', default=tempfile.mkdtemp()) parser_cov.add_argument( 'module', choices=cov_modules, help="Available coverage modules", action=covAction) parser_commands = subparsers.add_parser( 'mach-commands', help="list commands") parser_commands.add_argument( 'mach-commands', nargs='*', action=commandsAction) parser_wycheproof = subparsers.add_parser( 'wycheproof', help="generate wycheproof test headers") parser_wycheproof.add_argument( '--noroot', help='On linux, suppress the use of \'sudo\' for running docker.', action='store_true') parser_wycheproof.add_argument( 'wycheproof', nargs='*', action=wycheproofAction) commandsAction.commands = [c for c in subparsers.choices] return parser.parse_args() def main(): parse_arguments() if __name__ == '__main__': main()