From 7621bb07627d938fe8f828c9ebb6f48fc5236a43 Mon Sep 17 00:00:00 2001 From: Tim Taubert Date: Fri, 20 Jan 2017 14:45:55 +0100 Subject: [PATCH] Bug 1330655 - Make NSS build within OSS-Fuzz r=franziskus Differential Revision: https://nss-review.dev.mozaws.net/D161 --HG-- extra : amend_source : 875313e502f89c1488e945bd4f2b2aaf7b487575 --- .gitignore | 1 + .hgignore | 1 + automation/taskcluster/graph/src/extend.js | 2 +- automation/taskcluster/scripts/fuzz.sh | 2 +- build.sh | 22 +++- coreconf/config.gypi | 80 +++--------- coreconf/fuzz.sh | 22 ++-- coreconf/nspr.sh | 9 +- coreconf/sanitizers.py | 9 +- coreconf/sanitizers.sh | 11 +- fuzz/fuzz.gyp | 139 +++++++++++++-------- fuzz/warning.txt | 27 ++-- gtests/common/gtest.gypi | 2 +- lib/freebl/freebl.gyp | 4 +- lib/ssl/ssl.gyp | 2 +- nss.gyp | 8 +- 16 files changed, 178 insertions(+), 163 deletions(-) diff --git a/.gitignore b/.gitignore index 946f1418b8..079fcae258 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ GTAGS .ycm_extra_conf.py* fuzz/libFuzzer/* fuzz/corpus +fuzz/out diff --git a/.hgignore b/.hgignore index 1582595365..a8d626ecdd 100644 --- a/.hgignore +++ b/.hgignore @@ -17,3 +17,4 @@ GTAGS .ycm_extra_conf.py* fuzz/libFuzzer/* fuzz/corpus +fuzz/out diff --git a/automation/taskcluster/graph/src/extend.js b/automation/taskcluster/graph/src/extend.js index 25f2e3f2b3..a7e1d5c61b 100644 --- a/automation/taskcluster/graph/src/extend.js +++ b/automation/taskcluster/graph/src/extend.js @@ -285,7 +285,7 @@ async function scheduleFuzzing() { "/bin/bash", "-c", "bin/checkout.sh && " + - "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz" + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls" ], artifacts: { public: { diff --git a/automation/taskcluster/scripts/fuzz.sh b/automation/taskcluster/scripts/fuzz.sh index a1cb45830a..ed25a805ff 100755 --- a/automation/taskcluster/scripts/fuzz.sh +++ b/automation/taskcluster/scripts/fuzz.sh @@ -18,4 +18,4 @@ mkdir -p nss/fuzz/corpus/$type objdir=$(cat dist/latest) # Run nssfuzz. -LD_LIBRARY_PATH=$LD_LIBRARY_PATH:dist/$objdir/lib dist/$objdir/bin/nssfuzz-"$type" "$@" +dist/$objdir/bin/nssfuzz-"$type" "$@" diff --git a/build.sh b/build.sh index df0774a8cc..ab1774c4c5 100755 --- a/build.sh +++ b/build.sh @@ -15,9 +15,9 @@ show_help() { cat << EOF Usage: ${0##*/} [-hcv] [-j ] [--nspr] [--gyp|-g] [--opt|-o] [-m32] - [--test] [--fuzz] [--pprof] [--scan-build[=output]] + [--test] [--pprof] [--scan-build[=output]] [--ct-verif] [--asan] [--ubsan] [--msan] [--sancov[=edge|bb|func|...]] - [--ct-verif] [--disable-tests] + [--disable-tests] [--fuzz[=tls|oss]] This script builds NSS with gyp and ninja. @@ -35,7 +35,9 @@ NSS build tool options: --opt|-o do an opt build -m32 do a 32-bit build on a 64-bit system --test ignore map files and export everything we have - --fuzz enable fuzzing mode. this always enables test builds + --fuzz build fuzzing targets (this always enables test builds) + --fuzz=tls to enable TLS fuzzing mode + --fuzz=oss to build for OSS-Fuzz --pprof build with gperftool support --ct-verif build with valgrind for ct-verif --scan-build run the build with scan-build (scan-build has to be in the path) @@ -74,6 +76,8 @@ rebuild_nspr=0 target=Debug verbose=0 fuzz=0 +fuzz_tls=0 +fuzz_oss=0 gyp_params=(--depth="$cwd" --generator-output=".") nspr_params=() @@ -95,6 +99,8 @@ while [ $# -gt 0 ]; do -v) ninja_params+=(-v); verbose=1 ;; --test) gyp_params+=(-Dtest_build=1) ;; --fuzz) fuzz=1 ;; + --fuzz=oss) fuzz=1; fuzz_oss=1 ;; + --fuzz=tls) fuzz=1; fuzz_tls=1 ;; --scan-build) enable_scanbuild ;; --scan-build=?*) enable_scanbuild "${1#*=}" ;; --opt|-o) opt_build=1 ;; @@ -108,6 +114,7 @@ while [ $# -gt 0 ]; do --pprof) gyp_params+=(-Duse_pprof=1) ;; --ct-verif) gyp_params+=(-Dct_verif=1) ;; --disable-tests) gyp_params+=(-Ddisable_tests=1) ;; + --no-zdefs) gyp_params+=(-Dno_zdefs=1) ;; *) show_help; exit 2 ;; esac shift @@ -152,6 +159,7 @@ check_config() mkdir -p $(dirname "$newconf") echo CC="$CC" >"$newconf" echo CCC="$CCC" >>"$newconf" + echo CXX="$CXX" >>"$newconf" for i in "$@"; do echo $i; done | sort >>"$newconf" # Note: The following diff fails if $oldconf isn't there as well, which @@ -170,6 +178,9 @@ elif [ ! -d "$dist_dir"/$target ]; then rebuild_nspr=1 fi +# Update NSPR ${C,CXX,LD}FLAGS. +nspr_set_flags $sanitizer_flags + if check_config "$nspr_config" "${nspr_params[@]}" \ nspr_cflags="$nspr_cflags" \ nspr_cxxflags="$nspr_cxxflags" \ @@ -177,6 +188,11 @@ if check_config "$nspr_config" "${nspr_params[@]}" \ rebuild_nspr=1 fi +# Forward sanitizer flags. +if [ ! -z "$sanitizer_flags" ]; then + gyp_params+=(-Dsanitizer_flags="$sanitizer_flags") +fi + if check_config "$gyp_config" "${gyp_params[@]}"; then rebuild_gyp=1 fi diff --git a/coreconf/config.gypi b/coreconf/config.gypi index b531239cf9..a91d2a2035 100644 --- a/coreconf/config.gypi +++ b/coreconf/config.gypi @@ -97,12 +97,11 @@ 'moz_fold_libs%': 0, 'moz_folded_library_name%': '', 'ssl_enable_zlib%': 1, - 'use_asan%': 0, - 'use_ubsan%': 0, - 'use_msan%': 0, - 'use_sancov%': 0, + 'sanitizer_flags%': 0, 'test_build%': 0, + 'no_zdefs%': 0, 'fuzz%': 0, + 'fuzz_tls%': 0, 'sign_libs%': 1, 'use_pprof%': 0, 'ct_verif%': 0, @@ -135,7 +134,7 @@ '-lc', ], }], - [ 'use_asan==1 or use_ubsan!=0 or fuzz==1', { + [ 'fuzz==1', { 'variables': { 'debug_optimization_level%': '1', }, @@ -216,7 +215,6 @@ [ 'cc_use_gnu_ld==1', { 'ldflags': [ '-Wl,--gc-sections', - '-Wl,-z,defs', ], 'conditions': [ ['OS=="dragonfly" or OS=="freebsd" or OS=="netbsd" or OS=="openbsd"', { @@ -225,6 +223,11 @@ '-Wl,--warn-unresolved-symbols', ], }], + ['no_zdefs==0', { + 'ldflags': [ + '-Wl,-z,defs', + ], + }], ], }], ], @@ -348,73 +351,22 @@ '/dev/null; then make() { command gmake "$@"; } fi -nspr_sanitizer() +nspr_set_flags() { - local extra=$(python $cwd/coreconf/sanitizers.py "$@") - nspr_cflags="$nspr_cflags $extra" - nspr_cxxflags="$nspr_cxxflags $extra" - nspr_ldflags="$nspr_ldflags $extra" + nspr_cflags="$CFLAGS $@" + nspr_cxxflags="$CXXFLAGS $@" + nspr_ldflags="$LDFLAGS $@" } nspr_build() diff --git a/coreconf/sanitizers.py b/coreconf/sanitizers.py index ed00c49217..b1a77df64b 100644 --- a/coreconf/sanitizers.py +++ b/coreconf/sanitizers.py @@ -5,7 +5,7 @@ def main(): if len(sys.argv) < 2: - raise Exception('Specify either "ld", asan", "msan", "sancov" or "ubsan" as argument.') + raise Exception('Specify either "asan", "msan", "sancov" or "ubsan" as argument.') sanitizer = sys.argv[1] if sanitizer == "ubsan": @@ -27,12 +27,7 @@ def main(): print('-fsanitize-coverage='+sys.argv[2]+' ', end='') return - # We have to remove this from the ld flags when building asan. - if sanitizer == "ld": - print('-Wl,-z,defs ', end='') - return - - raise Exception('Specify either "ld", asan", "msan", "sancov" or "ubsan" as argument.') + raise Exception('Specify either "asan", "msan", "sancov" or "ubsan" as argument.') if __name__ == '__main__': main() diff --git a/coreconf/sanitizers.sh b/coreconf/sanitizers.sh index 1b4ff079ae..6315a1e866 100644 --- a/coreconf/sanitizers.sh +++ b/coreconf/sanitizers.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash # This file is used by build.sh to setup sanitizers. +sanitizer_flags="" + # This tracks what sanitizers are enabled, and their options. declare -A sanitizers enable_sanitizer() @@ -8,8 +10,13 @@ enable_sanitizer() local san="$1" [ -n "${sanitizers[$san]}" ] && return sanitizers[$san]="${2:-1}" - gyp_params+=(-Duse_"$san"="${2:-1}") - nspr_sanitizer "$san" "$2" + + if [ -z "$sanitizer_flags" ]; then + gyp_params+=(-Dno_zdefs=1) + fi + + local cflags=$(python $cwd/coreconf/sanitizers.py "$@") + sanitizer_flags="$sanitizer_flags $cflags" } enable_sancov() diff --git a/fuzz/fuzz.gyp b/fuzz/fuzz.gyp index d0e5e1a53a..3e4c62c2b0 100644 --- a/fuzz/fuzz.gyp +++ b/fuzz/fuzz.gyp @@ -4,39 +4,86 @@ { 'includes': [ '../coreconf/config.gypi', - '../cmd/platlibs.gypi' ], + 'variables': { + 'use_fuzzing_engine': '