Bug 1330655 - Make NSS build within OSS-Fuzz r=franziskus

Differential Revision: https://nss-review.dev.mozaws.net/D161 --HG-- extra : amend_source : 875313e502f89c1488e945bd4f2b2aaf7b487575
sailfishos-mirror · Jan 20, 2017 · 7621bb0 · 7621bb0
1 parent d8a4fc5
commit 7621bb0
Show file tree

Hide file tree

Showing 16 changed files with 141 additions and 126 deletions.
diff --git a/.gitignore b/.gitignore
@@ -17,3 +17,4 @@ GTAGS
 .ycm_extra_conf.py*
 fuzz/libFuzzer/*
 fuzz/corpus
+fuzz/out
diff --git a/.hgignore b/.hgignore
@@ -17,3 +17,4 @@ GTAGS
 .ycm_extra_conf.py*
 fuzz/libFuzzer/*
 fuzz/corpus
+fuzz/out
diff --git a/automation/taskcluster/graph/src/extend.js b/automation/taskcluster/graph/src/extend.js
@@ -285,7 +285,7 @@ async function scheduleFuzzing() {
       "/bin/bash",
       "-c",
       "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz"
+      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls"
     ],
     artifacts: {
       public: {

diff --git a/automation/taskcluster/scripts/fuzz.sh b/automation/taskcluster/scripts/fuzz.sh
@@ -18,4 +18,4 @@ mkdir -p nss/fuzz/corpus/$type
 objdir=$(cat dist/latest)
 
 # Run nssfuzz.
-LD_LIBRARY_PATH=$LD_LIBRARY_PATH:dist/$objdir/lib dist/$objdir/bin/nssfuzz-"$type" "$@"
+dist/$objdir/bin/nssfuzz-"$type" "$@"
diff --git a/build.sh b/build.sh
@@ -15,9 +15,9 @@ show_help()
 {
     cat << EOF
 Usage: ${0##*/} [-hcv] [-j <n>] [--nspr] [--gyp|-g] [--opt|-o] [-m32]
-                [--test] [--fuzz] [--pprof] [--scan-build[=output]]
+                [--test] [--pprof] [--scan-build[=output]] [--ct-verif]
                 [--asan] [--ubsan] [--msan] [--sancov[=edge|bb|func|...]]
-                [--ct-verif] [--disable-tests]
+                [--disable-tests] [--fuzz[=tls|oss]]
 
 This script builds NSS with gyp and ninja.
 
@@ -35,7 +35,9 @@ NSS build tool options:
     --opt|-o         do an opt build
     -m32             do a 32-bit build on a 64-bit system
     --test           ignore map files and export everything we have
-    --fuzz           enable fuzzing mode. this always enables test builds
+    --fuzz           build fuzzing targets (this always enables test builds)
+                     --fuzz=tls to enable TLS fuzzing mode
+                     --fuzz=oss to build for OSS-Fuzz
     --pprof          build with gperftool support
     --ct-verif       build with valgrind for ct-verif
     --scan-build     run the build with scan-build (scan-build has to be in the path)
@@ -74,6 +76,8 @@ rebuild_nspr=0
 target=Debug
 verbose=0
 fuzz=0
+fuzz_tls=0
+fuzz_oss=0
 
 gyp_params=(--depth="$cwd" --generator-output=".")
 nspr_params=()
@@ -95,6 +99,8 @@ while [ $# -gt 0 ]; do
         -v) ninja_params+=(-v); verbose=1 ;;
         --test) gyp_params+=(-Dtest_build=1) ;;
         --fuzz) fuzz=1 ;;
+        --fuzz=oss) fuzz=1; fuzz_oss=1 ;;
+        --fuzz=tls) fuzz=1; fuzz_tls=1 ;;
         --scan-build) enable_scanbuild  ;;
         --scan-build=?*) enable_scanbuild "${1#*=}" ;;
         --opt|-o) opt_build=1 ;;
@@ -108,6 +114,7 @@ while [ $# -gt 0 ]; do
         --pprof) gyp_params+=(-Duse_pprof=1) ;;
         --ct-verif) gyp_params+=(-Dct_verif=1) ;;
         --disable-tests) gyp_params+=(-Ddisable_tests=1) ;;
+        --no-zdefs) gyp_params+=(-Dno_zdefs=1) ;;
         *) show_help; exit 2 ;;
     esac
     shift
@@ -152,6 +159,7 @@ check_config()
     mkdir -p $(dirname "$newconf")
     echo CC="$CC" >"$newconf"
     echo CCC="$CCC" >>"$newconf"
+    echo CXX="$CXX" >>"$newconf"
     for i in "$@"; do echo $i; done | sort >>"$newconf"
 
     # Note: The following diff fails if $oldconf isn't there as well, which
@@ -170,13 +178,21 @@ elif [ ! -d "$dist_dir"/$target ]; then
     rebuild_nspr=1
 fi
 
+# Update NSPR ${C,CXX,LD}FLAGS.
+nspr_set_flags $sanitizer_flags
+
 if check_config "$nspr_config" "${nspr_params[@]}" \
                  nspr_cflags="$nspr_cflags" \
                  nspr_cxxflags="$nspr_cxxflags" \
                  nspr_ldflags="$nspr_ldflags"; then
     rebuild_nspr=1
 fi
 
+# Forward sanitizer flags.
+if [ ! -z "$sanitizer_flags" ]; then
+    gyp_params+=(-Dsanitizer_flags="$sanitizer_flags")
+fi
+
 if check_config "$gyp_config" "${gyp_params[@]}"; then
     rebuild_gyp=1
 fi

diff --git a/coreconf/config.gypi b/coreconf/config.gypi
@@ -97,12 +97,11 @@
     'moz_fold_libs%': 0,
     'moz_folded_library_name%': '',
     'ssl_enable_zlib%': 1,
-    'use_asan%': 0,
-    'use_ubsan%': 0,
-    'use_msan%': 0,
-    'use_sancov%': 0,
+    'sanitizer_flags%': 0,
     'test_build%': 0,
+    'no_zdefs%': 0,
     'fuzz%': 0,
+    'fuzz_tls%': 0,
     'sign_libs%': 1,
     'use_pprof%': 0,
     'ct_verif%': 0,
@@ -135,7 +134,7 @@
           '-lc',
         ],
       }],
-      [ 'use_asan==1 or use_ubsan!=0 or fuzz==1', {
+      [ 'fuzz==1', {
         'variables': {
           'debug_optimization_level%': '1',
         },
@@ -216,7 +215,6 @@
           [ 'cc_use_gnu_ld==1', {
             'ldflags': [
               '-Wl,--gc-sections',
-              '-Wl,-z,defs',
             ],
             'conditions': [
               ['OS=="dragonfly" or OS=="freebsd" or OS=="netbsd" or OS=="openbsd"', {
@@ -225,6 +223,11 @@
                   '-Wl,--warn-unresolved-symbols',
                 ],
               }],
+              ['no_zdefs==0', {
+                'ldflags': [
+                  '-Wl,-z,defs',
+                ],
+              }],
             ],
           }],
         ],
@@ -348,73 +351,22 @@
               '<!@(<(python) <(DEPTH)/coreconf/werror.py)',
             ],
           }],
-          [ 'fuzz==1', {
+          [ 'fuzz_tls==1', {
             'cflags': [
               '-Wno-unused-function',
-            ]
+            ],
           }],
-          [ 'use_asan==1', {
-            'variables': {
-              'asan_flags': '<!(<(python) <(DEPTH)/coreconf/sanitizers.py asan)',
-              'no_ldflags': '<!(<(python) <(DEPTH)/coreconf/sanitizers.py ld)',
-            },
-            'cflags': ['<@(asan_flags)'],
-            'ldflags': ['<@(asan_flags)'],
-            'ldflags!': ['<@(no_ldflags)'],
+          [ 'sanitizer_flags!=0', {
+            'cflags': ['<@(sanitizer_flags)'],
+            'ldflags': ['<@(sanitizer_flags)'],
             'xcode_settings': {
-              'OTHER_CFLAGS': ['<@(asan_flags)'],
-              'OTHER_LDFLAGS!': ['<@(no_ldflags)'],
+              'OTHER_CFLAGS': ['<@(sanitizer_flags)'],
               # We want to pass -fsanitize=... to our final link call,
               # but not to libtool. OTHER_LDFLAGS is passed to both.
               # To trick GYP into doing what we want, we'll piggyback on
               # LIBRARY_SEARCH_PATHS, producing "-L/usr/lib -fsanitize=...".
               # The -L/usr/lib is redundant but innocuous: it's a default path.
-              'LIBRARY_SEARCH_PATHS': ['/usr/lib <(asan_flags)'],
-            },
-          }],
-          [ 'use_ubsan!=0', {
-            'variables': {
-              'ubsan_flags': '<!(<(python) <(DEPTH)/coreconf/sanitizers.py ubsan <(use_ubsan))',
-              'no_ldflags': '<!(<(python) <(DEPTH)/coreconf/sanitizers.py ld)',
-            },
-            'cflags': ['<@(ubsan_flags)'],
-            'ldflags': ['<@(ubsan_flags)'],
-            'ldflags!': ['<@(no_ldflags)'],
-            'xcode_settings': {
-              'OTHER_CFLAGS': ['<@(ubsan_flags)'],
-              'OTHER_LDFLAGS!': ['<@(no_ldflags)'],
-              # See comment above.
-              'LIBRARY_SEARCH_PATHS': ['/usr/lib <(ubsan_flags)'],
-            },
-          }],
-          [ 'use_msan==1', {
-            'variables': {
-              'msan_flags': '<!(<(python) <(DEPTH)/coreconf/sanitizers.py msan)',
-              'no_ldflags': '<!(<(python) <(DEPTH)/coreconf/sanitizers.py ld)',
-            },
-            'cflags': ['<@(msan_flags)'],
-            'ldflags': ['<@(msan_flags)'],
-            'ldflags!': ['<@(no_ldflags)'],
-            'xcode_settings': {
-              'OTHER_CFLAGS': ['<@(msan_flags)'],
-              'OTHER_LDFLAGS!': ['<@(no_ldflags)'],
-              # See comment above.
-              'LIBRARY_SEARCH_PATHS': ['/usr/lib <(msan_flags)'],
-            },
-          }],
-          [ 'use_sancov!=0', {
-            'variables': {
-              'sancov_flags': '<!(<(python) <(DEPTH)/coreconf/sanitizers.py sancov <(use_sancov))',
-              'no_ldflags': '<!(<(python) <(DEPTH)/coreconf/sanitizers.py ld)',
-            },
-            'cflags': ['<@(sancov_flags)'],
-            'ldflags': ['<@(sancov_flags)'],
-            'ldflags!': ['<@(no_ldflags)'],
-            'xcode_settings': {
-              'OTHER_CFLAGS': ['<@(sancov_flags)'],
-              'OTHER_LDFLAGS!': ['<@(no_ldflags)'],
-              # See comment above.
-              'LIBRARY_SEARCH_PATHS': ['/usr/lib <(sancov_flags)'],
+              'LIBRARY_SEARCH_PATHS': ['/usr/lib <(sanitizer_flags)'],
             },
           }],
           [ 'OS=="android" and mozilla_client==0', {

diff --git a/coreconf/fuzz.sh b/coreconf/fuzz.sh
@@ -2,15 +2,23 @@
 # This file is used by build.sh to setup fuzzing.
 
 gyp_params+=(-Dtest_build=1 -Dfuzz=1)
+
+# Add debug symbols even for opt builds.
+nspr_params+=(--enable-debug-symbols)
+
+if [ "$fuzz_oss" = 1 ]; then
+  gyp_params+=(-Dno_zdefs=1)
+else
   enable_sanitizer asan
   enable_ubsan
   enable_sancov
+fi
 
-# Add debug symbols even for opt builds.
-nspr_params+=(--enable-debug-symbols)
+if [ "$fuzz_tls" = 1 ]; then
+  gyp_params+=(-Dfuzz_tls=1)
+fi
 
-echo "fuzz [1/2] Cloning libFuzzer files ..."
+if [ ! -f "/usr/lib/libFuzzingEngine.a" ]; then
+  echo "Cloning libFuzzer files ..."
   run_verbose "$cwd"/fuzz/clone_libfuzzer.sh
-
-echo "fuzz [2/2] Cloning fuzzing corpus ..."
-run_verbose  "$cwd"/fuzz/clone_corpus.sh
+fi
diff --git a/coreconf/nspr.sh b/coreconf/nspr.sh
@@ -14,12 +14,11 @@ if hash gmake 2>/dev/null; then
     make() { command gmake "$@"; }
 fi
 
-nspr_sanitizer()
+nspr_set_flags()
 {
-    local extra=$(python $cwd/coreconf/sanitizers.py "$@")
-    nspr_cflags="$nspr_cflags $extra"
-    nspr_cxxflags="$nspr_cxxflags $extra"
-    nspr_ldflags="$nspr_ldflags $extra"
+    nspr_cflags="$CFLAGS $@"
+    nspr_cxxflags="$CXXFLAGS $@"
+    nspr_ldflags="$LDFLAGS $@"
 }
 
 nspr_build()

diff --git a/coreconf/sanitizers.py b/coreconf/sanitizers.py
@@ -5,7 +5,7 @@
 
 def main():
     if len(sys.argv) < 2:
-        raise Exception('Specify either "ld", asan", "msan", "sancov" or "ubsan" as argument.')
+        raise Exception('Specify either "asan", "msan", "sancov" or "ubsan" as argument.')
 
     sanitizer = sys.argv[1]
     if sanitizer == "ubsan":
@@ -27,12 +27,7 @@ def main():
         print('-fsanitize-coverage='+sys.argv[2]+' ', end='')
         return
 
-    # We have to remove this from the ld flags when building asan.
-    if sanitizer == "ld":
-        print('-Wl,-z,defs ', end='')
-        return
-
-    raise Exception('Specify either "ld", asan", "msan", "sancov" or "ubsan" as argument.')
+    raise Exception('Specify either "asan", "msan", "sancov" or "ubsan" as argument.')
 
 if __name__ == '__main__':
     main()
diff --git a/coreconf/sanitizers.sh b/coreconf/sanitizers.sh
@@ -1,15 +1,22 @@
 #!/usr/bin/env bash
 # This file is used by build.sh to setup sanitizers.
 
+sanitizer_flags=""
+
 # This tracks what sanitizers are enabled, and their options.
 declare -A sanitizers
 enable_sanitizer()
 {
     local san="$1"
     [ -n "${sanitizers[$san]}" ] && return
     sanitizers[$san]="${2:-1}"
-    gyp_params+=(-Duse_"$san"="${2:-1}")
-    nspr_sanitizer "$san" "$2"
+
+    if [ -z "$sanitizer_flags" ]; then
+        gyp_params+=(-Dno_zdefs=1)
+    fi
+
+    local cflags=$(python $cwd/coreconf/sanitizers.py "$@")
+    sanitizer_flags="$sanitizer_flags $cflags"
 }
 
 enable_sancov()