Bug 1325089 - Use clang-4.0 and sancov=trace-pc-guard for fuzzing r=f…

…ranziskus

Differential Revision: https://nss-review.dev.mozaws.net/D128

--HG--
extra : amend_source : 6f274d5fedbe17df9346b1179c3306a6daeec324

automation/taskcluster/docker-fuzz/Dockerfile

@@ -0,0 +1,27 @@
+FROM ubuntu:16.04
+MAINTAINER Tim Taubert <ttaubert@mozilla.com>
+
+RUN useradd -d /home/worker -s /bin/bash -m worker
+WORKDIR /home/worker
+
+# Add build and test scripts.
+ADD bin /home/worker/bin
+RUN chmod +x /home/worker/bin/*
+
+# Install dependencies.
+ADD setup.sh /tmp/setup.sh
+RUN bash /tmp/setup.sh
+
+# Env variables.
+ENV HOME /home/worker
+ENV SHELL /bin/bash
+ENV USER worker
+ENV LOGNAME worker
+ENV HOSTNAME taskcluster-worker
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV HOST localhost
+ENV DOMSUF localdomain
+
+# Set a default command for debugging.
+CMD ["/bin/bash", "--login"]

automation/taskcluster/docker-fuzz/bin/checkout.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+if [ $(id -u) = 0 ]; then
+    # Drop privileges by re-running this script.
+    exec su worker $0
+fi
+
+# Default values for testing.
+REVISION=${NSS_HEAD_REVISION:-default}
+REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}
+
+# Clone NSS.
+for i in 0 2 5; do
+    sleep $i
+    hg clone -r $REVISION $REPOSITORY nss && exit 0
+    rm -rf nss
+done
+exit 1

automation/taskcluster/docker-fuzz/setup.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+# Update packages.
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y update && apt-get -y upgrade
+
+# Need this to add keys for PPAs below.
+apt-get install -y --no-install-recommends apt-utils
+
+apt_packages=()
+apt_packages+=('build-essential')
+apt_packages+=('ca-certificates')
+apt_packages+=('curl')
+apt_packages+=('git')
+apt_packages+=('gyp')
+apt_packages+=('ninja-build')
+apt_packages+=('pkg-config')
+apt_packages+=('zlib1g-dev')
+
+# ct-verif and sanitizers
+apt_packages+=('valgrind')
+
+# Latest Mercurial.
+apt_packages+=('mercurial')
+apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 41BD8711B1F0EC2B0D85B91CF59CE3A8323293EE
+echo "deb http://ppa.launchpad.net/mercurial-ppa/releases/ubuntu xenial main" > /etc/apt/sources.list.d/mercurial.list
+
+# Install packages.
+apt-get -y update
+apt-get install -y --no-install-recommends ${apt_packages[@]}
+
+# Install LLVM/clang-4.0.
+mkdir clang-tmp
+git clone -n --depth 1 https://chromium.googlesource.com/chromium/src/tools/clang clang-tmp/clang
+git -C clang-tmp/clang checkout HEAD scripts/update.py
+clang-tmp/clang/scripts/update.py
+rm -fr clang-tmp
+
+# Link to LLVM binaries.
+for b in clang clang++ llvm-symbolizer; do
+  ln -s /home/worker/third_party/llvm-build/Release+Asserts/bin/$b /usr/local/bin/$b
+done
+
+locale-gen en_US.UTF-8
+dpkg-reconfigure locales
+
+# Cleanup.
+rm -rf ~/.ccache ~/.cache
+apt-get autoremove -y
+apt-get clean
+apt-get autoclean
+rm $0

automation/taskcluster/graph/src/extend.js

@@ -6,6 +6,7 @@ import merge from "./merge";
 import * as queue from "./queue";
 
 const LINUX_IMAGE = {name: "linux", path: "automation/taskcluster/docker"};
+const FUZZ_IMAGE = {name: "fuzz", path: "automation/taskcluster/docker-fuzz"};
 
 const WINDOWS_CHECKOUT_CMD =
   "bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " +
@@ -270,7 +271,7 @@ async function scheduleFuzzing() {
     },
     platform: "linux64",
     collection: "fuzz",
-    image: LINUX_IMAGE
+    image: FUZZ_IMAGE
   };
 
   // Build base definition.
@@ -279,7 +280,7 @@ async function scheduleFuzzing() {
       "/bin/bash",
       "-c",
       "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz --ubsan"
+      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz"
     ],
     artifacts: {
       public: {

build.sh

@@ -36,6 +36,7 @@ NSS build tool options:
     --opt|-o      do an opt build
     --asan        do an asan build
     --ubsan       do an ubsan build
+                  --ubsan=bool,shift,... sets specific UB sanitizers
     --msan        do an msan build
     --sancov      do sanitize coverage builds
                   --sancov=func sets coverage to function level for example
@@ -54,7 +55,7 @@ rebuild_gyp=0
 target=Debug
 verbose=0
 fuzz=0
-sancov_default=edge,indirect-calls,8bit-counters
+ubsan_default=bool,signed-integer-overflow,shift,vptr
 
 # parse parameters to store in config
 params=$(echo "$*" | perl -pe 's/-c|-v|-g|-j [0-9]*|-h//g' | perl -pe 's/^\s*(.*?)\s*$/\1/')
@@ -74,13 +75,30 @@ gyp_params=()
 ninja_params=()
 scanbuild=()
 
+sancov_default()
+{
+    clang_version=$($CC --version | grep -oE 'clang version (3\.9\.|4\.)')
+    if [ -z "$clang_version" ]; then
+        echo "Need at least clang-3.9 (better 4.0) for sancov." 1>&2
+        exit 1
+    fi
+
+    if [ "$clang_version" = "clang version 3.9." ]; then
+        echo edge,indirect-calls,8bit-counters
+    else
+        echo trace-pc-guard
+    fi
+}
+
 enable_fuzz()
 {
     fuzz=1
     nspr_sanitizer asan
-    nspr_sanitizer sancov $sancov_default
+    nspr_sanitizer ubsan $ubsan_default
+    nspr_sanitizer sancov $(sancov_default)
     gyp_params+=(-Duse_asan=1)
-    gyp_params+=(-Duse_sancov=$sancov_default)
+    gyp_params+=(-Duse_ubsan=$ubsan_default)
+    gyp_params+=(-Duse_sancov=$(sancov_default))
 
     # Adding debug symbols even for opt builds.
     nspr_opt+=(--enable-debug-symbols)
@@ -100,8 +118,9 @@ while [ $# -gt 0 ]; do
         --opt|-o) opt_build=1 ;;
         -m32|--m32) build_64=0 ;;
         --asan) gyp_params+=(-Duse_asan=1); nspr_sanitizer asan ;;
-        --ubsan) gyp_params+=(-Duse_ubsan=1); nspr_sanitizer ubsan ;;
-        --sancov) gyp_params+=(-Duse_sancov=$sancov_default); nspr_sanitizer sancov $sancov_default ;;
+        --ubsan) gyp_params+=(-Duse_ubsan=$ubsan_default); nspr_sanitizer ubsan $ubsan_default ;;
+        --ubsan=?*) gyp_params+=(-Duse_ubsan="${1#*=}"); nspr_sanitizer ubsan "${1#*=}" ;;
+        --sancov) gyp_params+=(-Duse_sancov=$(sancov_default)); nspr_sanitizer sancov $(sancov_default) ;;
         --sancov=?*) gyp_params+=(-Duse_sancov="${1#*=}"); nspr_sanitizer sancov "${1#*=}" ;;
         --pprof) gyp_params+=(-Duse_pprof=1) ;;
         --msan) gyp_params+=(-Duse_msan=1); nspr_sanitizer msan ;;

coreconf/config.gypi

@@ -346,7 +346,7 @@
               '-Wno-unused-function',
             ]
           }],
-          [ 'fuzz==1 or use_asan==1 or use_ubsan==1', {
+          [ 'fuzz==1 or use_asan==1 or use_ubsan!=0', {
             'cflags': ['-O1'],
             'xcode_settings': {
               'GCC_OPTIMIZATION_LEVEL': '1', # -O1
@@ -371,9 +371,9 @@
               'LIBRARY_SEARCH_PATHS': ['/usr/lib <(asan_flags)'],
             },
           }],
-          [ 'use_ubsan==1', {
+          [ 'use_ubsan!=0', {
             'variables': {
-              'ubsan_flags': '<!(<(python) <(DEPTH)/coreconf/sanitizers.py ubsan)',
+              'ubsan_flags': '<!(<(python) <(DEPTH)/coreconf/sanitizers.py ubsan <(use_ubsan))',
               'no_ldflags': '<!(<(python) <(DEPTH)/coreconf/sanitizers.py ld)',
             },
             'cflags': ['<@(ubsan_flags)'],

coreconf/sanitizers.py

@@ -9,7 +9,9 @@ def main():
 
     sanitizer = sys.argv[1]
     if sanitizer == "ubsan":
-        print('-fsanitize=undefined -fno-sanitize-recover=undefined ', end='')
+        if len(sys.argv) < 3:
+            raise Exception('ubsan requires another argument.')
+        print('-fsanitize='+sys.argv[2]+' -fno-sanitize-recover=undefined ', end='')
         return
     if sanitizer == "asan":
         print('-fsanitize=address ', end='')