Bug 1329837 - Refactor build.sh further, r=ttaubert

--HG--
extra : rebase_source : cdffc4362cfc8c5df0f1f5e559c56073c0b238c5

build.sh

@@ -6,12 +6,14 @@
 
 set -e
 
-source $(dirname $0)/coreconf/nspr.sh
+cwd=$(cd $(dirname $0); pwd -P)
+source "$cwd"/coreconf/nspr.sh
+source "$cwd"/coreconf/sanitizers.sh
 
 # Usage info
-show_help() {
-cat << EOF
-
+show_help()
+{
+    cat << EOF
 Usage: ${0##*/} [-hcv] [-j <n>] [--nspr] [--gyp|-g] [--opt|-o] [-m32]
                 [--test] [--fuzz] [--pprof] [--scan-build[=output]]
                 [--asan] [--ubsan] [--msan] [--sancov[=edge|bb|func|...]]
@@ -45,6 +47,18 @@ NSS build tool options:
 EOF
 }
 
+run_verbose()
+{
+    if [ "$verbose" = 1 ]; then
+        echo "$@"
+        exec 3>&1
+    else
+        exec 3>/dev/null
+    fi
+    "$@" 1>&3 2>&3
+    exec 3>&-
+}
+
 if [ -n "$CCC" ] && [ -z "$CXX" ]; then
     export CXX="$CCC"
 fi
@@ -57,50 +71,17 @@ rebuild_nspr=0
 target=Debug
 verbose=0
 fuzz=0
-ubsan_default=bool,signed-integer-overflow,shift,vptr
-sancov_default=edge,indirect-calls,8bit-counters
-cwd=$(cd $(dirname $0); pwd -P)
 
 gyp_params=(--depth="$cwd" --generator-output=".")
 nspr_params=()
 ninja_params=()
-scanbuild=()
 
 # try to guess sensible defaults
 arch=$(python "$cwd"/coreconf/detect_host_arch.py)
 if [ "$arch" = "x64" -o "$arch" = "aarch64" ]; then
     build_64=1
 fi
 
-sancov_default()
-{
-    clang_version=$($CC --version | grep -oE 'clang version (3\.9\.|4\.)')
-    if [ -z "$clang_version" ]; then
-        echo "Need at least clang-3.9 (better 4.0) for sancov." 1>&2
-        exit 1
-    fi
-
-    if [ "$clang_version" = "clang version 3.9." ]; then
-        echo edge,indirect-calls,8bit-counters
-    else
-        echo trace-pc-guard
-    fi
-}
-
-enable_fuzz()
-{
-    fuzz=1
-    nspr_sanitizer asan
-    nspr_sanitizer ubsan $ubsan_default
-    nspr_sanitizer sancov $(sancov_default)
-    gyp_params+=(-Duse_asan=1)
-    gyp_params+=(-Duse_ubsan=$ubsan_default)
-    gyp_params+=(-Duse_sancov=$(sancov_default))
-
-    # Adding debug symbols even for opt builds.
-    nspr_params+=(--enable-debug-symbols)
-}
-
 # parse command line arguments
 while [ $# -gt 0 ]; do
     case $1 in
@@ -110,19 +91,19 @@ while [ $# -gt 0 ]; do
         -j) ninja_params+=(-j "$2"); shift ;;
         -v) ninja_params+=(-v); verbose=1 ;;
         --test) gyp_params+=(-Dtest_build=1) ;;
-        --fuzz) gyp_params+=(-Dtest_build=1 -Dfuzz=1); enable_fuzz ;;
-        --scan-build) scanbuild=(scan-build) ;;
-        --scan-build=?*) scanbuild=(scan-build -o "${1#*=}") ;;
+        --fuzz) fuzz=1 ;;
+        --scan-build) enable_scanbuild  ;;
+        --scan-build=?*) enable_scanbuild "${1#*=}" ;;
         --opt|-o) opt_build=1 ;;
         -m32|--m32) build_64=0 ;;
-        --asan) gyp_params+=(-Duse_asan=1); nspr_sanitizer asan ;;
-        --ubsan) gyp_params+=(-Duse_ubsan=$ubsan_default); nspr_sanitizer ubsan $ubsan_default ;;
-        --ubsan=?*) gyp_params+=(-Duse_ubsan="${1#*=}"); nspr_sanitizer ubsan "${1#*=}" ;;
-        --sancov) gyp_params+=(-Duse_sancov=$(sancov_default)); nspr_sanitizer sancov $(sancov_default) ;;
-        --sancov=?*) gyp_params+=(-Duse_sancov="${1#*=}"); nspr_sanitizer sancov "${1#*=}" ;;
+        --asan) enable_sanitizer asan ;;
+        --msan) enable_sanitizer msan ;;
+        --ubsan) enable_ubsan ;;
+        --ubsan=?*) enable_ubsan "${1#*=}" ;;
+        --sancov) enable_sancov ;;
+        --sancov=?*) enable_sancov "${1#*=}" ;;
         --pprof) gyp_params+=(-Duse_pprof=1) ;;
-        --msan) gyp_params+=(-Duse_msan=1); nspr_sanitizer msan ;;
-        *) show_help; exit ;;
+        *) show_help; exit 2 ;;
     esac
     shift
 done
@@ -137,18 +118,8 @@ if [ "$build_64" = 1 ]; then
 else
     gyp_params+=(-Dtarget_arch=ia32)
 fi
-
-# clone fuzzing stuff
 if [ "$fuzz" = 1 ]; then
-    [ $verbose = 0 ] && exec 3>/dev/null || exec 3>&1
-
-    echo "fuzz [1/2] Cloning libFuzzer files ..."
-    "$cwd"/fuzz/clone_libfuzzer.sh 1>&3 2>&3
-
-    echo "fuzz [2/2] Cloning fuzzing corpus ..."
-    "$cwd"/fuzz/clone_corpus.sh 1>&3 2>&3
-
-    exec 3>&-
+    source "$cwd"/coreconf/fuzz.sh
 fi
 
 # set paths
@@ -158,26 +129,29 @@ dist_dir="$cwd"/../dist
 dist_dir=$(mkdir -p "$dist_dir"; cd "$dist_dir"; pwd -P)
 gyp_params+=(-Dnss_dist_dir="$dist_dir")
 
-# pass on CC and CCC to scanbuild
-if [ "${#scanbuild[@]}" -gt 0 ]; then
-    if [ -n "$CC" ]; then
-       scanbuild+=(--use-cc="$CC")
-    fi
-    if [ -n "$CCC" ]; then
-       scanbuild+=(--use-c++="$CCC")
-    fi
+# -c = clean first
+if [ "$clean" = 1 ]; then
+    nspr_clean
+    rm -rf "$cwd"/out
+    rm -rf "$dist_dir"
 fi
 
 # This saves a canonical representation of arguments that we are passing to gyp
 # or the NSPR build so that we can work out if a rebuild is needed.
-normalize_config()
+# Caveat: This can fail for arguments that are position-dependent.
+# e.g., "-e 2 -f 1" and "-e 1 -f 2" canonicalize the same.
+check_config()
 {
-    conf="$1"
-    mkdir -p $(dirname "$conf")
+    local newconf="$1".new oldconf="$1"
     shift
-    echo CC="$CC" >"$conf"
-    echo CCC="$CCC" >>"$conf"
-    for i in "$@"; do echo $i; done | sort >>"$conf"
+    mkdir -p $(dirname "$newconf")
+    echo CC="$CC" >"$newconf"
+    echo CCC="$CCC" >>"$newconf"
+    for i in "$@"; do echo $i; done | sort >>"$newconf"
+
+    # Note: The following diff fails if $oldconf isn't there as well, which
+    # happens if we don't have a previous successful build.
+    ! diff -q "$newconf" "$oldconf" >/dev/null 2>&1
 }
 
 gyp_config="$cwd"/out/gyp_config
@@ -187,60 +161,49 @@ nspr_config="$cwd"/out/$target/nspr_config
 if [ ! -d "$target_dir" ]; then
     rebuild_nspr=1
     rebuild_gyp=1
+elif [ ! -d "$dist_dir"/$target ]; then
+    rebuild_nspr=1
 fi
 
-# -c = clean first
-if [ "$clean" = 1 ]; then
-    rebuild_gyp=1
+if check_config "$nspr_config" "${nspr_params[@]}" \
+                 nspr_cflags="$nspr_cflags" \
+                 nspr_cxxflags="$nspr_cxxflags" \
+                 nspr_ldflags="$nspr_ldflags"; then
     rebuild_nspr=1
-    nspr_clean
-    rm -rf "$cwd"/out
-    rm -rf "$dist_dir"
-    mkdir -p "$dist_dir"
 fi
 
-# save the chosen target
-echo $target > "$dist_dir"/latest
-
-normalize_config "$gyp_config".new "${gyp_params[@]}"
-if ! diff -q "$gyp_config".new "$gyp_config" >/dev/null 2>&1; then
+if check_config "$gyp_config" "${gyp_params[@]}"; then
     rebuild_gyp=1
 fi
 
-normalize_config "$nspr_config".new "${nspr_params[@]}" \
-                 nspr_cflags="$nspr_cflags" nspr_cxxflags="$nspr_cxxflags" \
-                 nspr_ldflags="$nspr_ldflags"
-if [ ! -d "$dist_dir"/$target ] || \
-   ! diff -q "$nspr_config".new "$nspr_config" >/dev/null 2>&1; then
-    rebuild_nspr=1
-fi
+# save the chosen target
+mkdir -p "$dist_dir"
+echo $target > "$dist_dir"/latest
 
 if [ "$rebuild_nspr" = 1 ]; then
     nspr_build "${nspr_params[@]}"
     mv -f "$nspr_config".new "$nspr_config"
 fi
 if [ "$rebuild_gyp" = 1 ]; then
-    if [ $verbose = 1 ]; then set -v -x; else echo gyp ...; fi
 
     # These extra arguments aren't used in determining whether to rebuild.
     obj_dir="$dist_dir"/$target
     gyp_params+=(-Dnss_dist_obj_dir=$obj_dir)
     gyp_params+=(-Dnspr_lib_dir=$obj_dir/lib)
     gyp_params+=(-Dnspr_include_dir=$obj_dir/include/nspr)
 
-    "${scanbuild[@]}" gyp -f ninja "${gyp_params[@]}" "$cwd"/nss.gyp
-    [ $verbose = 1 ] && set +v +x
+    run_verbose run_scanbuild gyp -f ninja "${gyp_params[@]}" "$cwd"/nss.gyp
 
     mv -f "$gyp_config".new "$gyp_config"
 fi
 
 # Run ninja.
 if hash ninja 2>/dev/null; then
     ninja=ninja
-elif which ninja-build 2>/dev/null; then
+elif hash ninja-build 2>/dev/null; then
     ninja=ninja-build
 else
     echo "Please install ninja" 1>&2
     exit 1
 fi
-"${scanbuild[@]}" $ninja -C "$target_dir" "${ninja_params[@]}"
+run_scanbuild $ninja -C "$target_dir" "${ninja_params[@]}"

coreconf/fuzz.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# This file is used by build.sh to setup fuzzing.
+
+gyp_params+=(-Dtest_build=1 -Dfuzz=1)
+enable_sanitizer asan
+enable_ubsan
+enable_sancov
+
+# Add debug symbols even for opt builds.
+nspr_params+=(--enable-debug-symbols)
+
+echo "fuzz [1/2] Cloning libFuzzer files ..."
+run_verbose "$cwd"/fuzz/clone_libfuzzer.sh
+
+echo "fuzz [2/2] Cloning fuzzing corpus ..."
+run_verbose  "$cwd"/fuzz/clone_corpus.sh

coreconf/nspr.sh

@@ -16,18 +16,16 @@ fi
 
 nspr_sanitizer()
 {
-    extra=$(python $cwd/coreconf/sanitizers.py "$@")
+    local extra=$(python $cwd/coreconf/sanitizers.py "$@")
     nspr_cflags="$nspr_cflags $extra"
     nspr_cxxflags="$nspr_cxxflags $extra"
     nspr_ldflags="$nspr_ldflags $extra"
 }
 
 nspr_build()
 {
-    [ "$verbose" = 0 ] && exec 3>/dev/null || exec 3>&1
-
-    mkdir -p "$cwd"/../nspr/$target
-    pushd "$cwd"/../nspr/$target >/dev/null
+    local nspr_dir="$cwd"/../nspr/$target
+    mkdir -p "$nspr_dir"
 
     # These NSPR options are directory-specific, so they don't need to be
     # included in nspr_opt and changing them doesn't force a rebuild of NSPR.
@@ -37,16 +35,15 @@ nspr_build()
     fi
 
     echo "NSPR [1/3] configure ..."
-    CFLAGS="$nspr_cflags" CXXFLAGS="$nspr_cxxflags" LDFLAGS="$nspr_ldflags" \
-          CC="$CC" CXX="$CCC" ../configure "${extra_params[@]}" "$@" 1>&3 2>&3
+    pushd "$nspr_dir" >/dev/null
+    CFLAGS="$nspr_cflags" CXXFLAGS="$nspr_cxxflags" \
+          LDFLAGS="$nspr_ldflags" CC="$CC" CXX="$CCC" \
+          run_verbose ../configure "${extra_params[@]}" "$@"
+    popd >/dev/null
     echo "NSPR [2/3] make ..."
-    make 1>&3 2>&3
+    run_verbose make -C "$nspr_dir"
     echo "NSPR [3/3] install ..."
-    make install 1>&3 2>&3
-
-    popd >/dev/null
-
-    exec 3>&-
+    run_verbose make -C "$nspr_dir" install
 }
 
 nspr_clean()

coreconf/sanitizers.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# This file is used by build.sh to setup sanitizers.
+
+# This tracks what sanitizers are enabled, and their options.
+declare -A sanitizers
+enable_sanitizer()
+{
+    local san="$1"
+    [ -n "${sanitizers[$san]}" ] && return
+    sanitizers[$san]="${2:-1}"
+    gyp_params+=(-Duse_"$san"="${2:-1}")
+    nspr_sanitizer "$san" "$2"
+}
+
+enable_sancov()
+{
+    local clang_version=$($CC --version | grep -oE 'clang version (3\.9\.|4\.)')
+    if [ -z "$clang_version" ]; then
+        echo "Need at least clang-3.9 (better 4.0) for sancov." 1>&2
+        exit 1
+    fi
+
+    local sancov
+    if [ -n "$1" ]; then
+        sancov="$1"
+    elif [ "$clang_version" = "clang version 3.9." ]; then
+        sancov=edge,indirect-calls,8bit-counters
+    else
+        sancov=trace-pc-guard
+    fi
+    enable_sanitizer sancov "$sancov"
+}
+
+enable_ubsan()
+{
+    local ubsan
+    if [ -n "$1" ]; then
+        ubsan="$1"
+    else
+        ubsan=bool,signed-integer-overflow,shift,vptr
+    fi
+    enable_sanitizer ubsan "$ubsan"
+}
+
+# Not strictly a sanitizer, but the pattern fits
+scanbuild=()
+enable_scanbuild()
+{
+    [ "${#scanbuild[@]}" -gt 0 ] && return
+
+    scanbuild=(scan-build)
+    if [ -n "$1" ]; then
+        scanbuild+=(-o "$1")
+    fi
+    # pass on CC and CCC to scanbuild
+    if [ -n "$CC" ]; then
+        scanbuild+=(--use-cc="$CC")
+    fi
+    if [ -n "$CCC" ]; then
+        scanbuild+=(--use-c++="$CCC")
+    fi
+}
+
+run_scanbuild()
+{
+    "${scanbuild[@]}" "$@"
+}