Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Bug 1316836 - sancov, ubsan, and asan for gyp, r=ttaubert
--HG--
extra : rebase_source : 0bf34de1e3c81a7b77e9f6f82a66034f97a4a005
extra : histedit_source : 9b56ebcfc88fdee376d0bab0418ad3a6f50596bb%2C172cdeea6e7a640c5cc6ac29f3de2745931b6169
  • Loading branch information
franziskuskiefer committed Nov 7, 2016
1 parent b8c8e0e commit cd02c38
Show file tree
Hide file tree
Showing 8 changed files with 187 additions and 80 deletions.
35 changes: 26 additions & 9 deletions automation/taskcluster/graph/src/extend.js
Expand Up @@ -17,7 +17,8 @@ const WINDOWS_CHECKOUT_CMD =
queue.filter(task => {
if (task.group == "Builds") {
// Remove extra builds on {A,UB}San and ARM.
if (task.collection == "asan" || task.collection == "arm-debug") {
if (task.collection == "asan" || task.collection == "arm-debug" ||
task.collection == "gyp-asan") {
return false;
}

Expand All @@ -41,24 +42,20 @@ queue.filter(task => {
}

// GYP builds with -Ddisable_libpkix=1 by default.
if (task.collection == "gyp" && task.tests == "chains") {
if ((task.collection == "gyp" || task.collection == "gyp-asan") &&
task.tests == "chains") {
return false;
}

return true;
});

queue.map(task => {
if (task.collection == "asan") {
if (task.collection == "asan" || task.collection == "gyp-asan") {
// CRMF and FIPS tests still leak, unfortunately.
if (task.tests == "crmf" || task.tests == "fips") {
task.env.ASAN_OPTIONS = "detect_leaks=0";
}

// SSL(standard) runs on ASan take some time.
if (task.tests == "ssl" && task.cycle == "standard") {
task.maxRunTime = 7200;
}
}

if (task.collection == "arm-debug") {
Expand Down Expand Up @@ -114,6 +111,25 @@ export default async function main() {
image: LINUX_IMAGE
});

await scheduleLinux("Linux 64 (debug, gyp, asan, ubsan)", {
command: [
"/bin/bash",
"-c",
"bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh -g -v --ubsan --asan"
],
env: {
ASAN_OPTIONS: "detect_odr_violation=0", // bug 1316276
UBSAN_OPTIONS: "print_stacktrace=1",
NSS_DISABLE_ARENA_FREE_LIST: "1",
NSS_DISABLE_UNLOAD: "1",
CC: "clang",
CCC: "clang++"
},
platform: "linux64",
collection: "gyp-asan",
image: LINUX_IMAGE
});

await scheduleLinux("Linux 64 (ASan, debug)", {
env: {
UBSAN_OPTIONS: "print_stacktrace=1",
Expand Down Expand Up @@ -241,7 +257,8 @@ async function scheduleLinux(name, base) {
async function scheduleFuzzing() {
let base = {
env: {
ASAN_OPTIONS: "allocator_may_return_null=1",
// bug 1316276
ASAN_OPTIONS: "allocator_may_return_null=1:detect_odr_violation=0",
UBSAN_OPTIONS: "print_stacktrace=1",
CC: "clang",
CCC: "clang++"
Expand Down
5 changes: 4 additions & 1 deletion automation/taskcluster/graph/src/try_syntax.js
Expand Up @@ -23,7 +23,7 @@ function parseOptions(opts) {

// Parse platforms.
let allPlatforms = ["linux", "linux64", "linux64-asan", "win64", "arm",
"linux64-gyp", "linux64-fuzz"];
"linux64-gyp", "linux64-gyp-asan", "linux64-fuzz"];
let platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms);

// If the given value is nonsense or "none" default to all platforms.
Expand Down Expand Up @@ -104,6 +104,7 @@ function filter(opts) {
"linux64-asan": "linux64",
"linux64-fuzz": "linux64",
"linux64-gyp": "linux64",
"linux64-gyp-asan": "linux64",
"win64": "windows2012-64",
"arm": "linux32"
};
Expand All @@ -118,6 +119,8 @@ function filter(opts) {
keep &= coll("arm-opt") || coll("arm-debug");
} else if (platform == "linux64-gyp") {
keep &= coll("gyp");
} else if (platform == "linux64-gyp-asan") {
keep &= coll("gyp-asan");
} else if (platform == "linux64-fuzz") {
keep &= coll("fuzz");
} else {
Expand Down
78 changes: 63 additions & 15 deletions build.sh
Expand Up @@ -6,12 +6,14 @@

set -e

source $(dirname $0)/coreconf/nspr.sh

# Usage info
show_help() {
cat << EOF
Usage: ${0##*/} [-hcgv] [-j <n>] [--test] [--fuzz] [--scan-build[=output]]
[-m32] [--opt|-o]
[-m32] [--opt|-o] [--asan] [--ubsan] [--sancov[=edge|bb|func]]
This script builds NSS with gyp and ninja.
Expand All @@ -31,6 +33,10 @@ NSS build tool options:
--scan-build run the build with scan-build (scan-build has to be in the path)
--scan-build=/out/path sets the output path for scan-build
--opt|-o do an opt build
--asan do an asan build
--ubsan do an ubsan build
--sancov do sanitize coverage builds
--sancov=func sets coverage to function level for example
EOF
}

Expand All @@ -43,17 +49,45 @@ build_64=0
clean=0
rebuild_gyp=0
target=Debug
params=$(echo "$* $CC $CCC" | tr " " "\n")

cwd=$(cd $(dirname $0); pwd -P)
dist_dir="$cwd/../dist"

# try to guess sensible defaults
arch=$(uname -m)
if [ "$arch" = "x86_64" -o "$arch" = "aarch64" ]; then
arch=$(python "$cwd/coreconf/detect_host_arch.py")
if [ "$arch" = "x64" -o "$arch" = "aarch64" ]; then
build_64=1
fi

gyp_params=()
ninja_params=()
scanbuild=()
nspr_env=()

check_sanitizer()
{
# We can't run sanitizers on Mac for now.
if [ $(uname) != "Linux" ]; then
echo "
#### #### #### #### #### #### #### #### #### #### #### ####
#### Sorry, sanitizers only work on Linux for now ####
#### #### #### #### #### #### #### #### #### #### #### ####"
show_help
exit
fi
}

enable_fuzz()
{
nspr_sanitizer asan
nspr_sanitizer ubsan
nspr_sanitizer sancov edge
gyp_params+=(-Duse_asan=1)
gyp_params+=(-Duse_ubsan=1)
gyp_params+=(-Duse_sancov=edge)
# Adding debug symbols even for opt builds.
nspr_opt+=(--enable-debug-symbols)
}

# parse command line arguments
while [ $# -gt 0 ]; do
Expand All @@ -63,31 +97,41 @@ while [ $# -gt 0 ]; do
-j) ninja_params+=(-j "$2"); shift ;;
-v) ninja_params+=(-v) ;;
--test) gyp_params+=(-Dtest_build=1) ;;
--fuzz) gyp_params+=(-Dtest_build=1 -Dfuzz=1) ;;
--fuzz) gyp_params+=(-Dtest_build=1 -Dfuzz=1); check_sanitizer; enable_fuzz ;;
--scan-build) scanbuild=(scan-build) ;;
--scan-build=?*) scanbuild=(scan-build -o "${1#*=}") ;;
--opt|-o) opt_build=1; nspr_env+=(BUILD_OPT=1) ;;
--opt|-o) opt_build=1 ;;
-m32|--m32) build_64=0 ;;
--asan) gyp_params+=(-Duse_asan=1); nspr_sanitizer asan ;;
--ubsan) gyp_params+=(-Duse_ubsan=1); nspr_sanitizer ubsan ;;
--sancov) gyp_params+=(-Duse_sancov=edge); nspr_sanitizer sancov edge ;;
--sancov=?*) gyp_params+=(-Duse_sancov="${1#*=}"); nspr_sanitizer sancov "${1#*=}" ;;
*) show_help; exit ;;
esac
shift
done

if [ "$opt_build" = "1" ]; then
target=Release
nspr_opt+=(--disable-debug --enable-optimize)
else
target=Debug
fi
if [ "$build_64" == "1" ]; then
target="${target}_x64"
nspr_env+=(USE_64=1)
nspr_opt+=(--enable-64bit)
else
gyp_params+=(-Dtarget_arch=ia32)
nspr_opt+=(--enable-x32)
fi

# check if we have to rebuild gyp
if [ "$params" != "$(cat $cwd/out/config 2>/dev/null)" -o "$rebuild_gyp" == 1 -o "$clean" == 1 ]; then
rebuild_gyp=1
rm -rf "$cwd/../nspr/$target" # force NSPR to rebuild
fi

# set paths
cwd=$(cd $(dirname $0); pwd -P)
dist_dir="$cwd/../dist"
target_dir="$cwd/out/$target"

# get the realpath of $dist_dir
Expand All @@ -100,14 +144,17 @@ echo $target > $dist_dir/latest
obj_dir="$dist_dir/$target"
gyp_params+=(-Dnss_dist_dir=$dist_dir)
gyp_params+=(-Dnss_dist_obj_dir=$obj_dir)
gyp_params+=(-Dnspr_lib_dir=$obj_dir/lib)
gyp_params+=(-Dnspr_include_dir=$obj_dir/include/nspr)

# -c = clean first
if [ "$clean" = 1 ]; then
rm -rf "$cwd/out"
rm -rf "$cwd/../nspr/$target"
rm -rf "$dist_dir"
fi

# figure out the scan-build string
# pass on CC and CCC
if [ "${#scanbuild[@]}" -gt 0 ]; then
if [ -n "$CC" ]; then
scanbuild+=(--use-cc="$CC")
Expand All @@ -120,13 +167,14 @@ if [ "${#scanbuild[@]}" -gt 0 ]; then
# These steps can take a while, so don't overdo them.
# Force a redo with -g.
if [ "$rebuild_gyp" = 1 -o ! -d "$target_dir" ]; then
# Build NSPR.
make "${nspr_env[@]}" -C "$cwd" NSS_GYP_PREFIX=$obj_dir install_nspr
build_nspr

# Run gyp.
PKG_CONFIG_PATH="$obj_dir/lib/pkgconfig" \
"${scanbuild[@]}" gyp -f ninja "${gyp_params[@]}" --depth="$cwd" \
--generator-output="." "$cwd/nss.gyp"
"${scanbuild[@]}" gyp -f ninja "${gyp_params[@]}" --depth="$cwd" \
--generator-output="." "$cwd/nss.gyp"

# Store used parameters for next run.
echo "$params" > "$cwd/out/config"
fi

# Run ninja.
Expand Down
42 changes: 33 additions & 9 deletions coreconf/config.gypi
Expand Up @@ -27,22 +27,17 @@
['OS=="win"', {
'use_system_zlib%': 0,
'nspr_libs%': ['nspr4.lib', 'plc4.lib', 'plds4.lib'],
#XXX: gyp breaks if these are empty!
'nspr_lib_dir%': ' ',
'nspr_include_dir%': ' ',
'zlib_libs%': [],
#TODO
'moz_debug_flags%': '',
'dll_prefix': '',
'dll_suffix': 'dll',
}, {
'nspr_libs%': ['-lplds4', '-lplc4', '-lnspr4'],
'nspr_lib_dir%': '<!(<(python) <(DEPTH)/coreconf/pkg_config.py . --libs nspr)',
'nspr_include_dir%': '<!(<(python) <(DEPTH)/coreconf/pkg_config.py . --cflags nspr)',
'use_system_zlib%': 1,
}],
['OS=="linux" or OS=="android"', {
'zlib_libs%': ['<!@(<(python) <(DEPTH)/coreconf/pkg_config.py raw --libs zlib)'],
'zlib_libs%': ['-lz'],
'moz_debug_flags%': '-gdwarf-2',
'optimize_flags%': '-O2',
'dll_prefix': 'lib',
Expand Down Expand Up @@ -97,6 +92,8 @@
'moz_folded_library_name%': '',
'ssl_enable_zlib%': 1,
'use_asan%': 0,
'use_ubsan%': 0,
'use_sancov%': 0,
'test_build%': 0,
'fuzz%': 0,
},
Expand Down Expand Up @@ -258,13 +255,11 @@
'-pipe',
'-ffunction-sections',
'-fdata-sections',
'<(moz_debug_flags)',
],
'cflags_cc': [
'-std=c++0x',
],
'conditions': [

[ 'target_arch=="ia32"', {
'cflags': ['-m32'],
'ldflags': ['-m32'],
Expand All @@ -281,7 +276,27 @@
],
}],
[ 'fuzz==1', {
'cflags': ['-Wno-unused-function']
'cflags': [
'-Wno-unused-function',
]
}],
[ 'use_asan==1 or use_ubsan==1', {
'cflags': ['-O1'],
}],
[ 'use_asan==1', {
'cflags': ['<!@(<(python) <(DEPTH)/coreconf/sanitizers.py asan)'],
'ldflags': ['<!@(<(python) <(DEPTH)/coreconf/sanitizers.py asan)'],
'ldflags!': ['<!@(<(python) <(DEPTH)/coreconf/sanitizers.py ld)'],
}],
[ 'use_ubsan==1', {
'cflags': ['<!@(<(python) <(DEPTH)/coreconf/sanitizers.py ubsan)'],
'ldflags': ['<!@(<(python) <(DEPTH)/coreconf/sanitizers.py ubsan)'],
'ldflags!': ['<!@(<(python) <(DEPTH)/coreconf/sanitizers.py ld)'],
}],
[ 'use_sancov!=0', {
'cflags': ['<!@(<(python) <(DEPTH)/coreconf/sanitizers.py sancov <(use_sancov))'],
'ldflags': ['<!@(<(python) <(DEPTH)/coreconf/sanitizers.py sancov <(use_sancov))'],
'ldflags!': ['<!@(<(python) <(DEPTH)/coreconf/sanitizers.py ld)'],
}],
[ 'OS=="android" and mozilla_client==0', {
'defines': [
Expand Down Expand Up @@ -377,11 +392,20 @@
# Common settings for debug should go here.
'Debug_Base': {
'abstract': 1,
'conditions': [
[ 'OS=="linux" or OS=="android"', {
'cflags': [
'-g',
'<(moz_debug_flags)',
],
}]
],
#TODO: DEBUG_$USER
'defines': ['DEBUG'],
'xcode_settings': {
'COPY_PHASE_STRIP': 'NO',
'GCC_OPTIMIZATION_LEVEL': '0',
'GCC_GENERATE_DEBUGGING_SYMBOLS': 'YES',
},
'msvs_settings': {
'VCCLCompilerTool': {
Expand Down
29 changes: 29 additions & 0 deletions coreconf/nspr.sh
@@ -0,0 +1,29 @@
#!/bin/bash
# This script builds NSPR for NSS.
#
# This build system is still under development. It does not yet support all
# the features or platforms that the regular NSPR build supports.

# variables
nspr_opt=()
nspr_cflags=
nspr_cxxflags=
nspr_ldflags=

nspr_sanitizer()
{
check_sanitizer
nspr_cflags="$nspr_cflags $(python $cwd/coreconf/sanitizers.py $1 $2)"
nspr_cxxflags="$nspr_cxxflags $(python $cwd/coreconf/sanitizers.py $1 $2)"
nspr_ldflags="$nspr_ldflags $(python $cwd/coreconf/sanitizers.py $1 $2)"
}

build_nspr()
{
mkdir -p "$cwd/../nspr/$target"
cd "$cwd/../nspr/$target"
CFLAGS=$nspr_cflags CXXFLAGS=$nspr_cxxflags LDFLAGS=$nspr_ldflags \
CC=$CC CXX=$CCC ../configure "${nspr_opt[@]}" --prefix="$obj_dir"
make -C "$cwd/../nspr/$target"
make -C "$cwd/../nspr/$target" install
}

0 comments on commit cd02c38

Please sign in to comment.