Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Bug 1594933 - disable libnssdbm by default; keep build on CI, r=jcj
Disale libnssdbm by default and add flag to enable it in builds.
On CI a build and certs test with enabled legacy DB are added.

Note that for some reason the coverage build fails. I have no idea why. I'm open for ideas.

Differential Revision: https://phabricator.services.mozilla.com/D54673

--HG--
extra : moz-landing-system : lando
  • Loading branch information
franziskuskiefer committed Dec 4, 2019
1 parent 77924fe commit eee8f82
Show file tree
Hide file tree
Showing 11 changed files with 73 additions and 31 deletions.
53 changes: 40 additions & 13 deletions automation/taskcluster/graph/src/extend.js
Expand Up @@ -110,6 +110,11 @@ queue.filter(task => {
return false;
}

// Don't run DBM builds on aarch64.
if (task.group == "DBM" && task.platform == "aarch64") {
return false;
}

return true;
});

Expand Down Expand Up @@ -500,7 +505,7 @@ async function scheduleLinux(name, overrides, args = "") {
}

// The task that generates certificates.
let task_cert = queue.scheduleTask(merge(build_base, {
let cert_base = merge(build_base, {
name: "Certificates",
command: [
"/bin/bash",
Expand All @@ -509,7 +514,8 @@ async function scheduleLinux(name, overrides, args = "") {
],
parent: task_build,
symbol: "Certs"
}));
});
let task_cert = queue.scheduleTask(cert_base);

// Schedule tests.
scheduleTests(task_build, task_cert, merge(base, {
Expand Down Expand Up @@ -592,6 +598,25 @@ async function scheduleLinux(name, overrides, args = "") {
symbol: "modular"
}));

if (base.collection != "make") {
let task_build_dbm = queue.scheduleTask(merge(extra_base, {
name: `${name} w/ legacy-db`,
command: [
"/bin/bash",
"-c",
checkout_and_gyp + "--enable-legacy-db"
],
symbol: "B",
group: "DBM",
}));

let task_cert_dbm = queue.scheduleTask(merge(cert_base, {
parent: task_build_dbm,
group: "DBM",
symbol: "Certs"
}));
}

return queue.submit();
}

Expand Down Expand Up @@ -830,11 +855,11 @@ async function scheduleWindows(name, base, build_script) {
workerType: "win2012r2",
env: {
PATH: "c:\\mozilla-build\\bin;c:\\mozilla-build\\python;" +
"c:\\mozilla-build\\msys\\local\\bin;c:\\mozilla-build\\7zip;" +
"c:\\mozilla-build\\info-zip;c:\\mozilla-build\\python\\Scripts;" +
"c:\\mozilla-build\\yasm;c:\\mozilla-build\\msys\\bin;" +
"c:\\Windows\\system32;c:\\mozilla-build\\upx391w;" +
"c:\\mozilla-build\\moztools-x64\\bin;c:\\mozilla-build\\wget",
"c:\\mozilla-build\\msys\\local\\bin;c:\\mozilla-build\\7zip;" +
"c:\\mozilla-build\\info-zip;c:\\mozilla-build\\python\\Scripts;" +
"c:\\mozilla-build\\yasm;c:\\mozilla-build\\msys\\bin;" +
"c:\\Windows\\system32;c:\\mozilla-build\\upx391w;" +
"c:\\mozilla-build\\moztools-x64\\bin;c:\\mozilla-build\\wget",
DOMSUF: "localdomain",
HOST: "localhost",
},
Expand Down Expand Up @@ -1039,12 +1064,6 @@ function scheduleTests(task_build, task_cert, test_base) {
queue.scheduleTask(merge(ssl_base, {
name: "SSL tests (pkix)", symbol: "pkix", cycle: "pkix"
}));
queue.scheduleTask(merge(ssl_base, {
name: "SSL tests (sharedb)", symbol: "sharedb", cycle: "sharedb"
}));
queue.scheduleTask(merge(ssl_base, {
name: "SSL tests (upgradedb)", symbol: "upgradedb", cycle: "upgradedb"
}));
queue.scheduleTask(merge(ssl_base, {
name: "SSL tests (stress)", symbol: "stress", cycle: "sharedb",
env: {NSS_SSL_RUN: "stress"}
Expand Down Expand Up @@ -1211,7 +1230,15 @@ async function scheduleTools() {
symbol: "Coverage",
name: "Coverage",
image: FUZZ_IMAGE,
type: "other",
features: ["allowPtrace"],
artifacts: {
public: {
expires: 24 * 7,
type: "directory",
path: "/home/worker/artifacts"
}
},
command: [
"/bin/bash",
"-c",
Expand Down
3 changes: 3 additions & 0 deletions automation/taskcluster/graph/src/queue.js
Expand Up @@ -220,6 +220,9 @@ export async function submit() {
maps.forEach(map => { task = map(merge({}, task)) });

let log_id = `${task.name} @ ${task.platform}[${task.collection || "opt"}]`;
if (task.group) {
log_id = `${task.group}::${log_id}`;
}
console.log(`+ Submitting ${log_id}.`);

// Index that task for each tag specified
Expand Down
3 changes: 2 additions & 1 deletion automation/taskcluster/scripts/check_abi.sh
Expand Up @@ -97,7 +97,8 @@ abi_diff()
rm -f ${ABI_REPORT}
PREVDIST=${HGDIR}/baseline/dist
NEWDIST=${HGDIR}/dist
ALL_SOs="libfreebl3.so libfreeblpriv3.so libnspr4.so libnss3.so libnssckbi.so libnssdbm3.so libnsssysinit.so libnssutil3.so libplc4.so libplds4.so libsmime3.so libsoftokn3.so libssl3.so"
# libnssdbm3.so isn't built by default anymore, skip it.
ALL_SOs="libfreebl3.so libfreeblpriv3.so libnspr4.so libnss3.so libnssckbi.so libnsssysinit.so libnssutil3.so libplc4.so libplds4.so libsmime3.so libsoftokn3.so libssl3.so"
for SO in ${ALL_SOs}; do
if [ ! -f ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt ]; then
touch ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt
Expand Down
5 changes: 3 additions & 2 deletions build.sh
Expand Up @@ -113,8 +113,8 @@ while [ $# -gt 0 ]; do
--fuzz) fuzz=1 ;;
--fuzz=oss) fuzz=1; fuzz_oss=1 ;;
--fuzz=tls) fuzz=1; fuzz_tls=1 ;;
--sancov) enable_sancov ;;
--sancov=?*) enable_sancov "${1#*=}" ;;
--sancov) enable_sancov; gyp_params+=(-Dcoverage=1) ;;
--sancov=?*) enable_sancov "${1#*=}"; gyp_params+=(-Dcoverage=1) ;;
--emit-llvm) gyp_params+=(-Demit_llvm=1 -Dsign_libs=0) ;;
--no-zdefs) gyp_params+=(-Dno_zdefs=1) ;;
--static) gyp_params+=(-Dstatic_libs=1) ;;
Expand All @@ -130,6 +130,7 @@ while [ $# -gt 0 ]; do
--enable-libpkix) gyp_params+=(-Ddisable_libpkix=0) ;;
--mozpkix-only) gyp_params+=(-Dmozpkix_only=1 -Ddisable_tests=1 -Dsign_libs=0) ;;
--disable-keylog) sslkeylogfile=0 ;;
--enable-legacy-db) gyp_params+=(-Ddisable_dbm=0) ;;
-D*) gyp_params+=("$1") ;;
*) show_help; exit 2 ;;
esac
Expand Down
3 changes: 2 additions & 1 deletion coreconf/config.gypi
Expand Up @@ -99,7 +99,7 @@
'disable_arm_hw_aes%': 0,
'disable_tests%': 0,
'disable_chachapoly%': 0,
'disable_dbm%': 0,
'disable_dbm%': 1,
'disable_libpkix%': 1,
'disable_werror%': 0,
'mozilla_client%': 0,
Expand All @@ -124,6 +124,7 @@
'only_dev_random%': 1,
'disable_fips%': 1,
'mozpkix_only%': 0,
'coverage%': 0,
},
'target_defaults': {
# Settings specific to targets should go here.
Expand Down
3 changes: 2 additions & 1 deletion help.txt
Expand Up @@ -7,7 +7,7 @@ Usage: build.sh [-h] [-c|-cc] [-v] [-j <n>] [--gyp|-g] [--opt|-o]
[--nspr|--with-nspr=<include>:<lib>|--system-nspr]
[--system-sqlite] [--enable-fips] [--enable-libpkix]
[--mozpkix-only] [-D<gyp-option>]
[--rebuild]
[--rebuild] [--enable-legacy-db]

This script builds NSS with gyp and ninja.

Expand Down Expand Up @@ -53,6 +53,7 @@ NSS build tool options:
--system-sqlite use system sqlite
--enable-fips enable FIPS checks
--enable-libpkix make libpkix part of the build
--enable-legacy-db enable the legacy db (libnssdbm)
--mozpkix-only build only static mozpkix and mozpkix-test libraries
support for this build option is limited
--disable-keylog disable support for logging key data to a file specified
Expand Down
2 changes: 1 addition & 1 deletion lib/freebl/freebl_base.gypi
Expand Up @@ -73,7 +73,7 @@
'mpi/mp_comba.c',
],
'conditions': [
[ 'cc_is_clang==1 and fuzz!=1', {
[ 'cc_is_clang==1 and fuzz!=1 and coverage!=1', {
'cflags': [
'-no-integrated-as',
],
Expand Down
16 changes: 12 additions & 4 deletions mach
Expand Up @@ -444,7 +444,8 @@ class covAction(argparse.Action):
def runSslGtests(self, outdir):
env = {
"GTESTFILTER": "*", # Prevent parallel test runs.
"ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir
"ASAN_OPTIONS": "coverage=1:coverage_dir=" + outdir,
"NSS_DEFAULT_DB_TYPE": "dbm"
}

run_tests("ssl_gtests", env=env, silent=True)
Expand All @@ -463,7 +464,7 @@ class covAction(argparse.Action):
print("\nBuild with coverage sanitizers...\n")
sancov_args = "edge,no-prune,trace-pc-guard,trace-cmp"
subprocess.check_call([
os.path.join(cwd, "build.sh"), "-c", "--clang", "--asan",
os.path.join(cwd, "build.sh"), "-c", "--clang", "--asan", "--enable-legacy-db",
"--sancov=" + sancov_args
])

Expand All @@ -478,16 +479,23 @@ class covAction(argparse.Action):

symcov_file = os.path.join(outdir, "ssl_gtest.symcov")
out = open(symcov_file, 'wb')
subprocess.check_call([
# Don't exit immediately on error
symbol_retcode = subprocess.call([
"sancov",
"-blacklist=" + os.path.join(cwd, ".sancov-blacklist"),
"-symbolize", sancov_file,
os.path.join(cwd, "../dist/Debug/bin/ssl_gtest")
], stdout=out)
out.close()

print("\nCoverage report: " + symcov_file)
print("\nCopying ssl_gtests to artifacts...")
shutil.copyfile(os.path.join(cwd, "../dist/Debug/bin/ssl_gtest"),
os.path.join(outdir, "ssl_gtest"))

print("\nCoverage report: " + symcov_file)
if symbol_retcode > 0:
print("sancov failed to symbolize with return code {}".format(symbol_retcode))
sys.exit(symbol_retcode)

class commandsAction(argparse.Action):
commands = []
Expand Down
10 changes: 5 additions & 5 deletions tests/all.sh
Expand Up @@ -51,10 +51,10 @@
# pkix - run test suites with PKIX enabled
# upgradedb - upgrade existing certificate databases to shareable
# format (creates them if doesn't exist yet) and run
# test suites with those databases
# test suites with those databases. Requires to enable libdm.
# sharedb - run test suites with shareable database format
# enabled (databases are created directly to this
# format)
# format). This is the default and doesn't need to be run separately.
#
# Mandatory environment variables (to be set before testing):
# -----------------------------------------------------------
Expand Down Expand Up @@ -135,7 +135,7 @@ run_tests()
}

########################## run_cycle_standard ##########################
# run test suites with dbm database (no PKIX, no sharedb)
# run test suites with sql database (no PKIX)
########################################################################
run_cycle_standard()
{
Expand All @@ -144,7 +144,7 @@ run_cycle_standard()
TESTS="${ALL_TESTS}"
TESTS_SKIP="cipher libpkix sdr ocsp pkits"

NSS_DEFAULT_DB_TYPE="dbm"
NSS_DEFAULT_DB_TYPE=${NSS_DEFAULT_DB_TYPE:-"sql"}
export NSS_DEFAULT_DB_TYPE

run_tests
Expand Down Expand Up @@ -288,7 +288,7 @@ if [ -z "${INIT_SOURCED}" -o "${INIT_SOURCED}" != "TRUE" ]; then
. ./init.sh
fi

cycles="standard pkix upgradedb sharedb"
cycles="standard pkix"
CYCLES=${NSS_CYCLES:-$cycles}

NO_INIT_SUPPORT=`certutil --build-flags |grep -cw NSS_NO_INIT_SUPPORT`
Expand Down
4 changes: 2 additions & 2 deletions tests/common/init.sh
Expand Up @@ -651,9 +651,9 @@ if [ -z "${INIT_SOURCED}" -o "${INIT_SOURCED}" != "TRUE" ]; then

RELOAD_CRL=1

# if test mode isn't set, test scripts default to expecting dbm
# if test mode isn't set, test scripts default to expecting sql
if [ "${TEST_MODE}" = "" ]; then
NSS_DEFAULT_DB_TYPE="dbm"
NSS_DEFAULT_DB_TYPE=${NSS_DEFAULT_DB_TYPE:-"sql"}
export NSS_DEFAULT_DB_TYPE
fi

Expand Down
2 changes: 1 addition & 1 deletion tests/remote/Makefile
Expand Up @@ -56,7 +56,7 @@ ifeq ($(OS_TARGET),Android)
TEST_SHELL?=$$HOME/bin/sh
ANDROID_PORT?="2222"
#Define the subset of tests that is known to work on Android
NSS_CYCLES?="standard pkix upgradedb sharedb"
NSS_CYCLES?="standard pkix sharedb"
NSS_TESTS?="cipher lowhash libpkix cert dbtests tools sdr crmf smime ssl ocsp merge pkits chains"
NSS_SSL_TESTS?="crl normal_normal iopr"
NSS_SSL_RUN?="cov auth stress"
Expand Down

0 comments on commit eee8f82

Please sign in to comment.