Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Bug 1485533 - Close gaps in taskcluster SSL testing. r=mt
This patch increases SSL testing on taskcluster, specifically, running an additional 395 tests on each SSL cycle (more for FIPS targets), and adding a new 'stress' cycle.

Notable changes:

1) This patch removes SSL stress tests from the default `NSS_SSL_RUN` list in all.sh and ssl.sh. If stress tests are needed, this variable must be set to include.

2) The "normal_normal" case is added to `NSS_SSL_TESTS` for all targets. FIPS targets also run "normal_fips", "fips_normal", and "fips_fips".

3) `--enable-libpkix` is now set for all taskcluster "build.sh" builds in order to support a number of OCSP tests that were previously not run.

Differential Revision: https://phabricator.services.mozilla.com/D43283

--HG--
extra : moz-landing-system : lando
  • Loading branch information
Kevin Jacobs committed Aug 27, 2019
1 parent a8313a6 commit 939f061
Show file tree
Hide file tree
Showing 8 changed files with 56 additions and 25 deletions.
26 changes: 20 additions & 6 deletions automation/taskcluster/graph/src/extend.js
Expand Up @@ -121,12 +121,22 @@ queue.map(task => {
}
}

// We don't run FIPS SSL tests
if (task.tests == "ssl") {
if (!task.env) {
task.env = {};
}
task.env.NSS_SSL_TESTS = "crl iopr policy";

// Stress tests to not include other SSL tests
if (task.symbol == "stress") {
task.env.NSS_SSL_TESTS = "normal_normal";
} else {
task.env.NSS_SSL_TESTS = "crl iopr policy normal_normal";
}

// FIPS runs
if (task.collection == "fips") {
task.env.NSS_SSL_TESTS += " fips_fips fips_normal normal_fips";
}

if (task.platform == "mac") {
task.maxRunTime = 7200;
Expand Down Expand Up @@ -574,7 +584,7 @@ async function scheduleFuzzing() {
"/bin/bash",
"-c",
"bin/checkout.sh && " +
"nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz"
"nss/automation/taskcluster/scripts/build_gyp.sh --fuzz"
],
artifacts: {
public: {
Expand All @@ -601,7 +611,7 @@ async function scheduleFuzzing() {
"/bin/bash",
"-c",
"bin/checkout.sh && " +
"nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls"
"nss/automation/taskcluster/scripts/build_gyp.sh --fuzz=tls"
],
}));

Expand Down Expand Up @@ -679,7 +689,7 @@ async function scheduleFuzzing32() {
"/bin/bash",
"-c",
"bin/checkout.sh && " +
"nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz -t ia32"
"nss/automation/taskcluster/scripts/build_gyp.sh --fuzz -t ia32"
],
artifacts: {
public: {
Expand All @@ -706,7 +716,7 @@ async function scheduleFuzzing32() {
"/bin/bash",
"-c",
"bin/checkout.sh && " +
"nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls -t ia32"
"nss/automation/taskcluster/scripts/build_gyp.sh --fuzz=tls -t ia32"
],
}));

Expand Down Expand Up @@ -958,6 +968,10 @@ function scheduleTests(task_build, task_cert, test_base) {
queue.scheduleTask(merge(ssl_base, {
name: "SSL tests (upgradedb)", symbol: "upgradedb", cycle: "upgradedb"
}));
queue.scheduleTask(merge(ssl_base, {
name: "SSL tests (stress)", symbol: "stress", cycle: "sharedb",
env: {NSS_SSL_RUN: "stress"}
}));
}

/*****************************************************************************/
Expand Down
2 changes: 1 addition & 1 deletion automation/taskcluster/scripts/build_gyp.sh
Expand Up @@ -6,7 +6,7 @@ source $(dirname "$0")/tools.sh
hg_clone https://hg.mozilla.org/projects/nspr ./nspr default

# Build.
nss/build.sh -g -v "$@"
nss/build.sh -g -v --enable-libpkix "$@"

# Package.
if [[ $(uname) = "Darwin" ]]; then
Expand Down
2 changes: 1 addition & 1 deletion automation/taskcluster/windows/build_gyp.sh
Expand Up @@ -32,7 +32,7 @@ export PATH="${PATH}:${PWD}/ninja/bin:${PWD}/gyp/test-env/Scripts"
hg_clone https://hg.mozilla.org/projects/nspr nspr default

# Build with gyp.
./nss/build.sh -g -v "$@"
./nss/build.sh -g -v --enable-libpkix "$@"

# Package.
7z a public/build/dist.7z dist
1 change: 1 addition & 0 deletions fuzz/fuzz.gyp
Expand Up @@ -43,6 +43,7 @@
'<(DEPTH)/lib/pkcs7/pkcs7.gyp:pkcs7',
# This is a static build of pk11wrap, softoken, and freebl.
'<(DEPTH)/lib/pk11wrap/pk11wrap.gyp:pk11wrap_static',
'<(DEPTH)/lib/libpkix/libpkix.gyp:libpkix',
],
'cflags_cc': [
'-Wno-error=shadow',
Expand Down
1 change: 1 addition & 0 deletions gtests/pk11_gtest/pk11_gtest.gyp
Expand Up @@ -47,6 +47,7 @@
'<(DEPTH)/lib/pk11wrap/pk11wrap.gyp:pk11wrap_static',
'<(DEPTH)/lib/pki/pki.gyp:nsspki',
'<(DEPTH)/lib/ssl/ssl.gyp:ssl',
'<(DEPTH)/lib/libpkix/libpkix.gyp:libpkix',
],
}, {
'dependencies': [
Expand Down
1 change: 1 addition & 0 deletions gtests/softoken_gtest/softoken_gtest.gyp
Expand Up @@ -30,6 +30,7 @@
'<(DEPTH)/lib/dev/dev.gyp:nssdev',
'<(DEPTH)/lib/pki/pki.gyp:nsspki',
'<(DEPTH)/lib/ssl/ssl.gyp:ssl',
'<(DEPTH)/lib/libpkix/libpkix.gyp:libpkix',
],
}, {
'dependencies': [
Expand Down
12 changes: 3 additions & 9 deletions tests/all.sh
Expand Up @@ -86,7 +86,7 @@
# +------------+------------+-----------+ ~ run_cycles
# | | | | |
# standard pkix upgradedb sharedb ~ run_cycle_*
# | |
# ... | ... ... |
# +------+------+------+-----> ~ run_tests
# | | | | |
# cert tools fips ssl ... ~ . *.sh
Expand Down Expand Up @@ -147,9 +147,6 @@ run_cycle_standard()
NSS_DEFAULT_DB_TYPE="dbm"
export NSS_DEFAULT_DB_TYPE

NSS_SSL_TESTS=`echo "${NSS_SSL_TESTS}" | sed -e "s/normal//g" -e "s/fips//g" -e "s/_//g"`
NSS_SSL_RUN=`echo "${NSS_SSL_RUN}" | sed -e "s/cov//g" -e "s/auth//g"`

run_tests
}

Expand All @@ -174,7 +171,6 @@ run_cycle_pkix()
TESTS="${ALL_TESTS}"
TESTS_SKIP="cipher dbtests sdr crmf smime merge multinit"

NSS_SSL_TESTS=`echo "${NSS_SSL_TESTS}" | sed -e "s/normal//g" -e "s/fips//g" -e "s/_//g"`
export -n NSS_SSL_RUN

# use the default format. (unset for the shell, export -n for binaries)
Expand Down Expand Up @@ -222,9 +218,6 @@ run_cycle_upgrade_db()
TESTS="${ALL_TESTS}"
TESTS_SKIP="cipher libpkix cert dbtests sdr ocsp pkits chains"

NSS_SSL_TESTS=`echo "${NSS_SSL_TESTS}" | sed -e "s/normal//g" -e "s/fips//g" -e "s/_//g"`
NSS_SSL_RUN=`echo "${NSS_SSL_RUN}" | sed -e "s/cov//g" -e "s/auth//g"`

run_tests
}

Expand Down Expand Up @@ -318,7 +311,8 @@ if [ $NO_INIT_SUPPORT -eq 0 ]; then
fi
NSS_SSL_TESTS="${NSS_SSL_TESTS:-$nss_ssl_tests}"

nss_ssl_run="cov auth stapling signed_cert_timestamps stress scheme"
# NOTE: 'stress' run is omitted by default
nss_ssl_run="cov auth stapling signed_cert_timestamps scheme"
NSS_SSL_RUN="${NSS_SSL_RUN:-$nss_ssl_run}"

# NOTE:
Expand Down
36 changes: 28 additions & 8 deletions tests/ssl/ssl.sh
Expand Up @@ -10,7 +10,27 @@
#
# Script to test NSS SSL
#
# needs to work on all Unix and Windows platforms
# Needs to work on all Unix and Windows platforms
#
# Testing schema:
# ---------------
# all.sh ~ (main)
# | |
# +------------+------------+-----------+ ~ run_cycles
# | | | | |
# standard pkix upgradedb sharedb ~ run_cycle_*
# ... | ... ... |
# +------+------+-----> ~ run_tests
# | | | |
# ... ssl ... ~ ssl.sh
# | |
# +-------+-------+-----------------+ ~ ssl_run_tests
# | | | | |
# crl iopr policy permute(normal,fips) ~ ssl_run_test_*
# | | | | |
# +------+------+------+------+---+-+-+-+----> ~ ssl_run
# | | | | | | |
# stapling cov auth stress dtls ... ~ ssl_run_*
#
# special strings
# ---------------
Expand Down Expand Up @@ -64,7 +84,7 @@ ssl_init()
PORT=$(($PORT + $padd))
fi
NSS_SSL_TESTS=${NSS_SSL_TESTS:-normal_normal}
nss_ssl_run="stapling signed_cert_timestamps cov auth stress dtls scheme"
nss_ssl_run="stapling signed_cert_timestamps cov auth dtls scheme"
NSS_SSL_RUN=${NSS_SSL_RUN:-$nss_ssl_run}

# Test case files
Expand Down Expand Up @@ -521,10 +541,10 @@ ssl_stapling_stress()
echo "${testname}"
start_selfserv

echo "strsclnt -q -p ${PORT} -d ${P_R_CLIENTDIR} ${CLIENT_OPTIONS} -w nss \\"
echo "strsclnt -4 -q -p ${PORT} -d ${P_R_CLIENTDIR} ${CLIENT_OPTIONS} -w nss \\"
echo " -c 1000 -V ssl3:tls1.2 -N -T $verbose ${HOSTADDR}"
echo "strsclnt started at `date`"
${PROFTOOL} ${BINDIR}/strsclnt -q -p ${PORT} -d ${P_R_CLIENTDIR} ${CLIENT_OPTIONS} -w nss \
${PROFTOOL} ${BINDIR}/strsclnt -4 -q -p ${PORT} -d ${P_R_CLIENTDIR} ${CLIENT_OPTIONS} -w nss \
-c 1000 -V ssl3:tls1.2 -N -T $verbose ${HOSTADDR}
ret=$?

Expand Down Expand Up @@ -651,10 +671,10 @@ ssl_stress()
dbdir=${P_R_CLIENTDIR}
fi

echo "strsclnt -q -p ${PORT} -d ${dbdir} ${CLIENT_OPTIONS} -w nss $cparam \\"
echo "strsclnt -4 -q -p ${PORT} -d ${dbdir} ${CLIENT_OPTIONS} -w nss $cparam \\"
echo " -V ssl3:tls1.2 $verbose ${HOSTADDR}"
echo "strsclnt started at `date`"
${PROFTOOL} ${BINDIR}/strsclnt -q -p ${PORT} -d ${dbdir} ${CLIENT_OPTIONS} -w nss $cparam \
${PROFTOOL} ${BINDIR}/strsclnt -4 -q -p ${PORT} -d ${dbdir} ${CLIENT_OPTIONS} -w nss $cparam \
-V ssl3:tls1.2 $verbose ${HOSTADDR}
ret=$?
echo "strsclnt completed at `date`"
Expand Down Expand Up @@ -1275,9 +1295,9 @@ ssl_scheme_stress()

start_selfserv -V tls1.2:tls1.2 -J "$sscheme"

echo "strsclnt -q -p ${PORT} -d ${P_R_CLIENTDIR} $verbose ${CLIENT_OPTIONS} \\"
echo "strsclnt -4 -q -p ${PORT} -d ${P_R_CLIENTDIR} $verbose ${CLIENT_OPTIONS} \\"
echo " -V tls1.2:tls1.2 -J "$cscheme" ${HOSTADDR} < ${REQUEST_FILE}"
${PROFTOOL} ${BINDIR}/strsclnt -q -p ${PORT} ${CLIENT_OPTIONS} \
${PROFTOOL} ${BINDIR}/strsclnt -4 -q -p ${PORT} ${CLIENT_OPTIONS} \
-d ${P_R_CLIENTDIR} $verbose -V tls1.2:tls1.2 -J "$cscheme" ${HOSTADDR} < ${REQUEST_FILE} 2>&1
ret=$?
# If both schemes include just one option and those options don't
Expand Down

0 comments on commit 939f061

Please sign in to comment.