Bug 1485533 - Close gaps in taskcluster SSL testing. r=mt

This patch increases SSL testing on taskcluster, specifically, running an additional 395 tests on each SSL cycle (more for FIPS targets), and adding a new 'stress' cycle.

Notable changes:

1) This patch removes SSL stress tests from the default `NSS_SSL_RUN` list in all.sh and ssl.sh. If stress tests are needed, this variable must be set to include.

2) The "normal_normal" case is added to `NSS_SSL_TESTS` for all targets. FIPS targets also run "normal_fips", "fips_normal", and "fips_fips".

3) `--enable-libpkix` is now set for all taskcluster "build.sh" builds in order to support a number of OCSP tests that were previously not run.

Differential Revision: https://phabricator.services.mozilla.com/D43283

--HG--
extra : moz-landing-system : lando

automation/taskcluster/graph/src/extend.js

@@ -121,12 +121,22 @@ queue.map(task => {
     }
   }
 
-  // We don't run FIPS SSL tests
   if (task.tests == "ssl") {
     if (!task.env) {
       task.env = {};
     }
-    task.env.NSS_SSL_TESTS = "crl iopr policy";
+
+    // Stress tests to not include other SSL tests
+    if (task.symbol == "stress") {
+      task.env.NSS_SSL_TESTS = "normal_normal";
+    } else {
+      task.env.NSS_SSL_TESTS = "crl iopr policy normal_normal";
+    }
+
+    // FIPS runs
+    if (task.collection == "fips") {
+      task.env.NSS_SSL_TESTS += " fips_fips fips_normal normal_fips";
+    }
 
     if (task.platform == "mac") {
       task.maxRunTime = 7200;
@@ -574,7 +584,7 @@ async function scheduleFuzzing() {
       "/bin/bash",
       "-c",
       "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz"
+      "nss/automation/taskcluster/scripts/build_gyp.sh --fuzz"
     ],
     artifacts: {
       public: {
@@ -601,7 +611,7 @@ async function scheduleFuzzing() {
       "/bin/bash",
       "-c",
       "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls"
+      "nss/automation/taskcluster/scripts/build_gyp.sh --fuzz=tls"
     ],
   }));
 
@@ -679,7 +689,7 @@ async function scheduleFuzzing32() {
       "/bin/bash",
       "-c",
       "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz -t ia32"
+      "nss/automation/taskcluster/scripts/build_gyp.sh --fuzz -t ia32"
     ],
     artifacts: {
       public: {
@@ -706,7 +716,7 @@ async function scheduleFuzzing32() {
       "/bin/bash",
       "-c",
       "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls -t ia32"
+      "nss/automation/taskcluster/scripts/build_gyp.sh --fuzz=tls -t ia32"
     ],
   }));
 
@@ -958,6 +968,10 @@ function scheduleTests(task_build, task_cert, test_base) {
   queue.scheduleTask(merge(ssl_base, {
     name: "SSL tests (upgradedb)", symbol: "upgradedb", cycle: "upgradedb"
   }));
+  queue.scheduleTask(merge(ssl_base, {
+    name: "SSL tests (stress)", symbol: "stress", cycle: "sharedb",
+    env: {NSS_SSL_RUN: "stress"}
+  }));
 }
 
 /*****************************************************************************/

automation/taskcluster/scripts/build_gyp.sh

@@ -6,7 +6,7 @@ source $(dirname "$0")/tools.sh
 hg_clone https://hg.mozilla.org/projects/nspr ./nspr default
 
 # Build.
-nss/build.sh -g -v "$@"
+nss/build.sh -g -v --enable-libpkix "$@"
 
 # Package.
 if [[ $(uname) = "Darwin" ]]; then

automation/taskcluster/windows/build_gyp.sh

@@ -32,7 +32,7 @@ export PATH="${PATH}:${PWD}/ninja/bin:${PWD}/gyp/test-env/Scripts"
 hg_clone https://hg.mozilla.org/projects/nspr nspr default
 
 # Build with gyp.
-./nss/build.sh -g -v "$@"
+./nss/build.sh -g -v --enable-libpkix "$@"
 
 # Package.
 7z a public/build/dist.7z dist

fuzz/fuzz.gyp

@@ -43,6 +43,7 @@
         '<(DEPTH)/lib/pkcs7/pkcs7.gyp:pkcs7',
         # This is a static build of pk11wrap, softoken, and freebl.
         '<(DEPTH)/lib/pk11wrap/pk11wrap.gyp:pk11wrap_static',
+        '<(DEPTH)/lib/libpkix/libpkix.gyp:libpkix',
       ],
       'cflags_cc': [
         '-Wno-error=shadow',

gtests/pk11_gtest/pk11_gtest.gyp

@@ -47,6 +47,7 @@
             '<(DEPTH)/lib/pk11wrap/pk11wrap.gyp:pk11wrap_static',
             '<(DEPTH)/lib/pki/pki.gyp:nsspki',
             '<(DEPTH)/lib/ssl/ssl.gyp:ssl',
+            '<(DEPTH)/lib/libpkix/libpkix.gyp:libpkix',
           ],
         }, {
           'dependencies': [

gtests/softoken_gtest/softoken_gtest.gyp

@@ -30,6 +30,7 @@
             '<(DEPTH)/lib/dev/dev.gyp:nssdev',
             '<(DEPTH)/lib/pki/pki.gyp:nsspki',
             '<(DEPTH)/lib/ssl/ssl.gyp:ssl',
+            '<(DEPTH)/lib/libpkix/libpkix.gyp:libpkix',
           ],
         }, {
           'dependencies': [

tests/all.sh

@@ -86,7 +86,7 @@
 #          +------------+------------+-----------+       ~  run_cycles
 #          |            |            |           |             |
 #      standard       pkix       upgradedb     sharedb   ~  run_cycle_*
-#                       |                                      |
+#         ...           |           ...         ...            |
 #                +------+------+------+----->            ~  run_tests
 #                |      |      |      |                        |
 #              cert   tools   fips   ssl   ...           ~  . *.sh
@@ -147,9 +147,6 @@ run_cycle_standard()
     NSS_DEFAULT_DB_TYPE="dbm"
     export NSS_DEFAULT_DB_TYPE
 
-    NSS_SSL_TESTS=`echo "${NSS_SSL_TESTS}" | sed -e "s/normal//g" -e "s/fips//g" -e "s/_//g"`
-    NSS_SSL_RUN=`echo "${NSS_SSL_RUN}" | sed -e "s/cov//g" -e "s/auth//g"`
-
     run_tests
 }
 
@@ -174,7 +171,6 @@ run_cycle_pkix()
     TESTS="${ALL_TESTS}"
     TESTS_SKIP="cipher dbtests sdr crmf smime merge multinit"
 
-    NSS_SSL_TESTS=`echo "${NSS_SSL_TESTS}" | sed -e "s/normal//g" -e "s/fips//g" -e "s/_//g"`
     export -n NSS_SSL_RUN
 
     # use the default format. (unset for the shell, export -n for binaries)
@@ -222,9 +218,6 @@ run_cycle_upgrade_db()
     TESTS="${ALL_TESTS}"
     TESTS_SKIP="cipher libpkix cert dbtests sdr ocsp pkits chains"
 
-    NSS_SSL_TESTS=`echo "${NSS_SSL_TESTS}" | sed -e "s/normal//g" -e "s/fips//g" -e "s/_//g"`
-    NSS_SSL_RUN=`echo "${NSS_SSL_RUN}" | sed -e "s/cov//g" -e "s/auth//g"`
-
     run_tests
 }
 
@@ -318,7 +311,8 @@ if [ $NO_INIT_SUPPORT -eq 0 ]; then
 fi
 NSS_SSL_TESTS="${NSS_SSL_TESTS:-$nss_ssl_tests}"
 
-nss_ssl_run="cov auth stapling signed_cert_timestamps stress scheme"
+# NOTE: 'stress' run is omitted by default
+nss_ssl_run="cov auth stapling signed_cert_timestamps scheme"
 NSS_SSL_RUN="${NSS_SSL_RUN:-$nss_ssl_run}"
 
 # NOTE:

tests/ssl/ssl.sh

@@ -10,7 +10,27 @@
 #
 # Script to test NSS SSL
 #
-# needs to work on all Unix and Windows platforms
+# Needs to work on all Unix and Windows platforms
+#
+# Testing schema:
+# ---------------
+#                           all.sh                       ~  (main)
+#                              |                               |
+#          +------------+------------+-----------+       ~  run_cycles
+#          |            |            |           |             |
+#      standard       pkix       upgradedb     sharedb   ~  run_cycle_*
+#         ...           |           ...         ...            |
+#                +------+------+----->                   ~  run_tests
+#                |      |      |                               |
+#               ...    ssl    ...                        ~   ssl.sh
+#                       |                                      |
+#          +-------+-------+-----------------+           ~  ssl_run_tests
+#          |       |       |                 |                 |
+#         crl     iopr   policy    permute(normal,fips)  ~  ssl_run_test_*
+#                                         | | | |              |
+#         +------+------+------+------+---+-+-+-+---->   ~  ssl_run
+#         |      |      |      |      |      |                 |
+#    stapling   cov   auth  stress  dtls    ...          ~  ssl_run_*
 #
 # special strings
 # ---------------
@@ -64,7 +84,7 @@ ssl_init()
     PORT=$(($PORT + $padd))
   fi
   NSS_SSL_TESTS=${NSS_SSL_TESTS:-normal_normal}
-  nss_ssl_run="stapling signed_cert_timestamps cov auth stress dtls scheme"
+  nss_ssl_run="stapling signed_cert_timestamps cov auth dtls scheme"
   NSS_SSL_RUN=${NSS_SSL_RUN:-$nss_ssl_run}
 
   # Test case files
@@ -521,10 +541,10 @@ ssl_stapling_stress()
     echo "${testname}"
     start_selfserv
 
-    echo "strsclnt -q -p ${PORT} -d ${P_R_CLIENTDIR} ${CLIENT_OPTIONS} -w nss \\"
+    echo "strsclnt -4 -q -p ${PORT} -d ${P_R_CLIENTDIR} ${CLIENT_OPTIONS} -w nss \\"
     echo "         -c 1000 -V ssl3:tls1.2 -N -T $verbose ${HOSTADDR}"
     echo "strsclnt started at `date`"
-    ${PROFTOOL} ${BINDIR}/strsclnt -q -p ${PORT} -d ${P_R_CLIENTDIR} ${CLIENT_OPTIONS} -w nss \
+    ${PROFTOOL} ${BINDIR}/strsclnt -4 -q -p ${PORT} -d ${P_R_CLIENTDIR} ${CLIENT_OPTIONS} -w nss \
             -c 1000 -V ssl3:tls1.2 -N -T $verbose ${HOSTADDR}
     ret=$?
 
@@ -651,10 +671,10 @@ ssl_stress()
               dbdir=${P_R_CLIENTDIR}
           fi
 
-          echo "strsclnt -q -p ${PORT} -d ${dbdir} ${CLIENT_OPTIONS} -w nss $cparam \\"
+          echo "strsclnt -4 -q -p ${PORT} -d ${dbdir} ${CLIENT_OPTIONS} -w nss $cparam \\"
           echo "         -V ssl3:tls1.2 $verbose ${HOSTADDR}"
           echo "strsclnt started at `date`"
-          ${PROFTOOL} ${BINDIR}/strsclnt -q -p ${PORT} -d ${dbdir} ${CLIENT_OPTIONS} -w nss $cparam \
+          ${PROFTOOL} ${BINDIR}/strsclnt -4 -q -p ${PORT} -d ${dbdir} ${CLIENT_OPTIONS} -w nss $cparam \
                    -V ssl3:tls1.2 $verbose ${HOSTADDR}
           ret=$?
           echo "strsclnt completed at `date`"
@@ -1275,9 +1295,9 @@ ssl_scheme_stress()
 
             start_selfserv -V tls1.2:tls1.2 -J "$sscheme"
 
-            echo "strsclnt -q -p ${PORT} -d ${P_R_CLIENTDIR} $verbose ${CLIENT_OPTIONS} \\"
+            echo "strsclnt -4 -q -p ${PORT} -d ${P_R_CLIENTDIR} $verbose ${CLIENT_OPTIONS} \\"
             echo "         -V tls1.2:tls1.2 -J "$cscheme" ${HOSTADDR} < ${REQUEST_FILE}"
-            ${PROFTOOL} ${BINDIR}/strsclnt -q -p ${PORT} ${CLIENT_OPTIONS} \
+            ${PROFTOOL} ${BINDIR}/strsclnt -4 -q -p ${PORT} ${CLIENT_OPTIONS} \
                         -d ${P_R_CLIENTDIR} $verbose -V tls1.2:tls1.2 -J "$cscheme" ${HOSTADDR} < ${REQUEST_FILE} 2>&1
             ret=$?
             # If both schemes include just one option and those options don't