From 4025271b203aec11ccb1249607c1d7ba65adea0d Mon Sep 17 00:00:00 2001 From: Daiki Ueno Date: Mon, 24 Sep 2018 17:28:24 +0200 Subject: [PATCH] Bug 1485989, integrate tlsfuzzer interop tests with Taskcluster, r=mt Summary: As a start, it only tests TLS 1.3 so far; 8 out of 18 are skipped because of failures. Reviewers: franziskus, HubertKario, mt Reviewed By: mt Subscribers: mt Bug #: 1485989 Differential Revision: https://phabricator.services.mozilla.com/D4217 --HG-- extra : amend_source : e657ab9197f0696fb3a85d86c85d3005362d65a0 --- automation/taskcluster/graph/src/extend.js | 5 +- .../taskcluster/graph/src/try_syntax.js | 2 +- tests/all.sh | 2 + tests/common/certsetup.sh | 57 ++++++++ tests/ssl_gtests/ssl_gtests.sh | 62 +------- tests/tlsfuzzer/config.json.in | 134 ++++++++++++++++++ tests/tlsfuzzer/tlsfuzzer.sh | 110 ++++++++++++++ 7 files changed, 312 insertions(+), 60 deletions(-) create mode 100644 tests/common/certsetup.sh create mode 100644 tests/tlsfuzzer/config.json.in create mode 100644 tests/tlsfuzzer/tlsfuzzer.sh diff --git a/automation/taskcluster/graph/src/extend.js b/automation/taskcluster/graph/src/extend.js index 4b74f593f0..352bdad214 100644 --- a/automation/taskcluster/graph/src/extend.js +++ b/automation/taskcluster/graph/src/extend.js @@ -75,7 +75,7 @@ queue.filter(task => { } } - if (task.tests == "bogo" || task.tests == "interop") { + if (task.tests == "bogo" || task.tests == "interop" || task.tests == "tlsfuzzer") { // No windows if (task.platform == "windows2012-64" || task.platform == "windows2012-32") { @@ -944,6 +944,9 @@ function scheduleTests(task_build, task_cert, test_base) { cycle: "standard", image: LINUX_INTEROP_IMAGE, })); + queue.scheduleTask(merge(no_cert_base, { + name: "tlsfuzzer tests", symbol: "tlsfuzzer", tests: "tlsfuzzer", cycle: "standard" + })); queue.scheduleTask(merge(no_cert_base, { name: "Chains tests", symbol: "Chains", tests: "chains" })); diff --git a/automation/taskcluster/graph/src/try_syntax.js b/automation/taskcluster/graph/src/try_syntax.js index 92cf9bb00b..f1772a658c 100644 --- a/automation/taskcluster/graph/src/try_syntax.js +++ b/automation/taskcluster/graph/src/try_syntax.js @@ -37,7 +37,7 @@ function parseOptions(opts) { let aliases = {"gtests": "gtest"}; let allUnitTests = ["bogo", "crmf", "chains", "cipher", "db", "ec", "fips", "gtest", "interop", "lowhash", "merge", "sdr", "smime", "tools", - "ssl", "mpi", "scert", "spki", "policy"]; + "ssl", "mpi", "scert", "spki", "policy", "tlsfuzzer"]; let unittests = intersect(opts.unittests.split(/\s*,\s*/).map(t => { return aliases[t] || t; }), allUnitTests); diff --git a/tests/all.sh b/tests/all.sh index 5a179e1f38..5ad0b522e4 100755 --- a/tests/all.sh +++ b/tests/all.sh @@ -42,6 +42,8 @@ # https://boringssl.googlesource.com/boringssl/+/master/ssl/test/PORTING.md # interop.sh - Interoperability tests (disabled by default) # https://github.com/ekr/tls_interop +# tlsfuzzer.sh - tlsfuzzer interop tests (disabled by default) +# https://github.com/tomato42/tlsfuzzer/ # # NSS testing is now devided to 4 cycles: # --------------------------------------- diff --git a/tests/common/certsetup.sh b/tests/common/certsetup.sh new file mode 100644 index 0000000000..2b5cef840b --- /dev/null +++ b/tests/common/certsetup.sh @@ -0,0 +1,57 @@ +# Generate input to certutil +certscript() { + ca=n + while [ $# -gt 0 ]; do + case $1 in + sign) echo 0 ;; + kex) echo 2 ;; + ca) echo 5;echo 6;ca=y ;; + esac; shift + done; + echo 9 + echo n + echo $ca + echo + echo n +} + +# $1: name +# $2: type +# $3+: usages: sign or kex +make_cert() { + name=$1 + type=$2 + + # defaults + type_args=() + trust=',,' + sign=(-x) + sighash=(-Z SHA256) + + case $type in + dsa) type_args=(-g 1024) ;; + rsa) type_args=(-g 1024) ;; + rsa2048) type_args=(-g 2048);type=rsa ;; + rsa8192) type_args=(-g 8192);type=rsa ;; + rsapss) type_args=(-g 1024 --pss);type=rsa ;; + rsapss384) type_args=(-g 1024 --pss);type=rsa;sighash=(-Z SHA384) ;; + rsapss512) type_args=(-g 2048 --pss);type=rsa;sighash=(-Z SHA512) ;; + rsapss_noparam) type_args=(-g 2048 --pss);type=rsa;sighash=() ;; + p256) type_args=(-q nistp256);type=ec ;; + p384) type_args=(-q secp384r1);type=ec ;; + p521) type_args=(-q secp521r1);type=ec ;; + rsa_ca) type_args=(-g 1024);trust='CT,CT,CT';type=rsa ;; + rsa_chain) type_args=(-g 1024);sign=(-c rsa_ca);type=rsa;; + rsapss_ca) type_args=(-g 1024 --pss);trust='CT,CT,CT';type=rsa ;; + rsapss_chain) type_args=(-g 1024);sign=(-c rsa_pss_ca);type=rsa;; + rsa_ca_rsapss_chain) type_args=(-g 1024 --pss-sign);sign=(-c rsa_ca);type=rsa;; + ecdh_rsa) type_args=(-q nistp256);sign=(-c rsa_ca);type=ec ;; + esac + shift 2 + counter=$(($counter + 1)) + certscript $@ | ${BINDIR}/certutil -S \ + -z ${R_NOISE_FILE} -d "${PROFILEDIR}" \ + -n $name -s "CN=$name" -t "$trust" "${sign[@]}" -m "$counter" \ + -w -2 -v 120 -k "$type" "${type_args[@]}" "${sighash[@]}" -1 -2 + html_msg $? 0 "create certificate: $@" +} diff --git a/tests/ssl_gtests/ssl_gtests.sh b/tests/ssl_gtests/ssl_gtests.sh index 30f83b0949..6c088d8a68 100755 --- a/tests/ssl_gtests/ssl_gtests.sh +++ b/tests/ssl_gtests/ssl_gtests.sh @@ -19,64 +19,6 @@ # ######################################################################## -# Generate input to certutil -certscript() { - ca=n - while [ $# -gt 0 ]; do - case $1 in - sign) echo 0 ;; - kex) echo 2 ;; - ca) echo 5;echo 6;ca=y ;; - esac; shift - done; - echo 9 - echo n - echo $ca - echo - echo n -} - -# $1: name -# $2: type -# $3+: usages: sign or kex -make_cert() { - name=$1 - type=$2 - - # defaults - type_args=() - trust=',,' - sign=(-x) - sighash=(-Z SHA256) - - case $type in - dsa) type_args=(-g 1024) ;; - rsa) type_args=(-g 1024) ;; - rsa2048) type_args=(-g 2048);type=rsa ;; - rsa8192) type_args=(-g 8192);type=rsa ;; - rsapss) type_args=(-g 1024 --pss);type=rsa ;; - rsapss384) type_args=(-g 1024 --pss);type=rsa;sighash=(-Z SHA384) ;; - rsapss512) type_args=(-g 2048 --pss);type=rsa;sighash=(-Z SHA512) ;; - rsapss_noparam) type_args=(-g 2048 --pss);type=rsa;sighash=() ;; - p256) type_args=(-q nistp256);type=ec ;; - p384) type_args=(-q secp384r1);type=ec ;; - p521) type_args=(-q secp521r1);type=ec ;; - rsa_ca) type_args=(-g 1024);trust='CT,CT,CT';type=rsa ;; - rsa_chain) type_args=(-g 1024);sign=(-c rsa_ca);type=rsa;; - rsapss_ca) type_args=(-g 1024 --pss);trust='CT,CT,CT';type=rsa ;; - rsapss_chain) type_args=(-g 1024);sign=(-c rsa_pss_ca);type=rsa;; - rsa_ca_rsapss_chain) type_args=(-g 1024 --pss-sign);sign=(-c rsa_ca);type=rsa;; - ecdh_rsa) type_args=(-q nistp256);sign=(-c rsa_ca);type=ec ;; - esac - shift 2 - counter=$(($counter + 1)) - certscript $@ | ${BINDIR}/certutil -S \ - -z ${R_NOISE_FILE} -d "${PROFILEDIR}" \ - -n $name -s "CN=$name" -t "$trust" "${sign[@]}" -m "$counter" \ - -w -2 -v 120 -k "$type" "${type_args[@]}" "${sighash[@]}" -1 -2 - html_msg $? 0 "create certificate: $@" -} - ssl_gtest_certs() { mkdir -p "${SSLGTESTDIR}" cd "${SSLGTESTDIR}" @@ -89,6 +31,10 @@ ssl_gtest_certs() { ${BINDIR}/certutil -N -d "${PROFILEDIR}" --empty-password 2>&1 html_msg $? 0 "create ssl_gtest database" + pushd "${QADIR}" + . common/certsetup.sh + popd + counter=0 make_cert client rsa sign make_cert rsa rsa sign kex diff --git a/tests/tlsfuzzer/config.json.in b/tests/tlsfuzzer/config.json.in new file mode 100644 index 0000000000..0cb6fa3bc8 --- /dev/null +++ b/tests/tlsfuzzer/config.json.in @@ -0,0 +1,134 @@ +[ + { + "server_command": [ + "@SELFSERV@", "-w", "nss", "-d", "@SERVERDIR@", + "-V", "tls1.0:", "-H", "1", + "-n", "rsa", + "-n", "rsa-pss", + "-J", "rsa_pss_rsae_sha256,rsa_pss_rsae_sha384,rsa_pss_rsae_sha512,rsa_pss_pss_sha256", + "-u", "-Z", "-p", "@PORT@" + ], + "server_hostname": "@HOSTADDR@", + "server_port": @PORT@, + "tests" : [ + { + "name" : "test-tls13-conversation.py", + "arguments": [ + "-p", "@PORT@" + ] + }, + { + "name" : "test-tls13-count-tickets.py", + "arguments": [ + "-p", "@PORT@", "-t", "1" + ] + }, + { + "name" : "test-tls13-empty-alert.py", + "arguments": [ + "-p", "@PORT@" + ], + "comment": "https://bugzilla.mozilla.org/show_bug.cgi?id=1471656", + "exp_pass": false + }, + { + "name" : "test-tls13-finished.py", + "arguments": [ + "-p", "@PORT@" + ], + "comment" : "https://bugzilla.mozilla.org/show_bug.cgi?id=1472747", + "exp_pass": false + }, + { + "name" : "test-tls13-0rtt-garbage.py", + "comment": "the disabled test timeouts because of https://bugzilla.mozilla.org/show_bug.cgi?id=1472747", + "arguments": [ + "-p", "@PORT@", "--cookie", + "-e", "undecryptable record later in handshake together with early_data" + ] + }, + { + "name" : "test-tls13-hrr.py", + "arguments": [ + "-p", "@PORT@", "--cookie" + ] + }, + { + "name" : "test-tls13-legacy-version.py", + "arguments": [ + "-p", "@PORT@" + ], + "comment": "https://bugzilla.mozilla.org/show_bug.cgi?id=1490006", + "exp_pass": false + }, + { + "name" : "test-tls13-nociphers.py", + "arguments": [ + "-p", "@PORT@" + ] + }, + { + "name" : "test-tls13-pkcs-signature.py", + "comment": "https://bugzilla.mozilla.org/show_bug.cgi?id=1489997", + "arguments": [ + "-p", "@PORT@", + "-e", "rsa_pkcs1_sha256 signature", + "-e", "rsa_pkcs1_sha384 signature", + "-e", "rsa_pkcs1_sha512 signature" + ] + }, + { + "name" : "test-tls13-rsa-signatures.py", + "comment": "selfserv can be set up to use multiple certs, but only one for each auth type", + "arguments": [ + "-p", "@PORT@", "-b", + "-e", "tls13 signature rsa_pss_pss_sha384", + "-e", "tls13 signature rsa_pss_pss_sha512" + ] + }, + { + "name" : "test-tls13-rsapss-signatures.py", + "comment": "selfserv can be set up to use multiple certs, but only one to each auth type", + "arguments": [ + "-p", "@PORT@", "-b", + "-e", "tls13 signature rsa_pss_pss_sha384", + "-e", "tls13 signature rsa_pss_pss_sha512" + ] + }, + { + "name" : "test-tls13-record-padding.py", + "arguments": [ + "-p", "@PORT@" + ] + }, + { + "name" : "test-tls13-session-resumption.py", + "arguments": [ + "-p", "@PORT@" + ] + }, + { + "name" : "test-tls13-signature-algorithms.py", + "arguments": [ + "-p", "@PORT@" + ], + "comment": "https://bugzilla.mozilla.org/show_bug.cgi?id=1482386", + "exp_pass": false + }, + { + "name" : "test-tls13-version-negotiation.py", + "comment": "the disabled test timeouts because of https://github.com/tomato42/tlsfuzzer/issues/452", + "arguments": [ + "-p", "@PORT@", + "-e", "SSL 2.0 ClientHello with TLS 1.3 version and TLS 1.3 only ciphersuites" + ] + }, + { + "name" : "test-tls13-zero-length-data.py", + "arguments": [ + "-p", "@PORT@" + ] + } + ] + } +] diff --git a/tests/tlsfuzzer/tlsfuzzer.sh b/tests/tlsfuzzer/tlsfuzzer.sh new file mode 100644 index 0000000000..000f1b6397 --- /dev/null +++ b/tests/tlsfuzzer/tlsfuzzer.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +######################################################################## +# +# tests/tlsfuzzer/tlsfuzzer.sh +# +# Script to drive the ssl tlsfuzzer interop unit tests +# +######################################################################## + +tlsfuzzer_certs() +{ + PROFILEDIR=`pwd` + + ${BINDIR}/certutil -N -d "${PROFILEDIR}" --empty-password 2>&1 + html_msg $? 0 "create tlsfuzzer database" + + pushd "${QADIR}" + . common/certsetup.sh + popd + + counter=0 + make_cert rsa rsa2048 sign kex + make_cert rsa-pss rsapss sign kex +} + +tlsfuzzer_init() +{ + SCRIPTNAME="tlsfuzzer.sh" + if [ -z "${INIT_SOURCED}" -o "${INIT_SOURCED}" != "TRUE" ] ; then + cd ../common + . ./init.sh + fi + + mkdir -p "${HOSTDIR}/tlsfuzzer" + pushd "${HOSTDIR}/tlsfuzzer" + tlsfuzzer_certs + + TLSFUZZER=${TLSFUZZER:=tlsfuzzer} + if [ ! -d "$TLSFUZZER" ]; then + # Can't use git-copy.sh here, as tlsfuzzer doesn't have any tags + git clone -q https://github.com/tomato42/tlsfuzzer/ "$TLSFUZZER" + git -C "$TLSFUZZER" checkout fc8fd0128643d6b2bcba344e276ba5f902445913 + + # We could use tlslite-ng from pip, but the pip command installed + # on TC is too old to support --pre + ${QADIR}/../fuzz/config/git-copy.sh https://github.com/tomato42/tlslite-ng/ v0.8.0-alpha18 tlslite-ng + + pushd "$TLSFUZZER" + ln -s ../tlslite-ng/tlslite tlslite + popd + + # Install tlslite-ng dependencies + ${QADIR}/../fuzz/config/git-copy.sh https://github.com/warner/python-ecdsa master python-ecdsa + ${QADIR}/../fuzz/config/git-copy.sh https://github.com/benjaminp/six master six + + pushd "$TLSFUZZER" + ln -s ../python-ecdsa/src/ecdsa ecdsa + ln -s ../six/six.py . + popd + fi + + # Find usable port + PORT=${PORT-8443} + while true; do + "${BINDIR}/selfserv" -w nss -d "${HOSTDIR}/tlsfuzzer" -n rsa \ + -p "${PORT}" -i selfserv.pid & + [ -f selfserv.pid ] || sleep 5 + if [ -f selfserv.pid ]; then + kill $(cat selfserv.pid) + wait $(cat selfserv.pid) + rm -f selfserv.pid + break + fi + PORT=$(($PORT + 1)) + done + + sed -e "s|@PORT@|${PORT}|g" \ + -e "s|@SELFSERV@|${BINDIR}/selfserv|g" \ + -e "s|@SERVERDIR@|${HOSTDIR}/tlsfuzzer|g" \ + -e "s|@HOSTADDR@|${HOSTADDR}|g" \ + ${QADIR}/tlsfuzzer/config.json.in > ${TLSFUZZER}/config.json + popd + + SCRIPTNAME="tlsfuzzer.sh" + html_head "tlsfuzzer test" +} + +tlsfuzzer_cleanup() +{ + cd ${QADIR} + . common/cleanup.sh +} + +tlsfuzzer_run_tests() +{ + pushd "${HOSTDIR}/tlsfuzzer/${TLSFUZZER}" + PYTHONPATH=. python tests/scripts_retention.py config.json "${BINDIR}/selfserv" + html_msg $? 0 "tlsfuzzer" "Run successfully" + popd +} + +cd "$(dirname "$0")" +tlsfuzzer_init +tlsfuzzer_run_tests +tlsfuzzer_cleanup