Merge NSS trunk to NSS_TLS13_DRAFT19_BRANCH, a=merge

Hg: changed gtests/nss_bogo_shim/nss_bogo_shim.gyp

--HG--
branch : NSS_TLS13_DRAFT19_BRANCH

.gitignore

@@ -18,3 +18,4 @@ GTAGS
 fuzz/libFuzzer/*
 fuzz/corpus
 fuzz/out
+.chk

.hgignore

@@ -18,3 +18,4 @@ GTAGS
 fuzz/libFuzzer/*
 fuzz/corpus
 fuzz/out
+.chk

Makefile

@@ -28,6 +28,7 @@ include $(CORE_DEPTH)/coreconf/config.mk
 
 ifdef NSS_DISABLE_GTESTS
 DIRS := $(filter-out gtests,$(DIRS))
+DIRS := $(filter-out cpputil,$(DIRS))
 endif
 
 #######################################################################

automation/clang-format/Dockerfile

@@ -0,0 +1,26 @@
+FROM ubuntu:16.04
+MAINTAINER Franziskus Kiefer <franziskuskiefer@gmail.com>
+
+RUN useradd -d /home/worker -s /bin/bash -m worker
+WORKDIR /home/worker
+
+# Install dependencies.
+ADD setup.sh /tmp/setup.sh
+RUN bash /tmp/setup.sh
+
+# Change user.
+USER worker
+
+# Env variables.
+ENV HOME /home/worker
+ENV SHELL /bin/bash
+ENV USER worker
+ENV LOGNAME worker
+ENV HOSTNAME taskcluster-worker
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV HOST localhost
+ENV DOMSUF localdomain
+
+# Entrypoint.
+ENTRYPOINT ["/home/worker/nss/automation/clang-format/run_clang_format.sh"]

automation/taskcluster/scripts/run_clang_format.sh → automation/clang-format/run_clang_format.sh

@@ -1,8 +1,10 @@
 #!/usr/bin/env bash
 
-source $(dirname "$0")/tools.sh
-
-set +x
+if [[ $(id -u) -eq 0 ]]; then
+    # Drop privileges by re-running this script.
+    # Note: this mangles arguments, better to avoid running scripts as root.
+    exec su worker -c "$0 $*"
+fi
 
 # Apply clang-format on the provided folder and verify that this doesn't change any file.
 # If any file differs after formatting, the script eventually exits with 1.
@@ -20,15 +22,16 @@ blacklist=(
      "./lib/sqlite" \
      "./gtests/google_test" \
      "./.hg" \
+     "./out" \
 )
 
-top="$PWD/$(dirname $0)/../../.."
+top="$(dirname $0)/../.."
 cd "$top"
 
 if [ $# -gt 0 ]; then
     dirs=("$@")
 else
-    dirs=($(find . ! -path . \( ! -regex '.*/' \) -maxdepth 2 -mindepth 1 -type d))
+    dirs=($(find . -maxdepth 2 -mindepth 1 -type d ! -path . \( ! -regex '.*/' \)))
 fi
 
 format_folder()

automation/clang-format/setup.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+# Update packages.
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y update && apt-get -y upgrade
+
+# Install packages.
+apt_packages=()
+apt_packages+=('ca-certificates')
+apt_packages+=('curl')
+apt_packages+=('xz-utils')
+apt_packages+=('mercurial')
+apt_packages+=('git')
+apt_packages+=('locales')
+apt-get install -y --no-install-recommends ${apt_packages[@]}
+
+# Download clang.
+curl -L http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -o clang.tar.xz
+curl -L http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig -o clang.tar.xz.sig
+# Verify the signature.
+gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D
+gpg --verify clang.tar.xz.sig
+# Install into /usr/local/.
+tar xJvf *.tar.xz -C /usr/local --strip-components=1
+
+# Cleanup.
+function cleanup() {
+  rm -f clang.tar.xz clang.tar.xz.sig
+}
+trap cleanup ERR EXIT
+
+locale-gen en_US.UTF-8
+dpkg-reconfigure locales
+
+# Cleanup.
+rm -rf ~/.ccache ~/.cache
+apt-get autoremove -y
+apt-get clean
+apt-get autoclean
+
+# We're done. Remove this script.
+rm $0

automation/taskcluster/graph/src/extend.js

@@ -746,7 +746,7 @@ async function scheduleTools() {
     command: [
       "/bin/bash",
       "-c",
-      "bin/checkout.sh && nss/automation/taskcluster/scripts/run_clang_format.sh"
+      "bin/checkout.sh && nss/automation/clang-format/run_clang_format.sh"
     ]
   }));
 

automation/taskcluster/graph/src/queue.js

@@ -25,10 +25,18 @@ function fromNow(hours) {
 }
 
 function parseRoutes(routes) {
-  return [
+  let rv = [
     `tc-treeherder.v2.${process.env.TC_PROJECT}.${process.env.NSS_HEAD_REVISION}.${process.env.NSS_PUSHLOG_ID}`,
     ...routes
   ];
+
+  // Notify about failures (except on try).
+  if (process.env.TC_PROJECT != "nss-try") {
+    rv.push(`notify.email.${process.env.TC_OWNER}.on-failed`,
+            `notify.email.${process.env.TC_OWNER}.on-exception`);
+  }
+
+  return rv;
 }
 
 function parseFeatures(list) {

automation/taskcluster/scripts/split.sh

@@ -24,14 +24,15 @@ split_util() {
   cp -R $nssdir $dstdir
 
   # Skip gtests when building.
-  sed '/^DIRS = /s/ gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn
+  sed '/^DIRS = /s/ cpputil gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn
 
   # Remove subdirectories that we don't want.
   rm -rf $dstdir/cmd
   rm -rf $dstdir/tests
   rm -rf $dstdir/lib
   rm -rf $dstdir/automation
   rm -rf $dstdir/gtests
+  rm -rf $dstdir/cpputil
   rm -rf $dstdir/doc
 
   # Start with an empty cmd lib directories to be filled selectively.
@@ -66,7 +67,7 @@ split_softoken() {
   cp -R $nssdir $dstdir
 
   # Skip gtests when building.
-  sed '/^DIRS = /s/ gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn
+  sed '/^DIRS = /s/ cpputil gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn
 
   # Remove subdirectories that we don't want.
   rm -rf $dstdir/cmd
@@ -75,6 +76,7 @@ split_softoken() {
   rm -rf $dstdir/pkg
   rm -rf $dstdir/automation
   rm -rf $dstdir/gtests
+  rm -rf $dstdir/cpputil
   rm -rf $dstdir/doc
 
   # Start with an empty lib directory and copy only what we need.

build.sh

@@ -93,6 +93,7 @@ while [ $# -gt 0 ]; do
         --system-sqlite) gyp_params+=(-Duse_system_sqlite=1) ;;
         --with-nspr=?*) set_nspr_path "${1#*=}"; no_local_nspr=1 ;;
         --system-nspr) set_nspr_path "/usr/include/nspr/:"; no_local_nspr=1 ;;
+        --enable-libpkix) gyp_params+=(-Ddisable_libpkix=0) ;;
         *) show_help; exit 2 ;;
     esac
     shift

cmd/certutil/keystuff.c

@@ -17,14 +17,6 @@
 #include <conio.h>
 #endif
 
-#if defined(__sun) && !defined(SVR4)
-extern int fclose(FILE *);
-extern int fprintf(FILE *, char *, ...);
-extern int isatty(int);
-extern char *sys_errlist[];
-#define strerror(errno) sys_errlist[errno]
-#endif
-
 #include "nspr.h"
 #include "prtypes.h"
 #include "prtime.h"

cmd/platlibs.gypi

@@ -33,6 +33,7 @@
           '<(DEPTH)/lib/base/base.gyp:nssb',
           '<(DEPTH)/lib/freebl/freebl.gyp:freebl',
           '<(DEPTH)/lib/sqlite/sqlite.gyp:sqlite3',
+          '<(DEPTH)/lib/libpkix/libpkix.gyp:libpkix',
         ],
         'conditions': [
           [ 'disable_dbm==0', {
@@ -41,21 +42,6 @@
               '<(DEPTH)/lib/softoken/legacydb/legacydb.gyp:nssdbm',
             ],
           }],
-          [ 'disable_libpkix==0', {
-            'dependencies': [
-              '<(DEPTH)/lib/libpkix/pkix/certsel/certsel.gyp:pkixcertsel',
-              '<(DEPTH)/lib/libpkix/pkix/checker/checker.gyp:pkixchecker',
-              '<(DEPTH)/lib/libpkix/pkix/params/params.gyp:pkixparams',
-              '<(DEPTH)/lib/libpkix/pkix/results/results.gyp:pkixresults',
-              '<(DEPTH)/lib/libpkix/pkix/top/top.gyp:pkixtop',
-              '<(DEPTH)/lib/libpkix/pkix/util/util.gyp:pkixutil',
-              '<(DEPTH)/lib/libpkix/pkix/crlsel/crlsel.gyp:pkixcrlsel',
-              '<(DEPTH)/lib/libpkix/pkix/store/store.gyp:pkixstore',
-              '<(DEPTH)/lib/libpkix/pkix_pl_nss/pki/pki.gyp:pkixpki',
-              '<(DEPTH)/lib/libpkix/pkix_pl_nss/system/system.gyp:pkixsystem',
-              '<(DEPTH)/lib/libpkix/pkix_pl_nss/module/module.gyp:pkixmodule'
-            ],
-          }],
         ]},{ # !use_static_libs
           'conditions': [
             ['moz_fold_libs==0', {

cmd/platlibs.mk

@@ -32,6 +32,12 @@ else
 DBMLIB = $(DIST)/lib/$(LIB_PREFIX)dbm.$(LIB_SUFFIX) 
 endif
 
+ifeq ($(NSS_BUILD_UTIL_ONLY),1)
+SECTOOL_LIB = $(NULL)
+else
+SECTOOL_LIB = $(DIST)/lib/$(LIB_PREFIX)sectool.$(LIB_SUFFIX)
+endif
+
 ifdef USE_STATIC_LIBS
 
 DEFINES += -DNSS_USE_STATIC_LIBS
@@ -70,20 +76,10 @@ endif
 endif
 
 NSS_LIBS_1=
-SECTOOL_LIB=
 NSS_LIBS_2=
 NSS_LIBS_3=
 NSS_LIBS_4=
 
-ifneq ($(NSS_BUILD_UTIL_ONLY),1)
-SECTOOL_LIB = \
-	$(DIST)/lib/$(LIB_PREFIX)sectool.$(LIB_SUFFIX) \
-	$(NULL)
-else
-SECTOOL_LIB = \
-	$(NULL)
-endif
-
 ifneq ($(NSS_BUILD_SOFTOKEN_ONLY),1)
 ifeq ($(OS_ARCH), WINNT)
 # breakdown for windows
@@ -121,9 +117,6 @@ NSS_LIBS_1 = \
 	$(DIST)/lib/$(LIB_PREFIX)ssl.$(LIB_SUFFIX) \
 	$(DIST)/lib/$(LIB_PREFIX)nss.$(LIB_SUFFIX) \
 	$(NULL)
-SECTOOL_LIB = \
-	$(DIST)/lib/$(LIB_PREFIX)sectool.$(LIB_SUFFIX) \
-	$(NULL)
 NSS_LIBS_2 = \
 	$(DIST)/lib/$(LIB_PREFIX)pkcs12.$(LIB_SUFFIX) \
 	$(DIST)/lib/$(LIB_PREFIX)pkcs7.$(LIB_SUFFIX) \
@@ -201,7 +194,7 @@ ifeq ($(OS_ARCH), WINNT)
 
 # $(PROGRAM) has explicit dependencies on $(EXTRA_LIBS)
 EXTRA_LIBS += \
-	$(DIST)/lib/$(LIB_PREFIX)sectool.$(LIB_SUFFIX) \
+	$(SECTOOL_LIB) \
 	$(NSSUTIL_LIB_DIR)/$(IMPORT_LIB_PREFIX)nssutil3$(IMPORT_LIB_SUFFIX) \
 	$(DIST)/lib/$(IMPORT_LIB_PREFIX)smime3$(IMPORT_LIB_SUFFIX) \
 	$(DIST)/lib/$(IMPORT_LIB_PREFIX)ssl3$(IMPORT_LIB_SUFFIX) \
@@ -220,7 +213,7 @@ else
 
 # $(PROGRAM) has explicit dependencies on $(EXTRA_LIBS)
 EXTRA_LIBS += \
-	$(DIST)/lib/$(LIB_PREFIX)sectool.$(LIB_SUFFIX) \
+	$(SECTOOL_LIB) \
 	$(NULL)
 
 ifeq ($(OS_ARCH), AIX) 
@@ -238,13 +231,15 @@ EXTRA_SHARED_LIBS += \
 	-lplds4 \
 	-lnspr4 \
 	$(NULL)
+ifndef NSS_BUILD_UTIL_ONLY
 ifndef NSS_BUILD_SOFTOKEN_ONLY
 EXTRA_SHARED_LIBS += \
 	-lssl3 \
 	-lsmime3 \
 	-lnss3
 endif
 endif
+endif
 
 ifdef SOFTOKEN_LIB_DIR
 ifdef NSS_USE_SYSTEM_FREEBL

cmd/symkeyutil/symkeyutil.c

@@ -233,6 +233,9 @@ BufToHex(SECItem *outbuf)
     unsigned int i;
 
     string = PORT_Alloc(len);
+    if (!string) {
+        return NULL;
+    }
 
     ptr = string;
     for (i = 0; i < outbuf->len; i++) {

cpputil/Makefile

@@ -0,0 +1,49 @@
+#! gmake
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#######################################################################
+# (1) Include initial platform-independent assignments (MANDATORY).   #
+#######################################################################
+
+include manifest.mn
+
+#######################################################################
+# (2) Include "global" configuration information. (OPTIONAL)          #
+#######################################################################
+
+include $(CORE_DEPTH)/coreconf/config.mk
+
+#######################################################################
+# (3) Include "component" configuration information. (OPTIONAL)       #
+#######################################################################
+
+ifeq (WINNT,$(OS_ARCH))
+OS_CFLAGS += -EHsc
+else
+CXXFLAGS += -std=c++0x
+endif
+
+#######################################################################
+# (4) Include "local" platform-dependent assignments (OPTIONAL).      #
+#######################################################################
+
+include config.mk
+
+#######################################################################
+# (5) Execute "global" rules. (OPTIONAL)                              #
+#######################################################################
+
+include $(CORE_DEPTH)/coreconf/rules.mk
+
+#######################################################################
+# (6) Execute "component" rules. (OPTIONAL)                           #
+#######################################################################
+
+
+
+#######################################################################
+# (7) Execute "local" rules. (OPTIONAL).                              #
+#######################################################################

cpputil/config.mk

@@ -0,0 +1,15 @@
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#
+#  Override TARGETS variable so that only static libraries
+#  are specifed as dependencies within rules.mk.
+#
+
+TARGETS        = $(LIBRARY)
+SHARED_LIBRARY =
+IMPORT_LIBRARY =
+PROGRAM        =
+