Commit 966fea27 authored by Martin Thomson's avatar Martin Thomson

Merge NSS trunk to NSS_TLS13_DRAFT19_BRANCH

--HG--
branch : NSS_TLS13_DRAFT19_BRANCH
rename : automation/taskcluster/docker/bin/checkout.sh => automation/taskcluster/docker-hacl/bin/checkout.sh
extra : rebase_source : 539f7521c7deb8f828cc1fbef51051568deedccb
parents 8c9ccbf1 5e8ecb16
......@@ -57,7 +57,7 @@ tasks:
- "tc-treeherder.v2.{{project}}.{{revision}}.{{pushlog_id}}"
payload:
image: ttaubert/nss-decision:0.0.2
image: nssdev/nss-decision:0.0.2
env:
TC_OWNER: {{owner}}
......
......@@ -236,11 +236,14 @@ check_abi()
BASE_NSPR=NSPR_$(head -1 ${HGDIR}/baseline/nss/automation/release/nspr-version.txt | cut -d . -f 1-2 | tr . _)_BRANCH
hg clone -u "${BASE_NSPR}" "${HGDIR}/nspr" "${HGDIR}/baseline/nspr"
if [ $? -ne 0 ]; then
echo "invalid tag ${BASE_NSPR} derived from ${BASE_NSS} automation/release/nspr-version.txt"
return 1
echo "nonexisting tag ${BASE_NSPR} derived from ${BASE_NSS} automation/release/nspr-version.txt"
# Assume that version hasn't been released yet, fall back to trunk
pushd "${HGDIR}/baseline/nspr"
hg update default
popd
fi
print_log "######## building older NSPR/NSS ########"
print_log "######## building baseline NSPR/NSS ########"
pushd ${HGDIR}/baseline/nss
print_log "$ ${MAKE} ${NSS_BUILD_TARGET}"
......
4.15
4.17
# The first line of this file must contain the human readable NSPR
# version number, which is the minimum required version of NSPR
......
......@@ -12,6 +12,9 @@ RUN chmod +x /home/worker/bin/*
ADD setup.sh /tmp/setup.sh
RUN bash /tmp/setup.sh
# Change user.
USER worker
# Env variables.
ENV HOME /home/worker
ENV SHELL /bin/bash
......
......@@ -2,11 +2,6 @@
set -v -e -x
if [ $(id -u) = 0 ]; then
# Drop privileges by re-running this script.
exec su worker $0
fi
# Default values for testing.
REVISION=${NSS_HEAD_REVISION:-default}
REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}
......
FROM ubuntu:14.04
MAINTAINER Tim Taubert <ttaubert@mozilla.com>
RUN useradd -d /home/worker -s /bin/bash -m worker
WORKDIR /home/worker
# Add build and test scripts.
ADD bin /home/worker/bin
RUN chmod +x /home/worker/bin/*
# Install dependencies.
ADD setup.sh /tmp/setup.sh
RUN bash /tmp/setup.sh
# Change user.
USER worker
# Env variables.
ENV HOME /home/worker
ENV SHELL /bin/bash
ENV USER worker
ENV LOGNAME worker
ENV HOSTNAME taskcluster-worker
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
ENV HOST localhost
ENV DOMSUF localdomain
# Set a default command for debugging.
CMD ["/bin/bash", "--login"]
#!/usr/bin/env bash
set -v -e -x
if [ $(id -u) = 0 ]; then
# Drop privileges by re-running this script.
exec su worker $0
fi
# Default values for testing.
REVISION=${NSS_HEAD_REVISION:-default}
REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}
# Clone NSS.
for i in 0 2 5; do
sleep $i
hg clone -r $REVISION $REPOSITORY nss && exit 0
rm -rf nss
done
exit 1
#!/usr/bin/env bash
set -v -e -x
# Update packages.
export DEBIAN_FRONTEND=noninteractive
apt-get -y update && apt-get -y upgrade
apt_packages=()
apt_packages+=('ca-certificates')
apt_packages+=('g++-4.4')
apt_packages+=('gcc-4.4')
apt_packages+=('locales')
apt_packages+=('make')
apt_packages+=('mercurial')
apt_packages+=('zlib1g-dev')
# Install packages.
apt-get -y update
apt-get install -y --no-install-recommends ${apt_packages[@]}
locale-gen en_US.UTF-8
dpkg-reconfigure locales
# Cleanup.
rm -rf ~/.ccache ~/.cache
apt-get autoremove -y
apt-get clean
apt-get autoclean
rm $0
FROM ubuntu:xenial
MAINTAINER Franziskus Kiefer <franziskuskiefer@gmail.com>
# Based on the HACL* image from Benjamin Beurdouche and
# the original F* formula with Daniel Fabian
# Pinned versions of HaCl* (F* and KreMLin are pinned as submodules)
ENV haclrepo https://github.com/mitls/hacl-star.git
# Define versions of dependencies
ENV opamv 4.04.2
ENV z3v 4.5.1.1f29cebd4df6-x64-ubuntu-14.04
ENV haclversion 0030539598cde15d1a0e5f93b32e121f7b7b5a1c
ENV haclbranch production-nss
# Install required packages and set versions
RUN apt-get -qq update
RUN apt-get install --yes sudo libssl-dev libsqlite3-dev g++-5 gcc-5 m4 make opam pkg-config python libgmp3-dev cmake curl libtool-bin autoconf
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 200
RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 200
# Create user
RUN useradd -ms /bin/bash worker
RUN echo "worker ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers
WORKDIR /home/worker
# Add build and test scripts.
ADD bin /home/worker/bin
RUN chmod +x /home/worker/bin/*
USER worker
# Add "known-good" version of Z3
RUN curl -LO https://github.com/FStarLang/binaries/raw/master/z3-tested/z3-${z3v}.zip
RUN unzip z3-${z3v}.zip
RUN rm z3-${z3v}.zip
RUN mv z3-${z3v} z3
ENV PATH "/home/worker/z3/bin:$PATH"
# Prepare build (OCaml packages)
ENV OPAMYES true
RUN opam init
RUN echo ". /home/worker/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true" >> .bashrc
RUN opam switch -v ${opamv}
RUN opam install ocamlfind batteries sqlite3 fileutils yojson ppx_deriving_yojson zarith pprint menhir ulex process fix wasm stdint
# Get the HaCl* code
RUN git clone ${haclrepo} hacl-star
RUN git -C hacl-star checkout ${haclversion}
# Prepare submodules, and build, verify, test, and extract c code
# This caches the extracted c code (pins the HaCl* version). All we need to do
# on CI now is comparing the code in this docker image with the one in NSS.
RUN opam config exec -- make -C hacl-star nss -j$(nproc)
# Get clang-format-3.9
RUN curl -LO http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz
RUN curl -LO http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig
# Verify the signature.
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D
RUN gpg --verify *.tar.xz.sig
# Install into /usr/local/.
RUN sudo tar xJvf *.tar.xz -C /usr/local --strip-components=1
# Cleanup.
RUN rm *.tar.xz*
# Cleanup
RUN rm -rf ~/.ccache ~/.cache
RUN sudo apt-get autoremove -y
RUN sudo apt-get clean
RUN sudo apt-get autoclean
#!/usr/bin/env bash
set -v -e -x
if [ $(id -u) = 0 ]; then
# Drop privileges by re-running this script.
exec su worker $0
fi
# Default values for testing.
REVISION=${NSS_HEAD_REVISION:-default}
REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}
# Clone NSS.
for i in 0 2 5; do
sleep $i
hg clone -r $REVISION $REPOSITORY nss && exit 0
rm -rf nss
done
exit 1
......@@ -27,14 +27,24 @@ function collectFilesInDirectory(dir) {
});
}
// Compute a context hash for the given context path.
export default function (context_path) {
// A list of hashes for each file in the given path.
function collectFileHashes(context_path) {
let root = path.join(__dirname, "../../../..");
let dir = path.join(root, context_path);
let files = collectFilesInDirectory(dir).sort();
let hashes = files.map(file => {
return files.map(file => {
return sha256(file + "|" + fs.readFileSync(file, "utf-8"));
});
}
// Compute a context hash for the given context path.
export default function (context_path) {
// Regenerate all images when the image_builder changes.
let hashes = collectFileHashes("automation/taskcluster/image_builder");
// Regenerate images when the image itself changes.
hashes = hashes.concat(collectFileHashes(context_path));
// Generate a new prefix every month to ensure the image stays buildable.
let now = new Date();
......
......@@ -15,11 +15,21 @@ const LINUX_CLANG39_IMAGE = {
path: "automation/taskcluster/docker-clang-3.9"
};
const LINUX_GCC44_IMAGE = {
name: "linux-gcc-4.4",
path: "automation/taskcluster/docker-gcc-4.4"
};
const FUZZ_IMAGE = {
name: "fuzz",
path: "automation/taskcluster/docker-fuzz"
};
const HACL_GEN_IMAGE = {
name: "hacl",
path: "automation/taskcluster/docker-hacl"
};
const WINDOWS_CHECKOUT_CMD =
"bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " +
"(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " +
......@@ -253,8 +263,7 @@ async function scheduleMac(name, base, args = "") {
},
provisioner: "localprovisioner",
workerType: "nss-macos-10-12",
platform: "mac",
tier: 3
platform: "mac"
});
// Build base definition.
......@@ -400,6 +409,26 @@ async function scheduleLinux(name, base, args = "") {
symbol: "clang-4.0"
}));
queue.scheduleTask(merge(extra_base, {
name: `${name} w/ gcc-4.4`,
image: LINUX_GCC44_IMAGE,
env: {
USE_64: "1",
CC: "gcc-4.4",
CCC: "g++-4.4",
// gcc-4.6 introduced nullptr.
NSS_DISABLE_GTESTS: "1",
},
// Use the old Makefile-based build system, GYP doesn't have a proper GCC
// version check for __int128 support. It's mainly meant to cover RHEL6.
command: [
"/bin/bash",
"-c",
"bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh",
],
symbol: "gcc-4.4"
}));
queue.scheduleTask(merge(extra_base, {
name: `${name} w/ gcc-4.8`,
env: {
......@@ -529,12 +558,13 @@ async function scheduleFuzzing() {
// Schedule MPI fuzzing runs.
let mpi_base = merge(run_base, {group: "MPI"});
let mpi_names = ["add", "addmod", "div", "expmod", "mod", "mulmod", "sqr",
let mpi_names = ["add", "addmod", "div", "mod", "mulmod", "sqr",
"sqrmod", "sub", "submod"];
for (let name of mpi_names) {
scheduleFuzzingRun(mpi_base, `MPI (${name})`, `mpi-${name}`, 4096, name);
}
scheduleFuzzingRun(mpi_base, `MPI (invmod)`, `mpi-invmod`, 256, "invmod");
scheduleFuzzingRun(mpi_base, `MPI (expmod)`, `mpi-expmod`, 2048, "expmod");
// Schedule TLS fuzzing runs (non-fuzzing mode).
let tls_base = merge(run_base, {group: "TLS"});
......@@ -934,5 +964,16 @@ async function scheduleTools() {
]
}));
queue.scheduleTask(merge(base, {
symbol: "hacl",
name: "hacl",
image: HACL_GEN_IMAGE,
command: [
"/bin/bash",
"-c",
"bin/checkout.sh && nss/automation/taskcluster/scripts/run_hacl.sh"
]
}));
return queue.submit();
}
......@@ -31,13 +31,11 @@ export async function buildTask({name, path}) {
return {
name: "Image Builder",
image: "taskcluster/image_builder:0.1.5",
image: "nssdev/image_builder:0.1.5",
routes: ["index." + ns],
env: {
HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
BASE_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
HEAD_REV: process.env.NSS_HEAD_REVISION,
HEAD_REF: process.env.NSS_HEAD_REVISION,
NSS_HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
NSS_HEAD_REVISION: process.env.NSS_HEAD_REVISION,
PROJECT: process.env.TC_PROJECT,
CONTEXT_PATH: path,
HASH: hash
......@@ -52,7 +50,7 @@ export async function buildTask({name, path}) {
command: [
"/bin/bash",
"-c",
"/home/worker/bin/build_image.sh"
"bin/checkout.sh && nss/automation/taskcluster/scripts/build_image.sh"
],
platform: "nss-decision",
features: ["dind"],
......
......@@ -51,7 +51,7 @@ function parseOptions(opts) {
}
// Parse tools.
let allTools = ["clang-format", "scan-build"];
let allTools = ["clang-format", "scan-build", "hacl"];
let tools = intersect(opts.tools.split(/\s*,\s*/), allTools);
// If the given value is "all" run all tools.
......
FROM ubuntu:16.04
MAINTAINER Tim Taubert <ttaubert@mozilla.com>
WORKDIR /home/worker
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && apt-get install -y apt-transport-https apt-utils
RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 && \
sh -c "echo deb https://get.docker.io/ubuntu docker main \
> /etc/apt/sources.list.d/docker.list"
RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 41BD8711B1F0EC2B0D85B91CF59CE3A8323293EE && \
sh -c "echo deb http://ppa.launchpad.net/mercurial-ppa/releases/ubuntu xenial main \
> /etc/apt/sources.list.d/mercurial.list"
RUN apt-get update && apt-get install -y \
lxc-docker-1.6.1 \
mercurial
ADD bin /home/worker/bin
RUN chmod +x /home/worker/bin/*
# Set a default command useful for debugging
CMD ["/bin/bash", "--login"]
#!/usr/bin/env bash
set -v -e -x
# Default values for testing.
REVISION=${NSS_HEAD_REVISION:-default}
REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}
# Clone NSS.
for i in 0 2 5; do
sleep $i
hg clone -r $REVISION $REPOSITORY nss && exit 0
rm -rf nss
done
exit 1
#!/bin/bash -vex
set -x -e -v
# Prefix errors with taskcluster error prefix so that they are parsed by Treeherder
raise_error() {
echo
echo "[taskcluster-image-build:error] $1"
exit 1
}
# Ensure that the PROJECT is specified so the image can be indexed
test -n "$PROJECT" || raise_error "Project must be provided."
test -n "$HASH" || raise_error "Context Hash must be provided."
CONTEXT_PATH=/home/worker/nss/$CONTEXT_PATH
test -d $CONTEXT_PATH || raise_error "Context Path $CONTEXT_PATH does not exist."
test -f "$CONTEXT_PATH/Dockerfile" || raise_error "Dockerfile must be present in $CONTEXT_PATH."
docker build -t $PROJECT:$HASH $CONTEXT_PATH
mkdir /artifacts
docker save $PROJECT:$HASH > /artifacts/image.tar
#!/usr/bin/env bash
if [[ $(id -u) -eq 0 ]]; then
# Drop privileges by re-running this script.
# Note: this mangles arguments, better to avoid running scripts as root.
exec su worker -c "$0 $*"
fi
set -e -x -v
# The docker image this is running in has the HACL* and NSS sources.
# The extracted C code from HACL* is already generated and the HACL* tests were
# successfully executed.
# Format the extracted C code.
cd ~/hacl-star/snapshots/nss-production
cp ~/nss/.clang-format .
find . -type f -name '*.[ch]' -exec clang-format -i {} \+
# These diff commands will return 1 if there are differences and stop the script.
files=($(find ~/nss/lib/freebl/verified/ -type f -name '*.[ch]'))
for f in "${files[@]}"; do
diff $f $(basename "$f")
done
This diff is collapsed.
This diff is collapsed.
......@@ -380,7 +380,6 @@ CERTUTIL_FileForRNG(const char *noise)
return SECSuccess;
}
#ifndef NSS_DISABLE_ECC
typedef struct curveNameTagPairStr {
char *curveName;
SECOidTag curveOidTag;
......@@ -495,9 +494,9 @@ getECParams(const char *curve)
ecparams = SECITEM_AllocItem(NULL, NULL, (2 + oidData->oid.len));
/*
/*
* ecparams->data needs to contain the ASN encoding of an object ID (OID)
* representing the named curve. The actual OID is in
* representing the named curve. The actual OID is in
* oidData->oid.data so we simply prepend 0x06 and OID length
*/
ecparams->data[0] = SEC_ASN1_OBJECT_ID;
......@@ -506,7 +505,6 @@ getECParams(const char *curve)
return ecparams;
}
#endif /* NSS_DISABLE_ECC */
SECKEYPrivateKey *
CERTUTIL_GeneratePrivateKey(KeyType keytype, PK11SlotInfo *slot, int size,
......@@ -564,14 +562,12 @@ CERTUTIL_GeneratePrivateKey(KeyType keytype, PK11SlotInfo *slot, int size,
params = (void *)&default_pqg_params;
}
break;
#ifndef NSS_DISABLE_ECC
case ecKey:
mechanism = CKM_EC_KEY_PAIR_GEN;
/* For EC keys, PQGFile determines EC parameters */
if ((params = (void *)getECParams(pqgFile)) == NULL)
return NULL;
break;
#endif /* NSS_DISABLE_ECC */
default:
return NULL;
}
......@@ -588,11 +584,9 @@ CERTUTIL_GeneratePrivateKey(KeyType keytype, PK11SlotInfo *slot, int size,
if (dsaparams)
CERTUTIL_DestroyParamsPQG(dsaparams);
break;
#ifndef NSS_DISABLE_ECC
case ecKey:
SECITEM_FreeItem((SECItem *)params, PR_TRUE);
break;
#endif
default: /* nothing to free */
break;
}
......
......@@ -35,13 +35,11 @@
#include "../../lib/freebl/mpi/mpi.h"
#endif
#ifndef NSS_DISABLE_ECC
extern SECStatus
EC_DecodeParams(const SECItem *encodedParams, ECParams **ecparams);
extern SECStatus
EC_CopyParams(PLArenaPool *arena, ECParams *dstParams,
const ECParams *srcParams);
#endif
#define ENCRYPT 1
#define DECRYPT 0
......@@ -2094,7 +2092,6 @@ get_next_line(FILE *req, char *key, char *val, FILE *rsp)
return (c == EOF) ? -1 : ignore;
}
#ifndef NSS_DISABLE_ECC
typedef struct curveNameTagPairStr {
char *curveName;
SECOidTag curveOidTag;
......@@ -2958,7 +2955,6 @@ loser:
}
fclose(ecdsareq);
}
#endif /* NSS_DISABLE_ECC */
PRBool
isblankline(char *b)
......@@ -6093,7 +6089,6 @@ main(int argc, char **argv)
/* Signature Verification Test */
dsa_sigver_test(argv[3]);
}
#ifndef NSS_DISABLE_ECC
/*************/
/* ECDSA */
/*************/
......@@ -6112,7 +6107,6 @@ main(int argc, char **argv)
/* Signature Verification Test */
ecdsa_sigver_test(argv[3]);
}
#endif /* NSS_DISABLE_ECC */
/*************/
/* RNG */
/*************/
......
......@@ -54,6 +54,10 @@ static char consoleName[] = {
static PRBool utf8DisplayEnabled = PR_FALSE;
/* The minimum password/pin length (in Unicode characters) in FIPS mode,
* defined in lib/softoken/pkcs11i.h. */
#define FIPS_MIN_PIN 7
void
SECU_EnableUtf8Display(PRBool enable)
{
......@@ -276,10 +280,25 @@ secu_InitSlotPassword(PK11SlotInfo *slot, PRBool retry, void *arg)
}
/* we have no password, so initialize database with one */
PR_fprintf(PR_STDERR,
"Enter a password which will be used to encrypt your keys.\n"
"The password should be at least 8 characters long,\n"
"and should contain at least one non-alphabetic character.\n\n");
if (PK11_IsFIPS()) {
PR_fprintf(PR_STDERR,
"Enter a password which will be used to encrypt your keys.\n"
"The password should be at least %d characters long,\n"
"and should consist of at least three character classes.\n"
"The available character classes are: digits (0-9), ASCII\n"
"lowercase letters, ASCII uppercase letters, ASCII\n"
"non-alphanumeric characters, and non-ASCII characters.\n\n"
"If an ASCII uppercase letter appears at the beginning of\n"
"the password, it is not counted toward its character class.\n"
"Similarly, if a digit appears at the end of the password,\n"
"it is not counted toward its character class.\n\n",
FIPS_MIN_PIN);
} else {
PR_fprintf(PR_STDERR,
"Enter a password which will be used to encrypt your keys.\n"
"The password should be at least 8 characters long,\n"
"and should contain at least one non-alphabetic character.\n\n");
}
output = fopen(consoleName, "w");
if (output == NULL) {
......@@ -465,48 +484,6 @@ SECU_ConfigDirectory(const char *base)
return buf;
}
/*Turn off SSL for now */
/* This gets called by SSL when server wants our cert & key */
int
SECU_GetClientAuthData(void *arg, PRFileDesc *fd,
struct CERTDistNamesStr *caNames,
struct CERTCertificateStr **pRetCert,
struct SECKEYPrivateKeyStr **pRetKey)
{
SECKEYPrivateKey *key;
CERTCertificate *cert;
int errsave;
if (arg == NULL) {
fprintf(stderr, "no key/cert name specified for client auth\n");
return -1;
}
cert = PK11_FindCertFromNickname(arg, NULL);
errsave = PORT_GetError();
if (!cert) {
if (errsave == SEC_ERROR_BAD_PASSWORD)
fprintf(stderr, "Bad password\n");
else if (errsave > 0)
fprintf(stderr, "Unable to read cert (error %d)\n", errsave);
else if (errsave == SEC_ERROR_BAD_DATABASE)
fprintf(stderr, "Unable to get cert from database (%d)\n", errsave);
else
fprintf(stderr, "SECKEY_FindKeyByName: internal error %d\n", errsave);
return -1;
}
key = PK11_FindKeyByAnyCert(arg, NULL);
if (!key) {
fprintf(stderr, "Unable to get key (%d)\n", PORT_GetError());
return -1;
}
*pRetCert = cert;
*pRetKey = key;
return 0;
}
SECStatus
SECU_ReadDERFromFile(SECItem *der, PRFileDesc *inFile, PRBool ascii,
PRBool warnOnPrivateKeyInAsciiFile)
......@@ -1390,7 +1367,6 @@ secu_PrintAttribute(FILE *out, SEC_PKCS7Attribute *attr, char *m, int level)