Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge NSS trunk to NSS_TLS13_DRAFT19_BRANCH
--HG--
branch : NSS_TLS13_DRAFT19_BRANCH
rename : automation/taskcluster/docker/bin/checkout.sh => automation/taskcluster/docker-hacl/bin/checkout.sh
extra : rebase_source : 539f7521c7deb8f828cc1fbef51051568deedccb
  • Loading branch information
martinthomson committed Oct 9, 2017
2 parents 8c9ccbf + 5e8ecb1 commit 966fea2
Show file tree
Hide file tree
Showing 143 changed files with 9,853 additions and 2,194 deletions.
2 changes: 1 addition & 1 deletion .taskcluster.yml
Expand Up @@ -57,7 +57,7 @@ tasks:
- "tc-treeherder.v2.{{project}}.{{revision}}.{{pushlog_id}}"

payload:
image: ttaubert/nss-decision:0.0.2
image: nssdev/nss-decision:0.0.2

env:
TC_OWNER: {{owner}}
Expand Down
2 changes: 1 addition & 1 deletion automation/abi-check/previous-nss-release
@@ -1 +1 @@
NSS_3_32_BRANCH
NSS_3_33_BRANCH
9 changes: 6 additions & 3 deletions automation/buildbot-slave/build.sh
Expand Up @@ -236,11 +236,14 @@ check_abi()
BASE_NSPR=NSPR_$(head -1 ${HGDIR}/baseline/nss/automation/release/nspr-version.txt | cut -d . -f 1-2 | tr . _)_BRANCH
hg clone -u "${BASE_NSPR}" "${HGDIR}/nspr" "${HGDIR}/baseline/nspr"
if [ $? -ne 0 ]; then
echo "invalid tag ${BASE_NSPR} derived from ${BASE_NSS} automation/release/nspr-version.txt"
return 1
echo "nonexisting tag ${BASE_NSPR} derived from ${BASE_NSS} automation/release/nspr-version.txt"
# Assume that version hasn't been released yet, fall back to trunk
pushd "${HGDIR}/baseline/nspr"
hg update default
popd
fi

print_log "######## building older NSPR/NSS ########"
print_log "######## building baseline NSPR/NSS ########"
pushd ${HGDIR}/baseline/nss

print_log "$ ${MAKE} ${NSS_BUILD_TARGET}"
Expand Down
2 changes: 1 addition & 1 deletion automation/release/nspr-version.txt
@@ -1,4 +1,4 @@
4.15
4.17

# The first line of this file must contain the human readable NSPR
# version number, which is the minimum required version of NSPR
Expand Down
3 changes: 3 additions & 0 deletions automation/taskcluster/docker-decision/Dockerfile
Expand Up @@ -12,6 +12,9 @@ RUN chmod +x /home/worker/bin/*
ADD setup.sh /tmp/setup.sh
RUN bash /tmp/setup.sh

# Change user.
USER worker

# Env variables.
ENV HOME /home/worker
ENV SHELL /bin/bash
Expand Down
5 changes: 0 additions & 5 deletions automation/taskcluster/docker-decision/bin/checkout.sh
Expand Up @@ -2,11 +2,6 @@

set -v -e -x

if [ $(id -u) = 0 ]; then
# Drop privileges by re-running this script.
exec su worker $0
fi

# Default values for testing.
REVISION=${NSS_HEAD_REVISION:-default}
REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}
Expand Down
30 changes: 30 additions & 0 deletions automation/taskcluster/docker-gcc-4.4/Dockerfile
@@ -0,0 +1,30 @@
FROM ubuntu:14.04
MAINTAINER Tim Taubert <ttaubert@mozilla.com>

RUN useradd -d /home/worker -s /bin/bash -m worker
WORKDIR /home/worker

# Add build and test scripts.
ADD bin /home/worker/bin
RUN chmod +x /home/worker/bin/*

# Install dependencies.
ADD setup.sh /tmp/setup.sh
RUN bash /tmp/setup.sh

# Change user.
USER worker

# Env variables.
ENV HOME /home/worker
ENV SHELL /bin/bash
ENV USER worker
ENV LOGNAME worker
ENV HOSTNAME taskcluster-worker
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
ENV HOST localhost
ENV DOMSUF localdomain

# Set a default command for debugging.
CMD ["/bin/bash", "--login"]
20 changes: 20 additions & 0 deletions automation/taskcluster/docker-gcc-4.4/bin/checkout.sh
@@ -0,0 +1,20 @@
#!/usr/bin/env bash

set -v -e -x

if [ $(id -u) = 0 ]; then
# Drop privileges by re-running this script.
exec su worker $0
fi

# Default values for testing.
REVISION=${NSS_HEAD_REVISION:-default}
REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}

# Clone NSS.
for i in 0 2 5; do
sleep $i
hg clone -r $REVISION $REPOSITORY nss && exit 0
rm -rf nss
done
exit 1
30 changes: 30 additions & 0 deletions automation/taskcluster/docker-gcc-4.4/setup.sh
@@ -0,0 +1,30 @@
#!/usr/bin/env bash

set -v -e -x

# Update packages.
export DEBIAN_FRONTEND=noninteractive
apt-get -y update && apt-get -y upgrade

apt_packages=()
apt_packages+=('ca-certificates')
apt_packages+=('g++-4.4')
apt_packages+=('gcc-4.4')
apt_packages+=('locales')
apt_packages+=('make')
apt_packages+=('mercurial')
apt_packages+=('zlib1g-dev')

# Install packages.
apt-get -y update
apt-get install -y --no-install-recommends ${apt_packages[@]}

locale-gen en_US.UTF-8
dpkg-reconfigure locales

# Cleanup.
rm -rf ~/.ccache ~/.cache
apt-get autoremove -y
apt-get clean
apt-get autoclean
rm $0
70 changes: 70 additions & 0 deletions automation/taskcluster/docker-hacl/Dockerfile
@@ -0,0 +1,70 @@
FROM ubuntu:xenial

MAINTAINER Franziskus Kiefer <franziskuskiefer@gmail.com>
# Based on the HACL* image from Benjamin Beurdouche and
# the original F* formula with Daniel Fabian

# Pinned versions of HaCl* (F* and KreMLin are pinned as submodules)
ENV haclrepo https://github.com/mitls/hacl-star.git

# Define versions of dependencies
ENV opamv 4.04.2
ENV z3v 4.5.1.1f29cebd4df6-x64-ubuntu-14.04
ENV haclversion 0030539598cde15d1a0e5f93b32e121f7b7b5a1c
ENV haclbranch production-nss

# Install required packages and set versions
RUN apt-get -qq update
RUN apt-get install --yes sudo libssl-dev libsqlite3-dev g++-5 gcc-5 m4 make opam pkg-config python libgmp3-dev cmake curl libtool-bin autoconf
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 200
RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 200

# Create user
RUN useradd -ms /bin/bash worker
RUN echo "worker ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers
WORKDIR /home/worker

# Add build and test scripts.
ADD bin /home/worker/bin
RUN chmod +x /home/worker/bin/*
USER worker

# Add "known-good" version of Z3
RUN curl -LO https://github.com/FStarLang/binaries/raw/master/z3-tested/z3-${z3v}.zip
RUN unzip z3-${z3v}.zip
RUN rm z3-${z3v}.zip
RUN mv z3-${z3v} z3
ENV PATH "/home/worker/z3/bin:$PATH"

# Prepare build (OCaml packages)
ENV OPAMYES true
RUN opam init
RUN echo ". /home/worker/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true" >> .bashrc
RUN opam switch -v ${opamv}
RUN opam install ocamlfind batteries sqlite3 fileutils yojson ppx_deriving_yojson zarith pprint menhir ulex process fix wasm stdint

# Get the HaCl* code
RUN git clone ${haclrepo} hacl-star
RUN git -C hacl-star checkout ${haclversion}

# Prepare submodules, and build, verify, test, and extract c code
# This caches the extracted c code (pins the HaCl* version). All we need to do
# on CI now is comparing the code in this docker image with the one in NSS.
RUN opam config exec -- make -C hacl-star nss -j$(nproc)

# Get clang-format-3.9
RUN curl -LO http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz
RUN curl -LO http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig
# Verify the signature.
RUN gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D
RUN gpg --verify *.tar.xz.sig
# Install into /usr/local/.
RUN sudo tar xJvf *.tar.xz -C /usr/local --strip-components=1
# Cleanup.
RUN rm *.tar.xz*

# Cleanup
RUN rm -rf ~/.ccache ~/.cache
RUN sudo apt-get autoremove -y
RUN sudo apt-get clean
RUN sudo apt-get autoclean
20 changes: 20 additions & 0 deletions automation/taskcluster/docker-hacl/bin/checkout.sh
@@ -0,0 +1,20 @@
#!/usr/bin/env bash

set -v -e -x

if [ $(id -u) = 0 ]; then
# Drop privileges by re-running this script.
exec su worker $0
fi

# Default values for testing.
REVISION=${NSS_HEAD_REVISION:-default}
REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}

# Clone NSS.
for i in 0 2 5; do
sleep $i
hg clone -r $REVISION $REPOSITORY nss && exit 0
rm -rf nss
done
exit 1
16 changes: 13 additions & 3 deletions automation/taskcluster/graph/src/context_hash.js
Expand Up @@ -27,14 +27,24 @@ function collectFilesInDirectory(dir) {
});
}

// Compute a context hash for the given context path.
export default function (context_path) {
// A list of hashes for each file in the given path.
function collectFileHashes(context_path) {
let root = path.join(__dirname, "../../../..");
let dir = path.join(root, context_path);
let files = collectFilesInDirectory(dir).sort();
let hashes = files.map(file => {

return files.map(file => {
return sha256(file + "|" + fs.readFileSync(file, "utf-8"));
});
}

// Compute a context hash for the given context path.
export default function (context_path) {
// Regenerate all images when the image_builder changes.
let hashes = collectFileHashes("automation/taskcluster/image_builder");

// Regenerate images when the image itself changes.
hashes = hashes.concat(collectFileHashes(context_path));

// Generate a new prefix every month to ensure the image stays buildable.
let now = new Date();
Expand Down
47 changes: 44 additions & 3 deletions automation/taskcluster/graph/src/extend.js
Expand Up @@ -15,11 +15,21 @@ const LINUX_CLANG39_IMAGE = {
path: "automation/taskcluster/docker-clang-3.9"
};

const LINUX_GCC44_IMAGE = {
name: "linux-gcc-4.4",
path: "automation/taskcluster/docker-gcc-4.4"
};

const FUZZ_IMAGE = {
name: "fuzz",
path: "automation/taskcluster/docker-fuzz"
};

const HACL_GEN_IMAGE = {
name: "hacl",
path: "automation/taskcluster/docker-hacl"
};

const WINDOWS_CHECKOUT_CMD =
"bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " +
"(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " +
Expand Down Expand Up @@ -253,8 +263,7 @@ async function scheduleMac(name, base, args = "") {
},
provisioner: "localprovisioner",
workerType: "nss-macos-10-12",
platform: "mac",
tier: 3
platform: "mac"
});

// Build base definition.
Expand Down Expand Up @@ -400,6 +409,26 @@ async function scheduleLinux(name, base, args = "") {
symbol: "clang-4.0"
}));

queue.scheduleTask(merge(extra_base, {
name: `${name} w/ gcc-4.4`,
image: LINUX_GCC44_IMAGE,
env: {
USE_64: "1",
CC: "gcc-4.4",
CCC: "g++-4.4",
// gcc-4.6 introduced nullptr.
NSS_DISABLE_GTESTS: "1",
},
// Use the old Makefile-based build system, GYP doesn't have a proper GCC
// version check for __int128 support. It's mainly meant to cover RHEL6.
command: [
"/bin/bash",
"-c",
"bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh",
],
symbol: "gcc-4.4"
}));

queue.scheduleTask(merge(extra_base, {
name: `${name} w/ gcc-4.8`,
env: {
Expand Down Expand Up @@ -529,12 +558,13 @@ async function scheduleFuzzing() {

// Schedule MPI fuzzing runs.
let mpi_base = merge(run_base, {group: "MPI"});
let mpi_names = ["add", "addmod", "div", "expmod", "mod", "mulmod", "sqr",
let mpi_names = ["add", "addmod", "div", "mod", "mulmod", "sqr",
"sqrmod", "sub", "submod"];
for (let name of mpi_names) {
scheduleFuzzingRun(mpi_base, `MPI (${name})`, `mpi-${name}`, 4096, name);
}
scheduleFuzzingRun(mpi_base, `MPI (invmod)`, `mpi-invmod`, 256, "invmod");
scheduleFuzzingRun(mpi_base, `MPI (expmod)`, `mpi-expmod`, 2048, "expmod");

// Schedule TLS fuzzing runs (non-fuzzing mode).
let tls_base = merge(run_base, {group: "TLS"});
Expand Down Expand Up @@ -934,5 +964,16 @@ async function scheduleTools() {
]
}));

queue.scheduleTask(merge(base, {
symbol: "hacl",
name: "hacl",
image: HACL_GEN_IMAGE,
command: [
"/bin/bash",
"-c",
"bin/checkout.sh && nss/automation/taskcluster/scripts/run_hacl.sh"
]
}));

return queue.submit();
}
10 changes: 4 additions & 6 deletions automation/taskcluster/graph/src/image_builder.js
Expand Up @@ -31,13 +31,11 @@ export async function buildTask({name, path}) {

return {
name: "Image Builder",
image: "taskcluster/image_builder:0.1.5",
image: "nssdev/image_builder:0.1.5",
routes: ["index." + ns],
env: {
HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
BASE_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
HEAD_REV: process.env.NSS_HEAD_REVISION,
HEAD_REF: process.env.NSS_HEAD_REVISION,
NSS_HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
NSS_HEAD_REVISION: process.env.NSS_HEAD_REVISION,
PROJECT: process.env.TC_PROJECT,
CONTEXT_PATH: path,
HASH: hash
Expand All @@ -52,7 +50,7 @@ export async function buildTask({name, path}) {
command: [
"/bin/bash",
"-c",
"/home/worker/bin/build_image.sh"
"bin/checkout.sh && nss/automation/taskcluster/scripts/build_image.sh"
],
platform: "nss-decision",
features: ["dind"],
Expand Down
2 changes: 1 addition & 1 deletion automation/taskcluster/graph/src/try_syntax.js
Expand Up @@ -51,7 +51,7 @@ function parseOptions(opts) {
}

// Parse tools.
let allTools = ["clang-format", "scan-build"];
let allTools = ["clang-format", "scan-build", "hacl"];
let tools = intersect(opts.tools.split(/\s*,\s*/), allTools);

// If the given value is "all" run all tools.
Expand Down

0 comments on commit 966fea2

Please sign in to comment.