Bug 1434943 - Support for MSVC in build.sh, r=jcj

Summary:
This adds basic support for MSVC to build.sh.  It uses the registry and vswhere
(which is part of the standard mozilla-build setup now) to work out paths and
set them properly.  It's probably a little fragile, but it's better than the
shoestring and tape we have in builds right now.

I took the liberty of sanitizing the command-line options a little here.  Mostly
that is sorting them, but I also deprecated the -m32 option in favour of
specifying target architecture with -t.  That turned out to be a lot cleaner.

Reviewers: jcj

Reviewed By: jcj

Bug #: 1434943

Differential Revision: https://phabricator.services.mozilla.com/D5125

--HG--
extra : rebase_source : 54465a06808f1164e31094773930831b8bb7e20b
extra : amend_source : f89a25ab6ab7b95fa6d54b8d55ebee88ec1dcc65

automation/taskcluster/graph/src/extend.js

@@ -152,13 +152,13 @@ export default async function main() {
   await scheduleLinux("Linux 32 (opt)", {
     platform: "linux32",
     image: LINUX_IMAGE
-  }, "-m32 --opt");
+  }, "-t ia32 --opt");
 
   await scheduleLinux("Linux 32 (debug)", {
     platform: "linux32",
     collection: "debug",
     image: LINUX_IMAGE
-  }, "-m32");
+  }, "-t ia32");
 
   await scheduleLinux("Linux 64 (opt)", {
     platform: "linux64",
@@ -248,12 +248,12 @@ export default async function main() {
 
   await scheduleWindows("Windows 2012 32 (opt)", {
     platform: "windows2012-32",
-  }, "build_gyp.sh --opt -m32");
+  }, "build_gyp.sh --opt -t ia32");
 
   await scheduleWindows("Windows 2012 32 (debug)", {
     platform: "windows2012-32",
     collection: "debug"
-  }, "build_gyp.sh -m32");
+  }, "build_gyp.sh -t ia32");
 
   await scheduleFuzzing();
   await scheduleFuzzing32();
@@ -679,7 +679,7 @@ async function scheduleFuzzing32() {
       "/bin/bash",
       "-c",
       "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz -m32"
+      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz -t ia32"
     ],
     artifacts: {
       public: {
@@ -706,7 +706,7 @@ async function scheduleFuzzing32() {
       "/bin/bash",
       "-c",
       "bin/checkout.sh && " +
-      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls -m32"
+      "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls -t ia32"
     ],
   }));
 
@@ -1105,7 +1105,7 @@ async function scheduleTools() {
     command: [
       "/bin/bash",
       "-c",
-      "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh --disable-tests --emit-llvm -m32"
+      "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh --disable-tests --emit-llvm -t ia32"
     ]
   }));
 

automation/taskcluster/windows/build.sh

@@ -2,12 +2,12 @@
 
 set -v -e -x
 
-# Set up the toolchain.
-if [ "$USE_64" = 1 ]; then
-  source $(dirname $0)/setup64.sh
+if [[ "$USE_64" == 1 ]]; then
+    m=x64
 else
-  source $(dirname $0)/setup32.sh
+    m=x86
 fi
+source "$(dirname "$0")/setup.sh"
 
 # Clone NSPR.
 hg_clone https://hg.mozilla.org/projects/nspr nspr default

automation/taskcluster/windows/build_gyp.sh

@@ -2,33 +2,37 @@
 
 set -v -e -x
 
-# Set up the toolchain.
-if [[ "$@" == *"-m32"* ]]; then
-  source $(dirname $0)/setup32.sh
-else
-  source $(dirname $0)/setup64.sh
-fi
+# Parse for the -t option.
+m=x64
+for i in "$@"; do
+    case "$i" in
+        -t|--target) m= ;;
+        --target=*) m="${i#*=}" ;;
+        *) [[ -z "$m" ]] && m="$i" ;;
+    esac
+done
+[[ "$m" == "ia32" ]] && m=x86
+source "$(dirname "$0")/setup.sh"
 
 # Install GYP.
-cd gyp
+pushd gyp
 python -m virtualenv test-env
 test-env/Scripts/python setup.py install
 test-env/Scripts/python -m pip install --upgrade pip
 test-env/Scripts/pip install --upgrade setuptools
-cd ..
-
-export GYP_MSVS_OVERRIDE_PATH="${VSPATH}"
-export GYP_MSVS_VERSION="2015"
-export GYP="${PWD}/gyp/test-env/Scripts/gyp"
-
 # Fool GYP.
 touch "${VSPATH}/VC/vcvarsall.bat"
+export GYP_MSVS_OVERRIDE_PATH="${VSPATH}"
+export GYP_MSVS_VERSION=2015
+popd
+
+export PATH="${PATH}:${PWD}/ninja/bin:${PWD}/gyp/test-env/Scripts"
 
 # Clone NSPR.
 hg_clone https://hg.mozilla.org/projects/nspr nspr default
 
 # Build with gyp.
-GYP=${GYP} ./nss/build.sh -g -v "$@"
+./nss/build.sh -g -v "$@"
 
 # Package.
 7z a public/build/dist.7z dist

automation/taskcluster/windows/setup.sh

@@ -2,13 +2,6 @@
 
 set -v -e -x
 
-export VSPATH="$(pwd)/vs2017_15.4.2"
-export NINJA_PATH="$(pwd)/ninja/bin"
-
-export WINDOWSSDKDIR="${VSPATH}/SDK"
-export VS90COMNTOOLS="${VSPATH}/VC"
-export INCLUDE="${VSPATH}/VC/include:${VSPATH}/SDK/Include/10.0.15063.0/ucrt:${VSPATH}/SDK/Include/10.0.15063.0/shared:${VSPATH}/SDK/Include/10.0.15063.0/um"
-
 # Usage: hg_clone repo dir [revision=@]
 hg_clone() {
     repo=$1
@@ -22,5 +15,42 @@ hg_clone() {
     exit 1
 }
 
-hg_clone https://hg.mozilla.org/build/tools tools default
-tools/scripts/tooltool/tooltool_wrapper.sh $(dirname $0)/releng.manifest https://tooltool.mozilla-releng.net/ non-existant-file.sh /c/mozilla-build/python/python.exe /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok -c /c/builds/tooltool_cache
+hg_clone https://hg.mozilla.org/build/tools tools b8d7c263dfc3
+tools/scripts/tooltool/tooltool_wrapper.sh \
+    $(dirname $0)/releng.manifest https://tooltool.mozilla-releng.net/ \
+    non-existant-file.sh /c/mozilla-build/python/python.exe \
+    /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok \
+    -c /c/builds/tooltool_cache
+
+# This needs $m to be set.
+[[ -n "$m" ]]
+
+# Setup MSVC paths.
+export VSPATH="${PWD}/vs2017_15.4.2"
+UCRTVersion="10.0.15063.0"
+
+export WINDOWSSDKDIR="${VSPATH}/SDK"
+export VS90COMNTOOLS="${VSPATH}/VC"
+export WIN32_REDIST_DIR="${VSPATH}/VC/redist/${m}/Microsoft.VC141.CRT"
+export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/${m}"
+
+if [ "$m" == "x86" ]; then
+    PATH="${PATH}:${VSPATH}/VC/bin/Hostx64/x86"
+    PATH="${PATH}:${VSPATH}/VC/bin/Hostx64/x64"
+fi
+PATH="${PATH}:${VSPATH}/VC/bin/Host${m}/${m}"
+PATH="${PATH}:${WIN32_REDIST_DIR}"
+PATH="${PATH}:${WIN_UCRT_REDIST_DIR}"
+PATH="${PATH}:${VSPATH}/SDK/bin/${UCRTVersion}/x64"
+export PATH
+
+LIB="${LIB}:${VSPATH}/VC/lib/${m}"
+LIB="${LIB}:${VSPATH}/SDK/lib/${UCRTVersion}/ucrt/${m}"
+LIB="${LIB}:${VSPATH}/SDK/lib/${UCRTVersion}/um/${m}"
+export LIB
+
+INCLUDE="${INCLUDE}:${VSPATH}/VC/include"
+INCLUDE="${INCLUDE}:${VSPATH}/SDK/Include/${UCRTVersion}/ucrt"
+INCLUDE="${INCLUDE}:${VSPATH}/SDK/Include/${UCRTVersion}/shared"
+INCLUDE="${INCLUDE}:${VSPATH}/SDK/Include/${UCRTVersion}/um"
+export INCLUDE

build.sh

@@ -50,76 +50,86 @@ fuzz=0
 fuzz_tls=0
 fuzz_oss=0
 no_local_nspr=0
-armhf=0
 
 gyp_params=(--depth="$cwd" --generator-output=".")
-nspr_params=()
 ninja_params=()
 
-# try to guess sensible defaults
-arch=$(python "$cwd"/coreconf/detect_host_arch.py)
-if [ "$arch" = "x64" -o "$arch" = "aarch64" ]; then
-    build_64=1
-elif [ "$arch" = "arm" ]; then
-    armhf=1
+# Assume that the target architecture is the same as the host by default.
+host_arch=$(python "$cwd"/coreconf/detect_host_arch.py)
+target_arch=$host_arch
+
+# Assume that MSVC is wanted if this is running on windows.
+platform=$(uname -s)
+if [ "${platform%-*}" = "MINGW32_NT" -o "${platform%-*}" = "MINGW64_NT" ]; then
+    msvc=1
 fi
 
-# parse command line arguments
+# Parse command line arguments.
 while [ $# -gt 0 ]; do
-    case $1 in
+    case "$1" in
         -c) clean=1 ;;
         -cc) clean_only=1 ;;
-        --gyp|-g) rebuild_gyp=1 ;;
-        --nspr) nspr_clean; rebuild_nspr=1 ;;
-        -j) ninja_params+=(-j "$2"); shift ;;
         -v) ninja_params+=(-v); verbose=1 ;;
-        --test) gyp_params+=(-Dtest_build=1) ;;
-        --clang) export CC=clang; export CCC=clang++; export CXX=clang++ ;;
-        --gcc) export CC=gcc; export CCC=g++; export CXX=g++ ;;
-        --fuzz) fuzz=1 ;;
-        --fuzz=oss) fuzz=1; fuzz_oss=1 ;;
-        --fuzz=tls) fuzz=1; fuzz_tls=1 ;;
+        -j) ninja_params+=(-j "$2"); shift ;;
+        --gyp|-g) rebuild_gyp=1 ;;
+        --opt|-o) opt_build=1 ;;
+        -m32|--m32) target_arch=ia32; echo 'Warning: use -t instead of -m32' 1>&2 ;;
+        -t|--target) target_arch="$2"; shift ;;
+        --target=*) target_arch="${1#*=}" ;;
+        --clang) export CC=clang; export CCC=clang++; export CXX=clang++; msvc=0 ;;
+        --gcc) export CC=gcc; export CCC=g++; export CXX=g++; msvc=0 ;;
+        --msvc) msvc=1 ;;
         --scan-build) enable_scanbuild  ;;
         --scan-build=?*) enable_scanbuild "${1#*=}" ;;
-        --opt|-o) opt_build=1 ;;
-        -m32|--m32) build_64=0 ;;
+        --disable-tests) gyp_params+=(-Ddisable_tests=1) ;;
+        --pprof) gyp_params+=(-Duse_pprof=1) ;;
         --asan) enable_sanitizer asan ;;
         --msan) enable_sanitizer msan ;;
         --ubsan) enable_ubsan ;;
         --ubsan=?*) enable_ubsan "${1#*=}" ;;
+        --fuzz) fuzz=1 ;;
+        --fuzz=oss) fuzz=1; fuzz_oss=1 ;;
+        --fuzz=tls) fuzz=1; fuzz_tls=1 ;;
         --sancov) enable_sancov ;;
         --sancov=?*) enable_sancov "${1#*=}" ;;
-        --pprof) gyp_params+=(-Duse_pprof=1) ;;
-        --ct-verif) gyp_params+=(-Dct_verif=1) ;;
         --emit-llvm) gyp_params+=(-Demit_llvm=1 -Dsign_libs=0) ;;
-        --disable-tests) gyp_params+=(-Ddisable_tests=1) ;;
         --no-zdefs) gyp_params+=(-Dno_zdefs=1) ;;
-        --system-sqlite) gyp_params+=(-Duse_system_sqlite=1) ;;
+        --test) gyp_params+=(-Dtest_build=1) ;;
+        --ct-verif) gyp_params+=(-Dct_verif=1) ;;
+        --nspr) nspr_clean; rebuild_nspr=1 ;;
         --with-nspr=?*) set_nspr_path "${1#*=}"; no_local_nspr=1 ;;
         --system-nspr) set_nspr_path "/usr/include/nspr/:"; no_local_nspr=1 ;;
-        --enable-libpkix) gyp_params+=(-Ddisable_libpkix=0) ;;
+        --system-sqlite) gyp_params+=(-Duse_system_sqlite=1) ;;
         --enable-fips) gyp_params+=(-Ddisable_fips=0) ;;
+        --enable-libpkix) gyp_params+=(-Ddisable_libpkix=0) ;;
         --mozpkix-only) gyp_params+=(-Dmozpkix_only=1 -Ddisable_tests=1 -Dsign_libs=0) ;;
         *) show_help; exit 2 ;;
     esac
     shift
 done
 
+# Set the target architecture and build type.
+gyp_params+=(-Dtarget_arch="$target_arch")
 if [ "$opt_build" = 1 ]; then
     target=Release
 else
     target=Debug
 fi
-if [ "$build_64" = 1 ]; then
-    nspr_params+=(--enable-64bit)
-elif [ ! "$armhf" = 1 ]; then
-    gyp_params+=(-Dtarget_arch=ia32)
-fi
+
+# Do special setup.
 if [ "$fuzz" = 1 ]; then
     source "$cwd"/coreconf/fuzz.sh
 fi
+nspr_set_flags $sanitizer_flags
+if [ ! -z "$sanitizer_flags" ]; then
+    gyp_params+=(-Dsanitizer_flags="$sanitizer_flags")
+fi
 
-# set paths
+if [ "$msvc" = 1 ]; then
+    source "$cwd"/coreconf/msvc.sh
+fi
+
+# Setup build paths.
 target_dir="$cwd"/out/$target
 mkdir -p "$target_dir"
 dist_dir="$cwd"/../dist
@@ -150,6 +160,7 @@ check_config()
     echo CC="$CC" >"$newconf"
     echo CCC="$CCC" >>"$newconf"
     echo CXX="$CXX" >>"$newconf"
+    echo target_arch="$target_arch" >>"$newconf"
     for i in "$@"; do echo $i; done | sort >>"$newconf"
 
     # Note: The following diff fails if $oldconf isn't there as well, which
@@ -160,6 +171,7 @@ check_config()
 gyp_config="$cwd"/out/gyp_config
 nspr_config="$cwd"/out/$target/nspr_config
 
+# Now check what needs to be rebuilt.
 # If we don't have a build directory make sure that we rebuild.
 if [ ! -d "$target_dir" ]; then
     rebuild_nspr=1
@@ -168,33 +180,28 @@ elif [ ! -d "$dist_dir"/$target ]; then
     rebuild_nspr=1
 fi
 
-# Update NSPR ${C,CXX,LD}FLAGS.
-nspr_set_flags $sanitizer_flags
-
-if check_config "$nspr_config" "${nspr_params[@]}" \
+if check_config "$nspr_config" \
                  nspr_cflags="$nspr_cflags" \
                  nspr_cxxflags="$nspr_cxxflags" \
                  nspr_ldflags="$nspr_ldflags"; then
     rebuild_nspr=1
 fi
 
-# Forward sanitizer flags.
-if [ ! -z "$sanitizer_flags" ]; then
-    gyp_params+=(-Dsanitizer_flags="$sanitizer_flags")
-fi
-
 if check_config "$gyp_config" "${gyp_params[@]}"; then
     rebuild_gyp=1
 fi
 
-# save the chosen target
+# Save the chosen target.
 mkdir -p "$dist_dir"
 echo $target > "$dist_dir"/latest
 
+# Build.
+# NSPR.
 if [[ "$rebuild_nspr" = 1 && "$no_local_nspr" = 0 ]]; then
-    nspr_build "${nspr_params[@]}"
+    nspr_build
     mv -f "$nspr_config".new "$nspr_config"
 fi
+# gyp.
 if [ "$rebuild_gyp" = 1 ]; then
     if ! hash ${GYP} 2> /dev/null; then
         echo "Please install gyp" 1>&2
@@ -212,11 +219,11 @@ if [ "$rebuild_gyp" = 1 ]; then
     mv -f "$gyp_config".new "$gyp_config"
 fi
 
-# Run ninja.
-if hash ninja 2>/dev/null; then
-    ninja=ninja
-elif hash ninja-build 2>/dev/null; then
+# ninja.
+if hash ninja-build 2>/dev/null; then
     ninja=ninja-build
+elif hash ninja 2>/dev/null; then
+    ninja=ninja
 else
     echo "Please install ninja" 1>&2
     exit 1